Skip to main content

bamboo_engine/session_app/
child_completion_coordinator.rs

1//! Child-session completion coordinator.
2//!
3//! Receives terminal child runner notifications from `bamboo-engine`, updates
4//! durable parent wait state, and resumes the parent when the configured wait
5//! policy is satisfied.
6
7use std::collections::HashMap;
8use std::sync::{Arc, Mutex as StdMutex, OnceLock, RwLock as StdRwLock, Weak};
9use std::time::Duration;
10
11use bamboo_domain::poison::PoisonRecover;
12use bamboo_domain::{
13    AgentHookPoint, HookPayload, HookToolOutcome, SessionChildOutcome, SessionMessageBody,
14    SessionMessageContent, SessionMessageEnvelope, SessionMessageId, SessionMessageKind,
15    SessionMessageSource, SessionProviderMessage,
16};
17
18use crate::execution::{
19    create_event_forwarder, finalize_runner, reserve_runner_core, reserve_session_execution,
20    spawn_session_execution, AgentRunner, AgentStatus, ChildCompletion, ChildCompletionHandler,
21    ReserveOutcome, SessionExecutionArgs, SessionExecutionReservation,
22    SessionExecutionReserveOutcome, SpawnJob, SpawnScheduler,
23};
24use crate::runtime::config::{BashResumeHook, GuardianSpawner, BASH_COMPLETION_RESUME_KIND};
25use crate::runtime::guardian_state::{
26    parse_guardian_verdict, read_guardian_config, read_guardian_state, write_guardian_state,
27    GuardianVerdict,
28};
29use crate::Agent;
30use async_trait::async_trait;
31use bamboo_agent_core::storage::Storage;
32use bamboo_agent_core::tools::ToolExecutor;
33use bamboo_agent_core::{
34    AgentEvent, BashCompletionInfo, BashCompletionSink, Message, Role, Session, SessionKind,
35};
36use bamboo_domain::session::runtime_state::{
37    AgentRuntimeState, AgentStatusState, ChildWaitPolicy, SuspensionState, WaitingForChildrenState,
38};
39use bamboo_llm::{Config, ProviderModelRouter, ProviderRegistry};
40use bamboo_storage::LockedSessionStore;
41use chrono::Utc;
42use sha2::{Digest, Sha256};
43use tokio::sync::{broadcast, RwLock};
44
45use crate::model_areas::resolve_global_area_models;
46use crate::model_config_helper::{
47    resolve_fast_model, resolve_gold_config, resolve_provider_routing_key, GOLD_CONFIG_METADATA_KEY,
48};
49use crate::session_activation::{
50    SessionActivationLaunch, SessionActivationReserveOutcome, SessionActivationSpawner,
51};
52use crate::session_app::execute::consume_pending_clarification_resume;
53use crate::session_app::provider_model::{persist_model_ref, session_effective_model_ref};
54use crate::session_app::resume::{
55    resume_session_execution, ResumeExecutionPort, ResumeSpawnRequest,
56};
57use crate::session_app::types::{ResumeConfigSnapshot, ResumeOutcome};
58
59const AGENT_RUNTIME_STATE_METADATA_KEY: &str = "agent.runtime.state";
60const RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: &str = "hidden_from_ui";
61const RUNTIME_RESUME_MESSAGE_KIND_KEY: &str = "runtime_kind";
62const CHILD_COMPLETION_INLINE_FIELD_BYTES: usize = 48 * 1024;
63const CHILD_COMPLETION_OVERSIZE_TAIL_BYTES: usize = 8 * 1024;
64
65fn read_runtime_state(session: &Session) -> AgentRuntimeState {
66    session
67        .agent_runtime_state
68        .clone()
69        .or_else(|| {
70            session
71                .metadata
72                .get(AGENT_RUNTIME_STATE_METADATA_KEY)
73                .and_then(|raw| serde_json::from_str::<AgentRuntimeState>(raw).ok())
74        })
75        .unwrap_or_else(|| AgentRuntimeState::new(format!("{}-child-wait", session.id)))
76}
77
78fn write_runtime_state(session: &mut Session, runtime_state: &AgentRuntimeState) {
79    session.agent_runtime_state = Some(runtime_state.clone());
80    if let Ok(serialized) = serde_json::to_string(runtime_state) {
81        session
82            .metadata
83            .insert(AGENT_RUNTIME_STATE_METADATA_KEY.to_string(), serialized);
84    }
85}
86
87/// Re-read and prepare an activation target under the same per-session
88/// persistence lock that commits the generic suspension clear.
89///
90/// The boolean is false for an unanswered human question or a respected
91/// child/Bash wait in the latest durable snapshot; leave activation unreserved.
92async fn prepare_session_inbox_activation(
93    persistence: &LockedSessionStore,
94    session_id: &str,
95    interrupt_specific_wait: bool,
96) -> std::io::Result<Option<(Session, bool)>> {
97    let ready = Arc::new(std::sync::atomic::AtomicBool::new(false));
98    let ready_for_mutation = ready.clone();
99    let saved = persistence
100        .update_runtime_config(session_id, move |latest| {
101            // Inbox steering never answers a human question. Check under the
102            // same final writer lock, including when an earlier user envelope
103            // has already advanced the prefix's interrupt watermark.
104            if latest.has_pending_question() {
105                return;
106            }
107            let mut runtime_state = read_runtime_state(latest);
108            let specifically_waiting = runtime_state.waiting_for_children.is_some()
109                || runtime_state.waiting_for_bash.is_some();
110            if specifically_waiting && !interrupt_specific_wait {
111                return;
112            }
113            // Explicit steering interrupts only this reasoning gate. The
114            // durable child/Bash wait remains owned so later terminal events
115            // still have exactly one coordinator and can authorize their
116            // staged outcomes. End-of-run bookkeeping re-suspends if that wait
117            // is still present.
118            runtime_state.status = AgentStatusState::Idle;
119            runtime_state.suspension = None;
120            write_runtime_state(latest, &runtime_state);
121            latest.metadata.remove("runtime.suspend_reason");
122            latest.updated_at = Utc::now();
123            ready_for_mutation.store(true, std::sync::atomic::Ordering::Release);
124        })
125        .await?;
126    Ok(saved.map(|session| (session, ready.load(std::sync::atomic::Ordering::Acquire))))
127}
128
129fn is_error_like(status: &str) -> bool {
130    matches!(status, "error" | "timeout" | "cancelled")
131}
132
133/// Terminal child run statuses, as mirrored into the session index.
134fn is_terminal_child_status(status: &str) -> bool {
135    matches!(
136        status,
137        "completed" | "error" | "timeout" | "cancelled" | "skipped"
138    )
139}
140
141fn child_completion_envelope(
142    completion: &ChildCompletion,
143    wait_registered_at: chrono::DateTime<Utc>,
144    result: Option<String>,
145    provider_message: &Message,
146) -> SessionMessageEnvelope {
147    fn bounded_terminal_field(
148        label: &str,
149        child_session_id: &str,
150        value: Option<String>,
151    ) -> (Option<String>, serde_json::Value, bool) {
152        let Some(value) = value else {
153            return (None, serde_json::Value::Null, false);
154        };
155        if value.len() <= CHILD_COMPLETION_INLINE_FIELD_BYTES {
156            return (Some(value.clone()), serde_json::Value::String(value), false);
157        }
158        let digest = hex::encode(Sha256::digest(value.as_bytes()));
159        let mut tail_start = value
160            .len()
161            .saturating_sub(CHILD_COMPLETION_OVERSIZE_TAIL_BYTES);
162        while tail_start < value.len() && !value.is_char_boundary(tail_start) {
163            tail_start += 1;
164        }
165        let tail = &value[tail_start..];
166        let summary = format!(
167            "Child {label} exceeded the durable inline limit ({} UTF-8 bytes, sha256={digest}). \
168             Retrieve the full child transcript with SubAgent.get(child_session_id=\"{child_session_id}\").\
169             \n\nBounded tail:\n{tail}",
170            value.len()
171        );
172        (
173            Some(summary),
174            serde_json::json!({
175                "oversized": true,
176                "utf8_bytes": value.len(),
177                "sha256": digest,
178            }),
179            true,
180        )
181    }
182
183    let (stored_result, result_identity, _result_oversized) =
184        bounded_terminal_field("result", &completion.child_session_id, result);
185    let (stored_error, error_identity, _error_oversized) = bounded_terminal_field(
186        "error",
187        &completion.child_session_id,
188        completion.error.clone(),
189    );
190    let mut bounded_provider_message = provider_message.clone();
191    let provider_oversized = serde_json::to_vec(provider_message)
192        .map(|bytes| bytes.len() > CHILD_COMPLETION_INLINE_FIELD_BYTES)
193        .unwrap_or(true);
194    if provider_oversized {
195        let mut content = format!(
196            "Runtime notification: child session `{}` finished with status `{}`.",
197            completion.child_session_id, completion.status
198        );
199        if let Some(result) = stored_result.as_deref() {
200            content.push_str("\n\n");
201            content.push_str(result);
202        }
203        if let Some(error) = stored_error.as_deref() {
204            content.push_str("\n\n");
205            content.push_str(error);
206        }
207        bounded_provider_message.content = content;
208        bounded_provider_message.content_parts = None;
209    }
210    let body = SessionMessageBody::ChildOutcome(SessionChildOutcome {
211        child_session_id: completion.child_session_id.clone(),
212        status: completion.status.clone(),
213        result: stored_result,
214        error: stored_error,
215        provider_message: Some(session_provider_message(&bounded_provider_message)),
216    });
217    let semantic = serde_json::json!({
218        "parent_session_id": completion.parent_session_id,
219        "child_session_id": completion.child_session_id,
220        "status": completion.status,
221        "error": error_identity,
222        "result": result_identity,
223        "wait_registered_at": wait_registered_at,
224    });
225    SessionMessageEnvelope {
226        id: SessionMessageId::stable("session_child_completion", &semantic),
227        source: SessionMessageSource::Runtime {
228            subsystem: "child_completion_coordinator".to_string(),
229        },
230        target_session_id: completion.parent_session_id.clone(),
231        kind: SessionMessageKind::ChildOutcome,
232        body,
233        created_at: completion.completed_at,
234        thread_id: None,
235        in_reply_to: None,
236        attempt: None,
237        correlation_id: Some(format!("child_completion:{}", completion.child_session_id)),
238    }
239}
240
241fn session_provider_message(message: &Message) -> SessionProviderMessage {
242    SessionProviderMessage {
243        content: SessionMessageContent {
244            text: message.content.clone(),
245            parts: message.content_parts.clone().unwrap_or_default(),
246        },
247        metadata: message
248            .metadata
249            .as_ref()
250            .and_then(serde_json::Value::as_object)
251            .cloned()
252            .unwrap_or_default(),
253        never_compress: message.never_compress,
254    }
255}
256
257/// Reconstruct the set of completed child session ids for a parent from the
258/// session index (the single source of truth), folding in the child whose
259/// completion event is being processed so a momentarily-lagging index can never
260/// stall the parent's resume.
261async fn derive_completed_child_ids(
262    storage: &Arc<dyn Storage>,
263    parent_session_id: &str,
264    just_completed_child_id: &str,
265) -> Vec<String> {
266    let mut completed: Vec<String> = storage
267        .list_child_run_statuses(parent_session_id)
268        .await
269        .unwrap_or_default()
270        .into_iter()
271        .filter(|(_, status)| status.as_deref().is_some_and(is_terminal_child_status))
272        .map(|(id, _)| id)
273        .collect();
274    if !completed.iter().any(|id| id == just_completed_child_id) {
275        completed.push(just_completed_child_id.to_string());
276    }
277    completed.sort();
278    completed.dedup();
279    completed
280}
281
282fn read_config_snapshot(config: &Arc<RwLock<Config>>, cached_config: &StdRwLock<Config>) -> Config {
283    if let Ok(config_guard) = config.try_read() {
284        let snapshot = config_guard.clone();
285
286        if let Ok(mut cached_guard) = cached_config.try_write() {
287            *cached_guard = snapshot.clone();
288        }
289
290        snapshot
291    } else {
292        cached_config
293            .try_read()
294            .map(|guard| guard.clone())
295            .unwrap_or_default()
296    }
297}
298
299/// Per-parent async locks that serialize concurrent `on_child_completed`
300/// invocations for the same parent session.
301///
302/// Race eliminated: when `wait_for=Any` and two child sessions complete
303/// simultaneously, both invocations load the parent with
304/// `waiting_for_children=Some` before either persists the cleared state, so
305/// both pass `wait_policy_satisfied`, both clear `waiting_for_children`, add a
306/// duplicate resume message, and call `resume_parent` — a double resume.
307/// Holding this per-parent `tokio::sync::Mutex` across the load-check-save
308/// critical section makes the second caller observe the already-cleared state.
309///
310/// The inner `std::sync::Mutex` guards only the brief HashMap lookup/insert
311/// (no await inside); the per-parent `tokio::sync::Mutex` is the one held
312/// across the async critical section. Entries exist only while a holder or
313/// waiter owns a `SessionResumeLock`; historical parent IDs are reclaimed.
314fn parent_locks() -> &'static std::sync::Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>> {
315    static LOCKS: OnceLock<std::sync::Mutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>> =
316        OnceLock::new();
317    LOCKS.get_or_init(|| std::sync::Mutex::new(HashMap::new()))
318}
319
320/// Fetch (or create) the per-session async lock from [`parent_locks`]. Held
321/// across the load-check-clear-resume critical section so the three resume
322/// sources for one session — child completion, the loop-facing bash **push**
323/// ([`BashCompletionSink::on_bash_completed`]), and the bash **backstop** poll
324/// ([`ChildCompletionCoordinator::bash_self_resume`]) — can never double-resume.
325/// The inner sync `Mutex` guards only the brief map lookup (no await inside).
326fn session_resume_lock(session_id: &str) -> SessionResumeLock {
327    let mut map = parent_locks().lock().recover_poison();
328    let lock = map
329        .entry(session_id.to_string())
330        .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
331        .clone();
332    SessionResumeLock {
333        session_id: session_id.to_string(),
334        lock: Some(lock),
335    }
336}
337
338/// The lease is constructed before awaiting the mutex, so cancelled waiters
339/// also reclaim their registration. Lookup and last-owner removal use the same
340/// brief registry lock; two live mutexes can never exist for the same ID.
341struct SessionResumeLock {
342    session_id: String,
343    lock: Option<Arc<tokio::sync::Mutex<()>>>,
344}
345
346impl std::ops::Deref for SessionResumeLock {
347    type Target = tokio::sync::Mutex<()>;
348    fn deref(&self) -> &Self::Target {
349        self.lock.as_ref().expect("live resume-lock lease")
350    }
351}
352
353impl Drop for SessionResumeLock {
354    fn drop(&mut self) {
355        self.lock.take();
356        let mut map = parent_locks().lock().recover_poison();
357        if map
358            .get(&self.session_id)
359            .is_some_and(|lock| Arc::strong_count(lock) == 1)
360        {
361            map.remove(&self.session_id);
362        }
363    }
364}
365
366fn wait_policy_satisfied(
367    policy: ChildWaitPolicy,
368    wait_child_ids: &[String],
369    completed_child_ids: &[String],
370    latest_child_id: &str,
371    latest_status: &str,
372) -> bool {
373    if wait_child_ids.is_empty() {
374        return false;
375    }
376
377    match policy {
378        ChildWaitPolicy::All => wait_child_ids
379            .iter()
380            .all(|id| completed_child_ids.iter().any(|completed| completed == id)),
381        ChildWaitPolicy::Any => completed_child_ids
382            .iter()
383            .any(|id| wait_child_ids.iter().any(|wait_id| wait_id == id)),
384        ChildWaitPolicy::FirstError => {
385            // The error short-circuit only counts a completion from a child
386            // this wait actually tracks (issue #546): a stray/duplicate
387            // completion from an untracked child — e.g. a frozen runner's
388            // task waking up after the watchdog already synthesized its
389            // timeout, in a later run's wait — must not resume the parent.
390            (is_error_like(latest_status) && wait_child_ids.iter().any(|id| id == latest_child_id))
391                || wait_child_ids
392                    .iter()
393                    .all(|id| completed_child_ids.iter().any(|completed| completed == id))
394        }
395    }
396}
397
398/// Extract the child session's last assistant content, if any. Returns `None`
399/// when the child produced no assistant message (e.g. errored before the first
400/// model response, or only emitted tool messages).
401fn child_final_assistant_text(child: &Session) -> Option<String> {
402    child
403        .messages
404        .iter()
405        .rev()
406        .find(|message| matches!(message.role, Role::Assistant))
407        .map(|message| message.content.clone())
408        .filter(|content| !content.trim().is_empty())
409}
410
411fn runtime_resume_message(
412    completion: &ChildCompletion,
413    remaining_children: usize,
414    child_final_response: Option<&str>,
415) -> Message {
416    let mut body = format!(
417        "Runtime notification: child session `{}` finished with status `{}`. Remaining child sessions: {}.",
418        completion.child_session_id, completion.status, remaining_children
419    );
420
421    // Fold the child's full final response back into the parent — no
422    // truncation. Sub-agents are first-class agents whose complete conclusion
423    // should be available to the parent without an extra `SubAgent.get` round
424    // trip. The message is left compressible (see `never_compress` below) so a
425    // long transcript can still be reclaimed under parent compaction.
426    let final_response = child_final_response.map(str::to_string);
427    if let Some(response) = final_response.as_deref() {
428        body.push_str("\n\nChild final response:\n");
429        body.push_str(response);
430    } else if let Some(error) = completion.error.as_deref() {
431        if !error.is_empty() {
432            body.push_str("\n\nChild error:\n");
433            body.push_str(error);
434        }
435    }
436
437    body.push_str(
438        "\n\nResume the parent task using this child result and continue from the previous plan. \
439         If you need the full child transcript, call SubAgent.get(child_session_id).",
440    );
441
442    let mut message = Message::user(body);
443    message.metadata = Some(serde_json::json!({
444        RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
445        RUNTIME_RESUME_MESSAGE_KIND_KEY: "child_completion_resume",
446        "child_session_id": completion.child_session_id,
447        "child_status": completion.status,
448        "child_error": completion.error,
449        "child_final_response_included": final_response.is_some(),
450    }));
451    // Allow parent-side compaction to reclaim this (now untruncated) message if
452    // the parent context grows — important once children nest and fold full
453    // results upward. The `SubAgent.get` hint preserves recoverability.
454    message.never_compress = false;
455    message
456}
457
458/// The hidden resume message for a completed **guardian** review: a directive,
459/// verdict-tailored note that carries the reviewer's findings straight into the
460/// parent (so it can act without a `SubAgent.get`), mirroring
461/// [`runtime_resume_message`]'s hidden/compressible shape.
462fn guardian_resume_message(completion: &ChildCompletion, verdict: &GuardianVerdict) -> Message {
463    let mut body = if verdict.approve {
464        String::from(
465            "Guardian review APPROVED: an independent reviewer verified the work and found no blocking issues. You may finalize the task.",
466        )
467    } else {
468        String::from(
469            "Guardian review REJECTED: an independent reviewer found issues. Address every finding below before completing — do NOT declare the task complete until they are resolved.",
470        )
471    };
472    if let Some(summary) = verdict.summary.as_deref().filter(|s| !s.trim().is_empty()) {
473        body.push_str("\n\nReviewer summary: ");
474        body.push_str(summary);
475    }
476    if !verdict.findings.is_empty() {
477        body.push_str("\n\nFindings:");
478        for (idx, finding) in verdict.findings.iter().enumerate() {
479            body.push_str(&format!("\n{}. {}", idx + 1, finding));
480        }
481    }
482    body.push_str(
483        "\n\nIf you need the full guardian transcript, call SubAgent.get(child_session_id).",
484    );
485
486    let mut message = Message::user(body);
487    message.metadata = Some(serde_json::json!({
488        RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
489        RUNTIME_RESUME_MESSAGE_KIND_KEY: "guardian_review_resume",
490        "child_session_id": completion.child_session_id,
491        "child_status": completion.status,
492        "guardian_approved": verdict.approve,
493    }));
494    message.never_compress = false;
495    message
496}
497
498#[derive(Clone)]
499pub struct ChildCompletionCoordinator {
500    storage: Arc<dyn Storage>,
501    persistence: Arc<bamboo_storage::LockedSessionStore>,
502    sessions: crate::SessionCache,
503    agent_runners: Arc<RwLock<HashMap<String, AgentRunner>>>,
504    session_event_senders: Arc<RwLock<HashMap<String, broadcast::Sender<AgentEvent>>>>,
505    agent: Arc<Agent>,
506    config: Arc<RwLock<Config>>,
507    provider_registry: Arc<ProviderRegistry>,
508    provider_router: Arc<ProviderModelRouter>,
509    app_data_dir: std::path::PathBuf,
510    account_feed_inbox: Option<crate::execution::AccountFeedInbox>,
511    root_tools: Arc<RwLock<Option<Arc<dyn ToolExecutor>>>>,
512    /// Late-bound guardian reviewer spawner, set post-construction by the server
513    /// (mirrors `root_tools`). Re-injected into resumed runs so a guardian's
514    /// reject→fix verdict can be re-reviewed across the suspend/resume boundary.
515    guardian_spawner: Arc<RwLock<Option<Arc<dyn GuardianSpawner>>>>,
516    /// Weak late binding avoids the scheduler -> completion handler ->
517    /// scheduler ownership cycle while still routing idle child activation
518    /// through the canonical placement-aware spawn core.
519    spawn_scheduler: Arc<RwLock<Weak<SpawnScheduler>>>,
520}
521
522impl ChildCompletionCoordinator {
523    #[allow(clippy::too_many_arguments)]
524    pub fn new(
525        storage: Arc<dyn Storage>,
526        persistence: Arc<LockedSessionStore>,
527        sessions: crate::SessionCache,
528        agent_runners: Arc<RwLock<HashMap<String, AgentRunner>>>,
529        session_event_senders: Arc<RwLock<HashMap<String, broadcast::Sender<AgentEvent>>>>,
530        agent: Arc<Agent>,
531        config: Arc<RwLock<Config>>,
532        provider_registry: Arc<ProviderRegistry>,
533        provider_router: Arc<ProviderModelRouter>,
534        app_data_dir: std::path::PathBuf,
535        account_feed_inbox: Option<crate::execution::AccountFeedInbox>,
536    ) -> Self {
537        Self {
538            storage,
539            persistence,
540            sessions,
541            agent_runners,
542            session_event_senders,
543            agent,
544            config,
545            provider_registry,
546            provider_router,
547            app_data_dir,
548            account_feed_inbox,
549            root_tools: Arc::new(RwLock::new(None)),
550            guardian_spawner: Arc::new(RwLock::new(None)),
551            spawn_scheduler: Arc::new(RwLock::new(Weak::new())),
552        }
553    }
554
555    pub async fn set_root_tools(&self, tools: Arc<dyn ToolExecutor>) {
556        *self.root_tools.write().await = Some(tools);
557    }
558
559    pub async fn set_spawn_scheduler(&self, scheduler: &Arc<SpawnScheduler>) {
560        *self.spawn_scheduler.write().await = Arc::downgrade(scheduler);
561    }
562
563    /// Wire the guardian reviewer spawner (server-provided), so resumed runs can
564    /// re-spawn a guardian to re-review a fix after a reject verdict.
565    pub async fn set_guardian_spawner(&self, spawner: Arc<dyn GuardianSpawner>) {
566        *self.guardian_spawner.write().await = Some(spawner);
567    }
568
569    fn build_resume_config(
570        &self,
571        session: &Session,
572        config_snapshot: &Config,
573    ) -> ResumeConfigSnapshot {
574        crate::session_app::resolution::resolve_resume_config_snapshot(
575            config_snapshot,
576            &self.provider_registry,
577            session,
578            None,
579        )
580    }
581
582    /// Drive a parent-resume and return the final [`ResumeOutcome`] so callers
583    /// can distinguish a successful spawn (`Started`) from a gate-blocked
584    /// attempt (`Completed`). The bash self-resume poll task uses this to
585    /// detect the finalize-clobber case — its appended resume message was
586    /// reverted by the suspending runner's final `merge_save_runtime`, so the
587    /// resume port's `has_pending_user_message` gate fails and nothing spawns —
588    /// and retry the clear→append→resume (see [`Self::bash_self_resume`]).
589    async fn resume_parent(&self, parent_session_id: String) -> ResumeOutcome {
590        for attempt in 0..=5u8 {
591            if attempt > 0 {
592                tokio::time::sleep(Duration::from_millis(250 * attempt as u64)).await;
593            }
594
595            let Some(session) = self.load_session(&parent_session_id).await else {
596                tracing::warn!(%parent_session_id, "cannot resume parent after child completion: session not found");
597                return ResumeOutcome::NotFound;
598            };
599            let config_snapshot = self.config.read().await.clone();
600            let resume_config = self.build_resume_config(&session, &config_snapshot);
601            let outcome = resume_session_execution(self, &parent_session_id, resume_config).await;
602            tracing::info!(
603                %parent_session_id,
604                attempt,
605                outcome = outcome.as_str(),
606                "child completion requested parent resume"
607            );
608
609            if !matches!(outcome, ResumeOutcome::AlreadyRunning { .. }) {
610                return outcome;
611            }
612        }
613        // Exhausted the AlreadyRunning retry budget; surface the final state.
614        // The wait state was already cleared and the resume message persisted,
615        // so nothing event-driven will retry — the child-wait watchdog is the
616        // backstop that picks this stranded parent up (it resumes suspended
617        // sessions that hold a pending runtime resume message but no runner).
618        tracing::error!(
619            %parent_session_id,
620            "parent resume gave up after AlreadyRunning retry budget; \
621             relying on the child-wait watchdog backstop"
622        );
623        ResumeOutcome::AlreadyRunning {
624            run_id: String::new(),
625        }
626    }
627
628    async fn save_and_cache(&self, session: &mut Session) {
629        if let Err(error) = self
630            .persistence
631            .merge_save_runtime_and_publish(session, |saved, _| {
632                self.sessions.insert(
633                    saved.id.clone(),
634                    Arc::new(crate::SessionSnapshot::new(saved.clone())),
635                );
636            })
637            .await
638        {
639            tracing::warn!(session_id = %session.id, %error, "failed to persist session");
640        }
641    }
642}
643
644#[async_trait]
645impl ChildCompletionHandler for ChildCompletionCoordinator {
646    async fn on_child_completed(&self, completion: ChildCompletion) {
647        // Terminality guard: a child that reports a NON-terminal status (e.g.
648        // "suspended" — awaiting parent approval, its own bash wait, or its own
649        // grandchildren) is not done. It must never satisfy the parent's wait:
650        // `derive_completed_child_ids` folds the just-reported child in
651        // unconditionally, so without this guard a suspending child would
652        // resume the parent with a premature "finished with status
653        // `suspended`" message. The child will publish a real terminal
654        // completion when it later resumes and finishes.
655        if !is_terminal_child_status(&completion.status) {
656            tracing::info!(
657                parent_session_id = %completion.parent_session_id,
658                child_session_id = %completion.child_session_id,
659                status = %completion.status,
660                "non-terminal child status; leaving the parent wait armed"
661            );
662            return;
663        }
664
665        // Acquire the per-session async lock to eliminate the concurrent
666        // double-resume race (see `parent_locks` for the full scenario). The
667        // inner std::sync::Mutex is released immediately so no sync lock is
668        // held across the await that follows.
669        let per_parent = session_resume_lock(&completion.parent_session_id);
670        let _per_parent_guard = per_parent.lock().await;
671
672        let Some(mut parent) = self.load_session(&completion.parent_session_id).await else {
673            tracing::warn!(
674                parent_session_id = %completion.parent_session_id,
675                child_session_id = %completion.child_session_id,
676                "child completion received for missing parent"
677            );
678            return;
679        };
680
681        // A parent may itself be a child (nested sub-agents): the rest of this
682        // handler is kind-agnostic — it operates on `completion.parent_session_id`,
683        // inspects that session's own `waiting_for_children` runtime state, and
684        // resumes it. (Previously this bailed unless the parent was Root, which
685        // silently dropped grandchild completions.)
686        let mut runtime_state = read_runtime_state(&parent);
687
688        // Single source of truth: reconstruct the completed-child set from the
689        // session index rather than from a denormalized copy on the parent file.
690        let completed_child_ids = derive_completed_child_ids(
691            &self.storage,
692            &completion.parent_session_id,
693            &completion.child_session_id,
694        )
695        .await;
696
697        let mut should_resume = false;
698        let mut remaining_children = 0usize;
699        let active_wait = runtime_state.waiting_for_children.clone();
700        if let Some(wait) = active_wait.as_ref() {
701            remaining_children = wait
702                .child_session_ids
703                .iter()
704                .filter(|id| !completed_child_ids.iter().any(|completed| completed == *id))
705                .count();
706            should_resume = wait_policy_satisfied(
707                wait.wait_for,
708                &wait.child_session_ids,
709                &completed_child_ids,
710                &completion.child_session_id,
711                &completion.status,
712            );
713        }
714
715        // READ-SIDE OWNERSHIP GUARD (issue #546): `SubAgent.wait` ids are
716        // model-provided and unvalidated, and the watchdog unstrands a wait
717        // over a FOREIGN/unknown id by publishing a synthetic completion
718        // here. We must resume the parent (so it is not stranded) but MUST
719        // NOT fold that foreign session's transcript into the parent — that
720        // would be a cross-session disclosure primitive. Decide ownership
721        // from the child's OWN parent linkage (control-plane only, no
722        // messages loaded), and only load its full content when it is truly
723        // this parent's child. An unowned id resumes with the neutral/error
724        // message (`runtime_resume_message` falls back to `completion.error`
725        // when no child content is supplied).
726        let reported_child_owned = match self
727            .storage
728            .load_runtime_control_plane(&completion.child_session_id)
729            .await
730        {
731            Ok(Some(control_plane)) => completion_child_is_owned(
732                &completion.parent_session_id,
733                control_plane.parent_session_id.as_deref(),
734            ),
735            _ => false,
736        };
737
738        // Load the completed child once, ONLY when owned. The guardian
739        // branch inspects its subagent_type + final verdict; the generic
740        // path folds its final assistant content into the hidden resume
741        // message (avoiding an extra `SubAgent.get` round trip after resume).
742        let loaded_child = if reported_child_owned {
743            match self
744                .storage
745                .load_session(&completion.child_session_id)
746                .await
747            {
748                Ok(child) => child,
749                Err(error) => {
750                    tracing::warn!(
751                        child_session_id = %completion.child_session_id,
752                        %error,
753                        "failed to load child session for runtime resume message"
754                    );
755                    None
756                }
757            }
758        } else {
759            tracing::warn!(
760                parent_session_id = %completion.parent_session_id,
761                child_session_id = %completion.child_session_id,
762                "completion child is not a child of this parent; resuming with a neutral \
763                 message and NOT folding its content"
764            );
765            None
766        };
767
768        let child_final_response = loaded_child.as_ref().and_then(child_final_assistant_text);
769        // Select the exact provider-facing resume message before durable
770        // admission. The typed body carries its content/parts and safe runtime
771        // metadata, so the canonical path is semantically identical to the
772        // rolling-upgrade transcript fallback.
773        let guardian_resume = if should_resume {
774            let reviewed_round = runtime_state.round.current_round;
775            loaded_child.as_ref().and_then(|child| {
776                if child.subagent_type().as_deref() != Some("guardian") {
777                    return None;
778                }
779                let mut guardian_state = read_guardian_state(&parent)?;
780                if guardian_state.guardian_child_id.as_deref()
781                    != Some(completion.child_session_id.as_str())
782                {
783                    // A *different* guardian is legitimately still in flight —
784                    // leave its Pending state intact and use the generic resume.
785                    tracing::warn!(
786                        parent_session_id = %completion.parent_session_id,
787                        child_session_id = %completion.child_session_id,
788                        expected = ?guardian_state.guardian_child_id,
789                        "guardian completion does not match recorded guardian_child_id; using generic resume"
790                    );
791                    return None;
792                }
793                // This IS the guardian we dispatched, so we MUST advance the
794                // phase out of `Pending` — otherwise the next terminal gate's
795                // `Pending => return None` would let the run complete unreviewed.
796                // A reviewer that errored or produced unparseable output is
797                // treated as a SYNTHETIC REJECT (never a silent pass), so the
798                // budgeted re-review loop governs the outcome: fail-closed, but
799                // still bounded by `max_reviews`.
800                let verdict = child_final_assistant_text(child)
801                    .and_then(|text| match parse_guardian_verdict(&text) {
802                        Ok(verdict) => Some(verdict),
803                        Err(error) => {
804                            tracing::warn!(
805                                child_session_id = %completion.child_session_id,
806                                %error,
807                                "guardian verdict unparseable; recording a synthetic reject"
808                            );
809                            None
810                        }
811                    })
812                    .unwrap_or_else(|| {
813                        GuardianVerdict::rejected(vec![
814                            "The guardian reviewer did not return a usable verdict (it errored or \
815                             emitted unparseable output); the work has NOT been independently \
816                             verified."
817                                .to_string(),
818                        ])
819                    });
820                let approved = verdict.approve;
821                let message = guardian_resume_message(&completion, &verdict);
822                guardian_state.record_verdict(verdict, reviewed_round);
823                write_guardian_state(&mut parent, guardian_state);
824                tracing::info!(
825                    parent_session_id = %completion.parent_session_id,
826                    child_session_id = %completion.child_session_id,
827                    approved,
828                    "guardian verdict recorded; resuming parent"
829                );
830                Some(message)
831            })
832        } else {
833            None
834        };
835        let resume_message = guardian_resume.unwrap_or_else(|| {
836            runtime_resume_message(
837                &completion,
838                remaining_children,
839                child_final_response.as_deref(),
840            )
841        });
842
843        // Stage the typed child outcome before clearing any durable wait. A
844        // crash after this admission leaves the parent suspended with an
845        // inspectable envelope; only a durably committed policy transition
846        // below is allowed to activate it.
847        let messenger = self.agent.session_messenger().cloned();
848        let child_admission =
849            if let (Some(wait), Some(messenger)) = (active_wait.as_ref(), messenger.as_ref()) {
850                let envelope = child_completion_envelope(
851                    &completion,
852                    wait.registered_at,
853                    child_final_response,
854                    &resume_message,
855                );
856                match messenger.admit(envelope).await {
857                    Ok(admission) => Some(admission),
858                    Err(error) => {
859                        tracing::warn!(
860                            parent_session_id = %completion.parent_session_id,
861                            child_session_id = %completion.child_session_id,
862                            %error,
863                            "child outcome SessionInbox admission failed; leaving parent wait armed"
864                        );
865                        return;
866                    }
867                }
868            } else {
869                None
870            };
871
872        if should_resume {
873            if let (Some(messenger), Some(admission)) =
874                (messenger.as_ref(), child_admission.as_ref())
875            {
876                if let Err(error) = messenger.prepare_activation(admission).await {
877                    tracing::warn!(
878                        parent_session_id = %completion.parent_session_id,
879                        child_session_id = %completion.child_session_id,
880                        %error,
881                        "child outcome activation watermark failed; leaving parent wait armed"
882                    );
883                    return;
884                }
885            }
886        }
887
888        if should_resume {
889            runtime_state.waiting_for_children = None;
890            runtime_state.status = AgentStatusState::Idle;
891            runtime_state.suspension = None;
892            parent.metadata.remove("runtime.suspend_reason");
893
894            if child_admission.is_none() {
895                // Rolling-upgrade fallback only. The canonical path keeps the
896                // child outcome solely in SessionInbox until the next safe
897                // reasoning boundary.
898                parent.add_message(resume_message);
899            }
900        } else if runtime_state.waiting_for_children.is_some() {
901            runtime_state.status = AgentStatusState::Suspended;
902            runtime_state.suspension = Some(SuspensionState {
903                reason: "waiting_for_children".to_string(),
904                suspended_at: Utc::now(),
905                resumable: true,
906                hook_point: Some("ChildCompletion".to_string()),
907            });
908        }
909
910        parent.updated_at = Utc::now();
911        write_runtime_state(&mut parent, &runtime_state);
912        if let Err(error) = self
913            .persistence
914            .checkpoint_runtime_session(&mut parent)
915            .await
916        {
917            tracing::warn!(
918                parent_session_id = %completion.parent_session_id,
919                child_session_id = %completion.child_session_id,
920                %error,
921                "child outcome is durable but parent wait transition failed; leaving activation deferred"
922            );
923            return;
924        }
925        self.sessions.insert(
926            parent.id.clone(),
927            Arc::new(crate::SessionSnapshot::new(parent.clone())),
928        );
929
930        // Capture before releasing the per-parent lock so the borrow checker
931        // is satisfied; `resume_parent` has its own retry loop and should not
932        // hold the per-parent lock (it would block other completions for the
933        // same parent, and the state is already durably settled above).
934        let resume_parent_id = parent.id.clone();
935        drop(_per_parent_guard);
936
937        if should_resume {
938            if let (Some(messenger), Some(admission)) = (messenger, child_admission) {
939                if let Err(error) = messenger.activate_prepared(&admission).await {
940                    tracing::warn!(
941                        parent_session_id = %resume_parent_id,
942                        %error,
943                        "child outcome and wait transition are durable but activation failed"
944                    );
945                }
946            } else {
947                self.resume_parent(resume_parent_id).await;
948            }
949        }
950    }
951}
952
953#[async_trait]
954impl ResumeExecutionPort for ChildCompletionCoordinator {
955    async fn load_session(&self, session_id: &str) -> Option<Session> {
956        match self.storage.load_session(session_id).await {
957            Ok(Some(session)) => Some(session),
958            Ok(None) => self
959                .sessions
960                .get(session_id)
961                .map(|e| e.value().clone())
962                .map(|arc| arc.read().clone()),
963            Err(error) => {
964                tracing::warn!(%session_id, %error, "failed to load session from storage");
965                self.sessions
966                    .get(session_id)
967                    .map(|e| e.value().clone())
968                    .map(|arc| arc.read().clone())
969            }
970        }
971    }
972
973    async fn save_and_cache_session(&self, session: &mut Session) {
974        self.save_and_cache(session).await;
975    }
976
977    async fn reserve_session_execution(
978        &self,
979        session_id: &str,
980        event_sender: &broadcast::Sender<AgentEvent>,
981    ) -> SessionExecutionReserveOutcome {
982        reserve_session_execution(
983            &self.agent,
984            &self.agent_runners,
985            &self.session_event_senders,
986            session_id,
987            event_sender,
988        )
989        .await
990    }
991
992    async fn get_or_create_event_sender(&self, session_id: &str) -> broadcast::Sender<AgentEvent> {
993        crate::execution::session_events::get_or_create_event_sender(
994            &self.session_event_senders,
995            session_id,
996        )
997        .await
998    }
999
1000    fn dispatch_resume_execution(
1001        &self,
1002        request: ResumeSpawnRequest,
1003    ) -> Result<(), ResumeSpawnRequest> {
1004        let owner = self.clone();
1005        tokio::spawn(async move {
1006            ResumeExecutionPort::spawn_resume_execution(&owner, request).await;
1007        });
1008        Ok(())
1009    }
1010
1011    async fn spawn_resume_execution(&self, request: ResumeSpawnRequest) {
1012        let ResumeSpawnRequest {
1013            session_id,
1014            mut session,
1015            mut execution_reservation,
1016            event_sender,
1017            config,
1018        } = request;
1019        if let Err(error) = execution_reservation.ensure_registered().await {
1020            tracing::warn!(
1021                %session_id,
1022                run_id = %execution_reservation.run_id(),
1023                %error,
1024                "cannot resume after child completion without exact router ownership"
1025            );
1026            return;
1027        }
1028
1029        let Some(root_tools) = self.root_tools.read().await.clone() else {
1030            tracing::error!(%session_id, "cannot resume parent after child completion: root tool surface is not initialized");
1031            return;
1032        };
1033
1034        let config_snapshot = self.config.read().await.clone();
1035        let model = session.model.clone();
1036        let session_model_ref = session_effective_model_ref(&session);
1037        let requested_provider = session_model_ref
1038            .as_ref()
1039            .map(|model_ref| model_ref.provider.as_str())
1040            .unwrap_or(config.provider_name.as_str());
1041        let resolved_provider_name = match resolve_provider_routing_key(
1042            &config_snapshot,
1043            requested_provider,
1044            &self.provider_registry,
1045        ) {
1046            Ok(provider) => provider,
1047            Err(error) => {
1048                tracing::error!(
1049                    session_id = %session_id,
1050                    provider = requested_provider,
1051                    %error,
1052                    "child-completion resume provider is unavailable; refusing to fall back"
1053                );
1054                execution_reservation.abandon().await;
1055                return;
1056            }
1057        };
1058        let provider_override = if let Some(mut model_ref) = session_model_ref {
1059            model_ref.provider = resolved_provider_name.clone();
1060            persist_model_ref(&mut session, &model_ref);
1061            match self.provider_router.route(&model_ref) {
1062                Ok(provider) => Some(provider),
1063                Err(error) => {
1064                    tracing::error!(
1065                        session_id = %session_id,
1066                        provider = %model_ref.provider,
1067                        model = %model_ref.model,
1068                        %error,
1069                        "child-completion resume provider routing failed closed"
1070                    );
1071                    execution_reservation.abandon().await;
1072                    return;
1073                }
1074            }
1075        } else {
1076            match self.provider_registry.get(&resolved_provider_name) {
1077                Some(provider) => Some(provider),
1078                None => {
1079                    tracing::error!(
1080                        session_id = %session_id,
1081                        provider = %resolved_provider_name,
1082                        "child-completion resume provider disappeared after resolution; refusing to fall back"
1083                    );
1084                    execution_reservation.abandon().await;
1085                    return;
1086                }
1087            }
1088        };
1089        let resolved_fast_provider = resolve_fast_model(
1090            &config_snapshot,
1091            &resolved_provider_name,
1092            &self.provider_registry,
1093        )
1094        .map(|model| model.provider);
1095        let reasoning_effort = session.reasoning_effort;
1096        let reasoning_effort_source = session
1097            .metadata
1098            .get("reasoning_effort_source")
1099            .cloned()
1100            .unwrap_or_default();
1101        let gold_config = resolve_gold_config(
1102            &config_snapshot,
1103            session
1104                .metadata
1105                .get(GOLD_CONFIG_METADATA_KEY)
1106                .map(String::as_str),
1107        )
1108        .or(config.gold_config.clone());
1109
1110        let (mpsc_tx, _forwarder) = create_event_forwarder(
1111            session_id.clone(),
1112            execution_reservation.run_id().to_string(),
1113            event_sender,
1114            self.agent_runners.clone(),
1115            self.account_feed_inbox.clone(),
1116        );
1117
1118        let config_handle = self.config.clone();
1119        let cached_config = Arc::new(StdRwLock::new(config_snapshot.clone()));
1120        let provider_registry = self.provider_registry.clone();
1121        let provider_name_for_aux = resolved_provider_name.clone();
1122        let auxiliary_model_resolver = std::sync::Arc::new(move || {
1123            let config_snapshot = read_config_snapshot(&config_handle, cached_config.as_ref());
1124            // Auxiliary models are global (config-derived), never session-bound.
1125            let areas = resolve_global_area_models(
1126                &config_snapshot,
1127                &provider_name_for_aux,
1128                &provider_registry,
1129            );
1130            crate::AuxiliaryModelConfig {
1131                fast_model_name: areas.fast.as_ref().map(|m| m.model_name.clone()),
1132                fast_model_provider: areas.fast.map(|m| m.provider),
1133                background_model_name: areas.background.as_ref().map(|m| m.model_name.clone()),
1134                planning_model_name: None,
1135                search_model_name: None,
1136                summarization_model_name: areas
1137                    .summarization
1138                    .as_ref()
1139                    .map(|m| m.model_name.clone()),
1140                background_model_provider: areas.background.map(|m| m.provider),
1141                summarization_model_provider: areas.summarization.map(|m| m.provider),
1142            }
1143        });
1144        let model_roster = crate::ModelRoster {
1145            model: Some(model),
1146            provider_name: Some(resolved_provider_name),
1147            provider_type: config.provider_type.clone(),
1148            fast: crate::RoleModel::from_parts(config.fast_model, resolved_fast_provider),
1149            background: crate::RoleModel::from_parts(
1150                config.background_model,
1151                config.background_model_provider,
1152            ),
1153            summarization: crate::RoleModel::from_parts(
1154                config.summarization_model,
1155                config.summarization_model_provider,
1156            ),
1157        };
1158
1159        // Re-inject guardian state on resume so a reject→fix verdict can be
1160        // re-reviewed: config from the session (persisted at first spawn),
1161        // spawner from the coordinator-held handle. Absent guardian config this
1162        // stays `None`, and the approve→complete path is unchanged.
1163        let guardian_config = read_guardian_config(&session);
1164        let guardian_spawner = self.guardian_spawner.read().await.clone();
1165
1166        consume_pending_clarification_resume(&mut session);
1167        spawn_session_execution(SessionExecutionArgs {
1168            agent: self.agent.clone(),
1169            session_id,
1170            session,
1171            execution_reservation,
1172            tools_override: Some(root_tools),
1173            provider_override,
1174            model_roster,
1175            reasoning_effort,
1176            reasoning_effort_source,
1177            auxiliary_model_resolver: Some(auxiliary_model_resolver),
1178            // Resumed child runs keep the spawn-time disabled snapshot (#136 lives
1179            // on the long-running main agent path; children are short-lived).
1180            disabled_filter_resolver: None,
1181            disabled_tools: Some(config.disabled_tools),
1182            disabled_skill_ids: Some(config.disabled_skill_ids),
1183            selected_skill_ids: None,
1184            selected_skill_mode: None,
1185            mpsc_tx,
1186            image_fallback: config.image_fallback,
1187            gold_config,
1188            guardian_config,
1189            guardian_spawner,
1190            bash_resume_hook: {
1191                let hook: Arc<dyn BashResumeHook> = Arc::new(self.clone());
1192                Some(hook)
1193            },
1194            bash_completion_sink: {
1195                // Resumed runs keep the push wired too, so a background shell
1196                // launched after resume still notifies the loop.
1197                let sink: Arc<dyn BashCompletionSink> = Arc::new(self.clone());
1198                Some(sink)
1199            },
1200            app_data_dir: Some(self.app_data_dir.clone()),
1201            // Resume does not carry a fresh per-request override; the
1202            // config-level default (issue #221) still applies.
1203            run_budget: None,
1204            runners: self.agent_runners.clone(),
1205            sessions_cache: self.sessions.clone(),
1206            on_complete: None,
1207            // A resumed session that is itself a CHILD (nested sub-agents)
1208            // must publish its terminal completion so ITS parent is woken
1209            // in turn (issue #546).
1210            child_completion_handler: Some(Arc::new(self.clone())),
1211        });
1212    }
1213}
1214
1215/// Real SessionInbox activation adapter. It reserves through the exact same
1216/// runner registry as every existing resume path, but deliberately bypasses
1217/// `has_pending_user_message`: the typed envelope is still in the durable inbox
1218/// and will be admitted by the loop's first safe turn boundary.
1219#[async_trait]
1220impl SessionActivationSpawner for ChildCompletionCoordinator {
1221    async fn reserve_activation(
1222        &self,
1223        target_session_id: &str,
1224        _inbox_generation: u64,
1225    ) -> Result<SessionActivationReserveOutcome, bamboo_domain::SessionActivationError> {
1226        let Some(inbox) = self.agent.session_inbox() else {
1227            return Err(bamboo_domain::SessionActivationError::Internal(
1228                "agent runtime has no SessionInbox".to_string(),
1229            ));
1230        };
1231        let backlog = inbox
1232            .inspect(target_session_id)
1233            .await
1234            .map_err(|error| bamboo_domain::SessionActivationError::Internal(error.to_string()))?;
1235        if !backlog.activation_pending() {
1236            return Ok(SessionActivationReserveOutcome::NoWork);
1237        }
1238        // Load and mutate under the persistence lock. A coordinator that armed
1239        // a child/Bash wait immediately before this activation therefore wins.
1240        let prepared = prepare_session_inbox_activation(
1241            &self.persistence,
1242            target_session_id,
1243            backlog.interrupt_pending(),
1244        )
1245        .await
1246        .map_err(|error| {
1247            bamboo_domain::SessionActivationError::Internal(format!(
1248                "persist resumable SessionInbox target: {error}"
1249            ))
1250        })?;
1251        let Some((session, ready)) = prepared else {
1252            return Ok(SessionActivationReserveOutcome::NotFound);
1253        };
1254        if !ready {
1255            tracing::info!(
1256                session_id = target_session_id,
1257                "SessionInbox backlog is activation-eligible but a specific durable wait remains armed"
1258            );
1259            return Ok(SessionActivationReserveOutcome::NoWork);
1260        }
1261
1262        enum LaunchPlan {
1263            Root(Box<ResumeConfigSnapshot>),
1264            Child {
1265                scheduler: Arc<SpawnScheduler>,
1266                parent_session_id: String,
1267                model: String,
1268                disabled_tools: Option<Vec<String>>,
1269            },
1270        }
1271
1272        // Derive every execution/security input from the latest locked session
1273        // returned above, never from a stale pre-lock snapshot.
1274        let launch_plan = match session.kind {
1275            SessionKind::Root => {
1276                if self.root_tools.read().await.is_none() {
1277                    return Err(bamboo_domain::SessionActivationError::Internal(
1278                        "root tool surface is not initialized".to_string(),
1279                    ));
1280                }
1281                let config_snapshot = self.config.read().await.clone();
1282                LaunchPlan::Root(Box::new(
1283                    self.build_resume_config(&session, &config_snapshot),
1284                ))
1285            }
1286            SessionKind::Child => {
1287                let scheduler = self.spawn_scheduler.read().await.upgrade().ok_or_else(|| {
1288                    bamboo_domain::SessionActivationError::Internal(
1289                        "child spawn scheduler is not initialized".to_string(),
1290                    )
1291                })?;
1292                let parent_session_id = session
1293                    .parent_session_id
1294                    .as_deref()
1295                    .map(str::trim)
1296                    .filter(|id| !id.is_empty())
1297                    .map(ToOwned::to_owned)
1298                    .ok_or_else(|| {
1299                        bamboo_domain::SessionActivationError::Internal(format!(
1300                            "child SessionInbox target {target_session_id} has no parent owner"
1301                        ))
1302                    })?;
1303                let parent = self
1304                    .storage
1305                    .load_session(&parent_session_id)
1306                    .await
1307                    .map_err(|error| {
1308                        bamboo_domain::SessionActivationError::Internal(format!(
1309                            "load parent owner {parent_session_id}: {error}"
1310                        ))
1311                    })?
1312                    .ok_or_else(|| {
1313                        bamboo_domain::SessionActivationError::Internal(format!(
1314                            "child SessionInbox parent owner {parent_session_id} disappeared"
1315                        ))
1316                    })?;
1317                let child_root = if session.root_session_id.trim().is_empty() {
1318                    parent_session_id.as_str()
1319                } else {
1320                    session.root_session_id.as_str()
1321                };
1322                let parent_root = if parent.root_session_id.trim().is_empty() {
1323                    parent.id.as_str()
1324                } else {
1325                    parent.root_session_id.as_str()
1326                };
1327                if child_root != parent_root {
1328                    return Err(bamboo_domain::SessionActivationError::Internal(format!(
1329                        "child SessionInbox target {target_session_id} does not share its parent owner's root"
1330                    )));
1331                }
1332                let model = if session.model.trim().is_empty() {
1333                    parent.model.clone()
1334                } else {
1335                    session.model.clone()
1336                };
1337                if model.trim().is_empty() {
1338                    return Err(bamboo_domain::SessionActivationError::Internal(format!(
1339                        "child SessionInbox target {target_session_id} has no executable model"
1340                    )));
1341                }
1342                let disabled_tools = match session.metadata.get("disabled_tools") {
1343                    None => None,
1344                    Some(raw) => {
1345                        let tools = serde_json::from_str::<std::collections::BTreeSet<String>>(raw)
1346                            .map_err(|error| {
1347                                bamboo_domain::SessionActivationError::Internal(format!(
1348                                    "child SessionInbox target {target_session_id} has malformed disabled_tools: {error}"
1349                                ))
1350                            })?;
1351                        (!tools.is_empty()).then(|| tools.into_iter().collect())
1352                    }
1353                };
1354                LaunchPlan::Child {
1355                    scheduler,
1356                    parent_session_id,
1357                    model,
1358                    disabled_tools,
1359                }
1360            }
1361        };
1362        let event_sender =
1363            ResumeExecutionPort::get_or_create_event_sender(self, target_session_id).await;
1364
1365        let reservation = match reserve_runner_core(
1366            &self.agent_runners,
1367            &self.session_event_senders,
1368            target_session_id,
1369            &event_sender,
1370        )
1371        .await
1372        {
1373            ReserveOutcome::Reserved(reservation) => reservation,
1374            ReserveOutcome::AlreadyRunning(run_id) => {
1375                return Ok(SessionActivationReserveOutcome::AlreadyRunning { run_id });
1376            }
1377        };
1378        let run_id = reservation.run_id.clone();
1379        let launch = match launch_plan {
1380            LaunchPlan::Root(config) => {
1381                let execution_reservation =
1382                    SessionExecutionReservation::from_activation_placeholder(
1383                        target_session_id,
1384                        reservation,
1385                        self.agent
1386                            .activation_router()
1387                            .expect("SessionInbox activation requires an activation router")
1388                            .clone(),
1389                        self.agent_runners.clone(),
1390                    );
1391                // Launch and rollback share one exact RAII reservation. Dropping
1392                // an unlaunched SessionActivationLaunch cannot race a raw slot
1393                // removal against the reservation's router-placeholder cleanup.
1394                let reservation_cell = Arc::new(StdMutex::new(Some(execution_reservation)));
1395                let launch_reservation = reservation_cell.clone();
1396                let rollback_reservation = reservation_cell;
1397                let coordinator = self.clone();
1398                let launch_sessions = self.sessions.clone();
1399                let launch_session_id = session.id.clone();
1400                let launch_session = session.clone();
1401                let request_session_id = target_session_id.to_string();
1402                SessionActivationLaunch::new_with_async_rollback(
1403                    run_id,
1404                    move || {
1405                        let mut execution_reservation = launch_reservation
1406                            .lock()
1407                            .unwrap_or_else(std::sync::PoisonError::into_inner)
1408                            .take()
1409                            .expect("activation reservation launches or rolls back exactly once");
1410                        execution_reservation.mark_activation_published();
1411                        // The router publishes the exact owner before invoking
1412                        // this closure, so only now may the prepared snapshot
1413                        // replace the shared cache entry.
1414                        launch_sessions.insert(
1415                            launch_session_id,
1416                            Arc::new(crate::SessionSnapshot::new(launch_session)),
1417                        );
1418                        let request = ResumeSpawnRequest {
1419                            session_id: request_session_id,
1420                            session,
1421                            execution_reservation,
1422                            event_sender,
1423                            config: *config,
1424                        };
1425                        tokio::spawn(async move {
1426                            ResumeExecutionPort::spawn_resume_execution(&coordinator, request)
1427                                .await;
1428                        });
1429                    },
1430                    move || async move {
1431                        let reservation = rollback_reservation
1432                            .lock()
1433                            .unwrap_or_else(std::sync::PoisonError::into_inner)
1434                            .take();
1435                        if let Some(reservation) = reservation {
1436                            reservation.rollback_unpublished_activation().await;
1437                        }
1438                    },
1439                )
1440            }
1441            LaunchPlan::Child {
1442                scheduler,
1443                parent_session_id,
1444                model,
1445                disabled_tools,
1446            } => {
1447                let execution_reservation =
1448                    SessionExecutionReservation::from_activation_placeholder(
1449                        target_session_id,
1450                        reservation,
1451                        self.agent
1452                            .activation_router()
1453                            .expect("SessionInbox activation requires an activation router")
1454                            .clone(),
1455                        self.agent_runners.clone(),
1456                    );
1457                let reservation_cell = Arc::new(StdMutex::new(Some(execution_reservation)));
1458                let launch_reservation = reservation_cell.clone();
1459                let rollback_reservation = reservation_cell;
1460                let job = SpawnJob {
1461                    parent_session_id,
1462                    child_session_id: target_session_id.to_string(),
1463                    model,
1464                    disabled_tools,
1465                };
1466                let launch_sessions = self.sessions.clone();
1467                let launch_session_id = session.id.clone();
1468                let launch_session = session;
1469                SessionActivationLaunch::new_with_async_rollback(
1470                    run_id,
1471                    move || {
1472                        let mut execution_reservation = launch_reservation
1473                            .lock()
1474                            .unwrap_or_else(std::sync::PoisonError::into_inner)
1475                            .take()
1476                            .expect("activation reservation launches or rolls back exactly once");
1477                        execution_reservation.mark_activation_published();
1478                        // As with root activation, publish the prepared cache
1479                        // snapshot only after router ownership commits.
1480                        launch_sessions.insert(
1481                            launch_session_id,
1482                            Arc::new(crate::SessionSnapshot::new(launch_session)),
1483                        );
1484                        // Dropping a JoinHandle detaches the task. The captured
1485                        // combined reservation remains RAII-protected if the
1486                        // task is later aborted or unwinds during setup.
1487                        drop(scheduler.launch_reserved(job, execution_reservation));
1488                    },
1489                    move || async move {
1490                        let reservation = rollback_reservation
1491                            .lock()
1492                            .unwrap_or_else(std::sync::PoisonError::into_inner)
1493                            .take();
1494                        if let Some(reservation) = reservation {
1495                            reservation.rollback_unpublished_activation().await;
1496                        }
1497                    },
1498                )
1499            }
1500        };
1501        Ok(SessionActivationReserveOutcome::Reserved(launch))
1502    }
1503}
1504
1505/// Hidden resume message for a bash-completion self-resume (issue #84 Phase 2b).
1506/// Mirrors [`runtime_resume_message`]'s hidden/compressible shape so the resume
1507/// port's `has_pending_user_message` gate is satisfied.
1508///
1509/// `timed_out` selects the wording: the normal path (all shells finished)
1510/// announces completion; the deadline path (the 6h+10m wait ceiling was hit with
1511/// shells STILL running) must NOT claim the shells completed — it says they may
1512/// still be running so the model verifies with BashOutput instead of assuming
1513/// success on a false premise.
1514fn bash_completion_resume_message(bash_ids: &[String], timed_out: bool) -> Message {
1515    let body = if timed_out {
1516        format!(
1517            "Runtime notification: the background-Bash wait ceiling was reached while one or more \
1518             shell(s) ({}) may still be running. The session is being resumed so it is not \
1519             stranded; verify their actual status with BashOutput before assuming completion.",
1520            bash_ids.join(", ")
1521        )
1522    } else {
1523        format!(
1524            "Runtime notification: all background Bash shell(s) ({}) have completed. \
1525             Review their output with BashOutput and resume the task from where you left off.",
1526            bash_ids.join(", ")
1527        )
1528    };
1529    let mut message = Message::user(body);
1530    message.metadata = Some(serde_json::json!({
1531        RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
1532        RUNTIME_RESUME_MESSAGE_KIND_KEY: BASH_COMPLETION_RESUME_KIND,
1533    }));
1534    message.never_compress = false;
1535    message
1536}
1537
1538/// Decide whether the bash self-resume should retry its clear→append→resume
1539/// sequence after a resume attempt returned `outcome`, given that the persisted
1540/// bash wait is (`true`) / is not (`false`) still set on reload.
1541///
1542/// Retry **only** when the resume did NOT spawn (`Completed` — no pending user
1543/// message, i.e. our resume message was dropped — or `AlreadyRunning`) AND the
1544/// persisted bash wait is still set: the signature of the finalize-clobber, where
1545/// the suspending runner's one-shot final `merge_save_runtime` lands after our
1546/// save and reverts `waiting_for_bash=Some` while dropping our resume message, so
1547/// `has_pending_user_message` fails and nothing spawns. `Started` (resume fired)
1548/// and `NotFound` (session gone) never retry. Pure helper so the clobber
1549/// detection is unit-testable in isolation from async I/O.
1550fn bash_resume_should_retry(outcome: &ResumeOutcome, persisted_waiting_for_bash: bool) -> bool {
1551    match outcome {
1552        ResumeOutcome::Started { .. } | ResumeOutcome::NotFound => false,
1553        ResumeOutcome::Completed | ResumeOutcome::AlreadyRunning { .. } => {
1554            persisted_waiting_for_bash
1555        }
1556    }
1557}
1558
1559/// Whether a background-shell completion push should **resume** the owning loop
1560/// (vs merely enqueue an injection). Resume only when the loop is actually
1561/// suspended on a bash wait AND every shell it was waiting on has now finished —
1562/// resuming while other waited shells are still running would drop them back into
1563/// a foreground turn prematurely. The last shell to finish (or the backstop)
1564/// drives the resume; earlier ones enqueue their notice. Pure so the invariant is
1565/// unit-testable in isolation.
1566fn bash_completion_should_resume(
1567    loop_suspended_on_bash: bool,
1568    all_waited_shells_done: bool,
1569) -> bool {
1570    loop_suspended_on_bash && all_waited_shells_done
1571}
1572
1573#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1574enum BashCompletionDeliveryPlan {
1575    /// Preserve the durable wait and do not reserve a successor. The last
1576    /// sibling completion (or the wait backstop) activates the accumulated
1577    /// inbox in order.
1578    DurableOnly,
1579    /// The loop is live/not waiting; notify its current owner after admission.
1580    Activate,
1581    /// The last waited shell finished: clear the wait durably, then activate.
1582    ClearWaitThenActivate,
1583}
1584
1585fn bash_completion_delivery_plan(
1586    loop_suspended_on_bash: bool,
1587    all_waited_shells_done: bool,
1588) -> BashCompletionDeliveryPlan {
1589    if bash_completion_should_resume(loop_suspended_on_bash, all_waited_shells_done) {
1590        BashCompletionDeliveryPlan::ClearWaitThenActivate
1591    } else if loop_suspended_on_bash {
1592        BashCompletionDeliveryPlan::DurableOnly
1593    } else {
1594        BashCompletionDeliveryPlan::Activate
1595    }
1596}
1597
1598fn background_bash_post_tool_payload(info: &BashCompletionInfo) -> HookPayload {
1599    let success = info.status == "completed" && info.exit_code == Some(0);
1600    let response = serde_json::json!({
1601        "bash_id": info.bash_id,
1602        "command": info.command,
1603        "exit_code": info.exit_code,
1604        "status": info.status,
1605        "output_tail": info.output_tail,
1606    });
1607    HookPayload::ToolResult {
1608        tool_name: "Bash".to_string(),
1609        tool_call_id: info.bash_id.clone(),
1610        outcome: HookToolOutcome {
1611            success,
1612            result: success.then(|| response.to_string()),
1613            error: (!success).then(|| response.to_string()),
1614            needs_human: false,
1615            duration_ms: 0,
1616        },
1617    }
1618}
1619
1620fn append_background_bash_hook_feedback(info: &mut BashCompletionInfo, feedback: Vec<String>) {
1621    let feedback = feedback
1622        .into_iter()
1623        .map(|text| text.trim().to_string())
1624        .filter(|text| !text.is_empty())
1625        .collect::<Vec<_>>();
1626    if feedback.is_empty() {
1627        return;
1628    }
1629    if !info.output_tail.is_empty() {
1630        info.output_tail.push_str("\n\n");
1631    }
1632    info.output_tail.push_str("<post_tool_use_feedback>\n");
1633    info.output_tail.push_str(&feedback.join("\n"));
1634    info.output_tail.push_str("\n</post_tool_use_feedback>");
1635}
1636
1637async fn run_background_bash_post_tool_hooks(
1638    config: &bamboo_config::LifecycleHooksConfig,
1639    fallback_cwd: Option<std::path::PathBuf>,
1640    session: &Session,
1641    info: &mut BashCompletionInfo,
1642) -> bool {
1643    let runner = crate::HookRunner::new().with_lifecycle_config(config, fallback_cwd);
1644    if !runner.has_hooks_for(AgentHookPoint::AfterToolExecution) {
1645        return false;
1646    }
1647
1648    let mut runtime_state = session
1649        .agent_runtime_state
1650        .clone()
1651        .unwrap_or_else(|| AgentRuntimeState::new(&session.id));
1652    let outcome = runner
1653        .run_observer_hooks(
1654            AgentHookPoint::AfterToolExecution,
1655            &background_bash_post_tool_payload(info),
1656            session,
1657            &mut runtime_state,
1658            None,
1659        )
1660        .await;
1661    append_background_bash_hook_feedback(info, outcome.injected_contexts);
1662    true
1663}
1664
1665/// Apply the bash-resume state transition to a loaded session **in place**: clear
1666/// the `waiting_for_bash` wait, mark the runtime Idle, drop the suspension +
1667/// `runtime.suspend_reason`, and append `resume_message`. Returns `false` (a
1668/// no-op) when the session was not actually waiting on bash — the double-resume
1669/// guard shared by the push and the backstop. Pure (no I/O) so both
1670/// [`ChildCompletionCoordinator::perform_bash_resume`] and unit tests exercise the
1671/// exact same transition.
1672fn apply_bash_resume_transition(session: &mut Session, resume_message: &Message) -> bool {
1673    let mut runtime_state = read_runtime_state(session);
1674    if runtime_state.waiting_for_bash.is_none() {
1675        return false;
1676    }
1677    runtime_state.waiting_for_bash = None;
1678    runtime_state.status = AgentStatusState::Idle;
1679    runtime_state.suspension = None;
1680    write_runtime_state(session, &runtime_state);
1681    session.metadata.remove("runtime.suspend_reason");
1682    session.add_message(resume_message.clone());
1683    true
1684}
1685
1686/// Bash self-resume support (issue #84 Phase 2b; push follow-up).
1687impl ChildCompletionCoordinator {
1688    /// **Backstop** for a session suspended on `waiting_for_bash`. The primary,
1689    /// event-driven wake is the loop-facing push
1690    /// ([`BashCompletionSink::on_bash_completed`] → [`Self::deliver_bash_completion`]):
1691    /// the shell's completion task fires it the instant the process exits, and it
1692    /// resumes the loop directly. This task exists ONLY to catch a **lost push** —
1693    /// the completion landing in the window before the suspend was persisted (so
1694    /// the push saw no `waiting_for_bash` and only queued an injection), or a
1695    /// configuration with no sink wired — and to honour the wait ceiling.
1696    ///
1697    /// So it is deliberately NOT a hot spin: a coarse backoff (1 s → 30 s) that
1698    /// **yields to the push**. In the happy path the push has already cleared
1699    /// `waiting_for_bash` before the first check fires, so this returns after one
1700    /// cheap load with no registry polling at all. It only performs a resume when
1701    /// the shell(s) have finished but the loop is somehow still suspended, or the
1702    /// 6 h wait ceiling is reached.
1703    async fn bash_self_resume(&self, session_id: String, bash_ids: Vec<String>) {
1704        let mut delay = Duration::from_secs(1);
1705        let max_delay = Duration::from_secs(30);
1706        // Hard ceiling: the wait lease (6 h) + the registry GC TTL (5 min) +
1707        // margin. After this the shells are gone from the registry regardless,
1708        // so force-resume to avoid stranding the session on a GC edge case.
1709        let max_poll = Duration::from_secs(6 * 3600 + 600);
1710        let deadline = tokio::time::Instant::now() + max_poll;
1711
1712        loop {
1713            tokio::time::sleep(delay).await;
1714
1715            let Some(session) = self.load_session(&session_id).await else {
1716                tracing::info!(%session_id, "bash self-resume backstop: session gone; nothing to do");
1717                return;
1718            };
1719            if read_runtime_state(&session).waiting_for_bash.is_none() {
1720                // The push (or another path) already resumed. This is the common
1721                // case — the backstop yields silently after a single load.
1722                return;
1723            }
1724
1725            let still_running =
1726                bamboo_tools::tools::bash_runtime::running_shells_for_session(&session_id);
1727            let timed_out = tokio::time::Instant::now() >= deadline;
1728            if still_running.is_empty() || timed_out {
1729                // The shell(s) finished but the loop is still suspended → the push
1730                // was lost (pre-persist window / no sink), or the ceiling hit.
1731                // Resume under the shared per-session lock so we never race the
1732                // push or a concurrent child-completion resume.
1733                let guard = session_resume_lock(&session_id);
1734                let _held = guard.lock().await;
1735                tracing::warn!(
1736                    %session_id,
1737                    shell_count = bash_ids.len(),
1738                    timed_out,
1739                    "bash self-resume backstop engaged (push lost or wait ceiling reached)"
1740                );
1741                self.perform_bash_resume(
1742                    &session_id,
1743                    bash_completion_resume_message(&bash_ids, timed_out),
1744                )
1745                .await;
1746                return;
1747            }
1748
1749            delay = (delay * 2).min(max_delay);
1750        }
1751    }
1752
1753    /// Clear a session's `waiting_for_bash` state, append `resume_message`, and
1754    /// drive the parent resume — the shared clear→append→resume used by BOTH the
1755    /// event-driven push ([`Self::deliver_bash_completion`]) and the backstop poll
1756    /// ([`Self::bash_self_resume`]).
1757    ///
1758    /// **The caller MUST hold the [`session_resume_lock`] for `session_id`** so the
1759    /// load-check-clear-resume critical section is serialized against every other
1760    /// resume source (no double resume). No-op when the persisted wait was already
1761    /// cleared (another source handled it first).
1762    ///
1763    /// The clear→append→resume is a **bounded retry loop** that closes the
1764    /// finalize-clobber strand. The suspending runner's `finalize_task_context`
1765    /// runs a full `save_runtime_session` (same `merge_save_runtime`, which
1766    /// overwrites the whole `messages` array) that can land AFTER our save,
1767    /// reverting `waiting_for_bash=Some` and dropping our resume message, so
1768    /// `has_pending_user_message` fails and `resume_parent` returns `Completed`
1769    /// without spawning. We detect that (persisted wait still set after a
1770    /// non-`Started` outcome) and re-clear/re-append/re-resume. It converges
1771    /// because the runner's finalize persist is one-shot: once landed, our retry's
1772    /// save is the last writer, the message sticks, and resume fires.
1773    async fn perform_bash_resume(&self, session_id: &str, resume_message: Message) {
1774        let retry_backoff = Duration::from_millis(200);
1775        const MAX_RESUME_ATTEMPTS: u8 = 5;
1776        for attempt in 0..MAX_RESUME_ATTEMPTS {
1777            if attempt > 0 {
1778                tokio::time::sleep(retry_backoff).await;
1779            }
1780
1781            let Some(mut session) = self.load_session(session_id).await else {
1782                tracing::warn!(%session_id, "bash resume: session not found; nothing to resume");
1783                return;
1784            };
1785
1786            if !apply_bash_resume_transition(&mut session, &resume_message) {
1787                // Double-resume guard: the wait was already cleared by another
1788                // source (the push, the backstop, or a user-driven resume). Do
1789                // not append a duplicate message or request a redundant resume.
1790                tracing::info!(
1791                    %session_id, attempt,
1792                    "bash resume: persisted bash wait already cleared; nothing to resume"
1793                );
1794                return;
1795            }
1796            session.updated_at = Utc::now();
1797            self.save_and_cache(&mut session).await;
1798            tracing::info!(
1799                %session_id, attempt,
1800                "bash resume: cleared bash wait and appended resume message"
1801            );
1802
1803            let outcome = self.resume_parent(session_id.to_string()).await;
1804            match outcome {
1805                ResumeOutcome::Started { .. } => {
1806                    tracing::info!(%session_id, attempt, "bash resume: resume fired");
1807                    return;
1808                }
1809                ResumeOutcome::NotFound => {
1810                    tracing::warn!(%session_id, "bash resume: session vanished during resume");
1811                    return;
1812                }
1813                _ => {
1814                    // Completed (no pending user message ⇒ our resume message was
1815                    // dropped by the runner's finalize persist) or AlreadyRunning.
1816                    // Decide via the persisted bash wait: still set ⇒
1817                    // finalize-clobber ⇒ retry; cleared ⇒ the session is being
1818                    // handled (by us or a concurrent resume) ⇒ stop.
1819                    let clobbered = match self.load_session(session_id).await {
1820                        Some(reloaded) => read_runtime_state(&reloaded).waiting_for_bash.is_some(),
1821                        None => {
1822                            tracing::warn!(%session_id, "bash resume: session vanished after resume");
1823                            return;
1824                        }
1825                    };
1826                    if bash_resume_should_retry(&outcome, clobbered) {
1827                        tracing::warn!(
1828                            %session_id, attempt,
1829                            outcome = outcome.as_str(),
1830                            "bash resume: persisted wait still set after resume (finalize-clobber); retrying"
1831                        );
1832                        continue;
1833                    }
1834                    tracing::info!(
1835                        %session_id, attempt,
1836                        outcome = outcome.as_str(),
1837                        "bash resume: wait cleared and resume handled; stopping"
1838                    );
1839                    return;
1840                }
1841            }
1842        }
1843
1844        tracing::warn!(
1845            %session_id,
1846            attempts = MAX_RESUME_ATTEMPTS,
1847            "bash resume: exhausted clobber-retry budget without confirming resume; giving up"
1848        );
1849    }
1850}
1851
1852impl BashResumeHook for ChildCompletionCoordinator {
1853    fn arrange_bash_self_resume(&self, session_id: String, bash_ids: Vec<String>) {
1854        let coordinator = Arc::new(self.clone());
1855        tokio::spawn(async move {
1856            coordinator.bash_self_resume(session_id, bash_ids).await;
1857        });
1858    }
1859}
1860
1861/// Build the injected user-message body for a completed background shell — a
1862/// concise notice plus a bounded output tail — so the model can act on the
1863/// result without a mandatory `BashOutput` round-trip (issue #84 Phase 2b
1864/// follow-up).
1865fn bash_completion_injection_body(info: &BashCompletionInfo) -> String {
1866    let exit = match info.exit_code {
1867        Some(code) => code.to_string(),
1868        None => "none (signal/killed)".to_string(),
1869    };
1870    let mut body = format!(
1871        "Runtime notification: background shell `{}` (`{}`) finished — status {}, exit code {}.",
1872        info.bash_id, info.command, info.status, exit
1873    );
1874    if info.output_tail.trim().is_empty() {
1875        body.push_str(" It produced no captured output.");
1876    } else {
1877        body.push_str("\n\nOutput tail:\n");
1878        body.push_str(&info.output_tail);
1879    }
1880    body.push_str(&format!(
1881        "\n\nUse BashOutput with bash_id=\"{}\" for the full output, then continue the task.",
1882        info.bash_id
1883    ));
1884    body
1885}
1886
1887fn bash_completion_envelope(info: &BashCompletionInfo) -> bamboo_domain::SessionMessageEnvelope {
1888    let provider_message = bash_resume_message_from_info(info);
1889    let data = serde_json::json!({
1890        "session_id": info.session_id,
1891        "bash_id": info.bash_id,
1892        "command": info.command,
1893        "exit_code": info.exit_code,
1894        "status": info.status,
1895        "output_tail": info.output_tail,
1896    });
1897    // The id covers the immutable completion snapshot. An exact delivery retry
1898    // (whose transport timestamp may differ) is idempotent; a changed status,
1899    // output tail, command, or exit code is a distinct correction rather than
1900    // silently reusing one id with different semantics.
1901    let identity = data.clone();
1902    bamboo_domain::SessionMessageEnvelope {
1903        id: bamboo_domain::SessionMessageId::stable("background_bash_completion", &identity),
1904        source: bamboo_domain::SessionMessageSource::Runtime {
1905            subsystem: "background_bash".to_string(),
1906        },
1907        target_session_id: info.session_id.clone(),
1908        kind: bamboo_domain::SessionMessageKind::RuntimeInstruction,
1909        body: bamboo_domain::SessionMessageBody::RuntimeInstruction(
1910            bamboo_domain::SessionRuntimeInstruction {
1911                instruction: "background_bash_completed".to_string(),
1912                content: Some(bamboo_domain::SessionMessageContent::text(
1913                    bash_completion_injection_body(info),
1914                )),
1915                data: Some(data),
1916                provider_message: Some(session_provider_message(&provider_message)),
1917            },
1918        ),
1919        created_at: Utc::now(),
1920        thread_id: None,
1921        in_reply_to: None,
1922        attempt: None,
1923        correlation_id: Some(info.bash_id.clone()),
1924    }
1925}
1926
1927/// Enqueue a completed shell's summary as a pending injected message on the
1928/// owning session for the rolling-upgrade polling backstop. New push delivery
1929/// uses [`SessionMessenger`]; this helper remains only so an older process that
1930/// has not wired the typed delivery plane can still resume persisted sessions.
1931/// Race-safe: `update_runtime_config` loads and saves under the per-session lock.
1932async fn enqueue_bash_completion_injection(
1933    persistence: &LockedSessionStore,
1934    info: &BashCompletionInfo,
1935) -> std::io::Result<Option<Session>> {
1936    let body = bash_completion_injection_body(info);
1937    let queued = serde_json::json!({
1938        "content": body,
1939        "created_at": Utc::now(),
1940    });
1941    persistence
1942        .update_runtime_config(&info.session_id, move |session| {
1943            let mut pending = session.pending_injected_messages().unwrap_or_default();
1944            pending.push(queued);
1945            session.set_pending_injected_messages(pending);
1946        })
1947        .await
1948}
1949
1950/// Build the hidden, compressible resume message for a completed background
1951/// shell — the same rich notice body used for a live-loop injection
1952/// ([`bash_completion_injection_body`]), but tagged as a resume message so it
1953/// satisfies the `has_pending_user_message` gate that lets a suspended session
1954/// spawn. This is what the **push** appends when it wakes a suspended loop, so
1955/// the model gets the shell's status + output tail in one shot without a
1956/// separate `BashOutput` round-trip.
1957fn bash_resume_message_from_info(info: &BashCompletionInfo) -> Message {
1958    let mut message = Message::user(bash_completion_injection_body(info));
1959    message.metadata = Some(serde_json::json!({
1960        RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
1961        RUNTIME_RESUME_MESSAGE_KIND_KEY: BASH_COMPLETION_RESUME_KIND,
1962    }));
1963    message.never_compress = false;
1964    message
1965}
1966
1967impl ChildCompletionCoordinator {
1968    /// Loop-facing delivery of a completed background shell. Two paths, chosen
1969    /// under the per-session resume lock so we never race the backstop poll or a
1970    /// concurrent child-completion resume:
1971    ///
1972    /// - **Suspended loop** (the model ended its turn with the shell running, so
1973    ///   `waiting_for_bash` is set) AND every waited shell has now finished →
1974    ///   **resume the loop directly**, event-driven, appending the rich completion
1975    ///   notice as the resume message. This is the push's whole point: no polling.
1976    /// - Every notice is then delivered through the same typed SessionMessenger
1977    ///   as peer/user steering. The durable inbox and activation router decide
1978    ///   whether to notify the current owner or reserve one successor.
1979    async fn deliver_bash_completion(&self, mut info: BashCompletionInfo) {
1980        // A background shell completes outside the originating tool call, so
1981        // fire its PostToolUse seam here before routing the completion into the
1982        // next round/resume message. Do not hold the resume lock while an
1983        // arbitrary command hook runs: its configured timeout must never block
1984        // the independent liveness backstop.
1985        if let Some(hook_session) = self.load_session(&info.session_id).await {
1986            let config_snapshot = self.config.read().await.clone();
1987            let _ = run_background_bash_post_tool_hooks(
1988                &config_snapshot.lifecycle_hooks,
1989                Some(self.app_data_dir.clone()),
1990                &hook_session,
1991                &mut info,
1992            )
1993            .await;
1994        }
1995
1996        let guard = session_resume_lock(&info.session_id);
1997        let _held = guard.lock().await;
1998
1999        let Some(session) = self.load_session(&info.session_id).await else {
2000            tracing::warn!(
2001                session_id = %info.session_id,
2002                bash_id = %info.bash_id,
2003                "background bash completion: owning session not found; nothing to notify"
2004            );
2005            return;
2006        };
2007
2008        let waiting = read_runtime_state(&session).waiting_for_bash.is_some();
2009        // The producer flips the shell's `running` flag false BEFORE firing this
2010        // push, so a now-empty per-session registry means every shell the loop was
2011        // waiting on has finished — safe to resume. If OTHER waited shells are
2012        // still running, fall through to the enqueue path and let the last one (or
2013        // the backstop) drive the resume.
2014        let all_shells_done =
2015            bamboo_tools::tools::bash_runtime::running_shells_for_session(&info.session_id)
2016                .is_empty();
2017        let delivery_plan = bash_completion_delivery_plan(waiting, all_shells_done);
2018
2019        // Stage 1: make the completion durable BEFORE clearing the wait. The
2020        // activation is intentionally deferred until the wait-state mutation
2021        // below is durably committed; otherwise a successor can observe the old
2022        // suspension, while clear-before-deliver can lose the only wake on
2023        // crash.
2024        let messenger = self.agent.session_messenger().cloned();
2025        let admission = match messenger.as_ref() {
2026            Some(messenger) => match messenger.admit(bash_completion_envelope(&info)).await {
2027                Ok(admission) => Some(admission),
2028                Err(error) => {
2029                    tracing::warn!(
2030                        session_id = %info.session_id,
2031                        bash_id = %info.bash_id,
2032                        %error,
2033                        "background bash completion: durable SessionInbox admission failed; leaving wait armed"
2034                    );
2035                    return;
2036                }
2037            },
2038            None => {
2039                tracing::warn!(
2040                    session_id = %info.session_id,
2041                    bash_id = %info.bash_id,
2042                    "SessionMessenger unavailable; using compatibility injection before clearing wait"
2043                );
2044                match enqueue_bash_completion_injection(&self.persistence, &info).await {
2045                    Ok(Some(_)) => None,
2046                    Ok(None) => {
2047                        tracing::warn!(
2048                            session_id = %info.session_id,
2049                            "background bash compatibility target disappeared"
2050                        );
2051                        return;
2052                    }
2053                    Err(error) => {
2054                        tracing::warn!(
2055                            session_id = %info.session_id,
2056                            %error,
2057                            "background bash compatibility admission failed; leaving wait armed"
2058                        );
2059                        return;
2060                    }
2061                }
2062            }
2063        };
2064
2065        if delivery_plan == BashCompletionDeliveryPlan::DurableOnly {
2066            tracing::info!(
2067                session_id = %info.session_id,
2068                bash_id = %info.bash_id,
2069                "background bash completion is durable; sibling shells still run, so wait remains armed"
2070            );
2071            return;
2072        }
2073
2074        if let (Some(messenger), Some(admission)) = (messenger.as_ref(), admission.as_ref()) {
2075            if let Err(error) = messenger.prepare_activation(admission).await {
2076                tracing::warn!(
2077                    session_id = %info.session_id,
2078                    bash_id = %info.bash_id,
2079                    %error,
2080                    "background bash completion activation watermark failed; leaving wait armed"
2081                );
2082                return;
2083            }
2084        }
2085
2086        if delivery_plan == BashCompletionDeliveryPlan::ClearWaitThenActivate {
2087            tracing::info!(
2088                session_id = %info.session_id,
2089                bash_id = %info.bash_id,
2090                status = %info.status,
2091                "background bash completion: push-resuming suspended loop (event-driven)"
2092            );
2093            let mut resumable = session.clone();
2094            let mut runtime_state = read_runtime_state(&resumable);
2095            runtime_state.waiting_for_bash = None;
2096            runtime_state.status = AgentStatusState::Idle;
2097            runtime_state.suspension = None;
2098            write_runtime_state(&mut resumable, &runtime_state);
2099            resumable.metadata.remove("runtime.suspend_reason");
2100            resumable.updated_at = Utc::now();
2101            if let Err(error) = self.persistence.merge_save_runtime(&mut resumable).await {
2102                tracing::warn!(
2103                    session_id = %info.session_id,
2104                    %error,
2105                    "background bash completion is durable but wait-state clear failed; leaving activation to the wait backstop"
2106                );
2107                return;
2108            }
2109            self.sessions.insert(
2110                resumable.id.clone(),
2111                Arc::new(crate::SessionSnapshot::new(resumable)),
2112            );
2113        }
2114
2115        // Stage 3: activation is allowed only after the durable wait-state
2116        // clear. A crash before here leaves either the wait backstop or startup
2117        // inbox reconciliation able to recover.
2118        let (Some(messenger), Some(admission)) = (messenger, admission) else {
2119            if delivery_plan == BashCompletionDeliveryPlan::ClearWaitThenActivate {
2120                let _ = self.resume_parent(info.session_id.clone()).await;
2121            }
2122            return;
2123        };
2124        match messenger.activate_prepared(&admission).await {
2125            Ok(receipt) => tracing::info!(
2126                session_id = %info.session_id,
2127                bash_id = %info.bash_id,
2128                status = %info.status,
2129                waiting,
2130                generation = receipt.delivery.generation,
2131                activation = ?receipt.activation,
2132                "background bash completion delivered through SessionMessenger"
2133            ),
2134            Err(error) => tracing::warn!(
2135                session_id = %info.session_id,
2136                bash_id = %info.bash_id,
2137                %error,
2138                "background bash completion: SessionMessenger delivery failed"
2139            ),
2140        }
2141    }
2142}
2143
2144impl BashCompletionSink for ChildCompletionCoordinator {
2145    fn on_bash_completed(&self, info: BashCompletionInfo) {
2146        // Best-effort, off the shell's completion-poll task: hand the delivery to
2147        // a detached task so the producer is never blocked (mirrors
2148        // `arrange_bash_self_resume`).
2149        let coordinator = Arc::new(self.clone());
2150        tokio::spawn(async move {
2151            coordinator.deliver_bash_completion(info).await;
2152        });
2153    }
2154}
2155
2156// ---------------------------------------------------------------------------
2157// Child-wait watchdog (issue #546)
2158// ---------------------------------------------------------------------------
2159
2160/// How often the child-wait watchdog sweeps suspended sessions.
2161const CHILD_WAIT_SWEEP_INTERVAL_SECS: u64 = 30;
2162/// Leave a freshly registered wait alone for this long so the event-driven
2163/// completion push always gets the first shot (and just-enqueued spawn jobs
2164/// have time to persist their running marker).
2165const CHILD_WAIT_REGISTRATION_GRACE_SECS: i64 = 60;
2166/// A waited child with NO live runner and a non-terminal index status is
2167/// declared dead once its control-plane has been quiet for this long.
2168const DEAD_CHILD_GRACE_SECS: i64 = 120;
2169/// Slack on top of the per-child liveness policy before a Running-but-frozen
2170/// runner entry (dead task) is force-finalized by the sweeper. The per-child
2171/// watchdog cancels at `max_idle`/`max_total` and the child then publishes its
2172/// own timeout; an entry frozen this far PAST those limits proves that
2173/// machinery is dead.
2174const STALE_RUNNER_SLACK_SECS: i64 = 600;
2175
2176/// Whether a waited child's index status means the sweeper must consider it
2177/// DEAD when nothing is driving it: non-terminal and not legitimately
2178/// suspended. `None` = never ran (created-but-never-started, or a spawn that
2179/// was lost before its running marker persisted).
2180fn is_dead_child_candidate_status(status: Option<&str>) -> bool {
2181    match status {
2182        // "suspended" children wait on a human / their own children / bash —
2183        // their wake has its own driver; never declare them dead here.
2184        Some(status) => !is_terminal_child_status(status) && status != "suspended",
2185        None => true,
2186    }
2187}
2188
2189/// Whether a reported completion's child id is genuinely a child of the parent
2190/// it claims (issue #546 read-side disclosure guard). `SubAgent.wait` ids are
2191/// model-provided, and the watchdog resolves an unowned id by publishing a
2192/// synthetic completion so the parent is unstranded — but the parent must NEVER
2193/// receive a FOREIGN session's content folded into its transcript.
2194/// `child_parent_linkage` is that session's own `parent_session_id` (`None`
2195/// when the session does not exist). Pure so the rule is unit-testable.
2196fn completion_child_is_owned(reported_parent: &str, child_parent_linkage: Option<&str>) -> bool {
2197    child_parent_linkage == Some(reported_parent)
2198}
2199
2200/// Pick which terminal child's completion to replay when the wait is already
2201/// satisfied but the parent is still suspended (lost wake). Prefer an
2202/// error-like child so a `FirstError` policy re-evaluates truthfully.
2203fn select_replay_child(terminal: &[(String, String)]) -> Option<&(String, String)> {
2204    terminal
2205        .iter()
2206        .find(|(_, status)| is_error_like(status))
2207        .or_else(|| terminal.last())
2208}
2209
2210fn child_wait_watchdog_resume_message(body: String) -> Message {
2211    let mut message = Message::user(body);
2212    message.metadata = Some(serde_json::json!({
2213        RUNTIME_RESUME_MESSAGE_HIDDEN_KEY: true,
2214        RUNTIME_RESUME_MESSAGE_KIND_KEY: "child_wait_watchdog_resume",
2215    }));
2216    message.never_compress = false;
2217    message
2218}
2219
2220fn empty_child_wait_message() -> Message {
2221    child_wait_watchdog_resume_message(
2222        "Runtime notification: this session was suspended waiting for child sessions, but the \
2223         wait tracked no children (internal inconsistency). The session has been resumed; use \
2224         SubAgent.list to inspect child state and continue the task."
2225            .to_string(),
2226    )
2227}
2228
2229fn child_wait_lease_expired_message(child_ids: &[String]) -> Message {
2230    child_wait_watchdog_resume_message(format!(
2231        "Runtime notification: the wait lease for child session(s) [{}] expired before they all \
2232         reported completion. They were NOT cancelled and may still be running or already \
2233         finished — verify their actual status with SubAgent.list / SubAgent.get before assuming \
2234         anything, then continue the task.",
2235        child_ids.join(", ")
2236    ))
2237}
2238
2239/// Child-wait watchdog (issue #546): the heartbeat backstop for parents
2240/// suspended on `waiting_for_children`.
2241///
2242/// The primary wake is the event-driven completion push (child terminal →
2243/// [`ChildCompletionHandler::on_child_completed`] → resume). This sweeper
2244/// exists because ANY break in that chain — a panicked child task, a dead
2245/// spawn scheduler, a process restart, a clobbered/exhausted resume, a wait
2246/// registered over an already-terminal child — previously stranded the parent
2247/// forever. It mirrors the bash backstop's philosophy: coarse, cheap, yields
2248/// to the push, and only acts when the durable state proves nothing else can.
2249///
2250/// All wake decisions funnel through the SAME machinery the push uses
2251/// (synthetic/replayed completions → `on_child_completed`, per-parent
2252/// serialization via [`session_resume_lock`]), so there is exactly one resume
2253/// implementation.
2254impl ChildCompletionCoordinator {
2255    /// Spawn the watchdog: one boot-time reconciliation pass, then a sweep
2256    /// every [`CHILD_WAIT_SWEEP_INTERVAL_SECS`]. Call once at server startup.
2257    pub fn spawn_child_wait_watchdog(self: &Arc<Self>) {
2258        let coordinator = Arc::clone(self);
2259        tokio::spawn(async move {
2260            use futures::FutureExt;
2261            if std::panic::AssertUnwindSafe(coordinator.reconcile_orphans_at_boot())
2262                .catch_unwind()
2263                .await
2264                .is_err()
2265            {
2266                tracing::error!("child-wait watchdog: boot reconciliation panicked");
2267            }
2268            let mut ticker = tokio::time::interval(std::time::Duration::from_secs(
2269                CHILD_WAIT_SWEEP_INTERVAL_SECS,
2270            ));
2271            ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
2272            // Skip the immediate tick (boot reconciliation just ran).
2273            ticker.tick().await;
2274            loop {
2275                ticker.tick().await;
2276                if std::panic::AssertUnwindSafe(coordinator.sweep_child_waits())
2277                    .catch_unwind()
2278                    .await
2279                    .is_err()
2280                {
2281                    tracing::error!("child-wait watchdog: sweep panicked; continuing");
2282                }
2283            }
2284        });
2285    }
2286
2287    /// One-shot startup reconciliation: a process restart kills every in-flight
2288    /// child task AND the in-memory bash backstop polls, but the durable state
2289    /// (child index status `running`, parents suspended on bash) survives —
2290    /// previously stranding those parents forever.
2291    async fn reconcile_orphans_at_boot(&self) {
2292        let cutoff = Utc::now();
2293
2294        // (1) Children left `running` by the previous process: no task in THIS
2295        // process is driving them, so no completion will ever fire. Mark them
2296        // terminal and wake their parents through the canonical path. (Root
2297        // sessions left `running` are user-visible and user-recoverable; only
2298        // children hold a suspended parent hostage.)
2299        let running = self
2300            .storage
2301            .list_sessions_by_run_status("running")
2302            .await
2303            .unwrap_or_default();
2304        for (child_id, parent_id) in running {
2305            let Some(parent_id) = parent_id else { continue };
2306            if self.runner_is_running(&child_id).await {
2307                continue;
2308            }
2309            let Some(control_plane) = self.load_control_plane(&child_id).await else {
2310                continue;
2311            };
2312            // A child that started AFTER boot is alive by definition — the
2313            // cutoff guards the tiny window where a fresh spawn races this scan.
2314            if control_plane.updated_at >= cutoff {
2315                continue;
2316            }
2317            tracing::warn!(
2318                child_session_id = %child_id,
2319                parent_session_id = %parent_id,
2320                "boot reconciliation: child was running when the process died; \
2321                 marking it error and waking the parent"
2322            );
2323            self.synthesize_child_completion(
2324                &parent_id,
2325                &child_id,
2326                "error",
2327                Some(
2328                    "orphaned by server restart: the process died while this child session \
2329                     was running"
2330                        .to_string(),
2331                ),
2332            )
2333            .await;
2334        }
2335
2336        // (2) Sessions suspended on `waiting_for_bash`: their backstop poll was
2337        // an in-memory task that died with the process. Re-arm it — with the
2338        // shell registry empty after a restart it resumes on its first check.
2339        let suspended = self
2340            .storage
2341            .list_sessions_by_run_status("suspended")
2342            .await
2343            .unwrap_or_default();
2344        for (session_id, _) in suspended {
2345            let Some(control_plane) = self.load_control_plane(&session_id).await else {
2346                continue;
2347            };
2348            if control_plane
2349                .metadata
2350                .get("runtime.suspend_reason")
2351                .map(String::as_str)
2352                != Some("waiting_for_bash")
2353            {
2354                continue;
2355            }
2356            if let Some(wait) = read_runtime_state(&control_plane).waiting_for_bash {
2357                tracing::warn!(
2358                    %session_id,
2359                    "boot reconciliation: re-arming bash self-resume backstop lost in restart"
2360                );
2361                let coordinator = self.clone();
2362                tokio::spawn(async move {
2363                    coordinator
2364                        .bash_self_resume(session_id, wait.bash_ids)
2365                        .await;
2366                });
2367            }
2368        }
2369    }
2370
2371    async fn runner_is_running(&self, session_id: &str) -> bool {
2372        let runners = self.agent_runners.read().await;
2373        runners
2374            .get(session_id)
2375            .is_some_and(|runner| matches!(runner.status, AgentStatus::Running))
2376    }
2377
2378    async fn load_control_plane(&self, session_id: &str) -> Option<Session> {
2379        match self.storage.load_runtime_control_plane(session_id).await {
2380            Ok(session) => session,
2381            Err(error) => {
2382                tracing::warn!(
2383                    %session_id,
2384                    %error,
2385                    "child-wait watchdog: failed to load session control plane"
2386                );
2387                None
2388            }
2389        }
2390    }
2391
2392    /// One sweep: inspect every session whose index status is `suspended`.
2393    /// Cheap in the common case — the candidate list is small and each check is
2394    /// a sidecar (control-plane) load.
2395    async fn sweep_child_waits(&self) {
2396        let suspended = match self.storage.list_sessions_by_run_status("suspended").await {
2397            Ok(entries) => entries,
2398            Err(error) => {
2399                tracing::warn!(%error, "child-wait watchdog: failed to list suspended sessions");
2400                return;
2401            }
2402        };
2403        for (session_id, _) in suspended {
2404            self.sweep_one_suspended_session(&session_id).await;
2405        }
2406    }
2407
2408    async fn sweep_one_suspended_session(&self, session_id: &str) {
2409        // A live runner means the session already resumed (the index status
2410        // only advances at its next terminal) — nothing to do.
2411        if self.runner_is_running(session_id).await {
2412            return;
2413        }
2414        let Some(session) = self.load_control_plane(session_id).await else {
2415            return;
2416        };
2417        let runtime_state = read_runtime_state(&session);
2418        let suspend_reason = session
2419            .metadata
2420            .get("runtime.suspend_reason")
2421            .map(String::as_str)
2422            .unwrap_or_default()
2423            .to_string();
2424        match (
2425            suspend_reason.as_str(),
2426            runtime_state.waiting_for_children.clone(),
2427        ) {
2428            // Human-gated or bash-owned waits: not ours to time out.
2429            ("waiting_for_bash", _)
2430            | ("awaiting_clarification", _)
2431            | ("awaiting_parent_approval", _) => {}
2432            (_, Some(wait)) => self.sweep_child_wait(session_id, wait).await,
2433            // Suspended with NO armed wait: either the coordinator cleared the
2434            // wait but its resume never spawned, or state is half-cleared.
2435            ("waiting_for_children", None) | ("", None) => {
2436                self.rescue_stranded_resume(session_id).await;
2437            }
2438            _ => {}
2439        }
2440    }
2441
2442    /// Evaluate one armed child wait against reality and act on what the
2443    /// durable state proves: dead children are synthesized terminal, an
2444    /// already-satisfied wait gets its lost wake replayed, an expired lease or
2445    /// empty wait force-resumes the parent.
2446    async fn sweep_child_wait(&self, parent_session_id: &str, wait: WaitingForChildrenState) {
2447        let now = Utc::now();
2448
2449        if wait.child_session_ids.is_empty() {
2450            tracing::warn!(
2451                %parent_session_id,
2452                "child-wait watchdog: wait armed over an empty child set; force-resuming"
2453            );
2454            self.force_resume_child_wait(parent_session_id, empty_child_wait_message())
2455                .await;
2456            return;
2457        }
2458
2459        // The 6h lease (previously written but never read). Expiry does not
2460        // kill children — child runners own child liveness; the parent is
2461        // resumed with a verify-don't-assume note.
2462        if wait.timeout_at.is_some_and(|deadline| now >= deadline) {
2463            tracing::warn!(
2464                %parent_session_id,
2465                "child-wait watchdog: wait lease expired; force-resuming parent"
2466            );
2467            self.force_resume_child_wait(
2468                parent_session_id,
2469                child_wait_lease_expired_message(&wait.child_session_ids),
2470            )
2471            .await;
2472            return;
2473        }
2474
2475        // Yield to the event-driven push on fresh waits.
2476        if now.signed_duration_since(wait.registered_at).num_seconds()
2477            < CHILD_WAIT_REGISTRATION_GRACE_SECS
2478        {
2479            return;
2480        }
2481
2482        let statuses: HashMap<String, Option<String>> = self
2483            .storage
2484            .list_child_run_statuses(parent_session_id)
2485            .await
2486            .unwrap_or_default()
2487            .into_iter()
2488            .collect();
2489
2490        struct DeadChild {
2491            child_id: String,
2492            status: String,
2493            reason: String,
2494            /// `false` = the id does not name a child of THIS parent (wait ids
2495            /// are model-provided and unvalidated): publish the wake but never
2496            /// persist onto / cancel / finalize the named session.
2497            owned: bool,
2498        }
2499
2500        let mut terminal: Vec<(String, String)> = Vec::new();
2501        let mut dead: Vec<DeadChild> = Vec::new();
2502        for child_id in &wait.child_session_ids {
2503            let status = statuses.get(child_id).and_then(|status| status.as_deref());
2504            if let Some(status) = status {
2505                if is_terminal_child_status(status) {
2506                    terminal.push((child_id.clone(), status.to_string()));
2507                    continue;
2508                }
2509            }
2510            if !is_dead_child_candidate_status(status) {
2511                continue;
2512            }
2513
2514            // Ownership check BEFORE anything destructive: `SubAgent.wait` ids
2515            // are model-provided and unvalidated, so an id absent from the
2516            // parent-scoped status map may name a REAL session in another tree
2517            // (a grandchild, a foreign root). Such a session must never be
2518            // mutated, cancelled, or finalized here — but the parent's bogus
2519            // wait entry still needs a synthetic completion to clear it.
2520            let control_plane = self.load_control_plane(child_id).await;
2521            let owned = control_plane
2522                .as_ref()
2523                .is_some_and(|cp| cp.parent_session_id.as_deref() == Some(parent_session_id));
2524            if !owned {
2525                dead.push(DeadChild {
2526                    child_id: child_id.clone(),
2527                    status: "error".to_string(),
2528                    reason: if control_plane.is_some() {
2529                        "waited-on session id is not a child of this session; clearing it \
2530                         from the wait without touching that session"
2531                            .to_string()
2532                    } else {
2533                        "waited-on child session does not exist".to_string()
2534                    },
2535                    owned: false,
2536                });
2537                continue;
2538            }
2539
2540            let runner = { self.agent_runners.read().await.get(child_id).cloned() };
2541            match runner {
2542                Some(runner) if matches!(runner.status, AgentStatus::Running) => {
2543                    // A live-looking runner: only intervene when it is frozen
2544                    // far PAST the per-child liveness limits — which proves the
2545                    // per-child watchdog machinery itself is dead (task
2546                    // panicked or lost), because it would have cancelled and
2547                    // published a timeout long before.
2548                    let last_activity = runner.last_activity_at().unwrap_or(runner.started_at);
2549                    let idle_secs = now.signed_duration_since(last_activity).num_seconds();
2550                    let total_secs = now.signed_duration_since(runner.started_at).num_seconds();
2551                    let policy = match &control_plane {
2552                        Some(child) => {
2553                            crate::runtime::execution::spawn::watchdog_policy_for_session(child)
2554                        }
2555                        None => Default::default(),
2556                    };
2557                    let idle_limit = policy.max_idle_secs.saturating_add(STALE_RUNNER_SLACK_SECS);
2558                    let total_limit = policy
2559                        .max_total_secs
2560                        .saturating_add(STALE_RUNNER_SLACK_SECS);
2561                    if idle_secs >= idle_limit || total_secs >= total_limit {
2562                        runner.cancel_token.cancel();
2563                        dead.push(DeadChild {
2564                            child_id: child_id.clone(),
2565                            status: "timeout".to_string(),
2566                            reason: format!(
2567                                "child runner stalled: no events for {idle_secs}s \
2568                                 (limit {idle_limit}s including watchdog slack); \
2569                                 force-finalized by the child-wait watchdog"
2570                            ),
2571                            owned: true,
2572                        });
2573                    }
2574                }
2575                _ => {
2576                    // Nothing is driving this child, yet its index status will
2577                    // never advance by itself. The grace covers the enqueue →
2578                    // running-marker window and slow spawn queues.
2579                    let quiet_secs = control_plane
2580                        .as_ref()
2581                        .map(|child| now.signed_duration_since(child.updated_at).num_seconds())
2582                        .unwrap_or(i64::MAX);
2583                    if quiet_secs >= DEAD_CHILD_GRACE_SECS {
2584                        dead.push(DeadChild {
2585                            child_id: child_id.clone(),
2586                            status: "error".to_string(),
2587                            reason: format!(
2588                                "child runner lost (crashed task, dropped spawn job, or \
2589                                 process restart): index status {status:?} with no live \
2590                                 runner driving it"
2591                            ),
2592                            owned: true,
2593                        });
2594                    }
2595                }
2596            }
2597        }
2598
2599        if !dead.is_empty() {
2600            for entry in dead {
2601                tracing::warn!(
2602                    %parent_session_id,
2603                    child_session_id = %entry.child_id,
2604                    status = %entry.status,
2605                    reason = %entry.reason,
2606                    owned = entry.owned,
2607                    "child-wait watchdog: synthesizing terminal completion for dead child"
2608                );
2609                if entry.owned {
2610                    self.synthesize_child_completion(
2611                        parent_session_id,
2612                        &entry.child_id,
2613                        &entry.status,
2614                        Some(entry.reason),
2615                    )
2616                    .await;
2617                } else {
2618                    // Foreign / nonexistent id: wake the parent only.
2619                    self.publish_synthetic_completion(
2620                        parent_session_id,
2621                        &entry.child_id,
2622                        &entry.status,
2623                        Some(entry.reason),
2624                    )
2625                    .await;
2626                }
2627            }
2628            // The publishes above re-evaluate the wait policy themselves.
2629            return;
2630        }
2631
2632        // No dead children — but if the terminal set ALREADY satisfies the
2633        // policy, the original wake was lost (clobbered resume / retry budget
2634        // exhausted / completion raced the wait registration). Replay one real
2635        // completion through the canonical path; `on_child_completed` is
2636        // idempotent for an already-cleared wait.
2637        let terminal_ids: Vec<String> = terminal.iter().map(|(id, _)| id.clone()).collect();
2638        if let Some((child_id, status)) = select_replay_child(&terminal) {
2639            if wait_policy_satisfied(
2640                wait.wait_for,
2641                &wait.child_session_ids,
2642                &terminal_ids,
2643                child_id,
2644                status,
2645            ) {
2646                tracing::warn!(
2647                    %parent_session_id,
2648                    child_session_id = %child_id,
2649                    "child-wait watchdog: wait already satisfied but parent still suspended \
2650                     (lost wake); replaying the completion"
2651                );
2652                let error = self
2653                    .load_control_plane(child_id)
2654                    .await
2655                    .and_then(|child| child.last_run_error());
2656                self.publish_synthetic_completion(parent_session_id, child_id, status, error)
2657                    .await;
2658            }
2659        }
2660    }
2661
2662    /// Persist a synthesized terminal status on the child (so the index flips
2663    /// and nothing re-detects or re-suspends on it), finalize any lingering
2664    /// runner entry (so a future re-run can reserve), then publish through the
2665    /// canonical completion path — broadcast + `on_child_completed`, exactly
2666    /// like a real child terminal.
2667    async fn synthesize_child_completion(
2668        &self,
2669        parent_session_id: &str,
2670        child_session_id: &str,
2671        status: &str,
2672        error: Option<String>,
2673    ) {
2674        match self.storage.load_session(child_session_id).await {
2675            Ok(Some(mut child)) => {
2676                // Ownership guard (defense in depth — callers check too): only
2677                // a session that IS a child of this parent may be mutated. An
2678                // arbitrary session id named in a wait still wakes the parent
2679                // via the publish below, but its own state stays untouched.
2680                if child.parent_session_id.as_deref() != Some(parent_session_id) {
2681                    tracing::warn!(
2682                        %parent_session_id,
2683                        child_session_id = %child.id,
2684                        "child-wait watchdog: refusing to synthesize status onto a session \
2685                         that is not a child of this parent"
2686                    );
2687                    self.publish_synthetic_completion(
2688                        parent_session_id,
2689                        child_session_id,
2690                        status,
2691                        error,
2692                    )
2693                    .await;
2694                    return;
2695                }
2696                child.set_last_run_status(status);
2697                match &error {
2698                    Some(message) => child.set_last_run_error(message.clone()),
2699                    None => child.clear_last_run_error(),
2700                }
2701                child.updated_at = Utc::now();
2702                if let Err(save_error) = self.persistence.merge_save_runtime(&mut child).await {
2703                    tracing::warn!(
2704                        child_session_id = %child.id,
2705                        %save_error,
2706                        "child-wait watchdog: failed to persist synthesized terminal status"
2707                    );
2708                }
2709                self.sessions.insert(
2710                    child.id.clone(),
2711                    Arc::new(crate::SessionSnapshot::new(child)),
2712                );
2713            }
2714            Ok(None) => {}
2715            Err(load_error) => {
2716                tracing::warn!(
2717                    %child_session_id,
2718                    %load_error,
2719                    "child-wait watchdog: failed to load child for synthesized terminal status"
2720                );
2721            }
2722        }
2723        finalize_runner(
2724            &self.agent_runners,
2725            child_session_id,
2726            &Err(bamboo_agent_core::AgentError::LLM(
2727                error
2728                    .clone()
2729                    .unwrap_or_else(|| format!("synthesized {status}")),
2730            )),
2731        )
2732        .await;
2733        self.publish_synthetic_completion(parent_session_id, child_session_id, status, error)
2734            .await;
2735    }
2736
2737    async fn publish_synthetic_completion(
2738        &self,
2739        parent_session_id: &str,
2740        child_session_id: &str,
2741        status: &str,
2742        error: Option<String>,
2743    ) {
2744        let publisher =
2745            crate::runtime::execution::session_events::ReplayableSessionEventPublisher::new(
2746                self.agent_runners.clone(),
2747                self.session_event_senders.clone(),
2748                self.account_feed_inbox.clone(),
2749            );
2750        let handler: Arc<dyn ChildCompletionHandler> = Arc::new(self.clone());
2751        crate::runtime::execution::spawn::publish_child_completion_parts(
2752            &publisher,
2753            Some(handler),
2754            parent_session_id.to_string(),
2755            child_session_id.to_string(),
2756            status.to_string(),
2757            error,
2758        )
2759        .await;
2760    }
2761
2762    /// A parent whose wait was already cleared (resume message appended) but
2763    /// whose resume never spawned — retry-budget exhaustion, root-tools not
2764    /// yet initialized, or a restart between clear and spawn. Detected by: no
2765    /// live runner, no armed wait, and a pending hidden runtime resume message
2766    /// as the LAST message. Resume is all that's left to do.
2767    async fn rescue_stranded_resume(&self, session_id: &str) {
2768        let Some(session) = self.load_session(session_id).await else {
2769            return;
2770        };
2771        let pending_runtime_resume = session.messages.last().is_some_and(|message| {
2772            matches!(message.role, Role::User)
2773                && message
2774                    .metadata
2775                    .as_ref()
2776                    .is_some_and(|meta| meta.get(RUNTIME_RESUME_MESSAGE_KIND_KEY).is_some())
2777        });
2778        if !pending_runtime_resume {
2779            return;
2780        }
2781        tracing::warn!(
2782            %session_id,
2783            "child-wait watchdog: stranded resume detected (wait cleared, resume never \
2784             spawned); resuming"
2785        );
2786        self.resume_parent(session_id.to_string()).await;
2787    }
2788
2789    /// Clear the parent's child wait, append `resume_message`, and drive the
2790    /// resume — with a bounded clobber-retry mirroring
2791    /// [`Self::perform_bash_resume`]: a suspending runner's one-shot finalize
2792    /// save can land after ours and revert the wait while dropping the
2793    /// message; we detect the re-armed wait and re-clear.
2794    async fn force_resume_child_wait(&self, session_id: &str, resume_message: Message) {
2795        const MAX_ATTEMPTS: u8 = 5;
2796        let lock = session_resume_lock(session_id);
2797        for attempt in 0..MAX_ATTEMPTS {
2798            if attempt > 0 {
2799                tokio::time::sleep(Duration::from_millis(200)).await;
2800            }
2801            {
2802                let _held = lock.lock().await;
2803                let Some(mut session) = self.load_session(session_id).await else {
2804                    return;
2805                };
2806                let mut runtime_state = read_runtime_state(&session);
2807                if runtime_state.waiting_for_children.is_none() {
2808                    // Another source already resumed this parent.
2809                    return;
2810                }
2811                runtime_state.waiting_for_children = None;
2812                runtime_state.status = AgentStatusState::Idle;
2813                runtime_state.suspension = None;
2814                write_runtime_state(&mut session, &runtime_state);
2815                session.metadata.remove("runtime.suspend_reason");
2816                session.add_message(resume_message.clone());
2817                session.updated_at = Utc::now();
2818                self.save_and_cache(&mut session).await;
2819            }
2820            let outcome = self.resume_parent(session_id.to_string()).await;
2821            match outcome {
2822                ResumeOutcome::Started { .. } | ResumeOutcome::NotFound => return,
2823                ResumeOutcome::Completed | ResumeOutcome::AlreadyRunning { .. } => {
2824                    // Only retry when the persisted wait was clobbered back to
2825                    // armed; if it stayed cleared with the message intact, the
2826                    // next sweep's stranded-resume rescue finishes the job.
2827                    let clobbered = self
2828                        .load_session(session_id)
2829                        .await
2830                        .map(|session| read_runtime_state(&session).waiting_for_children.is_some())
2831                        .unwrap_or(false);
2832                    if !clobbered {
2833                        return;
2834                    }
2835                }
2836            }
2837        }
2838        tracing::error!(
2839            %session_id,
2840            "child-wait watchdog: force-resume exhausted its clobber-retry budget"
2841        );
2842    }
2843}
2844
2845#[cfg(test)]
2846mod tests {
2847    use super::*;
2848
2849    #[tokio::test]
2850    async fn cancelled_resume_waiters_do_not_retain_historical_parent_ids() {
2851        for index in 0..512 {
2852            let id = format!("resume-lock-reclaim-{index}");
2853            let owner = session_resume_lock(&id);
2854            let held = owner.lock().await;
2855            let waiter = session_resume_lock(&id);
2856            let mut waiting = Box::pin(waiter.lock());
2857            assert!(futures::poll!(waiting.as_mut()).is_pending());
2858            drop(held);
2859            drop(owner);
2860            drop(waiting);
2861            drop(waiter);
2862            assert!(!parent_locks().lock().recover_poison().contains_key(&id));
2863        }
2864    }
2865
2866    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
2867    async fn resume_lock_reclamation_preserves_exclusive_parent_wake_ownership() {
2868        use std::sync::atomic::{AtomicUsize, Ordering};
2869        let active = Arc::new(AtomicUsize::new(0));
2870        let mut tasks = Vec::new();
2871        for _ in 0..32 {
2872            let active = active.clone();
2873            tasks.push(tokio::spawn(async move {
2874                for _ in 0..16 {
2875                    let lease = session_resume_lock("resume-lock-exclusive");
2876                    let _guard = lease.lock().await;
2877                    assert_eq!(active.fetch_add(1, Ordering::SeqCst), 0);
2878                    tokio::task::yield_now().await;
2879                    assert_eq!(active.fetch_sub(1, Ordering::SeqCst), 1);
2880                }
2881            }));
2882        }
2883        for task in tasks {
2884            task.await.unwrap();
2885        }
2886        assert!(!parent_locks()
2887            .lock()
2888            .recover_poison()
2889            .contains_key("resume-lock-exclusive"));
2890    }
2891    use bamboo_agent_core::Message;
2892    use bamboo_domain::SessionInboxPort;
2893    use futures::stream;
2894    use std::sync::atomic::{AtomicUsize, Ordering};
2895
2896    struct EmptyTools;
2897
2898    #[async_trait]
2899    impl bamboo_agent_core::tools::ToolExecutor for EmptyTools {
2900        async fn execute(
2901            &self,
2902            _call: &bamboo_agent_core::tools::ToolCall,
2903        ) -> Result<bamboo_agent_core::tools::ToolResult, bamboo_agent_core::tools::ToolError>
2904        {
2905            Err(bamboo_agent_core::tools::ToolError::NotFound(
2906                "no tools".to_string(),
2907            ))
2908        }
2909
2910        fn list_tools(&self) -> Vec<bamboo_agent_core::tools::ToolSchema> {
2911            Vec::new()
2912        }
2913    }
2914
2915    struct CompletedTestProvider;
2916
2917    #[async_trait]
2918    impl bamboo_llm::LLMProvider for CompletedTestProvider {
2919        async fn chat_stream(
2920            &self,
2921            _messages: &[Message],
2922            _tools: &[bamboo_agent_core::tools::ToolSchema],
2923            _max_output_tokens: Option<u32>,
2924            _model: &str,
2925        ) -> Result<bamboo_llm::LLMStream, bamboo_llm::LLMError> {
2926            let chunks: Vec<bamboo_llm::provider::Result<bamboo_llm::LLMChunk>> = vec![
2927                Ok(bamboo_llm::LLMChunk::Token("done".to_string())),
2928                Ok(bamboo_llm::LLMChunk::Done),
2929            ];
2930            Ok(Box::pin(stream::iter(chunks)))
2931        }
2932    }
2933
2934    struct CountingActivationSpawner {
2935        reservations: Arc<AtomicUsize>,
2936        launches: Arc<AtomicUsize>,
2937    }
2938
2939    #[async_trait]
2940    impl SessionActivationSpawner for CountingActivationSpawner {
2941        async fn reserve_activation(
2942            &self,
2943            target_session_id: &str,
2944            inbox_generation: u64,
2945        ) -> Result<SessionActivationReserveOutcome, bamboo_domain::SessionActivationError>
2946        {
2947            self.reservations.fetch_add(1, Ordering::SeqCst);
2948            let launches = self.launches.clone();
2949            Ok(SessionActivationReserveOutcome::Reserved(
2950                SessionActivationLaunch::new(
2951                    format!("{target_session_id}-{inbox_generation}"),
2952                    move || {
2953                        launches.fetch_add(1, Ordering::SeqCst);
2954                    },
2955                ),
2956            ))
2957        }
2958    }
2959
2960    async fn completion_inbox_fixture() -> (
2961        tempfile::TempDir,
2962        Arc<bamboo_storage::SessionStoreV2>,
2963        Arc<dyn SessionInboxPort>,
2964        Arc<ChildCompletionCoordinator>,
2965        Arc<AtomicUsize>,
2966        Arc<AtomicUsize>,
2967    ) {
2968        let temp = tempfile::tempdir().unwrap();
2969        let store = Arc::new(
2970            bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
2971                .await
2972                .unwrap(),
2973        );
2974        let storage: Arc<dyn Storage> = store.clone();
2975        let locked = Arc::new(LockedSessionStore::new(storage.clone()));
2976        let inbox: Arc<dyn SessionInboxPort> = Arc::new(bamboo_storage::FileSessionInbox::new(
2977            store.clone(),
2978            bamboo_domain::SessionInboxLimits::default(),
2979        ));
2980        let router = crate::SessionActivationRouter::new();
2981        let messenger = Arc::new(crate::SessionMessenger::new(
2982            storage.clone(),
2983            inbox.clone(),
2984            router.clone(),
2985        ));
2986        let reservations = Arc::new(AtomicUsize::new(0));
2987        let launches = Arc::new(AtomicUsize::new(0));
2988        router
2989            .set_spawner(Arc::new(CountingActivationSpawner {
2990                reservations: reservations.clone(),
2991                launches: launches.clone(),
2992            }))
2993            .await;
2994        let provider: Arc<dyn bamboo_llm::LLMProvider> = Arc::new(CompletedTestProvider);
2995        let config = Arc::new(RwLock::new(Config::default()));
2996        let metrics = bamboo_metrics::MetricsCollector::spawn(
2997            Arc::new(bamboo_metrics::SqliteMetricsStorage::new(
2998                temp.path().join("metrics.db"),
2999            )),
3000            7,
3001        );
3002        let tools: Arc<dyn ToolExecutor> = Arc::new(EmptyTools);
3003        let agent = Arc::new(
3004            Agent::builder()
3005                .storage(storage.clone())
3006                .persistence(locked.clone())
3007                .session_inbox(inbox.clone())
3008                .activation_router(router)
3009                .session_messenger(messenger)
3010                .attachment_reader(store.clone())
3011                .skill_manager(Arc::new(bamboo_skills::SkillManager::new()))
3012                .metrics_collector(metrics)
3013                .config(config.clone())
3014                .provider(provider.clone())
3015                .default_tools(tools)
3016                .build()
3017                .unwrap(),
3018        );
3019        let mut providers = HashMap::new();
3020        providers.insert("test".to_string(), provider);
3021        let registry = Arc::new(ProviderRegistry::new(providers, "test".to_string()));
3022        let provider_router = Arc::new(ProviderModelRouter::new(registry.clone()));
3023        let coordinator = Arc::new(ChildCompletionCoordinator::new(
3024            storage,
3025            locked,
3026            Arc::default(),
3027            Arc::new(RwLock::new(HashMap::new())),
3028            Arc::new(RwLock::new(HashMap::new())),
3029            agent,
3030            config,
3031            registry,
3032            provider_router,
3033            temp.path().to_path_buf(),
3034            None,
3035        ));
3036        (temp, store, inbox, coordinator, reservations, launches)
3037    }
3038
3039    #[tokio::test]
3040    async fn supervisor_resume_rejects_old_incarnation_without_replacing_cache() {
3041        let (_temp, store, _inbox, coordinator, _reservations, _launches) =
3042            completion_inbox_fixture().await;
3043        let original = store
3044            .get_or_create_default_supervisor("model")
3045            .await
3046            .unwrap();
3047        let mut stale = store
3048            .load_session(&original.session_id)
3049            .await
3050            .unwrap()
3051            .unwrap();
3052        store.delete_session(&original.session_id).await.unwrap();
3053        let recreated = store
3054            .get_or_create_default_supervisor("replacement")
3055            .await
3056            .unwrap();
3057        assert_ne!(original.incarnation_id, recreated.incarnation_id);
3058        let mut current = store
3059            .load_session(&recreated.session_id)
3060            .await
3061            .unwrap()
3062            .unwrap();
3063        coordinator.save_and_cache(&mut current).await;
3064        let cached = coordinator
3065            .sessions
3066            .get(&current.id)
3067            .unwrap()
3068            .value()
3069            .clone();
3070        coordinator.save_and_cache(&mut stale).await;
3071        assert!(Arc::ptr_eq(
3072            &cached,
3073            coordinator.sessions.get(&current.id).unwrap().value()
3074        ));
3075        assert_eq!(
3076            store
3077                .load_session(&current.id)
3078                .await
3079                .unwrap()
3080                .unwrap()
3081                .authority_identity,
3082            current.authority_identity
3083        );
3084    }
3085
3086    // ── child-wait watchdog pure helpers (issue #546) ────────────────────
3087
3088    #[test]
3089    fn dead_child_candidate_status_matrix() {
3090        // Never ran / lost before the running marker: dead candidate.
3091        assert!(is_dead_child_candidate_status(None));
3092        // Actively-reported non-terminal statuses: dead candidates when nothing
3093        // is driving them.
3094        assert!(is_dead_child_candidate_status(Some("running")));
3095        assert!(is_dead_child_candidate_status(Some("pending")));
3096        // Legitimately quiescent: waiting on a human / own children / bash.
3097        assert!(!is_dead_child_candidate_status(Some("suspended")));
3098        // Terminal statuses can never be "dead" — they are already done.
3099        for status in ["completed", "error", "timeout", "cancelled", "skipped"] {
3100            assert!(!is_dead_child_candidate_status(Some(status)), "{status}");
3101        }
3102    }
3103
3104    #[test]
3105    fn completion_child_ownership_gates_content_fold() {
3106        // Owned: the child's own parent linkage matches the reporting parent.
3107        assert!(completion_child_is_owned("parent-1", Some("parent-1")));
3108        // Foreign: a real session that belongs to a DIFFERENT parent — its
3109        // content must never be folded into parent-1's transcript.
3110        assert!(!completion_child_is_owned("parent-1", Some("parent-2")));
3111        // Root/unparented session, or a nonexistent id (linkage None).
3112        assert!(!completion_child_is_owned("parent-1", None));
3113    }
3114
3115    #[test]
3116    fn replay_child_prefers_error_like_for_first_error_policy() {
3117        let terminal = vec![
3118            ("c-ok".to_string(), "completed".to_string()),
3119            ("c-err".to_string(), "timeout".to_string()),
3120            ("c-late".to_string(), "completed".to_string()),
3121        ];
3122        let (id, status) = select_replay_child(&terminal).expect("non-empty");
3123        assert_eq!(id, "c-err");
3124        assert_eq!(status, "timeout");
3125
3126        let all_ok = vec![
3127            ("c-1".to_string(), "completed".to_string()),
3128            ("c-2".to_string(), "completed".to_string()),
3129        ];
3130        let (id, _) = select_replay_child(&all_ok).expect("non-empty");
3131        assert_eq!(id, "c-2");
3132
3133        assert!(select_replay_child(&[]).is_none());
3134    }
3135
3136    #[test]
3137    fn watchdog_resume_messages_are_hidden_runtime_messages() {
3138        for message in [
3139            empty_child_wait_message(),
3140            child_wait_lease_expired_message(&["c-1".to_string(), "c-2".to_string()]),
3141        ] {
3142            assert!(matches!(message.role, Role::User));
3143            let meta = message.metadata.expect("hidden runtime metadata");
3144            assert_eq!(meta[RUNTIME_RESUME_MESSAGE_HIDDEN_KEY], true);
3145            assert_eq!(
3146                meta[RUNTIME_RESUME_MESSAGE_KIND_KEY],
3147                "child_wait_watchdog_resume"
3148            );
3149        }
3150        let lease = child_wait_lease_expired_message(&["c-1".to_string()]);
3151        // The lease message must never claim the children finished.
3152        assert!(lease.content.contains("NOT cancelled"));
3153        assert!(lease.content.contains("c-1"));
3154    }
3155
3156    // ── on_child_completed terminality guard (issue #546) ────────────────
3157
3158    #[test]
3159    fn non_terminal_statuses_never_satisfy_wait_policies() {
3160        // The guard keys on `is_terminal_child_status`; "suspended" (and any
3161        // unknown non-terminal string) must not count toward any policy.
3162        assert!(!is_terminal_child_status("suspended"));
3163        assert!(!is_terminal_child_status("running"));
3164        assert!(!is_terminal_child_status("pending"));
3165    }
3166
3167    fn make_completion(status: &str) -> ChildCompletion {
3168        ChildCompletion {
3169            parent_session_id: "parent-1".to_string(),
3170            child_session_id: "child-1".to_string(),
3171            status: status.to_string(),
3172            error: None,
3173            completed_at: Utc::now(),
3174        }
3175    }
3176
3177    #[test]
3178    fn oversized_child_outcome_is_bounded_and_keeps_full_content_identity() {
3179        let mut completion = make_completion("completed");
3180        let wait_registered_at = Utc::now();
3181        let huge = format!("prefix-A-{}", "x".repeat(300 * 1024));
3182        let presentation = runtime_resume_message(&completion, 0, Some(&huge));
3183        let first = child_completion_envelope(
3184            &completion,
3185            wait_registered_at,
3186            Some(huge.clone()),
3187            &presentation,
3188        );
3189        assert!(
3190            serde_json::to_vec(&first).unwrap().len()
3191                < bamboo_domain::SessionInboxLimits::default().max_payload_bytes
3192        );
3193        let SessionMessageBody::ChildOutcome(outcome) = &first.body else {
3194            panic!("typed child outcome");
3195        };
3196        let stored = outcome.result.as_deref().unwrap();
3197        assert!(stored.contains("sha256="));
3198        assert!(stored.contains("SubAgent.get"));
3199        assert!(stored.len() < CHILD_COMPLETION_INLINE_FIELD_BYTES);
3200
3201        // Retry-only completion timestamps and provider presentation do not
3202        // change the logical id; full oversized content does.
3203        completion.completed_at += chrono::Duration::seconds(1);
3204        let exact_retry = child_completion_envelope(
3205            &completion,
3206            wait_registered_at,
3207            Some(huge.clone()),
3208            &runtime_resume_message(&completion, 9, Some(&huge)),
3209        );
3210        assert_eq!(exact_retry.id, first.id);
3211        let changed = format!("prefix-B-{}", "x".repeat(300 * 1024));
3212        let corrected = child_completion_envelope(
3213            &completion,
3214            wait_registered_at,
3215            Some(changed.clone()),
3216            &runtime_resume_message(&completion, 0, Some(&changed)),
3217        );
3218        assert_ne!(corrected.id, first.id);
3219    }
3220
3221    #[tokio::test]
3222    async fn oversized_child_completion_clears_wait_and_activates_exactly_once() {
3223        let (_temp, store, inbox, coordinator, reservations, launches) =
3224            completion_inbox_fixture().await;
3225        let parent_id = "oversized-parent";
3226        let child_id = "oversized-child";
3227        let now = Utc::now();
3228        let mut parent = Session::new(parent_id, "model");
3229        let mut parent_runtime = AgentRuntimeState::new("waiting-run");
3230        parent_runtime.status = AgentStatusState::Suspended;
3231        parent_runtime.waiting_for_children = Some(WaitingForChildrenState::for_children(
3232            vec![child_id.to_string()],
3233            ChildWaitPolicy::All,
3234            now,
3235        ));
3236        parent_runtime.suspension = Some(SuspensionState {
3237            reason: "waiting_for_children".to_string(),
3238            suspended_at: now,
3239            resumable: true,
3240            hook_point: Some("ChildCompletion".to_string()),
3241        });
3242        write_runtime_state(&mut parent, &parent_runtime);
3243        parent.metadata.insert(
3244            "runtime.suspend_reason".to_string(),
3245            "waiting_for_children".to_string(),
3246        );
3247        store.save_session(&parent).await.unwrap();
3248
3249        let mut child = Session::new_child(child_id, parent_id, "model", "Child");
3250        child.add_message(Message::assistant("z".repeat(300 * 1024), None));
3251        child.set_last_run_status("completed");
3252        store.save_session(&child).await.unwrap();
3253        let completion = ChildCompletion {
3254            parent_session_id: parent_id.to_string(),
3255            child_session_id: child_id.to_string(),
3256            status: "completed".to_string(),
3257            error: None,
3258            completed_at: Utc::now(),
3259        };
3260
3261        ChildCompletionHandler::on_child_completed(coordinator.as_ref(), completion.clone()).await;
3262        let durable_parent = store.load_session(parent_id).await.unwrap().unwrap();
3263        let durable_runtime = read_runtime_state(&durable_parent);
3264        assert!(durable_runtime.waiting_for_children.is_none());
3265        assert_eq!(durable_runtime.status, AgentStatusState::Idle);
3266        assert!(!durable_parent
3267            .metadata
3268            .contains_key("runtime.suspend_reason"));
3269        let backlog = inbox.inspect(parent_id).await.unwrap();
3270        assert_eq!(backlog.pending + backlog.claimed, 1);
3271        assert!(backlog.activation_pending());
3272        assert_eq!(reservations.load(Ordering::SeqCst), 1);
3273        assert_eq!(launches.load(Ordering::SeqCst), 1);
3274
3275        let claim = inbox.claim(parent_id, 1).await.unwrap().remove(0);
3276        assert!(
3277            serde_json::to_vec(&claim.envelope).unwrap().len()
3278                < bamboo_domain::SessionInboxLimits::default().max_payload_bytes
3279        );
3280        // Duplicate terminal notification after the wait was already cleared
3281        // cannot enqueue or activate a second outcome.
3282        ChildCompletionHandler::on_child_completed(coordinator.as_ref(), completion).await;
3283        assert_eq!(reservations.load(Ordering::SeqCst), 1);
3284        assert_eq!(launches.load(Ordering::SeqCst), 1);
3285        assert_eq!(inbox.inspect(parent_id).await.unwrap().claimed, 1);
3286    }
3287
3288    #[tokio::test]
3289    async fn latest_locked_activation_mutation_preserves_wait_armed_after_stale_load() {
3290        let temp = tempfile::tempdir().unwrap();
3291        let store = Arc::new(
3292            bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
3293                .await
3294                .unwrap(),
3295        );
3296        let storage: Arc<dyn Storage> = store.clone();
3297        let locked = LockedSessionStore::new(storage);
3298        let mut session = Session::new("activation-stale-wait", "model");
3299        session.agent_runtime_state = Some(AgentRuntimeState::new("old-run"));
3300        store.save_session(&session).await.unwrap();
3301
3302        // The activation path has already loaded this stale, wait-free snapshot.
3303        let stale = store.load_session(&session.id).await.unwrap().unwrap();
3304        assert!(read_runtime_state(&stale).waiting_for_children.is_none());
3305
3306        locked
3307            .update_runtime_config(&session.id, |latest| {
3308                let mut state = read_runtime_state(latest);
3309                state.status = AgentStatusState::Suspended;
3310                state.waiting_for_children = Some(WaitingForChildrenState::for_children(
3311                    vec!["child-new".to_string()],
3312                    ChildWaitPolicy::All,
3313                    Utc::now(),
3314                ));
3315                state.suspension = Some(SuspensionState {
3316                    reason: "waiting_for_children".to_string(),
3317                    suspended_at: Utc::now(),
3318                    resumable: true,
3319                    hook_point: None,
3320                });
3321                write_runtime_state(latest, &state);
3322                latest.metadata.insert(
3323                    "runtime.suspend_reason".to_string(),
3324                    "waiting_for_children".to_string(),
3325                );
3326            })
3327            .await
3328            .unwrap();
3329
3330        let (prepared, ready) = prepare_session_inbox_activation(&locked, &session.id, false)
3331            .await
3332            .unwrap()
3333            .unwrap();
3334        assert!(!ready, "latest durable specific wait must block activation");
3335        let state = read_runtime_state(&prepared);
3336        assert!(state.waiting_for_children.is_some());
3337        assert_eq!(state.status, AgentStatusState::Suspended);
3338        assert_eq!(
3339            prepared
3340                .metadata
3341                .get("runtime.suspend_reason")
3342                .map(String::as_str),
3343            Some("waiting_for_children")
3344        );
3345    }
3346
3347    #[tokio::test]
3348    async fn partial_child_and_bash_backlogs_remain_inert_after_inbox_reopen() {
3349        let temp = tempfile::tempdir().unwrap();
3350        let store = Arc::new(
3351            bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
3352                .await
3353                .unwrap(),
3354        );
3355        store
3356            .save_session(&Session::new("restart-child-parent", "model"))
3357            .await
3358            .unwrap();
3359        store
3360            .save_session(&Session::new("restart-bash-parent", "model"))
3361            .await
3362            .unwrap();
3363        let inbox = bamboo_storage::FileSessionInbox::new(
3364            store.clone(),
3365            bamboo_domain::SessionInboxLimits::default(),
3366        );
3367
3368        let completion = ChildCompletion {
3369            parent_session_id: "restart-child-parent".to_string(),
3370            child_session_id: "child-a".to_string(),
3371            status: "completed".to_string(),
3372            error: None,
3373            completed_at: Utc::now(),
3374        };
3375        let child_resume = runtime_resume_message(&completion, 1, Some("first child"));
3376        inbox
3377            .deliver(&child_completion_envelope(
3378                &completion,
3379                Utc::now(),
3380                Some("first child".to_string()),
3381                &child_resume,
3382            ))
3383            .await
3384            .unwrap();
3385        let bash = BashCompletionInfo {
3386            session_id: "restart-bash-parent".to_string(),
3387            bash_id: "bash-a".to_string(),
3388            command: "true".to_string(),
3389            exit_code: Some(0),
3390            status: "completed".to_string(),
3391            output_tail: String::new(),
3392        };
3393        inbox
3394            .deliver(&bash_completion_envelope(&bash))
3395            .await
3396            .unwrap();
3397
3398        let reopened = bamboo_storage::FileSessionInbox::new(
3399            store,
3400            bamboo_domain::SessionInboxLimits::default(),
3401        );
3402        for session_id in ["restart-child-parent", "restart-bash-parent"] {
3403            let backlog = reopened.inspect(session_id).await.unwrap();
3404            assert_eq!(backlog.pending, 1);
3405            assert_eq!(backlog.activation_generation, 0);
3406            assert!(
3407                !backlog.activation_pending(),
3408                "startup must not run a partial wait backlog for {session_id}"
3409            );
3410        }
3411    }
3412
3413    // ── ② derive completed children from the index ──────────────────────
3414
3415    struct StubChildIndex {
3416        children: Vec<(String, Option<String>)>,
3417    }
3418
3419    #[async_trait]
3420    impl Storage for StubChildIndex {
3421        async fn save_session(&self, _session: &Session) -> std::io::Result<()> {
3422            Ok(())
3423        }
3424        async fn load_session(&self, _id: &str) -> std::io::Result<Option<Session>> {
3425            Ok(None)
3426        }
3427        async fn delete_session(&self, _id: &str) -> std::io::Result<bool> {
3428            Ok(false)
3429        }
3430        async fn list_child_run_statuses(
3431            &self,
3432            _parent_session_id: &str,
3433        ) -> std::io::Result<Vec<(String, Option<String>)>> {
3434            Ok(self.children.clone())
3435        }
3436    }
3437
3438    #[tokio::test]
3439    async fn derive_completed_only_includes_terminal_children() {
3440        let storage: Arc<dyn Storage> = Arc::new(StubChildIndex {
3441            children: vec![
3442                ("a".into(), Some("completed".into())),
3443                ("b".into(), Some("running".into())),
3444                ("c".into(), Some("error".into())),
3445                ("d".into(), None),
3446            ],
3447        });
3448        let completed = derive_completed_child_ids(&storage, "parent-1", "b").await;
3449        // Terminal from index: a, c. Plus the just-completed child b folded in.
3450        assert_eq!(
3451            completed,
3452            vec!["a".to_string(), "b".to_string(), "c".to_string()]
3453        );
3454    }
3455
3456    #[tokio::test]
3457    async fn derive_completed_folds_in_just_completed_when_index_lags() {
3458        // Index hasn't caught up — reports the child as still running.
3459        let storage: Arc<dyn Storage> = Arc::new(StubChildIndex {
3460            children: vec![("only".into(), Some("running".into()))],
3461        });
3462        let completed = derive_completed_child_ids(&storage, "parent-1", "only").await;
3463        assert_eq!(completed, vec!["only".to_string()]);
3464    }
3465
3466    #[test]
3467    fn wait_policy_all_uses_derived_completed_set() {
3468        let waited = vec!["a".to_string(), "b".to_string()];
3469        assert!(!wait_policy_satisfied(
3470            ChildWaitPolicy::All,
3471            &waited,
3472            &["a".to_string()],
3473            "a",
3474            "completed"
3475        ));
3476        assert!(wait_policy_satisfied(
3477            ChildWaitPolicy::All,
3478            &waited,
3479            &["a".to_string(), "b".to_string()],
3480            "b",
3481            "completed"
3482        ));
3483    }
3484
3485    #[test]
3486    fn wait_policy_first_error_requires_tracked_membership() {
3487        let waited = vec!["a".to_string(), "b".to_string()];
3488        // An error from a TRACKED child resumes immediately.
3489        assert!(wait_policy_satisfied(
3490            ChildWaitPolicy::FirstError,
3491            &waited,
3492            &["a".to_string()],
3493            "a",
3494            "error"
3495        ));
3496        // An error-like completion from an UNTRACKED child (e.g. a zombie
3497        // task from an earlier run waking up late) must not resume the wait.
3498        assert!(!wait_policy_satisfied(
3499            ChildWaitPolicy::FirstError,
3500            &waited,
3501            &["a".to_string()],
3502            "stray-child",
3503            "timeout"
3504        ));
3505        // The all-complete fallback still applies regardless of the reporter.
3506        assert!(wait_policy_satisfied(
3507            ChildWaitPolicy::FirstError,
3508            &waited,
3509            &["a".to_string(), "b".to_string()],
3510            "stray-child",
3511            "completed"
3512        ));
3513    }
3514
3515    #[test]
3516    fn child_final_assistant_text_returns_last_assistant() {
3517        let mut session = Session::new("child-1", "gpt-4");
3518        session.messages.push(Message::user("hi"));
3519        session
3520            .messages
3521            .push(Message::assistant("first answer", None));
3522        session.messages.push(Message::user("again"));
3523        session
3524            .messages
3525            .push(Message::assistant("final answer", None));
3526
3527        assert_eq!(
3528            child_final_assistant_text(&session).as_deref(),
3529            Some("final answer")
3530        );
3531    }
3532
3533    #[test]
3534    fn child_final_assistant_text_returns_none_when_blank() {
3535        let mut session = Session::new("child-1", "gpt-4");
3536        session.messages.push(Message::assistant("   ", None));
3537        assert!(child_final_assistant_text(&session).is_none());
3538    }
3539
3540    #[test]
3541    fn child_final_assistant_text_returns_none_when_no_assistant() {
3542        let mut session = Session::new("child-1", "gpt-4");
3543        session.messages.push(Message::user("hi"));
3544        assert!(child_final_assistant_text(&session).is_none());
3545    }
3546
3547    #[test]
3548    fn runtime_resume_message_folds_full_response_without_truncation() {
3549        // A very long child final response is folded in verbatim (no 4000-char
3550        // cap, no truncation marker).
3551        let completion = make_completion("completed");
3552        let long: String = "a".repeat(10_000);
3553        let message = runtime_resume_message(&completion, 0, Some(&long));
3554        assert!(message.content.contains(&long));
3555        assert!(!message.content.contains("truncated"));
3556    }
3557
3558    #[test]
3559    fn runtime_resume_message_includes_child_response_when_provided() {
3560        let completion = make_completion("completed");
3561        let message = runtime_resume_message(&completion, 0, Some("the answer is 42"));
3562
3563        assert!(matches!(message.role, Role::User));
3564        // Folded child results are now compressible so the parent context can
3565        // reclaim them under compaction.
3566        assert!(!message.never_compress);
3567        assert!(message.content.contains("Child final response:"));
3568        assert!(message.content.contains("the answer is 42"));
3569
3570        let metadata = message.metadata.expect("metadata present");
3571        assert_eq!(
3572            metadata.get("hidden_from_ui").and_then(|v| v.as_bool()),
3573            Some(true)
3574        );
3575        assert_eq!(
3576            metadata.get("runtime_kind").and_then(|v| v.as_str()),
3577            Some("child_completion_resume")
3578        );
3579        assert_eq!(
3580            metadata
3581                .get("child_final_response_included")
3582                .and_then(|v| v.as_bool()),
3583            Some(true)
3584        );
3585    }
3586
3587    #[test]
3588    fn runtime_resume_message_falls_back_to_error_when_no_response() {
3589        let mut completion = make_completion("error");
3590        completion.error = Some("boom".to_string());
3591
3592        let message = runtime_resume_message(&completion, 1, None);
3593        assert!(message.content.contains("Child error:"));
3594        assert!(message.content.contains("boom"));
3595        let metadata = message.metadata.expect("metadata present");
3596        assert_eq!(
3597            metadata
3598                .get("child_final_response_included")
3599                .and_then(|v| v.as_bool()),
3600            Some(false)
3601        );
3602    }
3603
3604    #[test]
3605    fn runtime_resume_message_minimal_when_no_response_and_no_error() {
3606        let completion = make_completion("completed");
3607        let message = runtime_resume_message(&completion, 2, None);
3608        assert!(!message.content.contains("Child final response:"));
3609        assert!(!message.content.contains("Child error:"));
3610        assert!(message.content.contains("Resume the parent task"));
3611    }
3612
3613    #[test]
3614    fn read_config_snapshot_refreshes_cached_snapshot_from_live_config() {
3615        let runtime = tokio::runtime::Runtime::new().expect("runtime");
3616
3617        runtime.block_on(async {
3618            let config = Arc::new(RwLock::new(Config::default()));
3619            config.write().await.provider = "copilot".to_string();
3620            let cached_config = StdRwLock::new(Config::default());
3621
3622            let snapshot = read_config_snapshot(&config, &cached_config);
3623
3624            assert_eq!(snapshot.provider, "copilot");
3625            assert_eq!(
3626                cached_config.read().expect("cached snapshot lock").provider,
3627                "copilot"
3628            );
3629        });
3630    }
3631
3632    #[test]
3633    fn read_config_snapshot_uses_cached_snapshot_when_live_lock_is_busy() {
3634        let runtime = tokio::runtime::Runtime::new().expect("runtime");
3635
3636        runtime.block_on(async {
3637            let mut cached_snapshot = Config::default();
3638            cached_snapshot.provider = "cached-provider".to_string();
3639
3640            let config = Arc::new(RwLock::new(Config::default()));
3641            let cached_config = StdRwLock::new(cached_snapshot);
3642            let _write_guard = config.write().await;
3643
3644            let snapshot = read_config_snapshot(&config, &cached_config);
3645
3646            assert_eq!(snapshot.provider, "cached-provider");
3647        });
3648    }
3649
3650    // ── Bash self-resume (issue #84 Phase 2b): deadline message + clobber-retry ──
3651
3652    #[test]
3653    fn bash_completion_resume_message_normal_announces_completion() {
3654        let ids = vec!["bg-1".to_string(), "bg-2".to_string()];
3655        let message = bash_completion_resume_message(&ids, false);
3656        // Normal path: the shells genuinely finished.
3657        assert!(
3658            message.content.contains("have completed"),
3659            "normal resume message must announce completion: {}",
3660            message.content
3661        );
3662        // Hidden + compressible so the resume gate sees it but the UI hides it.
3663        let metadata = message.metadata.expect("metadata present");
3664        assert_eq!(
3665            metadata
3666                .get(RUNTIME_RESUME_MESSAGE_HIDDEN_KEY)
3667                .and_then(|v| v.as_bool()),
3668            Some(true),
3669            "resume message must be hidden from the UI"
3670        );
3671        assert_eq!(
3672            metadata
3673                .get(RUNTIME_RESUME_MESSAGE_KIND_KEY)
3674                .and_then(|v| v.as_str()),
3675            Some(BASH_COMPLETION_RESUME_KIND),
3676            "resume message must carry the bash-completion kind discriminant"
3677        );
3678    }
3679
3680    #[test]
3681    fn bash_completion_resume_message_deadline_does_not_claim_completion() {
3682        // The 6h+10m deadline force-breaks with shells STILL running. The message
3683        // must NOT say "have completed" — that would let the model assume success
3684        // on a false premise. It must direct the model to verify with BashOutput.
3685        let ids = vec!["bg-long".to_string()];
3686        let message = bash_completion_resume_message(&ids, true);
3687        assert!(
3688            !message.content.contains("have completed"),
3689            "deadline resume message must NOT claim the shells completed: {}",
3690            message.content
3691        );
3692        assert!(
3693            message.content.contains("may still be running"),
3694            "deadline resume message must warn shells may still be running: {}",
3695            message.content
3696        );
3697        assert!(
3698            message.content.contains("BashOutput"),
3699            "deadline resume message must direct verification via BashOutput: {}",
3700            message.content
3701        );
3702        // Same hidden/kind shape so the resume gate is satisfied identically.
3703        let metadata = message.metadata.expect("metadata present");
3704        assert_eq!(
3705            metadata
3706                .get(RUNTIME_RESUME_MESSAGE_KIND_KEY)
3707                .and_then(|v| v.as_str()),
3708            Some(BASH_COMPLETION_RESUME_KIND)
3709        );
3710    }
3711
3712    #[test]
3713    fn bash_resume_should_retry_matrix() {
3714        // The finalize-clobber retry predicate (issue #84 Phase 2b). Retry only
3715        // when the resume did NOT spawn (Completed / AlreadyRunning) AND the
3716        // persisted bash wait is still set on reload — the clobber signature.
3717
3718        // Started: the resume fired — never retry, regardless of persisted state.
3719        assert!(!bash_resume_should_retry(
3720            &ResumeOutcome::Started { run_id: "r".into() },
3721            true
3722        ));
3723        assert!(!bash_resume_should_retry(
3724            &ResumeOutcome::Started { run_id: "r".into() },
3725            false
3726        ));
3727
3728        // NotFound: session gone — never retry.
3729        assert!(!bash_resume_should_retry(&ResumeOutcome::NotFound, true));
3730        assert!(!bash_resume_should_retry(&ResumeOutcome::NotFound, false));
3731
3732        // Completed + persisted wait still set ⇒ finalize-clobber ⇒ retry.
3733        assert!(bash_resume_should_retry(&ResumeOutcome::Completed, true));
3734        // Completed + persisted wait cleared ⇒ handled (our message stuck, or a
3735        // concurrent resume finished) ⇒ stop.
3736        assert!(!bash_resume_should_retry(&ResumeOutcome::Completed, false));
3737
3738        // AlreadyRunning + persisted wait still set ⇒ clobbered while a runner is
3739        // (stale-)active ⇒ retry to re-establish the resume message.
3740        assert!(bash_resume_should_retry(
3741            &ResumeOutcome::AlreadyRunning { run_id: "r".into() },
3742            true
3743        ));
3744        // AlreadyRunning + wait cleared ⇒ a runner owns the session ⇒ stop.
3745        assert!(!bash_resume_should_retry(
3746            &ResumeOutcome::AlreadyRunning { run_id: "r".into() },
3747            false
3748        ));
3749    }
3750
3751    // ── bash completion injection body (Phase 2b follow-up) ──────────────
3752
3753    #[test]
3754    fn injection_body_includes_status_exit_command_and_tail() {
3755        let info = BashCompletionInfo {
3756            session_id: "s".into(),
3757            bash_id: "abc123".into(),
3758            command: "make build".into(),
3759            exit_code: Some(0),
3760            status: "completed".into(),
3761            output_tail: "BUILD OK".into(),
3762        };
3763        let body = bash_completion_injection_body(&info);
3764        assert!(body.contains("abc123"), "body: {body}");
3765        assert!(body.contains("make build"), "body: {body}");
3766        assert!(body.contains("completed"), "body: {body}");
3767        assert!(body.contains("exit code 0"), "body: {body}");
3768        assert!(body.contains("BUILD OK"), "body: {body}");
3769        // The model is pointed at BashOutput for the full log.
3770        assert!(body.contains("BashOutput"), "body: {body}");
3771        assert!(body.contains("bash_id=\"abc123\""), "body: {body}");
3772    }
3773
3774    #[test]
3775    fn injection_body_handles_no_output_and_signal_kill() {
3776        let info = BashCompletionInfo {
3777            session_id: "s".into(),
3778            bash_id: "xyz".into(),
3779            command: "sleep 99".into(),
3780            exit_code: None,
3781            status: "killed".into(),
3782            output_tail: String::new(),
3783        };
3784        let body = bash_completion_injection_body(&info);
3785        assert!(body.contains("killed"), "body: {body}");
3786        assert!(body.contains("none (signal/killed)"), "body: {body}");
3787        assert!(body.contains("no captured output"), "body: {body}");
3788        // No output tail section when there is nothing to show.
3789        assert!(!body.contains("Output tail:"), "body: {body}");
3790    }
3791
3792    #[test]
3793    fn background_completion_builds_post_tool_use_payload_and_feedback() {
3794        let mut info = BashCompletionInfo {
3795            session_id: "s".into(),
3796            bash_id: "bg-7".into(),
3797            command: "cargo test".into(),
3798            exit_code: Some(0),
3799            status: "completed".into(),
3800            output_tail: "test result: ok".into(),
3801        };
3802
3803        let payload = background_bash_post_tool_payload(&info);
3804        match payload {
3805            HookPayload::ToolResult {
3806                tool_name,
3807                tool_call_id,
3808                outcome,
3809            } => {
3810                assert_eq!(tool_name, "Bash");
3811                assert_eq!(tool_call_id, "bg-7");
3812                assert!(outcome.success);
3813                let response: serde_json::Value =
3814                    serde_json::from_str(outcome.result.as_deref().unwrap()).unwrap();
3815                assert_eq!(response["command"], "cargo test");
3816                assert_eq!(response["exit_code"], 0);
3817                assert_eq!(response["status"], "completed");
3818                assert_eq!(response["output_tail"], "test result: ok");
3819            }
3820            other => panic!("expected PostToolUse payload, got {other:?}"),
3821        }
3822
3823        append_background_bash_hook_feedback(
3824            &mut info,
3825            vec!["Run the formatter before continuing".to_string()],
3826        );
3827        assert!(info.output_tail.contains("<post_tool_use_feedback>"));
3828        assert!(info
3829            .output_tail
3830            .contains("Run the formatter before continuing"));
3831    }
3832
3833    #[tokio::test]
3834    async fn bash_completion_payload_identity_matches_file_inbox_idempotency() {
3835        let baseline = BashCompletionInfo {
3836            session_id: "session".into(),
3837            bash_id: "bg-7".into(),
3838            command: "cargo test".into(),
3839            exit_code: Some(0),
3840            status: "completed".into(),
3841            output_tail: "first tail".into(),
3842        };
3843        let mut retried = baseline.clone();
3844        retried.output_tail = "first tail\nlater bytes\n<hook feedback>".into();
3845        retried.status = "completed-after-hook".into();
3846        let baseline_envelope = bash_completion_envelope(&baseline);
3847        let exact_retry = bash_completion_envelope(&baseline);
3848        let corrected_envelope = bash_completion_envelope(&retried);
3849        assert_eq!(baseline_envelope.id, exact_retry.id);
3850        assert_ne!(
3851            baseline_envelope.id, corrected_envelope.id,
3852            "changed payload semantics must receive a distinct id"
3853        );
3854
3855        let mut other_shell = baseline.clone();
3856        other_shell.bash_id = "bg-8".into();
3857        assert_ne!(
3858            bash_completion_envelope(&baseline).id,
3859            bash_completion_envelope(&other_shell).id
3860        );
3861
3862        let temp = tempfile::tempdir().unwrap();
3863        let store = Arc::new(
3864            bamboo_storage::SessionStoreV2::new(temp.path().to_path_buf())
3865                .await
3866                .unwrap(),
3867        );
3868        store
3869            .save_session(&Session::new("session", "model"))
3870            .await
3871            .unwrap();
3872        let inbox = bamboo_storage::FileSessionInbox::new(
3873            store,
3874            bamboo_domain::SessionInboxLimits::default(),
3875        );
3876        let first = inbox.deliver(&baseline_envelope).await.unwrap();
3877        let duplicate = inbox.deliver(&exact_retry).await.unwrap();
3878        let corrected = inbox.deliver(&corrected_envelope).await.unwrap();
3879        assert_eq!(duplicate, first, "exact payload retry is idempotent");
3880        assert_ne!(corrected.id, first.id);
3881        assert_eq!(corrected.generation, first.generation + 1);
3882        assert_eq!(inbox.inspect("session").await.unwrap().pending, 2);
3883    }
3884
3885    #[cfg(unix)]
3886    #[tokio::test]
3887    async fn background_completion_fires_configured_post_tool_use_command() {
3888        use bamboo_config::{
3889            LifecycleHookGroup, LifecycleHookHandler, LifecycleHooksConfig,
3890            DEFAULT_LIFECYCLE_HOOK_TIMEOUT_MS,
3891        };
3892
3893        let dir = tempfile::tempdir().unwrap();
3894        let output = dir.path().join("background-post-tool.json");
3895        let command = format!(
3896            "cat > '{}'; printf '%s' '{{\"additional_context\":\"inspect the completed build log\"}}'",
3897            output.display()
3898        );
3899        let config = LifecycleHooksConfig {
3900            enabled: true,
3901            post_tool_use: vec![LifecycleHookGroup {
3902                enabled: true,
3903                matcher: Some("^Bash$".to_string()),
3904                hooks: vec![LifecycleHookHandler::command(
3905                    command,
3906                    DEFAULT_LIFECYCLE_HOOK_TIMEOUT_MS,
3907                )],
3908            }],
3909            ..Default::default()
3910        };
3911        let mut session = Session::new("session-bg-hook", "test-model");
3912        session.workspace = Some(dir.path().to_string_lossy().into_owned());
3913        let mut info = BashCompletionInfo {
3914            session_id: session.id.clone(),
3915            bash_id: "bg-9".into(),
3916            command: "cargo test".into(),
3917            exit_code: Some(0),
3918            status: "completed".into(),
3919            output_tail: "test result: ok".into(),
3920        };
3921
3922        assert!(run_background_bash_post_tool_hooks(&config, None, &session, &mut info).await);
3923        let envelope: serde_json::Value =
3924            serde_json::from_str(&std::fs::read_to_string(output).unwrap()).unwrap();
3925        assert_eq!(envelope["hook_event_name"], "PostToolUse");
3926        assert_eq!(envelope["tool_name"], "Bash");
3927        assert_eq!(envelope["payload"]["tool_call_id"], "bg-9");
3928        let response = envelope["tool_response"]["result"]
3929            .as_str()
3930            .map(serde_json::from_str::<serde_json::Value>)
3931            .transpose()
3932            .unwrap()
3933            .unwrap();
3934        assert_eq!(response["command"], "cargo test");
3935        assert_eq!(response["status"], "completed");
3936        assert!(info.output_tail.contains("inspect the completed build log"));
3937    }
3938
3939    async fn temp_store() -> (tempfile::TempDir, Arc<dyn Storage>, LockedSessionStore) {
3940        let temp = tempfile::tempdir().unwrap();
3941        let storage: Arc<dyn Storage> = Arc::new(
3942            bamboo_storage::v2::SessionStoreV2::new(temp.path().to_path_buf())
3943                .await
3944                .expect("storage init"),
3945        );
3946        let persistence = LockedSessionStore::new(storage.clone());
3947        (temp, storage, persistence)
3948    }
3949
3950    #[tokio::test]
3951    async fn enqueue_writes_pending_injection_and_preserves_messages() {
3952        let (_temp, storage, persistence) = temp_store().await;
3953
3954        let mut session = Session::new("sess-enq", "test-model");
3955        session.add_message(Message::user("do the build"));
3956        storage.save_session(&session).await.unwrap();
3957
3958        let info = BashCompletionInfo {
3959            session_id: "sess-enq".into(),
3960            bash_id: "sh-1".into(),
3961            command: "make".into(),
3962            exit_code: Some(0),
3963            status: "completed".into(),
3964            output_tail: "done".into(),
3965        };
3966        let saved = enqueue_bash_completion_injection(&persistence, &info)
3967            .await
3968            .expect("enqueue io ok")
3969            .expect("session exists");
3970
3971        let pending = saved
3972            .pending_injected_messages()
3973            .expect("pending injection present");
3974        assert_eq!(pending.len(), 1);
3975        let content = pending[0].get("content").and_then(|v| v.as_str()).unwrap();
3976        assert!(content.contains("sh-1"), "content: {content}");
3977        assert!(content.contains("make"), "content: {content}");
3978        assert!(content.contains("done"), "content: {content}");
3979        // The pre-existing conversation is untouched (no clobber).
3980        assert_eq!(saved.messages.len(), 1);
3981    }
3982
3983    #[tokio::test]
3984    async fn enqueue_returns_none_for_missing_session() {
3985        let (_temp, _storage, persistence) = temp_store().await;
3986        let info = BashCompletionInfo {
3987            session_id: "does-not-exist".into(),
3988            bash_id: "x".into(),
3989            command: "true".into(),
3990            exit_code: Some(0),
3991            status: "completed".into(),
3992            output_tail: String::new(),
3993        };
3994        let result = enqueue_bash_completion_injection(&persistence, &info)
3995            .await
3996            .expect("io ok");
3997        assert!(result.is_none(), "no session → nothing enqueued");
3998    }
3999
4000    // ── push-driven resume: the state transition + decision the push applies ──
4001
4002    /// A session suspended on `waiting_for_bash`, given the rich completion
4003    /// message, is transitioned to a resumable state: the wait is cleared, the
4004    /// runtime is Idle, the suspend-reason marker is gone, and the resume message
4005    /// is appended. This is exactly what the PUSH does to wake the loop
4006    /// event-driven (vs the old backstop poll).
4007    #[test]
4008    fn apply_bash_resume_transition_clears_wait_and_appends_message() {
4009        use bamboo_domain::session::runtime_state::WaitingForBashState;
4010
4011        let mut session = Session::new("sess-resume", "test-model");
4012        session.add_message(Message::user("kick off the build"));
4013        let mut rt = read_runtime_state(&session);
4014        rt.status = AgentStatusState::Running;
4015        rt.waiting_for_bash = Some(WaitingForBashState::for_bash(
4016            vec!["sh-1".into()],
4017            Utc::now(),
4018        ));
4019        write_runtime_state(&mut session, &rt);
4020        session.metadata.insert(
4021            "runtime.suspend_reason".to_string(),
4022            "waiting_for_bash".to_string(),
4023        );
4024
4025        let resume = bash_completion_resume_message(&["sh-1".to_string()], false);
4026        let did = apply_bash_resume_transition(&mut session, &resume);
4027
4028        assert!(did, "a suspended session must transition");
4029        let after = read_runtime_state(&session);
4030        assert!(
4031            after.waiting_for_bash.is_none(),
4032            "bash wait must be cleared"
4033        );
4034        assert_eq!(after.status, AgentStatusState::Idle, "runtime must be Idle");
4035        assert!(
4036            !session.metadata.contains_key("runtime.suspend_reason"),
4037            "suspend-reason marker must be removed"
4038        );
4039        assert_eq!(session.messages.len(), 2, "resume message must be appended");
4040        assert!(matches!(
4041            session.messages.last().map(|m| &m.role),
4042            Some(Role::User)
4043        ));
4044    }
4045
4046    /// The double-resume guard: a session NOT waiting on bash is a no-op — no
4047    /// message appended, nothing mutated. This is what makes the backstop poll
4048    /// harmlessly yield once the push has already resumed (and vice versa).
4049    #[test]
4050    fn apply_bash_resume_transition_noops_when_not_waiting() {
4051        let mut session = Session::new("sess-live", "test-model");
4052        session.add_message(Message::user("hi"));
4053
4054        let resume = bash_completion_resume_message(&["sh-1".to_string()], false);
4055        let did = apply_bash_resume_transition(&mut session, &resume);
4056
4057        assert!(!did, "a non-waiting session must not transition");
4058        assert_eq!(session.messages.len(), 1, "no resume message appended");
4059    }
4060
4061    /// The resume invariant: push-resume fires ONLY when the loop is suspended on
4062    /// bash AND every waited shell has finished. A still-running sibling shell
4063    /// keeps it on the enqueue path.
4064    #[test]
4065    fn bash_completion_should_resume_only_when_suspended_and_all_done() {
4066        assert!(bash_completion_should_resume(true, true));
4067        assert!(!bash_completion_should_resume(true, false)); // other shells still running
4068        assert!(!bash_completion_should_resume(false, true)); // live loop, not suspended
4069        assert!(!bash_completion_should_resume(false, false));
4070    }
4071
4072    #[test]
4073    fn two_shell_delivery_stages_backlog_before_one_final_activation() {
4074        let mut waiting = true;
4075        let mut durable_backlog = 0;
4076        let mut reservations = 0;
4077
4078        // First shell: its completion is durable, but a sibling still runs.
4079        durable_backlog += 1;
4080        let first = bash_completion_delivery_plan(waiting, false);
4081        assert_eq!(first, BashCompletionDeliveryPlan::DurableOnly);
4082        assert!(waiting);
4083        assert_eq!(durable_backlog, 1);
4084        assert_eq!(reservations, 0);
4085
4086        // Last shell: its own completion joins the same ordered backlog, then
4087        // the durable wait is cleared and exactly one activation is requested.
4088        durable_backlog += 1;
4089        let last = bash_completion_delivery_plan(waiting, true);
4090        assert_eq!(last, BashCompletionDeliveryPlan::ClearWaitThenActivate);
4091        waiting = false;
4092        reservations += 1;
4093        assert!(!waiting);
4094        assert_eq!(durable_backlog, 2);
4095        assert_eq!(reservations, 1);
4096    }
4097
4098    /// The push's resume message carries the shell's identity + status + output
4099    /// tail (so the model needs no `BashOutput` round-trip) and is tagged as a
4100    /// bash-completion resume so it satisfies the `has_pending_user_message` gate.
4101    #[test]
4102    fn bash_resume_message_from_info_carries_bashid_tail_and_kind() {
4103        let info = BashCompletionInfo {
4104            session_id: "s".into(),
4105            bash_id: "sh-42".into(),
4106            command: "cargo test".into(),
4107            exit_code: Some(0),
4108            status: "completed".into(),
4109            output_tail: "test result: ok".into(),
4110        };
4111        let msg = bash_resume_message_from_info(&info);
4112
4113        assert!(matches!(msg.role, Role::User));
4114        assert!(msg.content.contains("sh-42"), "content: {}", msg.content);
4115        assert!(
4116            msg.content.contains("cargo test"),
4117            "content: {}",
4118            msg.content
4119        );
4120        assert!(
4121            msg.content.contains("test result: ok"),
4122            "content: {}",
4123            msg.content
4124        );
4125        assert!(
4126            msg.content.contains("BashOutput"),
4127            "content: {}",
4128            msg.content
4129        );
4130        let meta = serde_json::to_string(&msg.metadata).unwrap();
4131        assert!(
4132            meta.contains(BASH_COMPLETION_RESUME_KIND),
4133            "resume message must be tagged as a bash-completion resume: {meta}"
4134        );
4135    }
4136}