Skip to main content

aft/subc/
mod.rs

1//! subc daemon attach — transport edge.
2//!
3//! When AFT is launched as `aft --subc <connection-file>`, it does NOT run the
4//! standalone NDJSON-over-stdin loop. Instead it connects to a running subc
5//! daemon over loopback TCP, authenticates with the pre-envelope HMAC handshake
6//! (`subc-transport`), then speaks the subc frame protocol (`subc-protocol`):
7//! ModuleHello → HelloAck (register provider capabilities), then a channel-0
8//! control loop (Ping/Pong, RouteBind) plus tool and management route calls.
9//!
10//! Concurrency: subc routes tool calls through the executor. The tokio
11//! edge never dispatches against `AppContext` inline; per-actor executor lanes
12//! own the reader/mutator epoch, while a writer task serializes outbound frames.
13
14pub mod blob_store;
15
16use std::collections::{HashMap, HashSet, VecDeque};
17use std::fmt;
18use std::io;
19use std::net::{IpAddr, SocketAddr};
20use std::ops::Deref;
21use std::path::{Path, PathBuf};
22use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
23use std::sync::{Arc, LazyLock, Mutex as StdMutex, OnceLock};
24use std::time::{Duration, Instant};
25
26use serde::Deserialize;
27use serde_json::{json, Value};
28
29use crate::config::Config;
30use crate::context::{App, AppContext, ProgressSender, RootHealthSnapshot};
31use crate::executor::{Executor, JobCancellation, Lane, PreExecutionCancelOutcome};
32use crate::fleet_status::{spawn_fleet_status_dial, FleetStatusClient};
33use crate::log_ctx;
34use crate::path_identity::ProjectRootId;
35use crate::protocol::{ProgressKind, PushFrame, RawRequest, Response};
36use crate::response_finalize::{DispatchOutcome, PendingResponse};
37use crate::run_tool_call::{
38    finish_tool_call_response, prepare_tool_call, run_tool_call, strip_agent_preview_arg_owned,
39    PhaseTrace, ToolCallContext, ToolCallOutcome, ToolCallResult,
40};
41use crate::runtime_drain;
42use crate::sandbox_spawn::{AuthenticatedPrincipal, PrincipalTrust};
43
44use subc_protocol::manifest::{
45    Bindings, Concurrency, ExecutionMode, IdentityBinding, IdentityScope, ManagementOperation,
46    ManagementOperationKind, ModuleManifest, ProviderRole, StorageBinding, StorageKind,
47    StorageScope, Tool, TrustTier,
48};
49use subc_protocol::session::{
50    HealthReport, HealthStatus, ModuleControlRequest, ModuleControlResponse,
51    MODULE_CONTROL_OP_HEALTH_CHECK,
52};
53use subc_protocol::{
54    ErrorBody, Flags, Frame, FrameType, ModuleHelloBody, Principal, Priority, RouteTarget,
55    MAX_FRAME_BODY_LEN, PROTOCOL_VERSION,
56};
57use subc_transport::{authenticate_client, connection_file, read_frame, write_frame};
58use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
59use tokio::net::TcpStream;
60use tokio::sync::{mpsc, oneshot, Notify};
61use tokio::task::JoinHandle;
62
63/// Per-attempt handshake deadline. The initial attach loop has a separate total
64/// budget so a stalled peer cannot consume an unbounded supervisor launch window.
65const AUTH_DEADLINE: Duration = Duration::from_secs(5);
66const ATTACH_RETRY_BUDGET: Duration = Duration::from_secs(60);
67const ATTACH_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(250);
68const ATTACH_RETRY_MAX_BACKOFF: Duration = Duration::from_secs(5);
69const ATTACH_RETRY_JITTER_PERCENT: u64 = 20;
70
71/// Correlation id for the initial ModuleHello (channel 0).
72const HELLO_CORR: u64 = 1;
73
74/// Per-session in-memory replay cap for must-deliver Push frames. This covers
75/// detach/re-attach while AFT stays alive; cross-restart replay is phased later.
76const PUSH_BUFFER_MAX_PER_KEY: usize = 256;
77
78/// Bounded guard for control-frame sends. If the daemon stops reading and the
79/// writer queue stays full, tear the subc edge down instead of stalling the
80/// route loop indefinitely.
81const CONTROL_SEND_TIMEOUT: Duration = Duration::from_millis(250);
82
83/// Cadence for the loop's deadline-driven drain work (retry-buffer flush,
84/// bg-wake emission, maintenance submission). Checked at the top of every
85/// loop turn so busy select arms cannot starve it.
86const DRAIN_TICK_PERIOD: Duration = Duration::from_millis(250);
87
88/// Fallback unbound-root artifact TTL when a root has no actor config yet.
89const IDLE_ROOT_TTL: Duration = Duration::from_secs(30 * 60);
90
91const WRITER_QUEUE_CAPACITY: usize = 256;
92
93/// Keep reliable Push bursts from monopolizing the current-thread subc loop;
94/// any remaining must-deliver frames stay queued for the next loop turn.
95const RELIABLE_PUSH_DRAIN_BUDGET: usize = 32;
96
97/// Limit maintenance submissions per tick so background drains cannot delay
98/// control-plane work such as completed RouteBind acknowledgements.
99///
100/// The decomposed maintenance pass charges this budget by Mutating job, not by
101/// root. Size the default burst for one maintenance pass over eight live roots,
102/// while follow-up batches still re-enter the capped queue instead of bypassing
103/// the budget.
104const MAINTENANCE_SUBMIT_BUDGET: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len() * 8;
105const INITIAL_MAINTENANCE_DRAIN_KINDS: [MaintenanceDrainKind; 4] = [
106    MaintenanceDrainKind::Watcher,
107    MaintenanceDrainKind::Lsp,
108    MaintenanceDrainKind::ConfigureTail,
109    MaintenanceDrainKind::CompletionDrains,
110];
111#[cfg(test)]
112const INITIAL_MAINTENANCE_JOB_COUNT: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len();
113
114const RELIABLE_WRITER_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(10);
115const RELIABLE_WRITER_RETRY_MAX_BACKOFF: Duration = Duration::from_millis(250);
116
117const DISPATCH_PATH_BIND_WARN_AFTER: Duration = Duration::from_secs(6);
118const ROUTE_BIND_DEADLINE: Duration = Duration::from_secs(12);
119
120/// Small bounded memory of completed task ids used to suppress stale lossy
121/// long-running reminders that arrive after their reliable completion event.
122const COMPLETED_TASK_SUPPRESSION_MAX: usize = 4096;
123
124/// Bash foreground orchestration polls detached tasks with short read-lane jobs.
125/// The sleep between polls is outside the executor so no read or write worker is
126/// pinned while a foreground command is still running.
127const PENDING_POLL_INTERVAL: Duration = Duration::from_millis(100);
128
129/// Host elicitation asks fail closed if the MCP facade does not answer promptly.
130const BASH_ELICITATION_TIMEOUT: Duration = Duration::from_secs(60);
131const BASH_ELICITATION_CREATE_METHOD: &str = "elicitation/create";
132
133#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
134struct RouteChannel {
135    channel: u16,
136    epoch: u32,
137}
138
139impl fmt::Display for RouteChannel {
140    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
141        write!(f, "{}@{}", self.channel, self.epoch)
142    }
143}
144
145type PushEnvelope = (ProjectRootId, PushFrame);
146type LossyPushEnvelope = (u64, ProjectRootId, PushFrame);
147type RetryBuffer = HashMap<RouteChannel, VecDeque<(push::ReplayKey, PushFrame)>>;
148mod bash;
149mod health;
150mod manifest;
151mod push;
152mod standing;
153mod wire;
154
155use self::health::{
156    build_health_report, warn_slow_pending_binds, warn_slow_running_interactive_jobs,
157    DeferredBashWaitGuard, DispatchPathMetrics, HealthRollupCache, HealthRollupWorker,
158    ReapBlockerCensus, ResponseTaskGuard,
159};
160pub(crate) use self::manifest::is_subc_native_plumbing_tool;
161use self::manifest::{
162    build_manifest, command_lane, control_flags, control_ops, is_bash_family_tool,
163    is_subc_agent_core_tool,
164};
165pub use self::wire::SubcError;
166
167/// Lifecycle milestones emitted only by the dedicated subc integration-test runner.
168///
169/// Production entry points never install a probe, so these notifications cannot
170/// affect routing or delivery behavior.
171#[doc(hidden)]
172#[derive(Clone, Debug, PartialEq, Eq)]
173pub enum SubcLifecycleEvent {
174    AttachDecision {
175        attempt: u32,
176        will_retry: bool,
177    },
178    RouteDetached {
179        route_channel: u16,
180        route_epoch: u32,
181        session_id: String,
182    },
183    ReliableCompletionRetained {
184        task_id: String,
185        session_id: String,
186    },
187    ReliableCompletionReplayed {
188        route_channel: u16,
189        route_epoch: u32,
190        task_id: String,
191        session_id: String,
192    },
193}
194
195/// Test-only observer for the detach/rebind lifecycle.
196#[doc(hidden)]
197#[derive(Clone)]
198pub struct SubcTestLifecycleProbe {
199    events_tx: mpsc::UnboundedSender<SubcLifecycleEvent>,
200}
201
202impl SubcTestLifecycleProbe {
203    #[doc(hidden)]
204    pub fn new(events_tx: mpsc::UnboundedSender<SubcLifecycleEvent>) -> Self {
205        Self { events_tx }
206    }
207
208    fn attach_decision(&self, attempt: u32, will_retry: bool) {
209        let _ = self.events_tx.send(SubcLifecycleEvent::AttachDecision {
210            attempt,
211            will_retry,
212        });
213    }
214
215    fn route_detached(&self, route: RouteChannel, session_id: &str) {
216        let _ = self.events_tx.send(SubcLifecycleEvent::RouteDetached {
217            route_channel: route.channel,
218            route_epoch: route.epoch,
219            session_id: session_id.to_string(),
220        });
221    }
222
223    fn reliable_completion_retained(&self, task_id: &str, session_id: &str) {
224        let _ = self
225            .events_tx
226            .send(SubcLifecycleEvent::ReliableCompletionRetained {
227                task_id: task_id.to_string(),
228                session_id: session_id.to_string(),
229            });
230    }
231
232    fn reliable_completion_replayed(&self, route: RouteChannel, task_id: &str, session_id: &str) {
233        let _ = self
234            .events_tx
235            .send(SubcLifecycleEvent::ReliableCompletionReplayed {
236                route_channel: route.channel,
237                route_epoch: route.epoch,
238                task_id: task_id.to_string(),
239                session_id: session_id.to_string(),
240            });
241    }
242}
243
244/// Test-only view of the fail-closed tool-call gate: would `name` be admitted
245/// on a bound route (as an agent tool or native plumbing)? Used by the
246/// plugin-send drift guard in `subc_plumbing_drift_test.rs`.
247pub fn is_tool_call_admitted_for_test(name: &str) -> bool {
248    manifest::is_subc_agent_core_tool(name) || manifest::is_subc_native_plumbing_tool(name)
249}
250use self::wire::{
251    build_error_frame, build_goodbye_frame, build_tool_response_frame,
252    build_tool_response_frame_with_limit, decrement_counted_channel, response_is_fatal_panic,
253    response_message, send_counted_channel, send_frame, send_reliable_writer_frame,
254    send_traced_tool_response_frame, ToolResponseWriteTrace, WriterFrame, WriterSender,
255};
256
257struct DecodedFrame {
258    frame: Frame,
259    phase_trace: PhaseTrace,
260}
261
262struct ToolCallCompletion {
263    text: String,
264    phase_trace: PhaseTrace,
265}
266
267#[derive(Clone, Copy, Debug, PartialEq, Eq)]
268enum RouteDetachPolicy {
269    RetainForReplay,
270    CancelOnDetach,
271}
272
273#[derive(Clone)]
274struct ActiveToolCall {
275    root_id: ProjectRootId,
276    cancellation: JobCancellation,
277    detach_policy: RouteDetachPolicy,
278}
279
280type ActiveToolCalls = Arc<StdMutex<HashMap<(RouteChannel, u64), ActiveToolCall>>>;
281
282struct PendingDeferredSetupGuard(Arc<AtomicUsize>);
283
284impl PendingDeferredSetupGuard {
285    fn new(count: Arc<AtomicUsize>) -> Self {
286        count.fetch_add(1, Ordering::SeqCst);
287        Self(count)
288    }
289}
290
291impl Drop for PendingDeferredSetupGuard {
292    fn drop(&mut self) {
293        self.0.fetch_sub(1, Ordering::SeqCst);
294    }
295}
296
297enum DeferredSetupOutcome {
298    Immediate {
299        text: String,
300        phase_trace: PhaseTrace,
301    },
302    Deferred {
303        pending: PendingResponse,
304        surface_downgraded: bool,
305        phase_trace: PhaseTrace,
306    },
307}
308
309struct PendingSubcResponse {
310    route: RouteChannel,
311    corr: u64,
312    flags: Flags,
313    ver: u8,
314    root: ProjectRootId,
315    session_id: String,
316    bare_name: String,
317    format_context: crate::subc_format::FormatContext,
318    bind_trust: BindTrust,
319    pending: PendingResponse,
320    surface_downgraded: bool,
321    phase_trace: PhaseTrace,
322}
323
324struct ResolvedSubcResponse {
325    entry: PendingSubcResponse,
326    response: Response,
327}
328
329#[derive(Default)]
330struct PendingSubcResponses {
331    entries: Vec<PendingSubcResponse>,
332}
333
334impl PendingSubcResponses {
335    fn register(&mut self, pending: PendingSubcResponse) {
336        self.entries.retain(|entry| {
337            let keep = entry.route != pending.route || entry.corr != pending.corr;
338            if !keep {
339                if let Some(cancellation) = &entry.pending.cancellation {
340                    cancellation.request_cancel();
341                }
342            }
343            keep
344        });
345        self.entries.push(pending);
346    }
347
348    fn poll_ready(&mut self, executor: &Executor) -> Vec<ResolvedSubcResponse> {
349        let mut ready = Vec::new();
350        let mut waiting = Vec::with_capacity(self.entries.len());
351        for mut entry in self.entries.drain(..) {
352            let response = executor
353                .actor_context(&entry.root)
354                .and_then(|ctx| (entry.pending.poll)(&ctx));
355            if let Some(response) = response {
356                ready.push(ResolvedSubcResponse { entry, response });
357            } else {
358                waiting.push(entry);
359            }
360        }
361        self.entries = waiting;
362        ready
363    }
364
365    fn cancel_request(&mut self, route: RouteChannel, corr: u64) -> bool {
366        let mut cancelled = false;
367        self.entries.retain(|entry| {
368            let keep = entry.route != route || entry.corr != corr;
369            if !keep {
370                cancelled = true;
371                if let Some(cancellation) = &entry.pending.cancellation {
372                    cancellation.request_cancel();
373                }
374            }
375            keep
376        });
377        cancelled
378    }
379
380    fn drain_route(
381        &mut self,
382        route: RouteChannel,
383        executor: &Executor,
384    ) -> Vec<ResolvedSubcResponse> {
385        self.drain_matching(executor, |entry| entry.route == route)
386    }
387
388    fn drain_on_shutdown(&mut self, executor: &Executor) -> Vec<ResolvedSubcResponse> {
389        self.drain_matching(executor, |_| true)
390    }
391
392    fn drain_matching(
393        &mut self,
394        executor: &Executor,
395        matches: impl Fn(&PendingSubcResponse) -> bool,
396    ) -> Vec<ResolvedSubcResponse> {
397        let mut resolved = Vec::new();
398        let mut waiting = Vec::with_capacity(self.entries.len());
399        for mut entry in self.entries.drain(..) {
400            if !matches(&entry) {
401                waiting.push(entry);
402                continue;
403            }
404            if let Some(cancellation) = &entry.pending.cancellation {
405                cancellation.request_cancel();
406            }
407            if let Some(ctx) = executor.actor_context(&entry.root) {
408                if let Some(on_shutdown) = entry.pending.on_shutdown.as_mut() {
409                    let response = on_shutdown(&ctx);
410                    resolved.push(ResolvedSubcResponse { entry, response });
411                }
412            }
413        }
414        self.entries = waiting;
415        resolved
416    }
417
418    fn is_empty(&self) -> bool {
419        self.entries.is_empty()
420    }
421}
422
423#[derive(Clone)]
424struct PushSenders {
425    lossy_tx: mpsc::Sender<LossyPushEnvelope>,
426    reliable_tx: mpsc::UnboundedSender<PushEnvelope>,
427    lossy_overflow: Arc<push::LossyOverflow>,
428    lossy_seq: Arc<AtomicU64>,
429    fleet_status_client: FleetStatusClient,
430}
431
432#[derive(Clone)]
433struct PersistentCancelSignal {
434    inner: Arc<PersistentCancelInner>,
435}
436
437struct PersistentCancelInner {
438    cancelled: AtomicBool,
439    notify: Notify,
440}
441
442impl PersistentCancelSignal {
443    fn new() -> Self {
444        Self {
445            inner: Arc::new(PersistentCancelInner {
446                cancelled: AtomicBool::new(false),
447                notify: Notify::new(),
448            }),
449        }
450    }
451
452    fn cancel(&self) {
453        if !self.inner.cancelled.swap(true, Ordering::SeqCst) {
454            self.inner.notify.notify_waiters();
455        }
456    }
457
458    fn is_cancelled(&self) -> bool {
459        self.inner.cancelled.load(Ordering::SeqCst)
460    }
461
462    async fn cancelled(&self) {
463        // `enable()` REGISTERS this waiter before we read the flag, closing the
464        // lost-wakeup window: `notify_waiters()` only wakes already-registered
465        // waiters and stores no permit, so without enable() a `cancel()` firing
466        // between the flag read and `.await` would be missed and the future
467        // would park forever (cancel() fires only once). With enable(), a cancel
468        // racing the flag read still wakes the registered waiter. The loop is a
469        // belt-and-suspenders re-check on spurious wakeups.
470        loop {
471            let notified = self.inner.notify.notified();
472            tokio::pin!(notified);
473            notified.as_mut().enable();
474            if self.is_cancelled() {
475                return;
476            }
477            notified.await;
478        }
479    }
480}
481
482fn submit_active_tool_call(
483    executor: &Executor,
484    active: &ActiveToolCalls,
485    route: RouteChannel,
486    corr: u64,
487    root_id: ProjectRootId,
488    lane: Lane,
489    request_id: String,
490    detach_policy: RouteDetachPolicy,
491    job: crate::executor::ExecutorJob,
492) -> oneshot::Receiver<Response> {
493    let (rx, cancellation) =
494        executor.submit_cancellable_async(root_id.clone(), lane, request_id, job);
495    active
496        .lock()
497        .unwrap_or_else(std::sync::PoisonError::into_inner)
498        .insert(
499            (route, corr),
500            ActiveToolCall {
501                root_id,
502                cancellation,
503                detach_policy,
504            },
505        );
506    rx
507}
508
509fn finish_active_tool_call(active: &ActiveToolCalls, route: RouteChannel, corr: u64) -> bool {
510    active
511        .lock()
512        .unwrap_or_else(std::sync::PoisonError::into_inner)
513        .remove(&(route, corr))
514        .is_some()
515}
516
517fn active_tool_call_is_registered(
518    active: &ActiveToolCalls,
519    route: RouteChannel,
520    corr: u64,
521) -> bool {
522    active
523        .lock()
524        .unwrap_or_else(std::sync::PoisonError::into_inner)
525        .contains_key(&(route, corr))
526}
527
528fn cancel_active_tool_call(
529    active: &ActiveToolCalls,
530    executor: &Executor,
531    route: RouteChannel,
532    corr: u64,
533    reason: &str,
534) -> bool {
535    let call = active
536        .lock()
537        .unwrap_or_else(std::sync::PoisonError::into_inner)
538        .remove(&(route, corr));
539    let Some(call) = call else {
540        return false;
541    };
542    let outcome = executor.cancel_job(&call.root_id, &call.cancellation);
543    log::debug!(
544        "subc attach: cancelled active tool call route={route} corr={corr} reason={reason} outcome={outcome:?}"
545    );
546    true
547}
548
549#[derive(Clone, Copy, Debug, PartialEq, Eq)]
550enum RouteWorkDisposition {
551    RetainForReplay,
552    RetainStartedForReplay,
553    Abandon,
554}
555
556fn apply_route_work_disposition(
557    active: &ActiveToolCalls,
558    executor: &Executor,
559    route: RouteChannel,
560    disposition: RouteWorkDisposition,
561    reason: &str,
562) -> usize {
563    if disposition != RouteWorkDisposition::Abandon {
564        let route_calls = active
565            .lock()
566            .unwrap_or_else(std::sync::PoisonError::into_inner)
567            .iter()
568            .filter(|((call_route, _), _)| *call_route == route)
569            .map(|(key, call)| (*key, call.clone()))
570            .collect::<Vec<_>>();
571        let mut retained = 0usize;
572        let mut cancelled_before_execution = 0usize;
573        let mut cancelled_terminal = 0usize;
574
575        for (key, call) in route_calls {
576            let remove = match (call.detach_policy, disposition) {
577                (RouteDetachPolicy::RetainForReplay, RouteWorkDisposition::RetainForReplay) => {
578                    retained += 1;
579                    false
580                }
581                (
582                    RouteDetachPolicy::RetainForReplay,
583                    RouteWorkDisposition::RetainStartedForReplay,
584                ) => {
585                    match executor.cancel_job_before_execution(&call.root_id, &call.cancellation) {
586                        PreExecutionCancelOutcome::AlreadyStarted => {
587                            retained += 1;
588                            false
589                        }
590                        PreExecutionCancelOutcome::QueuedRemoved
591                        | PreExecutionCancelOutcome::DispatchedCancelled => {
592                            cancelled_before_execution += 1;
593                            true
594                        }
595                    }
596                }
597                (RouteDetachPolicy::CancelOnDetach, _) => {
598                    executor.cancel_job(&call.root_id, &call.cancellation);
599                    cancelled_terminal += 1;
600                    true
601                }
602                (_, RouteWorkDisposition::Abandon) => unreachable!("handled below"),
603            };
604            if remove {
605                active
606                    .lock()
607                    .unwrap_or_else(std::sync::PoisonError::into_inner)
608                    .remove(&key);
609            }
610        }
611        log::debug!(
612            "subc attach: retained {retained} replayable tool call(s), cancelled {cancelled_before_execution} replayable call(s) before execution, and cancelled {cancelled_terminal} teardown-terminal call(s) route={route} reason={reason}"
613        );
614        return retained;
615    }
616
617    let cancelled = {
618        let mut calls = active
619            .lock()
620            .unwrap_or_else(std::sync::PoisonError::into_inner);
621        let mut cancelled = Vec::new();
622        calls.retain(|(call_route, _), call| {
623            if *call_route == route {
624                cancelled.push(call.clone());
625                false
626            } else {
627                true
628            }
629        });
630        cancelled
631    };
632    for call in &cancelled {
633        let outcome = executor.cancel_job(&call.root_id, &call.cancellation);
634        log::debug!(
635            "subc attach: cancelled active tool call route={route} reason={reason} outcome={outcome:?}"
636        );
637    }
638    cancelled.len()
639}
640
641fn cancel_all_active_tool_calls(
642    active: &ActiveToolCalls,
643    executor: &Executor,
644    reason: &str,
645) -> usize {
646    let cancelled = {
647        let mut calls = active
648            .lock()
649            .unwrap_or_else(std::sync::PoisonError::into_inner);
650        calls.drain().map(|(_, call)| call).collect::<Vec<_>>()
651    };
652    for call in &cancelled {
653        let outcome = executor.cancel_job(&call.root_id, &call.cancellation);
654        log::debug!("subc attach: cancelled active tool call reason={reason} outcome={outcome:?}");
655    }
656    cancelled.len()
657}
658
659#[derive(Debug, Clone, Copy, PartialEq, Eq)]
660pub enum BindTrust {
661    FirstParty,
662    Untrusted,
663}
664
665impl BindTrust {
666    fn allows_bash_observation(self) -> bool {
667        matches!(self, Self::FirstParty)
668    }
669
670    fn label(self) -> &'static str {
671        match self {
672            Self::FirstParty => "first_party",
673            Self::Untrusted => "untrusted",
674        }
675    }
676
677    fn sandbox_trust(self) -> PrincipalTrust {
678        match self {
679            Self::FirstParty => PrincipalTrust::FirstParty,
680            Self::Untrusted => PrincipalTrust::Untrusted,
681        }
682    }
683}
684
685pub(super) fn trust_for_principal(principal: &Option<Principal>) -> BindTrust {
686    match principal {
687        Some(Principal::Direct) => BindTrust::FirstParty,
688        // Module renames are flag-days: the daemon registry refuses duplicate
689        // active ids, so a renaming module cannot advertise both names during
690        // its transition. This allowlist is DIALLED, not dialling — it must
691        // accept a module's NEW name in a released binary before the module
692        // starts using it, and the old name stays until the flip has settled.
693        // That is why transitional pairs appear here: llm-runner/broca was the
694        // previous rename, alfonso-core/prefrontal is the current one. When
695        // retiring an old name, confirm the fleet no longer spawns it — a
696        // stale entry here is inert, but a missing one silently downgrades a
697        // first-party module to Untrusted and revokes its bash access.
698        Some(Principal::Reserved { module_id })
699            if module_id == "llm-runner"
700                || module_id == "aft"
701                || module_id == "broca"
702                || module_id == "alfonso-core"
703                || module_id == "prefrontal"
704                || module_id == "prefrontal-core" =>
705        {
706            BindTrust::FirstParty
707        }
708        Some(Principal::Reserved { .. }) | Some(Principal::Unverified) | None => {
709            BindTrust::Untrusted
710        }
711    }
712}
713
714fn harness_forces_untrusted(harness: &str) -> bool {
715    harness.starts_with("fed:")
716}
717
718pub(super) fn trust_for_bind(harness: &str, principal: &Option<Principal>) -> BindTrust {
719    if harness_forces_untrusted(harness) {
720        BindTrust::Untrusted
721    } else {
722        trust_for_principal(principal)
723    }
724}
725
726fn principal_id(principal: &Option<Principal>) -> Option<String> {
727    match principal {
728        Some(Principal::Direct) => Some("direct".to_string()),
729        Some(Principal::Reserved { module_id }) => Some(format!("reserved:{module_id}")),
730        Some(Principal::Unverified) => Some("unverified".to_string()),
731        None => None,
732    }
733}
734
735fn principal_label(principal: &Option<Principal>) -> String {
736    principal_id(principal).unwrap_or_else(|| "absent".to_string())
737}
738
739#[derive(Debug)]
740/// Per-root route metadata owned by the subc loop. The `active_bash_waits` field
741/// counts detached bash processes that are still being observed for this root.
742/// Any future logic that evicts roots based on idle time must not evict a root
743/// while this count is greater than zero, because a foreground bash response may
744/// still arrive later.
745struct RootMeta {
746    maintenance_pending: bool,
747    maintenance_jobs_in_flight: usize,
748    maintenance_queued_kinds: VecDeque<MaintenanceDrainKind>,
749    maintenance_last_submitted: Option<Instant>,
750    maintenance_poisoned: bool,
751    last_touched: Instant,
752    diagnostics_on_edit: bool,
753    active_bash_waits: usize,
754    idle_artifacts_evicted: bool,
755    unbound_quiesced: bool,
756    consecutive_missing_sweeps: u8,
757}
758
759#[derive(Debug)]
760struct PendingBind {
761    bind_root_id: ProjectRootId,
762    inserted_new_actor: bool,
763    cancelled: bool,
764    configure_request_id: String,
765    started_at: Instant,
766    warned_half_deadline: bool,
767    deadline_reported: bool,
768    corr: u64,
769    ver: u8,
770    flags: Flags,
771    /// Exact-job cancellation for the submitted configure: Goodbye and
772    /// deadline expiry cancel the executor job operationally (queued jobs are
773    /// removed, running configures return at their next checkpoint) instead of
774    /// only marking bookkeeping.
775    cancellation: crate::executor::JobCancellation,
776}
777
778struct RouteBindCompletion {
779    route: RouteChannel,
780    identity: RouteIdentity,
781    bind_root_id: ProjectRootId,
782    inserted_new_actor: bool,
783    configure_response: Response,
784    diagnostics_on_edit: bool,
785    ver: u8,
786    corr: u64,
787    flags: Flags,
788}
789
790#[derive(Debug, Clone)]
791struct RouteIdentity(Arc<RouteIdentityData>);
792
793#[derive(Debug)]
794struct RouteIdentityData {
795    root: ProjectRootId,
796    project_root: PathBuf,
797    harness: String,
798    session: String,
799    trust: BindTrust,
800    spawn_principal: AuthenticatedPrincipal,
801    consumer_elicitation_capable: bool,
802}
803
804impl Deref for RouteIdentity {
805    type Target = RouteIdentityData;
806
807    fn deref(&self) -> &Self::Target {
808        &self.0
809    }
810}
811
812#[derive(Debug, Clone)]
813struct RetainedSessionIdentity {
814    harness: String,
815    trust: BindTrust,
816}
817
818#[derive(Clone)]
819struct BgSub {
820    corr: u64,
821    ver: u8,
822    flags: Flags,
823    root: ProjectRootId,
824    session: String,
825}
826
827#[derive(Clone, Copy, Debug)]
828struct BgWakeState {
829    next_nudge_at: Instant,
830    nudges_sent: u32,
831}
832
833impl BgWakeState {
834    fn armed(now: Instant) -> Self {
835        Self {
836            next_nudge_at: now,
837            nudges_sent: 0,
838        }
839    }
840}
841
842type BgWakePending = HashMap<RouteChannel, BgWakeState>;
843
844// A session can be observed by multiple long-lived consumer records. Retain
845// every route so each wake uses the correlation captured by that route's BgSub.
846type BgSubsBySession = HashMap<(ProjectRootId, String), HashSet<RouteChannel>>;
847
848struct MaintenanceCompletion {
849    root_id: ProjectRootId,
850    kind: MaintenanceDrainKind,
851    response: Response,
852    empty_bg_sessions: Vec<(String, u64)>,
853    unacked_bg_keys: Option<HashSet<String>>,
854    requeue_kind: Option<MaintenanceDrainKind>,
855}
856
857#[derive(Clone, Copy, Debug, PartialEq, Eq)]
858enum MaintenanceDrainKind {
859    Watcher,
860    Lsp,
861    ConfigureTail,
862    CompletionDrains,
863}
864
865impl MaintenanceDrainKind {
866    fn label(self) -> &'static str {
867        match self {
868            Self::Watcher => "watcher",
869            Self::Lsp => "lsp",
870            Self::ConfigureTail => "configure-tail",
871            Self::CompletionDrains => "completion-drains",
872        }
873    }
874}
875
876#[derive(Debug, Default)]
877struct MaintenanceJobOutcome {
878    empty_bg_sessions: Vec<(String, u64)>,
879    unacked_bg_keys: Option<HashSet<String>>,
880    requeue_kind: Option<MaintenanceDrainKind>,
881}
882
883#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
884struct ReverseCorrKey {
885    route: RouteChannel,
886    corr: u64,
887}
888
889struct PendingBashAsk {
890    route: RouteChannel,
891    tool_corr: u64,
892    tool_flags: Flags,
893    tool_ver: u8,
894    root: ProjectRootId,
895    project_root: PathBuf,
896    session_id: String,
897    spawn_principal: AuthenticatedPrincipal,
898    edit_slot_survives: Option<bool>,
899    request_id: String,
900    arguments: Value,
901    format_context: crate::subc_format::FormatContext,
902    cancel: bash::BashWaitCancel,
903    grants: Vec<String>,
904    expires_at: Instant,
905}
906
907impl RootMeta {
908    fn new(now: Instant) -> Self {
909        Self {
910            maintenance_pending: false,
911            maintenance_jobs_in_flight: 0,
912            maintenance_queued_kinds: VecDeque::new(),
913            maintenance_last_submitted: None,
914            maintenance_poisoned: false,
915            last_touched: now,
916            diagnostics_on_edit: false,
917            active_bash_waits: 0,
918            idle_artifacts_evicted: false,
919            unbound_quiesced: false,
920            consecutive_missing_sweeps: 0,
921        }
922    }
923
924    fn note_activity(&mut self) {
925        self.last_touched = Instant::now();
926    }
927
928    fn reactivate_bound(&mut self) {
929        self.note_activity();
930        self.idle_artifacts_evicted = false;
931        self.unbound_quiesced = false;
932    }
933}
934
935fn due_maintenance_jobs(
936    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
937    executor: Option<&Executor>,
938    bg_sub_by_session: &BgSubsBySession,
939    bg_wake_pending: &BgWakePending,
940    budget: usize,
941    pending_bind_roots: &HashSet<ProjectRootId>,
942) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
943    let mut jobs = Vec::new();
944    let mut deferred = false;
945    let mut roots = live_roots.keys().cloned().collect::<Vec<_>>();
946    roots.sort_by(|left, right| {
947        let left_last = live_roots
948            .get(left)
949            .and_then(|meta| meta.maintenance_last_submitted);
950        let right_last = live_roots
951            .get(right)
952            .and_then(|meta| meta.maintenance_last_submitted);
953        left_last
954            .cmp(&right_last)
955            .then_with(|| left.as_path().cmp(right.as_path()))
956    });
957
958    for root_id in roots {
959        let Some(meta) = live_roots.get_mut(&root_id) else {
960            continue;
961        };
962        if meta.maintenance_poisoned {
963            continue;
964        }
965
966        if pending_bind_roots.contains(&root_id) {
967            if meta.maintenance_pending || !meta.maintenance_queued_kinds.is_empty() {
968                deferred = true;
969            }
970            continue;
971        }
972
973        if !meta.maintenance_pending {
974            if jobs.len() >= budget {
975                deferred = true;
976                continue;
977            }
978            // Only enqueue kinds with pending work. Probes are cheap and
979            // fail-open (contended sources count as pending), so an idle root
980            // costs four probes per tick instead of four dispatched jobs.
981            let executor_actor_context =
982                executor.and_then(|executor| executor.actor_context(&root_id));
983            let root_has_pending_bg_wake =
984                bg_sub_by_session.iter().any(|((sub_root, _), channels)| {
985                    sub_root == &root_id
986                        && channels
987                            .iter()
988                            .any(|channel| bg_wake_pending.contains_key(channel))
989                });
990            let kinds_with_work: Vec<MaintenanceDrainKind> = match executor_actor_context {
991                Some(ctx) => INITIAL_MAINTENANCE_DRAIN_KINDS
992                    .into_iter()
993                    .filter(|kind| {
994                        if meta.unbound_quiesced && !matches!(kind, MaintenanceDrainKind::Lsp) {
995                            return false;
996                        }
997                        match kind {
998                            MaintenanceDrainKind::Watcher => ctx.watcher_drain_has_work(),
999                            MaintenanceDrainKind::Lsp => ctx.lsp_drain_has_work(),
1000                            MaintenanceDrainKind::ConfigureTail => ctx.configure_tail_has_work(),
1001                            // Every CompletionDrains source is visible at this enqueue site:
1002                            // AppContext probes completion queues, this loop owns bg wakes,
1003                            // and queued continuations bypass probing via maintenance_pending.
1004                            // New drain sources must expose a probe here rather than making
1005                            // every subscribed root fail open again.
1006                            MaintenanceDrainKind::CompletionDrains => {
1007                                root_has_pending_bg_wake || ctx.completion_drains_have_work()
1008                            }
1009                        }
1010                    })
1011                    .collect(),
1012                None if meta.unbound_quiesced => Vec::new(),
1013                // No context handle (actor gone mid-tick): enqueue everything.
1014                None => INITIAL_MAINTENANCE_DRAIN_KINDS.to_vec(),
1015            };
1016            if kinds_with_work.is_empty() {
1017                continue;
1018            }
1019            meta.maintenance_pending = true;
1020            meta.maintenance_queued_kinds.extend(kinds_with_work);
1021        }
1022
1023        while let Some(kind) = meta.maintenance_queued_kinds.pop_front() {
1024            if jobs.len() >= budget {
1025                meta.maintenance_queued_kinds.push_front(kind);
1026                deferred = true;
1027                break;
1028            }
1029            meta.maintenance_jobs_in_flight += 1;
1030            meta.maintenance_last_submitted = Some(Instant::now());
1031            jobs.push((root_id.clone(), kind));
1032        }
1033
1034        meta.maintenance_pending =
1035            meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
1036    }
1037
1038    (jobs, deferred)
1039}
1040
1041fn eviction_estimate_label(estimate: &crate::memory::MemoryEstimate) -> String {
1042    match estimate.estimated_bytes {
1043        Some(bytes) => format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
1044        None if estimate.status == "busy" => "busy".to_string(),
1045        None => "not estimated".to_string(),
1046    }
1047}
1048
1049fn optional_memory_label(bytes: Option<u64>) -> String {
1050    bytes.map_or_else(
1051        || "not estimated".to_string(),
1052        |bytes| format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
1053    )
1054}
1055
1056fn pressure_relief_label(relief: &crate::memory::AllocatorPressureRelief) -> String {
1057    format!(
1058        "; allocator pressure relief: RSS {} -> {}, in-use {} -> {}, allocated {} -> {}, slack {} -> {}, allocator reported {:.1} MB released",
1059        optional_memory_label(relief.rss_before_bytes),
1060        optional_memory_label(relief.rss_after_bytes),
1061        optional_memory_label(relief.allocator_before.bytes_in_use),
1062        optional_memory_label(relief.allocator_after.bytes_in_use),
1063        optional_memory_label(relief.allocator_before.size_allocated),
1064        optional_memory_label(relief.allocator_after.size_allocated),
1065        optional_memory_label(relief.allocator_before.retained_slack_bytes),
1066        optional_memory_label(relief.allocator_after.retained_slack_bytes),
1067        relief.bytes_released as f64 / (1024.0 * 1024.0),
1068    )
1069}
1070
1071fn idle_root_eviction_message(
1072    root_id: &ProjectRootId,
1073    memory: &crate::memory::RootMemorySnapshot,
1074    pressure_relief: Option<&crate::memory::AllocatorPressureRelief>,
1075) -> String {
1076    // Bash, LSP, and parser state remain resident. The freed total is deliberately
1077    // only the known-byte portion of handles eviction actually drops.
1078    let freed_bytes = [
1079        &memory.semantic,
1080        &memory.trigram,
1081        &memory.symbols,
1082        &memory.callgraph,
1083        &memory.inspect,
1084    ]
1085    .iter()
1086    .filter_map(|estimate| estimate.estimated_bytes)
1087    .fold(0u64, u64::saturating_add);
1088    let mut message = format!(
1089        "evicted idle root {}: freed ~{:.1} MB (semantic {}, trigram {}, symbols {}, callgraph {}, inspect {}; retained: bash {}, lsp {}, parser_pool {})",
1090        root_id.as_path().display(),
1091        freed_bytes as f64 / (1024.0 * 1024.0),
1092        eviction_estimate_label(&memory.semantic),
1093        eviction_estimate_label(&memory.trigram),
1094        eviction_estimate_label(&memory.symbols),
1095        eviction_estimate_label(&memory.callgraph),
1096        eviction_estimate_label(&memory.inspect),
1097        eviction_estimate_label(&memory.bash),
1098        eviction_estimate_label(&memory.lsp),
1099        eviction_estimate_label(&memory.parser_pool),
1100    );
1101    if let Some(pressure_relief) = pressure_relief {
1102        message.push_str(&pressure_relief_label(pressure_relief));
1103    }
1104    message
1105}
1106
1107fn root_idle_ttl(executor: &Executor, root_id: &ProjectRootId) -> Duration {
1108    executor
1109        .actor_context(root_id)
1110        .map(|ctx| ctx.config().idle.root_ttl())
1111        .unwrap_or(IDLE_ROOT_TTL)
1112}
1113
1114fn process_has_been_idle(
1115    now: Instant,
1116    live_roots: &HashMap<ProjectRootId, RootMeta>,
1117    executor: &Executor,
1118) -> bool {
1119    !live_roots.is_empty()
1120        && live_roots.iter().all(|(root_id, meta)| {
1121            now.saturating_duration_since(meta.last_touched) >= root_idle_ttl(executor, root_id)
1122                && meta.active_bash_waits == 0
1123                && !meta.maintenance_pending
1124                && meta.maintenance_queued_kinds.is_empty()
1125        })
1126}
1127
1128fn allocator_pressure_relief_after_idle_sweep(
1129    now: Instant,
1130    live_roots: &HashMap<ProjectRootId, RootMeta>,
1131    executor: &Executor,
1132) -> Option<crate::memory::AllocatorPressureRelief> {
1133    if !process_has_been_idle(now, live_roots, executor)
1134        || live_roots.keys().any(|root_id| {
1135            executor
1136                .actor_context(root_id)
1137                .is_some_and(|ctx| ctx.artifact_eviction_blocked())
1138        })
1139    {
1140        return None;
1141    }
1142
1143    #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu")))]
1144    {
1145        Some(crate::memory::relieve_allocator_pressure())
1146    }
1147    #[cfg(not(any(target_os = "macos", all(target_os = "linux", target_env = "gnu"))))]
1148    {
1149        None
1150    }
1151}
1152
1153fn quiesce_unbound_root(
1154    root_id: &ProjectRootId,
1155    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1156    executor: &Arc<Executor>,
1157) {
1158    let Some(meta) = live_roots.get_mut(root_id) else {
1159        return;
1160    };
1161
1162    let ctx = executor.actor_context(root_id);
1163    if let Some(ctx) = ctx.as_ref() {
1164        // Close lifecycle admission before touching scheduler queues. A running
1165        // ConfigureTail cannot release gates, install a watcher, or reserve a
1166        // callgraph build after this transition becomes visible.
1167        ctx.mark_subc_unbound();
1168        ctx.bash_background()
1169            .replace_live_delivery_sessions(HashSet::new());
1170    }
1171    let cancelled = executor.cancel_queued_maintenance(root_id);
1172    // Transient unbind keeps the root WARM: the watcher stays running (its
1173    // events accumulate and replay on rebind, so no unobserved gap exists) and
1174    // resident artifacts stay resident. Host restarts unbind every root and
1175    // rebind seconds later; stopping the watcher here would force strict
1176    // re-verification plus a full callgraph rebuild on every restart. The
1177    // expensive teardown (watcher stop + gap invalidation) belongs to the
1178    // idle-TTL reaper and the root-deleted path.
1179    let discarded = ctx
1180        .map(|ctx| crate::commands::configure::cancel_deferred_configure_maintenance(&ctx))
1181        .unwrap_or(0);
1182    meta.unbound_quiesced = true;
1183    meta.maintenance_queued_kinds.clear();
1184    meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
1185    log::info!(
1186        "subc attach: quiesced unbound root {} (cancelled {} queued maintenance job(s), cancelled {} configure maintenance job(s)); cause=goodbye_unbound",
1187        root_id.as_path().display(),
1188        cancelled,
1189        discarded
1190    );
1191}
1192
1193#[allow(clippy::too_many_arguments)]
1194fn quiesce_connection_roots(
1195    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1196    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
1197    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1198    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1199    installed_route_epochs: &mut HashMap<u16, u32>,
1200    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1201    active_tool_calls: &ActiveToolCalls,
1202    executor: &Arc<Executor>,
1203) {
1204    cancel_all_active_tool_calls(active_tool_calls, executor, "connection teardown");
1205    for cancel in route_bash_cancels.values() {
1206        cancel.token.cancel();
1207    }
1208    route_bash_cancels.clear();
1209
1210    let mut roots = live_roots.keys().cloned().collect::<HashSet<_>>();
1211    for pending in pending_binds.values_mut() {
1212        pending.cancelled = true;
1213        roots.insert(pending.bind_root_id.clone());
1214        let _ = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
1215    }
1216
1217    // A connection exit abandons every installed route at once. Close lifecycle
1218    // admission before cancelling maintenance so no deferred worker can restore
1219    // root activity after the loop-owned route tables disappear.
1220    for root_id in roots {
1221        if live_roots.contains_key(&root_id) {
1222            quiesce_unbound_root(&root_id, live_roots, executor);
1223        } else if let Some(ctx) = executor.actor_context(&root_id) {
1224            ctx.mark_subc_unbound();
1225            executor.cancel_queued_maintenance(&root_id);
1226            crate::commands::configure::cancel_deferred_configure_maintenance(&ctx);
1227        }
1228    }
1229
1230    routes.clear();
1231    root_channels.clear();
1232    installed_route_epochs.clear();
1233}
1234
1235/// Per-channel epoch watermarks for roots reclaimed without a client Goodbye.
1236/// The 16-bit channel space bounds this map, and no root identity or resource
1237/// handle is retained. It exists only so late requests receive a typed error.
1238#[derive(Debug, Default)]
1239struct ReclaimedRoutes {
1240    highest_epoch_by_channel: HashMap<u16, u32>,
1241}
1242
1243impl ReclaimedRoutes {
1244    fn insert(&mut self, route: RouteChannel) {
1245        self.highest_epoch_by_channel
1246            .entry(route.channel)
1247            .and_modify(|epoch| *epoch = (*epoch).max(route.epoch))
1248            .or_insert(route.epoch);
1249    }
1250
1251    fn contains(&self, route: RouteChannel) -> bool {
1252        self.highest_epoch_by_channel
1253            .get(&route.channel)
1254            .is_some_and(|epoch| route.epoch <= *epoch)
1255    }
1256}
1257
1258#[derive(Debug, Default)]
1259struct IdleReapOutcome {
1260    evicted: usize,
1261    forgotten_deleted_roots: Vec<ProjectRootId>,
1262}
1263
1264fn reap_idle_roots(
1265    now: Instant,
1266    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1267    pending_binds: &HashMap<RouteChannel, PendingBind>,
1268    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1269    executor: &Arc<Executor>,
1270    metrics: &DispatchPathMetrics,
1271) -> IdleReapOutcome {
1272    let pending_bind_roots = pending_binds
1273        .values()
1274        .map(|pending| pending.bind_root_id.clone())
1275        .collect::<HashSet<_>>();
1276    let mut census = ReapBlockerCensus::default();
1277    let mut candidates = Vec::new();
1278
1279    for (root_id, meta) in live_roots.iter_mut() {
1280        let deleted = !root_id.as_path().exists();
1281        if deleted {
1282            // A missing directory makes a bound route obsolete, but one failed
1283            // lookup is not enough evidence to tear down a client-visible actor.
1284            // Requiring two maintenance sweeps protects atomic replacement and
1285            // transient filesystem failures; observing the path resets the proof.
1286            // Absence also covers renames: a task's cwd handle can follow the
1287            // moved directory while the registered path disappears. The old
1288            // path is deliberately treated as a retired root identity, so the
1289            // reaper accepts killing such tasks; rename a project only with no
1290            // live tasks rather than relying on cwd-resolution heuristics.
1291            meta.consecutive_missing_sweeps = meta.consecutive_missing_sweeps.saturating_add(1);
1292        } else {
1293            meta.consecutive_missing_sweeps = 0;
1294        }
1295        let deletion_confirmed = meta.consecutive_missing_sweeps >= 2;
1296        let has_bound_route = root_channels
1297            .get(root_id)
1298            .is_some_and(|channels| !channels.is_empty());
1299        let has_pending_bind = pending_bind_roots.contains(root_id);
1300
1301        if deleted {
1302            let mut retained = false;
1303            if !deletion_confirmed {
1304                census.absence_unconfirmed += 1;
1305                retained = true;
1306            }
1307            // Once absence is confirmed, the directory cannot serve this route
1308            // again. Neither a stale route nor the lack of normal unbind cleanup
1309            // justifies retaining the root; purge removes the route after retirement.
1310            if meta.active_bash_waits > 0 {
1311                census.bash_waits += 1;
1312                retained = true;
1313            }
1314            if meta.maintenance_pending {
1315                census.maintenance_pending += 1;
1316                retained = true;
1317            }
1318            if !meta.maintenance_queued_kinds.is_empty() {
1319                census.maintenance_queued += 1;
1320                retained = true;
1321            }
1322            if has_pending_bind {
1323                census.pending_binds += 1;
1324                retained = true;
1325            }
1326            match executor.try_actor_is_idle(root_id) {
1327                Some(true) => {}
1328                Some(false) => {
1329                    census.actor_busy += 1;
1330                    retained = true;
1331                }
1332                None => {
1333                    census.actor_state_busy += 1;
1334                    retained = true;
1335                }
1336            }
1337            if retained {
1338                census.deleted_retained += 1;
1339                continue;
1340            }
1341        } else {
1342            // Route teardown marks the lifecycle admission gate before the last
1343            // channel disappears. Requiring zero bound channels and a quiesced
1344            // lifecycle prevents a still-bound root from losing its watcher.
1345            if has_bound_route
1346                || !meta.unbound_quiesced
1347                || meta.idle_artifacts_evicted
1348                || now.saturating_duration_since(meta.last_touched)
1349                    < root_idle_ttl(executor, root_id)
1350                || meta.active_bash_waits > 0
1351                || meta.maintenance_pending
1352                || !meta.maintenance_queued_kinds.is_empty()
1353                || has_pending_bind
1354                || !executor.actor_is_idle(root_id)
1355            {
1356                continue;
1357            }
1358        }
1359        candidates.push((root_id.clone(), deleted));
1360    }
1361
1362    let mut reaped = Vec::new();
1363    let mut forgotten_deleted_roots = Vec::new();
1364    for (root_id, deleted) in candidates {
1365        let Some(ctx) = executor.actor_context(&root_id) else {
1366            if deleted {
1367                census.deleted_retained += 1;
1368                census.actor_busy += 1;
1369            }
1370            continue;
1371        };
1372        // A TTL-aged unbound root retained its watcher-derived pending paths
1373        // across the transient-unbind window. Strict gap invalidation subsumes
1374        // them, but every abort path must restore them because a rebind can
1375        // still happen until eviction commits.
1376        //
1377        // After two consecutive directory-absence scans confirm that the
1378        // root is gone, terminate its background task before checking the
1379        // artifact-eviction gate. The task can otherwise keep the root's
1380        // artifacts in use; cleanup first lets confirmed reclamation finish
1381        // without weakening the gate for unrelated active work.
1382        if deleted {
1383            ctx.bash_background()
1384                .kill_running_tasks_for_root(root_id.as_path());
1385        }
1386        let taken_pending = Some(ctx.take_pending_reconciliation_state());
1387        if ctx.artifact_eviction_blocked() {
1388            if let Some(pending) = taken_pending {
1389                ctx.restore_pending_reconciliation_state(pending);
1390            }
1391            if deleted {
1392                census.deleted_retained += 1;
1393                census.artifact_eviction_blocked += 1;
1394            }
1395            continue;
1396        }
1397        let memory_before = ctx.memory_root_snapshot();
1398        if !ctx.evict_idle_artifacts() {
1399            if let Some(pending) = taken_pending {
1400                ctx.restore_pending_reconciliation_state(pending);
1401            }
1402            if deleted {
1403                census.deleted_retained += 1;
1404                census.artifact_eviction_failed += 1;
1405            }
1406            continue;
1407        }
1408        drop(taken_pending);
1409        ctx.stop_watcher_runtime_in_background();
1410        // Edits during watcher downtime are unobserved. Advance publication
1411        // epochs and force strict verification before any later warm reload.
1412        ctx.invalidate_artifacts_after_watcher_gap();
1413
1414        if deleted {
1415            if executor.retire_idle_actor_in_background(&root_id) {
1416                live_roots.remove(&root_id);
1417                forgotten_deleted_roots.push(root_id.clone());
1418            } else {
1419                census.deleted_retained += 1;
1420                census.actor_busy += 1;
1421            }
1422        } else {
1423            if let Some(meta) = live_roots.get_mut(&root_id) {
1424                meta.idle_artifacts_evicted = true;
1425            }
1426            ctx.release_idle_reopenable_resources_in_background();
1427        }
1428        reaped.push((root_id, memory_before));
1429    }
1430
1431    // Emit-on-change: the census is on the health surface every sweep; the log
1432    // line is for transitions, including the one back to zero retained.
1433    let census_changed = metrics.record_reap(census);
1434    if census_changed {
1435        log::info!(
1436            "subc attach: retained {} deleted root(s) during idle reap; blockers={}",
1437            census.deleted_retained,
1438            census.blocker_histogram()
1439        );
1440    }
1441
1442    let pressure_relief = (!reaped.is_empty())
1443        .then(|| allocator_pressure_relief_after_idle_sweep(now, live_roots, executor))
1444        .flatten();
1445    for (root_id, memory_before) in &reaped {
1446        log::info!(
1447            "{}",
1448            idle_root_eviction_message(root_id, memory_before, pressure_relief.as_ref())
1449        );
1450    }
1451    IdleReapOutcome {
1452        evicted: reaped.len(),
1453        forgotten_deleted_roots,
1454    }
1455}
1456
1457/// Shut down language servers for roots that have had no request for the
1458/// configured LSP idle window. This is independent of artifact eviction and
1459/// runs even while the root is still bound.
1460///
1461/// `meta.last_touched` moves on every inbound tool call for a bound root
1462/// (`reactivate_bound` in the route-request handler) and on response delivery,
1463/// so an active harness does not look idle.
1464fn reap_idle_lsp_servers(
1465    now: Instant,
1466    live_roots: &HashMap<ProjectRootId, RootMeta>,
1467    executor: &Executor,
1468) {
1469    for (root_id, meta) in live_roots {
1470        let Some(ctx) = executor.actor_context(root_id) else {
1471            continue;
1472        };
1473        crate::runtime_drain::shutdown_idle_lsp_at(&ctx, now, meta.last_touched);
1474    }
1475}
1476
1477#[allow(clippy::too_many_arguments)]
1478fn purge_deleted_root_residents(
1479    root_id: &ProjectRootId,
1480    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1481    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1482    installed_route_epochs: &mut HashMap<u16, u32>,
1483    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1484    active_tool_calls: &ActiveToolCalls,
1485    executor: &Executor,
1486    retry_buffer: &mut RetryBuffer,
1487    reclaimed_routes: &mut ReclaimedRoutes,
1488    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1489    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1490    bg_subs: &mut HashMap<RouteChannel, BgSub>,
1491    bg_sub_by_session: &mut BgSubsBySession,
1492    bg_wake_pending: &mut BgWakePending,
1493    bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
1494    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1495    metrics: &DispatchPathMetrics,
1496) {
1497    let mut stale_routes = root_channels.get(root_id).cloned().unwrap_or_default();
1498    stale_routes.extend(
1499        routes
1500            .iter()
1501            .filter_map(|(route, identity)| (&identity.root == root_id).then_some(*route)),
1502    );
1503    stale_routes.extend(
1504        bg_sub_by_session
1505            .iter()
1506            .filter(|((root, _), _)| root == root_id)
1507            .flat_map(|(_, routes)| routes.iter().copied()),
1508    );
1509    stale_routes.extend(
1510        pending_bash_asks
1511            .values()
1512            .filter_map(|ask| (&ask.root == root_id).then_some(ask.route)),
1513    );
1514
1515    for route in stale_routes {
1516        reclaimed_routes.insert(route);
1517        remove_installed_route(installed_route_epochs, route);
1518        remove_route_channel(routes, root_channels, route);
1519        if let Some(cancel) = route_bash_cancels.remove(&route) {
1520            cancel.token.cancel();
1521        }
1522        apply_route_work_disposition(
1523            active_tool_calls,
1524            executor,
1525            route,
1526            RouteWorkDisposition::Abandon,
1527            "root reclaim",
1528        );
1529        retry_buffer.remove(&route);
1530        if let Some(sub) = bg_subs.remove(&route) {
1531            metrics.record_bg_subscription_ended(&sub.root, &sub.session, route, "root-reclaim");
1532        }
1533        bg_wake_pending.remove(&route);
1534    }
1535    root_channels.remove(root_id);
1536    session_identity.retain(|(root, _), _| root != root_id);
1537    push_buffer.retain(|key, _| &key.root != root_id);
1538    bg_wake_epoch.retain(|(root, _), _| root != root_id);
1539    pending_bash_asks.retain(|_, ask| &ask.root != root_id);
1540    bg_sub_by_session.retain(|(root, _), _| root != root_id);
1541    sync_bg_live_delivery_sessions(executor, routes, Some(root_id));
1542
1543    log::info!(
1544        "subc attach: fully forgot deleted root {}; cause=absence_reclaim",
1545        root_id.as_path().display()
1546    );
1547}
1548
1549#[allow(clippy::too_many_arguments)]
1550fn submit_due_maintenance_jobs(
1551    executor: &Arc<Executor>,
1552    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1553    pending_binds: &HashMap<RouteChannel, PendingBind>,
1554    bg_sub_by_session: &BgSubsBySession,
1555    bg_wake_pending: &BgWakePending,
1556    bg_wake_epoch: &HashMap<(ProjectRootId, String), u64>,
1557    maintenance_tx: &mpsc::Sender<MaintenanceCompletion>,
1558    metrics: &Arc<DispatchPathMetrics>,
1559) {
1560    let pending_bind_roots = pending_binds
1561        .values()
1562        .map(|pending| pending.bind_root_id.clone())
1563        .collect::<HashSet<_>>();
1564    let (due_jobs, deferred_jobs) = due_maintenance_jobs(
1565        live_roots,
1566        Some(executor),
1567        bg_sub_by_session,
1568        bg_wake_pending,
1569        MAINTENANCE_SUBMIT_BUDGET,
1570        &pending_bind_roots,
1571    );
1572    if deferred_jobs {
1573        metrics
1574            .maintenance_budget_deferrals
1575            .fetch_add(1, Ordering::Relaxed);
1576    }
1577    for (root_id, kind) in due_jobs {
1578        let bg_sessions_to_check = if kind == MaintenanceDrainKind::CompletionDrains {
1579            bg_sub_by_session
1580                .iter()
1581                .filter_map(|((root, session), _)| {
1582                    if root == &root_id {
1583                        Some((
1584                            session.clone(),
1585                            bg_wake_epoch
1586                                .get(&(root_id.clone(), session.clone()))
1587                                .copied()
1588                                .unwrap_or(0),
1589                        ))
1590                    } else {
1591                        None
1592                    }
1593                })
1594                .collect()
1595        } else {
1596            Vec::new()
1597        };
1598        submit_maintenance_job(
1599            executor,
1600            root_id,
1601            kind,
1602            bg_sessions_to_check,
1603            maintenance_tx,
1604            metrics,
1605        );
1606    }
1607}
1608
1609fn should_requiesce_after_maintenance(
1610    meta: &RootMeta,
1611    completed_kind: MaintenanceDrainKind,
1612    bind_pending: bool,
1613) -> bool {
1614    meta.unbound_quiesced && completed_kind != MaintenanceDrainKind::Lsp && !bind_pending
1615}
1616
1617fn note_maintenance_completion(
1618    meta: &mut RootMeta,
1619    requeue_kind: Option<MaintenanceDrainKind>,
1620    fatal: bool,
1621    defer_requeue: bool,
1622) {
1623    if fatal {
1624        meta.maintenance_poisoned = true;
1625    }
1626
1627    if let Some(kind) = requeue_kind.filter(|_| !meta.maintenance_poisoned && !defer_requeue) {
1628        meta.maintenance_queued_kinds.push_back(kind);
1629    }
1630
1631    meta.maintenance_jobs_in_flight = meta.maintenance_jobs_in_flight.saturating_sub(1);
1632    meta.maintenance_pending =
1633        meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
1634}
1635
1636fn route_key(channel: u16, epoch: u32) -> RouteChannel {
1637    RouteChannel { channel, epoch }
1638}
1639
1640fn remove_installed_route(installed_epochs: &mut HashMap<u16, u32>, route: RouteChannel) {
1641    if installed_epochs.get(&route.channel).copied() == Some(route.epoch) {
1642        installed_epochs.remove(&route.channel);
1643    }
1644}
1645
1646fn ingress_route_should_be_processed(
1647    installed_epochs: &HashMap<u16, u32>,
1648    reclaimed_routes: &ReclaimedRoutes,
1649    frame: &Frame,
1650) -> bool {
1651    if frame.header.channel == 0
1652        || installed_epochs.get(&frame.header.channel).copied() == Some(frame.header.epoch)
1653    {
1654        return true;
1655    }
1656
1657    // A late request for a reclaimed root reaches the normal unknown-route
1658    // handler, which returns the typed `route_not_bound` error. Other stale or
1659    // never-installed generations remain silent so they cannot affect a newer
1660    // route or change the protocol's rejected-bind behavior.
1661    frame.header.ty == FrameType::Request
1662        && reclaimed_routes.contains(route_key(frame.header.channel, frame.header.epoch))
1663}
1664
1665fn bash_elicitation_timeout() -> Duration {
1666    if cfg!(debug_assertions) {
1667        if let Ok(raw) = std::env::var("AFT_TEST_SUBC_BASH_ELICITATION_TTL_MS") {
1668            if let Ok(ms) = raw.parse::<u64>() {
1669                if ms > 0 {
1670                    return Duration::from_millis(ms);
1671                }
1672            }
1673        }
1674    }
1675    BASH_ELICITATION_TIMEOUT
1676}
1677
1678fn allocate_reverse_corr(
1679    pending_bash_asks: &HashMap<ReverseCorrKey, PendingBashAsk>,
1680    route: RouteChannel,
1681    next_corr: &mut u64,
1682) -> u64 {
1683    loop {
1684        let corr = *next_corr;
1685        *next_corr = (*next_corr).wrapping_add(1).max(1);
1686        if !pending_bash_asks.contains_key(&ReverseCorrKey { route, corr }) {
1687            return corr;
1688        }
1689    }
1690}
1691
1692fn bash_permission_kind_label(kind: &crate::bash_permissions::PermissionKind) -> &'static str {
1693    match kind {
1694        crate::bash_permissions::PermissionKind::ExternalDirectory => "external directory",
1695        crate::bash_permissions::PermissionKind::Bash => "bash",
1696    }
1697}
1698
1699fn bash_elicitation_patterns(asks: &[crate::bash_permissions::PermissionAsk]) -> Vec<String> {
1700    let mut patterns = Vec::new();
1701    let mut seen = HashSet::new();
1702    for ask in asks {
1703        for pattern in ask.patterns.iter().chain(ask.always.iter()) {
1704            if seen.insert(pattern.clone()) {
1705                patterns.push(pattern.clone());
1706            }
1707        }
1708    }
1709    patterns
1710}
1711
1712fn bash_elicitation_message(
1713    command: &str,
1714    asks: &[crate::bash_permissions::PermissionAsk],
1715) -> String {
1716    let command = command.split_whitespace().collect::<Vec<_>>().join(" ");
1717    let patterns = bash_elicitation_patterns(asks);
1718    let pattern_text = if patterns.is_empty() {
1719        "no matched permission patterns".to_string()
1720    } else {
1721        patterns.join(", ")
1722    };
1723    let ask_kinds = asks
1724        .iter()
1725        .map(|ask| bash_permission_kind_label(&ask.kind))
1726        .collect::<HashSet<_>>()
1727        .into_iter()
1728        .collect::<Vec<_>>()
1729        .join(", ");
1730    if ask_kinds.is_empty() {
1731        format!("Allow bash command `{command}`? Matched patterns: {pattern_text}")
1732    } else {
1733        format!("Allow bash command `{command}`? Matched {ask_kinds} patterns: {pattern_text}")
1734    }
1735}
1736
1737fn bash_elicitation_request_body(
1738    command: &str,
1739    asks: &[crate::bash_permissions::PermissionAsk],
1740) -> Value {
1741    json!({
1742        "method": BASH_ELICITATION_CREATE_METHOD,
1743        "params": {
1744            "mode": "form",
1745            "message": bash_elicitation_message(command, asks),
1746            "requestedSchema": {
1747                "type": "object",
1748                "properties": {
1749                    "decision": {
1750                        "type": "string",
1751                        "enum": ["allow", "deny"],
1752                        "description": "Choose allow to run this bash command once, or deny to block it."
1753                    }
1754                },
1755                "required": ["decision"],
1756                "additionalProperties": false
1757            },
1758            "_meta": {
1759                "aft": {
1760                    "tool": "bash",
1761                    "command": command,
1762                    "asks": asks
1763                }
1764            }
1765        }
1766    })
1767}
1768
1769fn build_bash_elicitation_request_frame(
1770    ver: u8,
1771    route: RouteChannel,
1772    corr: u64,
1773    flags: Flags,
1774    command: &str,
1775    asks: &[crate::bash_permissions::PermissionAsk],
1776) -> Result<Frame, SubcError> {
1777    let body = bash_elicitation_request_body(command, asks);
1778    Frame::build_with_version(
1779        ver,
1780        FrameType::Request,
1781        flags,
1782        route.channel,
1783        route.epoch,
1784        corr,
1785        serde_json::to_vec(&body).map_err(SubcError::Json)?,
1786    )
1787    .map_err(SubcError::FrameBuild)
1788}
1789
1790fn bash_elicitation_reply_is_allow(body: &[u8]) -> bool {
1791    let Ok(value) = serde_json::from_slice::<Value>(body) else {
1792        return false;
1793    };
1794    flat_bash_elicitation_reply_is_allow(&value) || mcp_bash_elicitation_reply_is_allow(&value)
1795}
1796
1797fn flat_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1798    let Some(object) = value.as_object() else {
1799        return false;
1800    };
1801    object.len() == 1 && object.get("decision").and_then(Value::as_str) == Some("allow")
1802}
1803
1804fn mcp_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1805    let Some(object) = value.as_object() else {
1806        return false;
1807    };
1808    if object.len() != 2 || object.get("action").and_then(Value::as_str) != Some("accept") {
1809        return false;
1810    }
1811    let Some(content) = object.get("content").and_then(Value::as_object) else {
1812        return false;
1813    };
1814    content.len() == 1 && content.get("decision").and_then(Value::as_str) == Some("allow")
1815}
1816
1817#[allow(clippy::too_many_arguments)]
1818async fn settle_pending_bash_ask_denied(
1819    tx: &WriterSender,
1820    pending: PendingBashAsk,
1821    routes: &HashMap<RouteChannel, RouteIdentity>,
1822    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1823    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1824    shutdown: &Arc<Notify>,
1825    metrics: &DispatchPathMetrics,
1826) -> Result<(), SubcError> {
1827    let completion = bash::bash_denied_untrusted_completion(
1828        pending.route,
1829        pending.tool_corr,
1830        pending.tool_flags,
1831        pending.tool_ver,
1832        pending.root,
1833        pending.request_id,
1834        pending.format_context,
1835    );
1836    bash::handle_bash_deferred_completion(
1837        tx,
1838        completion,
1839        routes,
1840        live_roots,
1841        route_bash_cancels,
1842        shutdown,
1843        metrics,
1844    )
1845    .await
1846}
1847
1848fn take_pending_bash_asks_for_route(
1849    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1850    route: RouteChannel,
1851) -> Vec<PendingBashAsk> {
1852    let keys = pending_bash_asks
1853        .keys()
1854        .copied()
1855        .filter(|key| key.route == route)
1856        .collect::<Vec<_>>();
1857    keys.into_iter()
1858        .filter_map(|key| pending_bash_asks.remove(&key))
1859        .collect()
1860}
1861
1862#[allow(clippy::too_many_arguments)]
1863async fn settle_pending_bash_asks_for_route(
1864    tx: &WriterSender,
1865    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1866    route: RouteChannel,
1867    routes: &HashMap<RouteChannel, RouteIdentity>,
1868    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1869    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1870    shutdown: &Arc<Notify>,
1871    metrics: &DispatchPathMetrics,
1872) -> Result<(), SubcError> {
1873    for pending in take_pending_bash_asks_for_route(pending_bash_asks, route) {
1874        settle_pending_bash_ask_denied(
1875            tx,
1876            pending,
1877            routes,
1878            live_roots,
1879            route_bash_cancels,
1880            shutdown,
1881            metrics,
1882        )
1883        .await?;
1884    }
1885    Ok(())
1886}
1887
1888#[allow(clippy::too_many_arguments)]
1889async fn settle_all_pending_bash_asks(
1890    tx: &WriterSender,
1891    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1892    routes: &HashMap<RouteChannel, RouteIdentity>,
1893    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1894    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1895    shutdown: &Arc<Notify>,
1896    metrics: &DispatchPathMetrics,
1897) -> Result<(), SubcError> {
1898    let pending = pending_bash_asks
1899        .drain()
1900        .map(|(_, pending)| pending)
1901        .collect::<Vec<_>>();
1902    for pending in pending {
1903        settle_pending_bash_ask_denied(
1904            tx,
1905            pending,
1906            routes,
1907            live_roots,
1908            route_bash_cancels,
1909            shutdown,
1910            metrics,
1911        )
1912        .await?;
1913    }
1914    Ok(())
1915}
1916
1917#[allow(clippy::too_many_arguments)]
1918async fn expire_pending_bash_asks(
1919    tx: &WriterSender,
1920    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1921    routes: &HashMap<RouteChannel, RouteIdentity>,
1922    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1923    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1924    shutdown: &Arc<Notify>,
1925    metrics: &DispatchPathMetrics,
1926) -> Result<(), SubcError> {
1927    let now = Instant::now();
1928    let expired = pending_bash_asks
1929        .iter()
1930        .filter_map(|(key, pending)| (pending.expires_at <= now).then_some(*key))
1931        .collect::<Vec<_>>();
1932    for key in expired {
1933        if let Some(pending) = pending_bash_asks.remove(&key) {
1934            log::debug!(
1935                "subc attach: bash elicitation request {} on route {} expired fail-closed",
1936                key.corr,
1937                pending.route
1938            );
1939            settle_pending_bash_ask_denied(
1940                tx,
1941                pending,
1942                routes,
1943                live_roots,
1944                route_bash_cancels,
1945                shutdown,
1946                metrics,
1947            )
1948            .await?;
1949        }
1950    }
1951    Ok(())
1952}
1953
1954#[allow(clippy::too_many_arguments)]
1955async fn handle_bash_elicitation_reply(
1956    tx: &WriterSender,
1957    frame: &Frame,
1958    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1959    routes: &HashMap<RouteChannel, RouteIdentity>,
1960    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1961    executor: &Arc<Executor>,
1962    shutdown: &Arc<Notify>,
1963    bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
1964    bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
1965    metrics: &Arc<DispatchPathMetrics>,
1966    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1967    dispatch: DispatchFn,
1968) -> Result<(), SubcError> {
1969    let key = ReverseCorrKey {
1970        route: route_key(frame.header.channel, frame.header.epoch),
1971        corr: frame.header.corr,
1972    };
1973    let Some(pending) = pending_bash_asks.remove(&key) else {
1974        return Ok(());
1975    };
1976
1977    if frame.header.ty == FrameType::Response && bash_elicitation_reply_is_allow(&frame.body) {
1978        if routes.contains_key(&key.route) {
1979            bash::submit_deferred_bash(
1980                executor,
1981                bash_deferred_tx,
1982                bash_poll_touch_tx,
1983                metrics,
1984                dispatch,
1985                pending.root,
1986                pending.project_root,
1987                pending.session_id,
1988                pending.request_id,
1989                pending.route,
1990                pending.tool_corr,
1991                pending.tool_flags,
1992                pending.tool_ver,
1993                pending.arguments,
1994                pending.format_context,
1995                pending.cancel,
1996                BindTrust::Untrusted,
1997                pending.spawn_principal,
1998                pending.edit_slot_survives,
1999                Some(pending.grants),
2000            );
2001            return Ok(());
2002        }
2003        log::debug!(
2004            "subc attach: dropping allowed bash elicitation reply {} for unbound route {}",
2005            key.corr,
2006            pending.route
2007        );
2008    }
2009
2010    settle_pending_bash_ask_denied(
2011        tx,
2012        pending,
2013        routes,
2014        live_roots,
2015        route_bash_cancels,
2016        shutdown,
2017        metrics,
2018    )
2019    .await
2020}
2021
2022#[allow(clippy::too_many_arguments)]
2023async fn cancel_pending_bash_ask_for_tool_call(
2024    tx: &WriterSender,
2025    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
2026    route: RouteChannel,
2027    tool_corr: u64,
2028    routes: &HashMap<RouteChannel, RouteIdentity>,
2029    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2030    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
2031    shutdown: &Arc<Notify>,
2032    metrics: &DispatchPathMetrics,
2033) -> Result<(), SubcError> {
2034    let keys = pending_bash_asks
2035        .iter()
2036        .filter_map(|(key, pending)| {
2037            (key.route == route && pending.tool_corr == tool_corr).then_some(*key)
2038        })
2039        .collect::<Vec<_>>();
2040    for key in keys {
2041        if let Some(pending) = pending_bash_asks.remove(&key) {
2042            settle_pending_bash_ask_denied(
2043                tx,
2044                pending,
2045                routes,
2046                live_roots,
2047                route_bash_cancels,
2048                shutdown,
2049                metrics,
2050            )
2051            .await?;
2052        }
2053    }
2054    Ok(())
2055}
2056
2057fn remove_root_channel(
2058    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2059    root: &ProjectRootId,
2060    channel: RouteChannel,
2061) {
2062    let remove_root = if let Some(channels) = root_channels.get_mut(root) {
2063        channels.remove(&channel);
2064        channels.is_empty()
2065    } else {
2066        false
2067    };
2068    if remove_root {
2069        root_channels.remove(root);
2070    }
2071}
2072
2073fn remove_route_channel(
2074    routes: &mut HashMap<RouteChannel, RouteIdentity>,
2075    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2076    channel: RouteChannel,
2077) -> Option<RouteIdentity> {
2078    let removed = routes.remove(&channel);
2079    if let Some(identity) = &removed {
2080        remove_root_channel(root_channels, &identity.root, channel);
2081    }
2082    removed
2083}
2084
2085fn insert_route_channel(
2086    routes: &mut HashMap<RouteChannel, RouteIdentity>,
2087    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2088    channel: RouteChannel,
2089    identity: RouteIdentity,
2090) {
2091    if let Some(previous) = routes.insert(channel, identity.clone()) {
2092        remove_root_channel(root_channels, &previous.root, channel);
2093    }
2094    root_channels
2095        .entry(identity.root.clone())
2096        .or_default()
2097        .insert(channel);
2098}
2099
2100fn sync_bg_live_delivery_sessions(
2101    executor: &Executor,
2102    routes: &HashMap<RouteChannel, RouteIdentity>,
2103    additional_root: Option<&ProjectRootId>,
2104) {
2105    // The loop-owned installed-route table is the lifecycle source of truth:
2106    // an originating session is live exactly while the daemon has an installed,
2107    // bash-observation-capable route whose identity names that session.
2108    let sessions = routes
2109        .values()
2110        .filter(|identity| identity.trust.allows_bash_observation())
2111        .map(|identity| identity.session.clone())
2112        .collect::<HashSet<_>>();
2113    let mut roots = routes
2114        .values()
2115        .map(|identity| identity.root.clone())
2116        .collect::<HashSet<_>>();
2117    roots.extend(additional_root.cloned());
2118    for root in roots {
2119        if let Some(ctx) = executor.actor_context(&root) {
2120            ctx.bash_background()
2121                .replace_live_delivery_sessions(sessions.clone());
2122        }
2123    }
2124}
2125
2126fn insert_bg_subscription_index(
2127    bg_sub_by_session: &mut BgSubsBySession,
2128    root: ProjectRootId,
2129    session: String,
2130    channel: RouteChannel,
2131) {
2132    bg_sub_by_session
2133        .entry((root, session))
2134        .or_default()
2135        .insert(channel);
2136}
2137
2138fn remove_bg_subscription_index(
2139    bg_sub_by_session: &mut BgSubsBySession,
2140    channel: RouteChannel,
2141    identity: Option<&RouteIdentity>,
2142) {
2143    if let Some(identity) = identity {
2144        let key = (identity.root.clone(), identity.session.clone());
2145        let remove_key = bg_sub_by_session.get_mut(&key).is_some_and(|channels| {
2146            channels.remove(&channel);
2147            channels.is_empty()
2148        });
2149        if remove_key {
2150            bg_sub_by_session.remove(&key);
2151        }
2152    } else {
2153        bg_sub_by_session.retain(|_, channels| {
2154            channels.remove(&channel);
2155            !channels.is_empty()
2156        });
2157    }
2158}
2159
2160fn route_removal_will_quiesce_root(
2161    root: &ProjectRootId,
2162    route: RouteChannel,
2163    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
2164    has_pending_bind: bool,
2165    replacement_root: Option<&ProjectRootId>,
2166) -> bool {
2167    let removes_last_route = root_channels
2168        .get(root)
2169        .is_some_and(|channels| channels.len() == 1 && channels.contains(&route));
2170    removes_last_route && !has_pending_bind && replacement_root != Some(root)
2171}
2172
2173fn should_quiesce_removed_root(
2174    root: &ProjectRootId,
2175    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
2176    has_pending_bind: bool,
2177    replacement_root: Option<&ProjectRootId>,
2178) -> bool {
2179    !root_channels.contains_key(root) && !has_pending_bind && replacement_root != Some(root)
2180}
2181
2182async fn end_bg_subscription(
2183    writer_tx: &WriterSender,
2184    metrics: &DispatchPathMetrics,
2185    bg_subs: &mut HashMap<RouteChannel, BgSub>,
2186    bg_sub_by_session: &mut BgSubsBySession,
2187    bg_wake_pending: &mut BgWakePending,
2188    channel: RouteChannel,
2189    identity: Option<&RouteIdentity>,
2190    cause: &str,
2191) -> Result<(), SubcError> {
2192    if let Some(sub) = bg_subs.remove(&channel) {
2193        bg_wake_pending.remove(&channel);
2194        remove_bg_subscription_index(bg_sub_by_session, channel, identity);
2195        metrics.record_bg_subscription_ended(&sub.root, &sub.session, channel, cause);
2196        push::send_reliable_bg_stream_end(writer_tx, metrics, channel, &sub).await?;
2197    }
2198    Ok(())
2199}
2200
2201#[allow(clippy::too_many_arguments)]
2202async fn teardown_installed_route(
2203    tx: &WriterSender,
2204    metrics: &DispatchPathMetrics,
2205    executor: &Arc<Executor>,
2206    channel: RouteChannel,
2207    cancellation_reason: &str,
2208    replacement_root: Option<&ProjectRootId>,
2209    installed_route_epochs: &mut HashMap<u16, u32>,
2210    routes: &mut HashMap<RouteChannel, RouteIdentity>,
2211    management_routes: &mut HashSet<RouteChannel>,
2212    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2213    bg_subs: &mut HashMap<RouteChannel, BgSub>,
2214    bg_sub_by_session: &mut BgSubsBySession,
2215    bg_wake_pending: &mut BgWakePending,
2216    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
2217    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2218    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
2219    active_tool_calls: &ActiveToolCalls,
2220    pending_responses: &mut PendingSubcResponses,
2221    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2222    retry_buffer: &mut RetryBuffer,
2223    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2224    shutdown: &Arc<Notify>,
2225    tool_response_body_limit: usize,
2226    lifecycle_probe: Option<&SubcTestLifecycleProbe>,
2227) -> Result<(), SubcError> {
2228    remove_installed_route(installed_route_epochs, channel);
2229    management_routes.remove(&channel);
2230    let bg_end_cause = match cancellation_reason {
2231        "Goodbye" => "goodbye",
2232        "higher-epoch RouteBind" => "higher-epoch",
2233        other => other,
2234    };
2235    end_bg_subscription(
2236        tx,
2237        metrics,
2238        bg_subs,
2239        bg_sub_by_session,
2240        bg_wake_pending,
2241        channel,
2242        routes.get(&channel),
2243        bg_end_cause,
2244    )
2245    .await?;
2246    settle_pending_bash_asks_for_route(
2247        tx,
2248        pending_bash_asks,
2249        channel,
2250        routes,
2251        live_roots,
2252        route_bash_cancels,
2253        shutdown,
2254        metrics,
2255    )
2256    .await?;
2257    if let Some(cancel) = route_bash_cancels.remove(&channel) {
2258        cancel.token.cancel();
2259    }
2260    for resolved in pending_responses.drain_route(channel, executor) {
2261        deliver_resolved_subc_response(
2262            tx,
2263            resolved,
2264            routes,
2265            live_roots,
2266            executor.as_ref(),
2267            active_tool_calls,
2268            shutdown,
2269            metrics,
2270            tool_response_body_limit,
2271        )
2272        .await?;
2273    }
2274    // A higher-epoch replacement keeps replayable work because the logical
2275    // route remains live. A route that actually closes keeps only calls that
2276    // already started; pre-execution work has no response to replay and must
2277    // not retain scheduler capacity or an epoch-reader admission.
2278    let work_disposition = if replacement_root.is_some() {
2279        RouteWorkDisposition::RetainForReplay
2280    } else {
2281        RouteWorkDisposition::RetainStartedForReplay
2282    };
2283    apply_route_work_disposition(
2284        active_tool_calls,
2285        executor,
2286        channel,
2287        work_disposition,
2288        cancellation_reason,
2289    );
2290    if let Some(pending) = pending_binds.get_mut(&channel) {
2291        pending.cancelled = true;
2292        let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
2293        log::debug!(
2294            "subc attach: cancelled pending RouteBind for route {} on {cancellation_reason} (configure job: {outcome:?})",
2295            channel.channel
2296        );
2297    }
2298    let migrated = push::migrate_retry_buffer_to_push_buffer(retry_buffer, channel, push_buffer);
2299    if let Some(identity) = routes.get(&channel) {
2300        let has_pending_bind = pending_binds
2301            .values()
2302            .any(|pending| pending.bind_root_id == identity.root);
2303        if route_removal_will_quiesce_root(
2304            &identity.root,
2305            channel,
2306            root_channels,
2307            has_pending_bind,
2308            replacement_root,
2309        ) {
2310            if let Some(ctx) = executor.actor_context(&identity.root) {
2311                // Fence deferred admissions before the final route disappears
2312                // from the loop-owned routing tables.
2313                ctx.mark_subc_unbound();
2314            }
2315        }
2316    }
2317    // This test-only delay lets the lifecycle probe verify that a queued rebind
2318    // cannot run before the route is removed and a completion is recorded for replay.
2319    delay_route_detach_for_test(lifecycle_probe).await;
2320    if let Some(identity) = remove_route_channel(routes, root_channels, channel) {
2321        sync_bg_live_delivery_sessions(executor, routes, Some(&identity.root));
2322        if let Some(probe) = lifecycle_probe {
2323            probe.route_detached(channel, &identity.session);
2324        }
2325        let session_still_routed = routes
2326            .values()
2327            .any(|route| route.root == identity.root && route.session == identity.session);
2328        if !session_still_routed {
2329            if let Some(ctx) = executor.actor_context(&identity.root) {
2330                ctx.hashline_bindings()
2331                    .teardown(identity.root.as_path(), &identity.session);
2332            }
2333        }
2334        if migrated > 0 {
2335            log::debug!(
2336                "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from route {} into detach replay",
2337                channel.channel
2338            );
2339        }
2340        if let Some(meta) = live_roots.get_mut(&identity.root) {
2341            let idle_for = meta.last_touched.elapsed();
2342            meta.note_activity();
2343            log::debug!(
2344                "subc attach: route {} torn down for root {} harness {} session {} (last touched {:?} ago)",
2345                channel.channel,
2346                identity.root.as_path().display(),
2347                identity.harness,
2348                identity.session,
2349                idle_for
2350            );
2351        } else {
2352            log::debug!(
2353                "subc attach: route {} torn down for root {} harness {} session {}",
2354                channel.channel,
2355                identity.root.as_path().display(),
2356                identity.harness,
2357                identity.session
2358            );
2359        }
2360        let has_pending_bind = pending_binds
2361            .values()
2362            .any(|pending| pending.bind_root_id == identity.root);
2363        if should_quiesce_removed_root(
2364            &identity.root,
2365            root_channels,
2366            has_pending_bind,
2367            replacement_root,
2368        ) {
2369            quiesce_unbound_root(&identity.root, live_roots, executor);
2370        }
2371    } else {
2372        if migrated > 0 {
2373            log::debug!(
2374                "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from unbound route {} into detach replay",
2375                channel.channel
2376            );
2377        }
2378        log::debug!("subc attach: unbound route {} torn down", channel.channel);
2379    }
2380    Ok(())
2381}
2382
2383async fn delay_route_detach_for_test(lifecycle_probe: Option<&SubcTestLifecycleProbe>) {
2384    if lifecycle_probe.is_none() {
2385        return;
2386    }
2387    let Some(delay) = std::env::var("AFT_TEST_SUBC_ROUTE_DETACH_DELAY_MS")
2388        .ok()
2389        .and_then(|raw| raw.parse::<u64>().ok())
2390    else {
2391        return;
2392    };
2393    tokio::time::sleep(Duration::from_millis(delay)).await;
2394}
2395
2396fn remember_session_identity(
2397    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2398    identity: &RouteIdentity,
2399) {
2400    let key = (identity.root.clone(), identity.session.clone());
2401    if matches!(identity.trust, BindTrust::Untrusted)
2402        && session_identity
2403            .get(&key)
2404            .is_some_and(|retained| matches!(retained.trust, BindTrust::FirstParty))
2405    {
2406        return;
2407    }
2408
2409    // Retained after route Goodbye so reliable session-scoped frames emitted while
2410    // the session is detached can still be keyed by the full (root,harness,session)
2411    // replay triple. Untrusted binds never overwrite a retained first-party
2412    // session identity, because bash completion replay is an observation channel.
2413    session_identity.insert(
2414        key,
2415        RetainedSessionIdentity {
2416            harness: identity.harness.clone(),
2417            trust: identity.trust,
2418        },
2419    );
2420}
2421
2422fn replay_key_for_session(
2423    session_identity: &HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2424    root: &ProjectRootId,
2425    session: &str,
2426) -> Option<(push::ReplayKey, BindTrust)> {
2427    let retained = session_identity.get(&(root.clone(), session.to_string()))?;
2428    Some((
2429        push::ReplayKey {
2430            root: root.clone(),
2431            harness: retained.harness.clone(),
2432            session: session.to_string(),
2433        },
2434        retained.trust,
2435    ))
2436}
2437/// Sync command dispatch, passed in from `main` (the binary owns the command
2438/// table). Invoked only inside executor jobs in subc mode.
2439pub type DispatchFn = fn(RawRequest, &AppContext) -> Response;
2440
2441#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2442enum ModuleLoopExit {
2443    /// The daemon asked us to stop (channel-0 Goodbye). Exit 0 is correct:
2444    /// the supervisor treats a clean exit as "stopped on request".
2445    Graceful,
2446    /// The connection ended without a Goodbye. Indexes still flush, but the
2447    /// process must exit non-zero so the supervisor restarts it.
2448    ConnectionLost,
2449    /// An actor went fatal (worker panic on a mutating job) and the loop tore
2450    /// the module down. Index flushes are skipped because the state that
2451    /// panicked cannot be trusted, and the process exits non-zero for the
2452    /// same reason as `ConnectionLost`.
2453    SkipSearchFlush,
2454}
2455
2456/// Entry point for `aft --subc <connection-file>`. Synchronous on the outside;
2457/// owns an isolated current-thread tokio runtime for the async transport.
2458/// Returns `Err` (fail-loud) on any connect/auth/protocol failure — we never
2459/// fall back to the standalone loop, to avoid split-brain index state.
2460pub fn run_subc_mode(
2461    connection_file_path: &Path,
2462    ctx: Arc<AppContext>,
2463    executor: Arc<Executor>,
2464    dispatch: DispatchFn,
2465    user_config_path: Option<PathBuf>,
2466) -> Result<(), SubcError> {
2467    // Production NEVER allows non-manifest tool names on route channels: AFT
2468    // fails closed and does not trust subc to enforce the manifest. The
2469    // test-only harness sets this through `run_subc_mode_for_test`.
2470    run_subc_mode_inner(
2471        connection_file_path,
2472        ctx,
2473        executor,
2474        dispatch,
2475        user_config_path,
2476        false,
2477        MAX_FRAME_BODY_LEN as usize,
2478        None,
2479    )
2480}
2481
2482fn run_subc_mode_inner(
2483    connection_file_path: &Path,
2484    ctx: Arc<AppContext>,
2485    executor: Arc<Executor>,
2486    dispatch: DispatchFn,
2487    user_config_path: Option<PathBuf>,
2488    allow_native_passthrough: bool,
2489    tool_response_body_limit: usize,
2490    lifecycle_probe: Option<SubcTestLifecycleProbe>,
2491) -> Result<(), SubcError> {
2492    let runtime = tokio::runtime::Builder::new_current_thread()
2493        .enable_all()
2494        .build()
2495        .map_err(SubcError::Runtime)?;
2496
2497    let executor_for_loop = Arc::clone(&executor);
2498    let loop_result = runtime.block_on(async move {
2499        let shared_app = ctx.app();
2500        drop(ctx);
2501        let stream =
2502            connect_and_authenticate(connection_file_path, lifecycle_probe.as_ref()).await?;
2503        log::info!(
2504            "subc attach: authenticated to daemon via {}",
2505            connection_file_path.display()
2506        );
2507        let (read_half, write_half) = tokio::io::split(stream);
2508        run_module_loop(
2509            read_half,
2510            write_half,
2511            connection_file_path,
2512            shared_app,
2513            executor_for_loop,
2514            dispatch,
2515            user_config_path,
2516            allow_native_passthrough,
2517            tool_response_body_limit,
2518            lifecycle_probe,
2519        )
2520        .await
2521    });
2522
2523    let actor_contexts = executor.actor_contexts();
2524    if matches!(
2525        loop_result,
2526        Ok(ModuleLoopExit::Graceful | ModuleLoopExit::ConnectionLost)
2527    ) {
2528        // EOF/Goodbye teardown flushes each root's index deltas and queued
2529        // callgraph refreshes. Fatal/panic teardown skips this best-effort work.
2530        flush_actor_indexes_on_graceful_shutdown(&actor_contexts);
2531    }
2532    for actor_ctx in &actor_contexts {
2533        actor_ctx.lsp().shutdown_all();
2534        actor_ctx.bash_background().detach();
2535    }
2536
2537    match loop_result {
2538        Ok(exit) => module_loop_exit_result(exit),
2539        Err(error) => Err(error),
2540    }
2541}
2542
2543/// Maps how the module loop ended onto the process outcome the supervisor
2544/// reads. Only a daemon-requested stop may exit 0: the supervisor never
2545/// respawns a clean exit, so every other ending must surface as an error.
2546fn module_loop_exit_result(exit: ModuleLoopExit) -> Result<(), SubcError> {
2547    match exit {
2548        ModuleLoopExit::Graceful => Ok(()),
2549        ModuleLoopExit::ConnectionLost => Err(SubcError::ConnectionLost),
2550        ModuleLoopExit::SkipSearchFlush => Err(SubcError::ActorFatal),
2551    }
2552}
2553
2554/// Records a fatal panic response in the module log before the teardown it
2555/// triggers. The panic text otherwise lives only in the error frame sent to
2556/// the caller, which leaves the crash undiagnosable from the host afterwards.
2557fn note_fatal_panic_response(response: &Response) -> bool {
2558    let fatal = response_is_fatal_panic(response);
2559    if fatal {
2560        log::error!(
2561            "subc attach: request {} returned a fatal panic response; tearing the module down: {}",
2562            response.id,
2563            response
2564                .data
2565                .get("message")
2566                .and_then(Value::as_str)
2567                .unwrap_or("(no message)")
2568        );
2569    }
2570    fatal
2571}
2572
2573fn flush_actor_indexes_on_graceful_shutdown(actor_contexts: &[Arc<AppContext>]) {
2574    for actor_ctx in actor_contexts {
2575        let _ = actor_ctx.flush_search_index_on_graceful_shutdown();
2576    }
2577    let _ = crate::callgraph_store::flush_callgraph_store_refreshes_on_graceful_shutdown();
2578}
2579
2580/// Test-only entry that enables the non-manifest native-command passthrough on
2581/// route channels. Integration tests drive synthetic native commands (`glob`,
2582/// `callers`, `subc_test_echo_session`, …) through the executor to exercise
2583/// mechanics; production callers use [`run_subc_mode`], which fails closed.
2584#[doc(hidden)]
2585pub fn run_subc_mode_for_test(
2586    connection_file_path: &Path,
2587    ctx: Arc<AppContext>,
2588    executor: Arc<Executor>,
2589    dispatch: DispatchFn,
2590    user_config_path: Option<PathBuf>,
2591) -> Result<(), SubcError> {
2592    run_subc_mode_inner(
2593        connection_file_path,
2594        ctx,
2595        executor,
2596        dispatch,
2597        user_config_path,
2598        true,
2599        MAX_FRAME_BODY_LEN as usize,
2600        None,
2601    )
2602}
2603
2604/// Test-only entry that observes detach/rebind lifecycle milestones.
2605#[doc(hidden)]
2606pub fn run_subc_mode_for_test_with_lifecycle_probe(
2607    connection_file_path: &Path,
2608    ctx: Arc<AppContext>,
2609    executor: Arc<Executor>,
2610    dispatch: DispatchFn,
2611    user_config_path: Option<PathBuf>,
2612    lifecycle_probe: SubcTestLifecycleProbe,
2613) -> Result<(), SubcError> {
2614    run_subc_mode_inner(
2615        connection_file_path,
2616        ctx,
2617        executor,
2618        dispatch,
2619        user_config_path,
2620        true,
2621        MAX_FRAME_BODY_LEN as usize,
2622        Some(lifecycle_probe),
2623    )
2624}
2625
2626/// Test-only entry that lowers the effective tool-response body limit without
2627/// allocating a 64 MiB fixture. The fixed fallback envelope needs 4 KiB of room.
2628#[doc(hidden)]
2629pub fn run_subc_mode_for_test_with_response_body_limit(
2630    connection_file_path: &Path,
2631    ctx: Arc<AppContext>,
2632    executor: Arc<Executor>,
2633    dispatch: DispatchFn,
2634    user_config_path: Option<PathBuf>,
2635    tool_response_body_limit: usize,
2636) -> Result<(), SubcError> {
2637    assert!((4 * 1_024..=MAX_FRAME_BODY_LEN as usize).contains(&tool_response_body_limit));
2638    run_subc_mode_inner(
2639        connection_file_path,
2640        ctx,
2641        executor,
2642        dispatch,
2643        user_config_path,
2644        true,
2645        tool_response_body_limit,
2646        None,
2647    )
2648}
2649
2650#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2651enum AttachErrorClass {
2652    Transient,
2653    Permanent,
2654}
2655
2656impl fmt::Display for AttachErrorClass {
2657    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2658        match self {
2659            Self::Transient => f.write_str("transient"),
2660            Self::Permanent => f.write_str("permanent"),
2661        }
2662    }
2663}
2664
2665#[derive(Clone, Copy)]
2666struct AttachRetryPolicy {
2667    budget: Duration,
2668    initial_backoff: Duration,
2669    max_backoff: Duration,
2670    jitter_percent: u64,
2671}
2672
2673const ATTACH_RETRY_POLICY: AttachRetryPolicy = AttachRetryPolicy {
2674    budget: ATTACH_RETRY_BUDGET,
2675    initial_backoff: ATTACH_RETRY_INITIAL_BACKOFF,
2676    max_backoff: ATTACH_RETRY_MAX_BACKOFF,
2677    jitter_percent: ATTACH_RETRY_JITTER_PERCENT,
2678};
2679
2680/// Retry only failures that can be caused by a daemon bounce or an interrupted
2681/// handshake. Protocol and credential failures are permanent for this process.
2682fn classify_attach_error(error: &SubcError) -> AttachErrorClass {
2683    let transient = match error {
2684        SubcError::Connect { source, .. } => is_transient_attach_io(source.kind()),
2685        SubcError::Auth { source, .. } => match source {
2686            subc_transport::AuthError::Timeout { .. }
2687            | subc_transport::AuthError::UnexpectedEof { .. } => true,
2688            subc_transport::AuthError::Io { source, .. } => is_transient_attach_io(source.kind()),
2689            _ => false,
2690        },
2691        _ => false,
2692    };
2693    if transient {
2694        AttachErrorClass::Transient
2695    } else {
2696        AttachErrorClass::Permanent
2697    }
2698}
2699
2700fn is_transient_attach_io(kind: io::ErrorKind) -> bool {
2701    matches!(
2702        kind,
2703        io::ErrorKind::ConnectionRefused
2704            | io::ErrorKind::TimedOut
2705            | io::ErrorKind::ConnectionReset
2706            | io::ErrorKind::ConnectionAborted
2707            | io::ErrorKind::BrokenPipe
2708            | io::ErrorKind::UnexpectedEof
2709    )
2710}
2711
2712/// Read the connection file → resolve the first endpoint → TCP connect → HMAC
2713/// handshake. Transient initial-attach failures retry on fresh sockets and reread
2714/// the file so a daemon bounce can publish a new endpoint or authentication key.
2715async fn connect_and_authenticate(
2716    connection_file_path: &Path,
2717    lifecycle_probe: Option<&SubcTestLifecycleProbe>,
2718) -> Result<TcpStream, SubcError> {
2719    connect_and_authenticate_with_policy(connection_file_path, ATTACH_RETRY_POLICY, lifecycle_probe)
2720        .await
2721}
2722
2723async fn connect_and_authenticate_with_policy(
2724    connection_file_path: &Path,
2725    policy: AttachRetryPolicy,
2726    lifecycle_probe: Option<&SubcTestLifecycleProbe>,
2727) -> Result<TcpStream, SubcError> {
2728    let started_at = Instant::now();
2729    let deadline = started_at + policy.budget;
2730    let mut attempt = 0_u32;
2731    let mut backoff = policy.initial_backoff;
2732    let mut history = Vec::new();
2733
2734    loop {
2735        attempt = attempt.saturating_add(1);
2736        let error = match connect_and_authenticate_once(connection_file_path, deadline).await {
2737            Ok(stream) => return Ok(stream),
2738            Err(error) => error,
2739        };
2740        let class = classify_attach_error(&error);
2741        let will_retry = class != AttachErrorClass::Permanent;
2742        if let Some(probe) = lifecycle_probe {
2743            probe.attach_decision(attempt, will_retry);
2744        }
2745        let error_text = error.to_string().lines().collect::<Vec<_>>().join(" ");
2746        history.push(format!("attempt {attempt} [{class}]: {error_text}"));
2747
2748        if !will_retry {
2749            log_attach_final_failure(started_at.elapsed(), &history);
2750            return Err(error);
2751        }
2752
2753        let remaining = deadline.saturating_duration_since(Instant::now());
2754        if remaining.is_zero() {
2755            log_attach_final_failure(started_at.elapsed(), &history);
2756            return Err(error);
2757        }
2758
2759        let delay = jittered_attach_delay(backoff, policy.jitter_percent, attempt).min(remaining);
2760        log::info!(
2761            "subc attach retry: attempt {attempt} failed; error_class={class}; error={error_text}; next_delay={delay:?}"
2762        );
2763        tokio::time::sleep(delay).await;
2764
2765        if Instant::now() >= deadline {
2766            log_attach_final_failure(started_at.elapsed(), &history);
2767            return Err(error);
2768        }
2769        backoff = backoff.saturating_mul(2).min(policy.max_backoff);
2770    }
2771}
2772
2773fn jittered_attach_delay(base: Duration, jitter_percent: u64, attempt: u32) -> Duration {
2774    let jitter_percent = jitter_percent.min(100);
2775    if jitter_percent == 0 {
2776        return base;
2777    }
2778
2779    let mut random_bytes = [0_u8; 8];
2780    let random = if getrandom::fill(&mut random_bytes).is_ok() {
2781        u64::from_le_bytes(random_bytes)
2782    } else {
2783        let timestamp = std::time::SystemTime::now()
2784            .duration_since(std::time::UNIX_EPOCH)
2785            .unwrap_or_default()
2786            .subsec_nanos();
2787        u64::from(timestamp) ^ u64::from(attempt)
2788    };
2789    let span = jitter_percent.saturating_mul(2).saturating_add(1);
2790    let multiplier_percent = 100 - jitter_percent + random % span;
2791    let base_millis = u64::try_from(base.as_millis()).unwrap_or(u64::MAX);
2792    Duration::from_millis(base_millis.saturating_mul(multiplier_percent) / 100)
2793}
2794
2795fn log_attach_final_failure(elapsed: Duration, history: &[String]) {
2796    log::error!(
2797        "subc initial attach failed after {} attempt(s) in {elapsed:?}; attempt history: {}",
2798        history.len(),
2799        history.join(" | ")
2800    );
2801}
2802
2803async fn connect_and_authenticate_once(
2804    connection_file_path: &Path,
2805    deadline: Instant,
2806) -> Result<TcpStream, SubcError> {
2807    // This read intentionally lives inside the per-attempt function. The daemon
2808    // publishes connection files atomically and may change both port and key.
2809    let conn = connection_file::read_for_client(connection_file_path).map_err(|source| {
2810        SubcError::ConnectionFile {
2811            path: connection_file_path.to_path_buf(),
2812            source,
2813        }
2814    })?;
2815
2816    let endpoint = conn
2817        .endpoints
2818        .first()
2819        .ok_or_else(|| SubcError::NoEndpoint {
2820            path: connection_file_path.to_path_buf(),
2821        })?;
2822    let endpoint_label = format!("{}:{}", endpoint.host, endpoint.port);
2823    let ip = endpoint
2824        .host
2825        .parse::<IpAddr>()
2826        .map_err(|_| SubcError::InvalidEndpoint {
2827            path: connection_file_path.to_path_buf(),
2828            endpoint: endpoint_label.clone(),
2829        })?;
2830    let addr = SocketAddr::new(ip, endpoint.port);
2831
2832    let connect_budget = deadline.saturating_duration_since(Instant::now());
2833    let mut stream = tokio::time::timeout(connect_budget, TcpStream::connect(addr))
2834        .await
2835        .map_err(|_| SubcError::Connect {
2836            endpoint: endpoint_label.clone(),
2837            source: io::Error::new(
2838                io::ErrorKind::TimedOut,
2839                "initial subc attach retry budget elapsed during TCP connect",
2840            ),
2841        })?
2842        .map_err(|source| SubcError::Connect {
2843            endpoint: endpoint_label.clone(),
2844            source,
2845        })?;
2846    stream
2847        .set_nodelay(true)
2848        .map_err(|source| SubcError::Connect {
2849            endpoint: endpoint_label.clone(),
2850            source,
2851        })?;
2852
2853    let auth_budget = AUTH_DEADLINE.min(deadline.saturating_duration_since(Instant::now()));
2854    authenticate_client(&mut stream, &conn, auth_budget)
2855        .await
2856        .map_err(|source| SubcError::Auth {
2857            endpoint: endpoint_label,
2858            source,
2859        })?;
2860
2861    Ok(stream)
2862}
2863
2864#[allow(clippy::too_many_arguments)]
2865async fn process_route_bind_completion(
2866    writer_tx: &WriterSender,
2867    completion: RouteBindCompletion,
2868    routes: &mut HashMap<RouteChannel, RouteIdentity>,
2869    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2870    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2871    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2872    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2873    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2874    installed_route_epochs: &mut HashMap<u16, u32>,
2875    executor: &Arc<Executor>,
2876    standing_actor: &standing::StandingActor,
2877    shutdown: &Arc<Notify>,
2878    metrics: &Arc<DispatchPathMetrics>,
2879    lifecycle_probe: Option<&SubcTestLifecycleProbe>,
2880) -> Result<(), SubcError> {
2881    decrement_counted_channel(&metrics.control_completion_queued);
2882    handle_route_bind_completion(
2883        writer_tx,
2884        completion,
2885        routes,
2886        root_channels,
2887        session_identity,
2888        push_buffer,
2889        live_roots,
2890        pending_binds,
2891        installed_route_epochs,
2892        executor,
2893        standing_actor,
2894        shutdown,
2895        metrics,
2896        lifecycle_probe,
2897    )
2898    .await
2899}
2900
2901#[allow(clippy::too_many_arguments)]
2902async fn drain_pending_route_bind_completions(
2903    control_completion_rx: &mut mpsc::Receiver<RouteBindCompletion>,
2904    writer_tx: &WriterSender,
2905    routes: &mut HashMap<RouteChannel, RouteIdentity>,
2906    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2907    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2908    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2909    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2910    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2911    installed_route_epochs: &mut HashMap<u16, u32>,
2912    executor: &Arc<Executor>,
2913    standing_actor: &standing::StandingActor,
2914    shutdown: &Arc<Notify>,
2915    metrics: &Arc<DispatchPathMetrics>,
2916    lifecycle_probe: Option<&SubcTestLifecycleProbe>,
2917) -> Result<usize, SubcError> {
2918    let mut drained = 0;
2919    while let Ok(completion) = control_completion_rx.try_recv() {
2920        process_route_bind_completion(
2921            writer_tx,
2922            completion,
2923            routes,
2924            root_channels,
2925            session_identity,
2926            push_buffer,
2927            live_roots,
2928            pending_binds,
2929            installed_route_epochs,
2930            executor,
2931            standing_actor,
2932            shutdown,
2933            metrics,
2934            lifecycle_probe,
2935        )
2936        .await?;
2937        drained += 1;
2938    }
2939    Ok(drained)
2940}
2941
2942/// ModuleHello → HelloAck → control/route loop. Runs until the daemon closes
2943/// the connection (EOF), sends channel-0 Goodbye, or a fatal mutating executor
2944/// response requests whole-connection teardown.
2945async fn run_module_loop<R, W>(
2946    mut read: R,
2947    mut write: W,
2948    connection_file_path: &Path,
2949    shared_app: Arc<App>,
2950    executor: Arc<Executor>,
2951    dispatch: DispatchFn,
2952    user_config_path: Option<PathBuf>,
2953    allow_native_passthrough: bool,
2954    tool_response_body_limit: usize,
2955    lifecycle_probe: Option<SubcTestLifecycleProbe>,
2956) -> Result<ModuleLoopExit, SubcError>
2957where
2958    R: AsyncRead + Unpin + Send + 'static,
2959    W: AsyncWrite + Unpin + Send + 'static,
2960{
2961    // ModuleHello registers the tool and management providers and advertises
2962    // the separate channel-0 control operations.
2963    // Echo the one-time launch nonce the daemon injected via SUBC_LAUNCH_NONCE so a
2964    // reserved module_id's HELLO is accepted; absent for non-reserved/self-connect.
2965    let hello = ModuleHelloBody {
2966        manifest: build_manifest(),
2967        protocol_ver: PROTOCOL_VERSION,
2968        control_ops: control_ops(),
2969        launch_nonce: std::env::var("SUBC_LAUNCH_NONCE").ok(),
2970    };
2971    let hello_frame = Frame::build(
2972        FrameType::Hello,
2973        control_flags(),
2974        0,
2975        0,
2976        HELLO_CORR,
2977        serde_json::to_vec(&hello).map_err(SubcError::Json)?,
2978    )
2979    .map_err(SubcError::FrameBuild)?;
2980    write_frame(&mut write, &hello_frame)
2981        .await
2982        .map_err(SubcError::FrameIo)?;
2983
2984    // Expect HelloAck (registered) or a channel-0 Error (manifest/version reject).
2985    match read_frame(&mut read).await.map_err(SubcError::FrameIo)? {
2986        None => return Err(SubcError::ClosedBeforeHelloAck),
2987        Some(frame) => match frame.header.ty {
2988            FrameType::HelloAck => {
2989                log::info!("subc attach: registered (HelloAck received)");
2990            }
2991            FrameType::Error => {
2992                let body = serde_json::from_slice::<ErrorBody>(&frame.body).ok();
2993                return Err(SubcError::HelloRejected { body });
2994            }
2995            other => return Err(SubcError::UnexpectedFrame { ty: other }),
2996        },
2997    }
2998
2999    let dispatch_path_metrics = Arc::new(DispatchPathMetrics::new());
3000    let (writer_tx, writer_rx) = mpsc::channel::<WriterFrame>(WRITER_QUEUE_CAPACITY);
3001    let writer_task = spawn_writer_task(write, writer_rx, Arc::clone(&dispatch_path_metrics));
3002    // `read_frame` is NOT cancellation-safe, so it must never sit directly inside
3003    // the `select!` below: a drain-interval tick (or shutdown) firing while a
3004    // frame is mid-transit would drop the partially-consumed bytes and desync the
3005    // stream (the next read would parse a body byte as a frame header). A
3006    // dedicated reader task owns the socket, reads whole frames sequentially, and
3007    // forwards them over a channel; the loop selects on the cancel-safe `recv()`.
3008    let (reader_tx, mut reader_rx) = mpsc::channel::<Result<DecodedFrame, SubcError>>(256);
3009    let reader_task = spawn_reader_task(read, reader_tx);
3010    let shutdown = Arc::new(Notify::new());
3011    // Drain-tick deadline is tracked manually and checked at the TOP of every
3012    // loop turn rather than as an Interval select arm: the select below is
3013    // `biased` (bind completions first), and biased polling means a saturated
3014    // higher arm (sustained lossy push traffic keeps lossy_rx always-ready)
3015    // would starve every arm below it, including a timer arm — leaving
3016    // backpressured reliable frames parked in the retry buffer past their
3017    // delivery deadline. The pre-turn check cannot be starved by arm order;
3018    // the sleep_until arm below only exists to wake an otherwise-idle loop.
3019    let mut next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
3020    let mut next_maintenance_at = next_drain_at;
3021    let standing_actor =
3022        standing::StandingActor::new(Arc::clone(&shared_app), Arc::clone(&executor));
3023    // Startup reconciliation is intentionally direct; subsequent passes use
3024    // this existing maintenance timer arm and never create a standing timer.
3025    standing_actor.reconcile_at_startup();
3026    let mut next_standing_pass_at = tokio::time::Instant::now();
3027    let (maintenance_tx, mut maintenance_rx) = mpsc::channel::<MaintenanceCompletion>(256);
3028    let (bash_deferred_tx, mut bash_deferred_rx) =
3029        mpsc::channel::<bash::BashDeferredCompletion>(256);
3030    let (deferred_response_tx, mut deferred_response_rx) =
3031        mpsc::unbounded_channel::<PendingSubcResponse>();
3032    let (bash_poll_touch_tx, mut bash_poll_touch_rx) = mpsc::channel::<ProjectRootId>(256);
3033    let (control_completion_tx, mut control_completion_rx) =
3034        mpsc::channel::<RouteBindCompletion>(256);
3035    let (lossy_tx, mut lossy_rx) = mpsc::channel::<LossyPushEnvelope>(1024);
3036    let lossy_overflow = Arc::new(push::LossyOverflow::default());
3037    let lossy_seq = Arc::new(AtomicU64::new(0));
3038    let (reliable_tx, mut reliable_rx) = mpsc::unbounded_channel::<PushEnvelope>();
3039    let (fleet_status_client, fleet_status_task) =
3040        spawn_fleet_status_dial(connection_file_path, 64);
3041    let push_senders = PushSenders {
3042        lossy_tx,
3043        reliable_tx,
3044        lossy_overflow: Arc::clone(&lossy_overflow),
3045        lossy_seq,
3046        fleet_status_client: fleet_status_client.clone(),
3047    };
3048    let connection_cancel = PersistentCancelSignal::new();
3049    let mut installed_route_epochs: HashMap<u16, u32> = HashMap::new();
3050    let mut routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
3051    let mut management_routes: HashSet<RouteChannel> = HashSet::new();
3052    let mut bg_subs: HashMap<RouteChannel, BgSub> = HashMap::new();
3053    let mut bg_sub_by_session: BgSubsBySession = HashMap::new();
3054    let mut bg_wake_pending = BgWakePending::new();
3055    let mut bg_wake_epoch: HashMap<(ProjectRootId, String), u64> = HashMap::new();
3056    let mut bg_unacked_keys_by_root: HashMap<ProjectRootId, HashSet<String>> = HashMap::new();
3057    let mut root_channels: HashMap<ProjectRootId, HashSet<RouteChannel>> = HashMap::new();
3058    let mut session_identity: HashMap<(ProjectRootId, String), RetainedSessionIdentity> =
3059        HashMap::new();
3060    let mut push_buffer: HashMap<push::ReplayKey, VecDeque<PushFrame>> = HashMap::new();
3061    let mut retry_buffer: RetryBuffer = HashMap::new();
3062    let mut reclaimed_routes = ReclaimedRoutes::default();
3063    let mut completed_tasks = push::CompletedTaskIds::default();
3064    let mut live_roots: HashMap<ProjectRootId, RootMeta> = HashMap::new();
3065    let mut pending_binds: HashMap<RouteChannel, PendingBind> = HashMap::new();
3066    let mut pending_bash_asks: HashMap<ReverseCorrKey, PendingBashAsk> = HashMap::new();
3067    let mut next_bash_ask_corr: u64 = 1;
3068    let mut route_bash_cancels: HashMap<RouteChannel, bash::RouteBashCancel> = HashMap::new();
3069    let active_tool_calls: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
3070    let pending_deferred_setups = Arc::new(AtomicUsize::new(0));
3071    let mut pending_responses = PendingSubcResponses::default();
3072    let health_rollup_cache = Arc::new(HealthRollupCache::new());
3073    let health_rollup_worker = HealthRollupWorker::start(
3074        Arc::clone(&health_rollup_cache),
3075        Arc::clone(&executor),
3076        Arc::clone(&shared_app),
3077    );
3078
3079    let loop_result: Result<ModuleLoopExit, SubcError> = 'module_loop: loop {
3080        shared_app.set_open_route_count(routes.len() + management_routes.len());
3081        crate::logging::perf_tick(Some(&executor));
3082        dispatch_path_metrics.mark_frame_loop_tick();
3083        let ready_inspects = pending_responses.poll_ready(executor.as_ref());
3084        for resolved in ready_inspects {
3085            if let Err(error) = deliver_resolved_subc_response(
3086                &writer_tx,
3087                resolved,
3088                &routes,
3089                &mut live_roots,
3090                executor.as_ref(),
3091                &active_tool_calls,
3092                &shutdown,
3093                &dispatch_path_metrics,
3094                tool_response_body_limit,
3095            )
3096            .await
3097            {
3098                break 'module_loop Err(error);
3099            }
3100        }
3101        if let Err(error) = expire_pending_bash_asks(
3102            &writer_tx,
3103            &mut pending_bash_asks,
3104            &routes,
3105            &mut live_roots,
3106            &mut route_bash_cancels,
3107            &shutdown,
3108            &dispatch_path_metrics,
3109        )
3110        .await
3111        {
3112            break Err(error);
3113        }
3114
3115        // RouteBind completions are control-plane unblockers. Drain any completed
3116        // binds before entering other branch work so Push and maintenance bursts
3117        // can only add one loop-turn of latency.
3118        match drain_pending_route_bind_completions(
3119            &mut control_completion_rx,
3120            &writer_tx,
3121            &mut routes,
3122            &mut root_channels,
3123            &mut session_identity,
3124            &mut push_buffer,
3125            &mut live_roots,
3126            &mut pending_binds,
3127            &mut installed_route_epochs,
3128            &executor,
3129            &standing_actor,
3130            &shutdown,
3131            &dispatch_path_metrics,
3132            lifecycle_probe.as_ref(),
3133        )
3134        .await
3135        {
3136            Ok(drained) => {
3137                if drained > 0 {
3138                    next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
3139                    health_rollup_worker.request_refresh();
3140                }
3141            }
3142            Err(error) => break Err(error),
3143        }
3144
3145        if tokio::time::Instant::now() >= next_drain_at {
3146            push::emit_bg_event_wakes(
3147                &writer_tx,
3148                &dispatch_path_metrics,
3149                &bg_subs,
3150                &mut bg_wake_pending,
3151            );
3152            dispatch_path_metrics.warn_stuck_pending_watches(&executor, &bg_sub_by_session);
3153            warn_slow_pending_binds(&mut pending_binds, &executor);
3154            warn_slow_running_interactive_jobs(&executor);
3155            if let Err(error) = expire_overdue_route_binds(
3156                &writer_tx,
3157                &executor,
3158                &mut pending_binds,
3159                &mut installed_route_epochs,
3160                &dispatch_path_metrics,
3161            )
3162            .await
3163            {
3164                break Err(error);
3165            }
3166
3167            let retried = push::drain_retry_buffers_for_bound_routes(
3168                &writer_tx,
3169                &dispatch_path_metrics,
3170                &routes,
3171                &mut retry_buffer,
3172            );
3173            if retried > 0 {
3174                log::debug!(
3175                    "subc attach: retried {retried} reliable Push frame(s) after writer backpressure"
3176                );
3177            }
3178
3179            next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
3180        }
3181
3182        // A lossy emitter may place its newest update in the overflow buffer
3183        // when the bounded channel is full, while this receive loop is draining
3184        // the channel. Drain overflow before selecting again so that raced
3185        // update is delivered on the next timer tick instead of waiting for
3186        // another lossy enqueue.
3187        let overflow_batch = lossy_overflow.drain();
3188        if !overflow_batch.is_empty() {
3189            let (_, deferred) = push::drain_reliable_push_turn(
3190                &writer_tx,
3191                &dispatch_path_metrics,
3192                &routes,
3193                &root_channels,
3194                &session_identity,
3195                &mut retry_buffer,
3196                &mut push_buffer,
3197                &mut completed_tasks,
3198                &bg_sub_by_session,
3199                &mut bg_wake_pending,
3200                &mut bg_wake_epoch,
3201                &mut reliable_rx,
3202                None,
3203                lifecycle_probe.as_ref(),
3204            );
3205            if deferred {
3206                tokio::task::yield_now().await;
3207            }
3208
3209            let mut batch = Vec::new();
3210            while let Ok(item) = lossy_rx.try_recv() {
3211                batch.push(item);
3212            }
3213            batch.extend(overflow_batch);
3214            push::process_lossy_push_envelope_batch(
3215                &writer_tx,
3216                &dispatch_path_metrics,
3217                &routes,
3218                &root_channels,
3219                &completed_tasks,
3220                batch,
3221            );
3222        }
3223
3224        tokio::select! {
3225            biased;
3226            Some(completion) = control_completion_rx.recv() => {
3227                if let Err(error) = process_route_bind_completion(
3228                    &writer_tx,
3229                    completion,
3230                    &mut routes,
3231                    &mut root_channels,
3232                    &mut session_identity,
3233                    &mut push_buffer,
3234                    &mut live_roots,
3235                    &mut pending_binds,
3236                    &mut installed_route_epochs,
3237                    &executor,
3238                    &standing_actor,
3239                    &shutdown,
3240                    &dispatch_path_metrics,
3241                    lifecycle_probe.as_ref(),
3242                )
3243                .await
3244                {
3245                    break Err(error);
3246                }
3247                next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
3248                health_rollup_worker.request_refresh();
3249            }
3250            _ = shutdown.notified() => {
3251                log::warn!("subc attach: fatal executor response requested teardown");
3252                break Ok(ModuleLoopExit::SkipSearchFlush);
3253            }
3254            maybe_frame = reader_rx.recv() => {
3255                let frame = match maybe_frame {
3256                    None => {
3257                        log::warn!(
3258                            "subc attach: daemon connection ended without Goodbye; exiting for restart"
3259                        );
3260                        break Ok(ModuleLoopExit::ConnectionLost);
3261                    }
3262                    Some(Err(error)) => break Err(error),
3263                    Some(Ok(frame)) => frame,
3264                };
3265                let phase_trace = frame.phase_trace;
3266                let frame = frame.frame;
3267
3268                if !ingress_route_should_be_processed(
3269                    &installed_route_epochs,
3270                    &reclaimed_routes,
3271                    &frame,
3272                ) {
3273                    log::debug!(
3274                        "subc attach: silently dropping {:?} for uninstalled route {}@{}",
3275                        frame.header.ty,
3276                        frame.header.channel,
3277                        frame.header.epoch
3278                    );
3279                    continue;
3280                }
3281
3282                match frame.header.ty {
3283                    FrameType::Ping if frame.header.channel == 0 => {
3284                        let pong = match Frame::build_with_version(
3285                            frame.header.ver,
3286                            FrameType::Pong,
3287                            frame.header.flags,
3288                            0,
3289                            0,
3290                            frame.header.corr,
3291                            Vec::new(),
3292                        ) {
3293                            Ok(pong) => pong,
3294                            Err(error) => break Err(SubcError::FrameBuild(error)),
3295                        };
3296                        if let Err(error) = send_frame(&writer_tx, &dispatch_path_metrics, pong).await {
3297                            break Err(error);
3298                        }
3299                    }
3300                    FrameType::Goodbye if frame.header.channel == 0 => {
3301                        log::info!("subc attach: received channel-0 Goodbye");
3302                        break Ok(ModuleLoopExit::Graceful);
3303                    }
3304                    FrameType::Goodbye => {
3305                        let channel = route_key(frame.header.channel, frame.header.epoch);
3306                        if let Err(error) = teardown_installed_route(
3307                            &writer_tx,
3308                            &dispatch_path_metrics,
3309                            &executor,
3310                            channel,
3311                            "Goodbye",
3312                            None,
3313                            &mut installed_route_epochs,
3314                            &mut routes,
3315                            &mut management_routes,
3316                            &mut root_channels,
3317                            &mut bg_subs,
3318                            &mut bg_sub_by_session,
3319                            &mut bg_wake_pending,
3320                            &mut pending_bash_asks,
3321                            &mut live_roots,
3322                            &mut route_bash_cancels,
3323                            &active_tool_calls,
3324                            &mut pending_responses,
3325                            &mut pending_binds,
3326                            &mut retry_buffer,
3327                            &mut push_buffer,
3328                            &shutdown,
3329                            tool_response_body_limit,
3330                            lifecycle_probe.as_ref(),
3331                        )
3332                        .await
3333                        {
3334                            break Err(error);
3335                        }
3336                    }
3337                    FrameType::Response | FrameType::Error if frame.header.channel != 0 => {
3338                        if let Err(error) = handle_bash_elicitation_reply(
3339                            &writer_tx,
3340                            &frame,
3341                            &mut pending_bash_asks,
3342                            &routes,
3343                            &mut live_roots,
3344                            &executor,
3345                            &shutdown,
3346                            &bash_deferred_tx,
3347                            &bash_poll_touch_tx,
3348                            &dispatch_path_metrics,
3349                            &mut route_bash_cancels,
3350                            dispatch,
3351                        )
3352                        .await
3353                        {
3354                            break Err(error);
3355                        }
3356                    }
3357                    FrameType::Request if frame.header.channel == 0 => {
3358                        if let Err(error) = handle_control_request(
3359                            &writer_tx,
3360                            &frame,
3361                            &shared_app,
3362                            &executor,
3363                            &mut live_roots,
3364                            &mut pending_binds,
3365                            &mut installed_route_epochs,
3366                            &mut routes,
3367                            &mut management_routes,
3368                            &mut root_channels,
3369                            &mut bg_subs,
3370                            &mut bg_sub_by_session,
3371                            &mut bg_wake_pending,
3372                            &mut pending_bash_asks,
3373                            &mut route_bash_cancels,
3374                            &active_tool_calls,
3375                            &mut pending_responses,
3376                            &mut retry_buffer,
3377                            &mut push_buffer,
3378                            &shutdown,
3379                            &control_completion_tx,
3380                            &dispatch_path_metrics,
3381                            lifecycle_probe.as_ref(),
3382                            &health_rollup_cache,
3383                            &push_senders,
3384                            dispatch,
3385                            user_config_path.as_deref(),
3386                            tool_response_body_limit,
3387                        )
3388                        .await
3389                        {
3390                            break Err(error);
3391                        }
3392                    }
3393                    FrameType::Request => {
3394                        let route = route_key(frame.header.channel, frame.header.epoch);
3395                        let result = if management_routes.contains(&route) {
3396                            handle_management_request(
3397                                &writer_tx,
3398                                &frame,
3399                                &shared_app,
3400                                &executor,
3401                                &live_roots,
3402                                &root_channels,
3403                                &health_rollup_cache,
3404                                &dispatch_path_metrics,
3405                            )
3406                            .await
3407                        } else {
3408                            handle_tool_call(
3409                                &writer_tx,
3410                                &frame,
3411                                phase_trace,
3412                                &routes,
3413                                &pending_binds,
3414                                &mut live_roots,
3415                                &executor,
3416                                &active_tool_calls,
3417                                &pending_deferred_setups,
3418                                &shutdown,
3419                                &connection_cancel,
3420                                &bash_deferred_tx,
3421                                &bash_poll_touch_tx,
3422                                &dispatch_path_metrics,
3423                                &mut route_bash_cancels,
3424                                &mut pending_bash_asks,
3425                                &mut next_bash_ask_corr,
3426                                &mut bg_subs,
3427                                &mut bg_sub_by_session,
3428                                &mut bg_wake_pending,
3429                                &mut bg_wake_epoch,
3430                                dispatch,
3431                                &deferred_response_tx,
3432                                allow_native_passthrough,
3433                                tool_response_body_limit,
3434                            )
3435                            .await
3436                        };
3437                        if let Err(error) = result {
3438                            break Err(error);
3439                        }
3440                    }
3441                    FrameType::Cancel => {
3442                        let channel = route_key(frame.header.channel, frame.header.epoch);
3443                        cancel_active_tool_call(
3444                            &active_tool_calls,
3445                            executor.as_ref(),
3446                            channel,
3447                            frame.header.corr,
3448                            "Cancel frame",
3449                        );
3450                        pending_responses.cancel_request(channel, frame.header.corr);
3451                        if bg_subs.contains_key(&channel) {
3452                            if let Err(error) = end_bg_subscription(
3453                                &writer_tx,
3454                                &dispatch_path_metrics,
3455                                &mut bg_subs,
3456                                &mut bg_sub_by_session,
3457                                &mut bg_wake_pending,
3458                                channel,
3459                                routes.get(&channel),
3460                                "cancel",
3461                            )
3462                            .await
3463                            {
3464                                break Err(error);
3465                            }
3466                        }
3467                        if let Err(error) = cancel_pending_bash_ask_for_tool_call(
3468                            &writer_tx,
3469                            &mut pending_bash_asks,
3470                            channel,
3471                            frame.header.corr,
3472                            &routes,
3473                            &mut live_roots,
3474                            &mut route_bash_cancels,
3475                            &shutdown,
3476                            &dispatch_path_metrics,
3477                        )
3478                        .await
3479                        {
3480                            break Err(error);
3481                        }
3482                    }
3483                    // Incoming push messages are ignored here. Cancel frames are
3484                    // handled above for active and deferred tool calls plus pending
3485                    // bash elicitation requests.
3486                    _ => {}
3487                }
3488            }
3489            Some(pending) = deferred_response_rx.recv() => {
3490                if routes.contains_key(&pending.route)
3491                    && active_tool_call_is_registered(
3492                        &active_tool_calls,
3493                        pending.route,
3494                        pending.corr,
3495                    )
3496                {
3497                    pending_responses.register(pending);
3498                } else {
3499                    if let Some(cancellation) = &pending.pending.cancellation {
3500                        cancellation.request_cancel();
3501                    }
3502                    finish_active_tool_call(&active_tool_calls, pending.route, pending.corr);
3503                }
3504            }
3505            Some((root_id, frame)) = reliable_rx.recv() => {
3506                // Reliable Push frames are FIFO and must-deliver, but draining an
3507                // unbounded burst in one current-thread turn can starve RouteBind
3508                // completions. The budget defers excess frames, never drops them.
3509                let (_, deferred) = push::drain_reliable_push_turn(
3510                    &writer_tx,
3511                    &dispatch_path_metrics,
3512                    &routes,
3513                    &root_channels,
3514                    &session_identity,
3515                    &mut retry_buffer,
3516                    &mut push_buffer,
3517                    &mut completed_tasks,
3518                    &bg_sub_by_session,
3519                    &mut bg_wake_pending,
3520                    &mut bg_wake_epoch,
3521                    &mut reliable_rx,
3522                    Some((root_id, frame)),
3523                    lifecycle_probe.as_ref(),
3524                );
3525                if deferred {
3526                    tokio::task::yield_now().await;
3527                }
3528            }
3529            Some((order, root_id, frame)) = lossy_rx.recv() => {
3530                // When both push lanes have work, handle a small reliable slice before lossy work.
3531                // That ordering lets completed task ids suppress stale BashLongRunning frames.
3532                // The slice stays bounded so reliable bursts cannot monopolize this loop turn.
3533                let (_, deferred) = push::drain_reliable_push_turn(
3534                    &writer_tx,
3535                    &dispatch_path_metrics,
3536                    &routes,
3537                    &root_channels,
3538                    &session_identity,
3539                    &mut retry_buffer,
3540                    &mut push_buffer,
3541                    &mut completed_tasks,
3542                    &bg_sub_by_session,
3543                    &mut bg_wake_pending,
3544                    &mut bg_wake_epoch,
3545                    &mut reliable_rx,
3546                    None,
3547                    lifecycle_probe.as_ref(),
3548                );
3549                if deferred {
3550                    tokio::task::yield_now().await;
3551                }
3552
3553                // Drain the currently queued burst in one loop turn so lossy
3554                // status/progress updates can be merged before reaching subc's
3555                // shared egress queue. Each lossy frame gets a sequence number
3556                // before it goes to the channel or overflow buffer, so the
3557                // combined batch is sorted back into producer order before
3558                // coalescing drops stale updates for the same key.
3559                let mut batch = vec![(order, root_id, frame)];
3560                while let Ok(item) = lossy_rx.try_recv() {
3561                    batch.push(item);
3562                }
3563                batch.extend(lossy_overflow.drain());
3564                push::process_lossy_push_envelope_batch(
3565                    &writer_tx,
3566                    &dispatch_path_metrics,
3567                    &routes,
3568                    &root_channels,
3569                    &completed_tasks,
3570                    batch,
3571                );
3572            }
3573            Some(done) = bash_deferred_rx.recv() => {
3574                decrement_counted_channel(&dispatch_path_metrics.bash_deferred_queued);
3575                if let Err(error) = bash::handle_bash_deferred_completion(
3576                    &writer_tx,
3577                    done,
3578                    &routes,
3579                    &mut live_roots,
3580                    &mut route_bash_cancels,
3581                    &shutdown,
3582                    &dispatch_path_metrics,
3583                )
3584                .await
3585                {
3586                    break Err(error);
3587                }
3588            }
3589            Some(root_id) = bash_poll_touch_rx.recv() => {
3590                decrement_counted_channel(&dispatch_path_metrics.bash_poll_touch_queued);
3591                if let Some(meta) = live_roots.get_mut(&root_id) {
3592                    meta.note_activity();
3593                }
3594            }
3595            Some(completion) = maintenance_rx.recv() => {
3596                decrement_counted_channel(&dispatch_path_metrics.maintenance_queued);
3597                let root_id = completion.root_id.clone();
3598                let response = completion.response;
3599                let response_is_fatal = response_is_fatal_panic(&response);
3600                let bind_pending = pending_binds
3601                    .values()
3602                    .any(|pending| pending.bind_root_id == root_id);
3603                let requiesce = if let Some(meta) = live_roots.get_mut(&root_id) {
3604                    let defer_requeue = meta.unbound_quiesced || bind_pending;
3605                    note_maintenance_completion(
3606                        meta,
3607                        completion.requeue_kind,
3608                        response_is_fatal,
3609                        defer_requeue,
3610                    );
3611                    should_requiesce_after_maintenance(meta, completion.kind, bind_pending)
3612                } else {
3613                    false
3614                };
3615                if requiesce {
3616                    quiesce_unbound_root(&root_id, &mut live_roots, &executor);
3617                }
3618                push::clear_stale_bg_wakes_for_empty_sessions(
3619                    &root_id,
3620                    &completion.empty_bg_sessions,
3621                    &bg_sub_by_session,
3622                    &mut bg_wake_pending,
3623                    &bg_wake_epoch,
3624                );
3625                if let Some(keys) = completion.unacked_bg_keys {
3626                    bg_unacked_keys_by_root.insert(root_id.clone(), keys);
3627                }
3628                record_bg_runtime_from_snapshots(
3629                    &dispatch_path_metrics,
3630                    bg_subs.len(),
3631                    bg_wake_pending.len(),
3632                    &bg_unacked_keys_by_root,
3633                );
3634                if response_is_fatal {
3635                    if let Some(meta) = live_roots.get_mut(&root_id) {
3636                        meta.maintenance_poisoned = true;
3637                    }
3638                    log::warn!(
3639                        "subc attach: maintenance drain observed a fatal actor; deferring teardown until a route request can receive actor_fatal"
3640                    );
3641                }
3642            }
3643            _ = tokio::time::sleep(PENDING_POLL_INTERVAL), if !pending_responses.is_empty() => {
3644                // The next loop turn polls detached inspect completions. Keeping
3645                // the timer here lets already-ready control frames run first.
3646            }
3647            _ = tokio::time::sleep_until(next_drain_at) => {
3648                // Wakes an otherwise-idle loop so the pre-turn drain check
3649                // above runs on schedule; the drain work itself lives there.
3650            }
3651            _ = tokio::time::sleep_until(next_maintenance_at) => {
3652                // Delay cache-draining maintenance until any already-ready
3653                // inbound route/control messages and push completions have run,
3654                // so maintenance does not block the actor from handling the
3655                // first request that arrives after a route bind is acknowledged.
3656                crate::logging::maybe_sweep_logs();
3657                let retention_registries = shared_app.try_memory_contexts().map(|contexts| {
3658                    contexts
3659                        .into_iter()
3660                        .map(|(_, context)| context.bash_background().clone())
3661                        .collect()
3662                });
3663                crate::db::compression_events::maybe_spawn_retention(
3664                    shared_app.db(),
3665                    retention_registries,
3666                );
3667                let reaped_lsp_children = shared_app
3668                    .lsp_child_registry()
3669                    .reap_children_with_gone_cwd_or_reclaimed_root();
3670                if reaped_lsp_children > 0 {
3671                    log::warn!(
3672                        "subc attach: reaped {reaped_lsp_children} orphaned LSP child process group(s)"
3673                    );
3674                }
3675                let now = Instant::now();
3676                reap_idle_lsp_servers(now, &live_roots, &executor);
3677                let reap = reap_idle_roots(
3678                    now,
3679                    &mut live_roots,
3680                    &pending_binds,
3681                    &root_channels,
3682                    &executor,
3683                    &dispatch_path_metrics,
3684                );
3685                for root_id in &reap.forgotten_deleted_roots {
3686                    bg_unacked_keys_by_root.remove(root_id);
3687                    purge_deleted_root_residents(
3688                        root_id,
3689                        &mut routes,
3690                        &mut root_channels,
3691                        &mut installed_route_epochs,
3692                        &mut route_bash_cancels,
3693                        &active_tool_calls,
3694                        executor.as_ref(),
3695                        &mut retry_buffer,
3696                        &mut reclaimed_routes,
3697                        &mut session_identity,
3698                        &mut push_buffer,
3699                        &mut bg_subs,
3700                        &mut bg_sub_by_session,
3701                        &mut bg_wake_pending,
3702                        &mut bg_wake_epoch,
3703                        &mut pending_bash_asks,
3704                        &dispatch_path_metrics,
3705                    );
3706                }
3707                if reap.evicted > 0 {
3708                    log::debug!("subc attach: reaped {} idle root(s)", reap.evicted);
3709                }
3710                record_bg_runtime_from_snapshots(
3711                    &dispatch_path_metrics,
3712                    bg_subs.len(),
3713                    bg_wake_pending.len(),
3714                    &bg_unacked_keys_by_root,
3715                );
3716                submit_due_maintenance_jobs(
3717                    &executor,
3718                    &mut live_roots,
3719                    &pending_binds,
3720                    &bg_sub_by_session,
3721                    &bg_wake_pending,
3722                    &bg_wake_epoch,
3723                    &maintenance_tx,
3724                    &dispatch_path_metrics,
3725                );
3726                if tokio::time::Instant::now() >= next_standing_pass_at {
3727                    standing_actor.tick();
3728                    next_standing_pass_at = tokio::time::Instant::now()
3729                        + standing::STANDING_MAINTENANCE_INTERVAL;
3730                }
3731                // Opportunistic allocator relief, independent of the idle
3732                // sweep: the sweep's whole-process idle gate never opens while
3733                // any session stays active, which let freed warm-up arenas sit
3734                // resident for the process lifetime (5.1 GB RSS over ~600 MB
3735                // live). Slack threshold + spacing live in memory.rs; the pass
3736                // itself runs on a detached thread.
3737                #[cfg(any(target_os = "macos", target_os = "linux"))]
3738                {
3739                    let now_std = std::time::Instant::now();
3740                    let _ = crate::memory::spawn_allocator_slack_relief_if_due(now_std);
3741                }
3742                next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
3743            }
3744        }
3745    };
3746
3747    shared_app.set_open_route_count(0);
3748    health_rollup_worker.shutdown();
3749
3750    connection_cancel.cancel();
3751    cancel_all_active_tool_calls(&active_tool_calls, executor.as_ref(), "connection teardown");
3752    let setup_drain_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
3753    while pending_deferred_setups.load(Ordering::SeqCst) != 0
3754        && tokio::time::Instant::now() < setup_drain_deadline
3755    {
3756        tokio::select! {
3757            biased;
3758            Some(pending) = deferred_response_rx.recv() => pending_responses.register(pending),
3759            _ = tokio::time::sleep(Duration::from_millis(5)) => {}
3760        }
3761    }
3762    if pending_deferred_setups.load(Ordering::SeqCst) != 0 {
3763        log::warn!(
3764            "subc attach: timed out waiting for deferred response setup registration during shutdown"
3765        );
3766    }
3767    while let Ok(pending) = deferred_response_rx.try_recv() {
3768        pending_responses.register(pending);
3769    }
3770    for resolved in pending_responses.drain_on_shutdown(executor.as_ref()) {
3771        if let Err(error) = deliver_resolved_subc_response(
3772            &writer_tx,
3773            resolved,
3774            &routes,
3775            &mut live_roots,
3776            executor.as_ref(),
3777            &active_tool_calls,
3778            &shutdown,
3779            &dispatch_path_metrics,
3780            tool_response_body_limit,
3781        )
3782        .await
3783        {
3784            log::warn!("subc attach: failed to emit deferred shutdown terminal: {error}");
3785        }
3786    }
3787    // Channel-0 Goodbye, EOF, and fatal exits bypass per-route Goodbye. Settle
3788    // their root lifecycle state before loop-owned routing metadata is dropped.
3789    quiesce_connection_roots(
3790        &mut live_roots,
3791        &mut pending_binds,
3792        &mut routes,
3793        &mut root_channels,
3794        &mut installed_route_epochs,
3795        &mut route_bash_cancels,
3796        &active_tool_calls,
3797        &executor,
3798    );
3799
3800    fleet_status_client.set_route_live(false);
3801    fleet_status_task.abort();
3802    let _ = fleet_status_task.await;
3803
3804    let mut loop_result = loop_result;
3805    if !pending_bash_asks.is_empty() {
3806        let no_routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
3807        if let Err(error) = settle_all_pending_bash_asks(
3808            &writer_tx,
3809            &mut pending_bash_asks,
3810            &no_routes,
3811            &mut live_roots,
3812            &mut route_bash_cancels,
3813            &shutdown,
3814            &dispatch_path_metrics,
3815        )
3816        .await
3817        {
3818            loop_result = loop_result.and(Err(error));
3819        }
3820    }
3821
3822    // The reader task may be parked on `read_frame`; abort it (we are done with
3823    // the connection) and flush the writer.
3824    reader_task.abort();
3825    drop(writer_tx);
3826    let writer_result = finish_writer_task(writer_task).await;
3827    loop_result.and_then(|exit| writer_result.map(|_| exit))
3828}
3829
3830fn spawn_writer_task<W>(
3831    mut write: W,
3832    mut rx: mpsc::Receiver<WriterFrame>,
3833    metrics: Arc<DispatchPathMetrics>,
3834) -> JoinHandle<Result<(), subc_transport::FrameIoError>>
3835where
3836    W: AsyncWrite + Unpin + Send + 'static,
3837{
3838    tokio::spawn(async move {
3839        let mut write_buffer = Vec::new();
3840        while let Some(mut queued) = rx.recv().await {
3841            let measure = queued.tool_response_trace.is_some();
3842            let dequeued = measure.then(Instant::now);
3843            metrics.writer_active.store(true, Ordering::Relaxed);
3844            decrement_counted_channel(&metrics.writer_queued);
3845            let write_timing = write_frame_contiguous(
3846                &mut write,
3847                queued.frame(),
3848                queued.body(),
3849                &mut write_buffer,
3850                measure,
3851            )
3852            .await;
3853            metrics.writer_active.store(false, Ordering::Relaxed);
3854            let write_timing = write_timing?;
3855
3856            if let (Some(trace), Some(dequeued), Some(write_timing)) =
3857                (queued.tool_response_trace.take(), dequeued, write_timing)
3858            {
3859                if let Some(completed) = trace.finish(
3860                    dequeued,
3861                    write_timing.write_started,
3862                    write_timing.write_finished,
3863                    write_timing.frame_bytes,
3864                ) {
3865                    log_ctx::with_session(Some(completed.session), || {
3866                        crate::logging::note_tool_call_trace(
3867                            &completed.name,
3868                            &completed.root,
3869                            completed.channel,
3870                            completed.corr,
3871                            completed.phases,
3872                        );
3873                    });
3874                }
3875            }
3876        }
3877        Ok(())
3878    })
3879}
3880
3881struct FrameWriteTiming {
3882    write_started: Instant,
3883    write_finished: Instant,
3884    frame_bytes: usize,
3885}
3886
3887/// Encode one complete frame into the existing reusable buffer and write it
3888/// without interleaving bytes from another channel. Timing is collected only
3889/// for tool responses, so Push and control frames add no clock reads.
3890async fn write_frame_contiguous<W>(
3891    writer: &mut W,
3892    frame: &Frame,
3893    body: &[u8],
3894    buffer: &mut Vec<u8>,
3895    measure: bool,
3896) -> Result<Option<FrameWriteTiming>, subc_transport::FrameIoError>
3897where
3898    W: AsyncWrite + Unpin,
3899{
3900    if frame.header.len as usize != body.len() {
3901        return Err(subc_transport::FrameIoError::BodyLengthMismatch {
3902            header_len: frame.header.len,
3903            body_len: body.len(),
3904        });
3905    }
3906
3907    let header = frame.header.encode();
3908    buffer.clear();
3909    buffer.reserve(header.len() + body.len());
3910    buffer.extend_from_slice(&header);
3911    buffer.extend_from_slice(body);
3912    let write_started = measure.then(Instant::now);
3913    writer
3914        .write_all(buffer)
3915        .await
3916        .map_err(subc_transport::FrameIoError::Io)?;
3917    Ok(write_started.map(|write_started| FrameWriteTiming {
3918        write_started,
3919        write_finished: Instant::now(),
3920        frame_bytes: buffer.len(),
3921    }))
3922}
3923
3924fn spawn_reader_task<R>(
3925    mut read: R,
3926    tx: mpsc::Sender<Result<DecodedFrame, SubcError>>,
3927) -> JoinHandle<()>
3928where
3929    R: AsyncRead + Unpin + Send + 'static,
3930{
3931    tokio::spawn(async move {
3932        loop {
3933            match read_frame(&mut read).await {
3934                Ok(Some(frame)) => {
3935                    let decoded = DecodedFrame {
3936                        frame,
3937                        phase_trace: PhaseTrace::new(Instant::now()),
3938                    };
3939                    if tx.send(Ok(decoded)).await.is_err() {
3940                        return;
3941                    }
3942                }
3943                Ok(None) => {
3944                    // EOF: let the loop observe channel close as "daemon closed".
3945                    return;
3946                }
3947                Err(error) => {
3948                    // A killed daemon surfaces as ConnectionReset (RST) on
3949                    // Windows where Unix delivers a clean EOF (FIN); a
3950                    // mid-teardown daemon can also abort the socket. Both mean
3951                    // "daemon went away", not a wire fault — normalize them to
3952                    // the clean-close path so module exit behavior matches
3953                    // across platforms (same class subc-core fixed in d33d9a71).
3954                    if let subc_transport::FrameIoError::Io(io_error) = &error {
3955                        if matches!(
3956                            io_error.kind(),
3957                            std::io::ErrorKind::ConnectionReset
3958                                | std::io::ErrorKind::ConnectionAborted
3959                        ) {
3960                            log::info!(
3961                                "subc attach: connection reset by daemon; treating as close"
3962                            );
3963                            return;
3964                        }
3965                    }
3966                    let _ = tx.send(Err(SubcError::FrameIo(error))).await;
3967                    return;
3968                }
3969            }
3970        }
3971    })
3972}
3973
3974async fn finish_writer_task(
3975    mut writer_task: JoinHandle<Result<(), subc_transport::FrameIoError>>,
3976) -> Result<(), SubcError> {
3977    match tokio::time::timeout(Duration::from_millis(100), &mut writer_task).await {
3978        Ok(Ok(Ok(()))) => Ok(()),
3979        Ok(Ok(Err(error))) => Err(SubcError::FrameIo(error)),
3980        Ok(Err(error)) => Err(SubcError::WriterJoin(error)),
3981        Err(_) => {
3982            writer_task.abort();
3983            Ok(())
3984        }
3985    }
3986}
3987
3988fn register_actor_for_bind(
3989    shared_app: &Arc<App>,
3990    executor: &Arc<Executor>,
3991    push_senders: &PushSenders,
3992    bind_root_id: &ProjectRootId,
3993    route_channel: u16,
3994    root_was_live: bool,
3995) -> bool {
3996    if executor.actor_registered(bind_root_id) {
3997        log::debug!(
3998            "subc attach: reusing actor for route {} root {}",
3999            route_channel,
4000            bind_root_id.as_path().display()
4001        );
4002        return false;
4003    }
4004
4005    if root_was_live {
4006        log::warn!(
4007            "subc attach: recreating missing actor for live root {} on route {}",
4008            bind_root_id.as_path().display(),
4009            route_channel
4010        );
4011    }
4012
4013    let actor_ctx = Arc::new(AppContext::from_app(
4014        Arc::clone(shared_app),
4015        Config::default(),
4016    ));
4017    install_bash_compressor(&actor_ctx);
4018    actor_ctx.install_fleet_status_client(Some(push_senders.fleet_status_client.clone()));
4019    actor_ctx.set_progress_sender(Some(push::progress_sender_for_root(
4020        push_senders.clone(),
4021        bind_root_id.clone(),
4022    )));
4023    let inserted = executor.register_actor(bind_root_id.clone(), Arc::clone(&actor_ctx));
4024    drop(actor_ctx);
4025    if inserted {
4026        // Do not insert into live_roots until configure succeeds: live_roots
4027        // drives maintenance, and a half-configured new actor must not be
4028        // maintenance-eligible before its route/session identity exists.
4029        log::debug!(
4030            "subc attach: registered actor for route {} root {}",
4031            route_channel,
4032            bind_root_id.as_path().display()
4033        );
4034    } else {
4035        log::debug!(
4036            "subc attach: actor appeared while binding route {} root {}; reusing it",
4037            route_channel,
4038            bind_root_id.as_path().display()
4039        );
4040    }
4041    inserted
4042}
4043
4044fn rollback_pending_bind_actor(
4045    executor: &Arc<Executor>,
4046    live_roots: &HashMap<ProjectRootId, RootMeta>,
4047    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
4048    root_id: &ProjectRootId,
4049    inserted_new_actor: bool,
4050) {
4051    if !inserted_new_actor || live_roots.contains_key(root_id) {
4052        return;
4053    }
4054
4055    if let Some((route, pending)) = pending_binds
4056        .iter_mut()
4057        .find(|(_, pending)| &pending.bind_root_id == root_id)
4058    {
4059        pending.inserted_new_actor = true;
4060        log::debug!(
4061            "subc attach: transferred rollback ownership for root {} to pending route {}",
4062            root_id.as_path().display(),
4063            route
4064        );
4065        return;
4066    }
4067
4068    executor.remove_actor(root_id);
4069}
4070
4071fn route_bind_error_code_for_configure_response(response: &Response) -> &'static str {
4072    match response.data.get("code").and_then(|code| code.as_str()) {
4073        // Preserve typed configure rejections across the bind boundary: a
4074        // malformed fed fingerprint means a federation-module bug or
4075        // fingerprint-format drift, and the fed side matches on the code rather
4076        // than parsing prose.
4077        Some("bad_harness_fingerprint") => "bad_harness_fingerprint",
4078        // Cache-key probe failures are transient (fd pressure, git spawn
4079        // contention); the client retries the bind rather than treating the
4080        // root as permanently divergent.
4081        Some("cache_key_probe_failed") => "cache_key_probe_failed",
4082        // Actor lifecycle gaps are transient from the daemon/client viewpoint:
4083        // a fresh bind can create or join a healthy actor, so do not classify
4084        // them as permanent config divergence.
4085        Some("actor_not_registered" | "actor_fatal") => "actor_not_ready",
4086        _ => "config_divergence",
4087    }
4088}
4089
4090fn queue_post_bind_configure_and_completion_maintenance(
4091    root_id: &ProjectRootId,
4092    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4093) {
4094    let Some(meta) = live_roots.get_mut(root_id) else {
4095        return;
4096    };
4097    if meta.maintenance_poisoned || meta.maintenance_pending {
4098        return;
4099    }
4100
4101    meta.maintenance_pending = true;
4102    meta.maintenance_queued_kinds
4103        .push_back(MaintenanceDrainKind::ConfigureTail);
4104    meta.maintenance_queued_kinds
4105        .push_back(MaintenanceDrainKind::CompletionDrains);
4106}
4107
4108#[allow(clippy::too_many_arguments)]
4109async fn handle_route_bind_completion(
4110    tx: &WriterSender,
4111    completion: RouteBindCompletion,
4112    routes: &mut HashMap<RouteChannel, RouteIdentity>,
4113    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
4114    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
4115    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
4116    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4117    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
4118    installed_route_epochs: &mut HashMap<u16, u32>,
4119    executor: &Arc<Executor>,
4120    standing_actor: &standing::StandingActor,
4121    shutdown: &Arc<Notify>,
4122    metrics: &Arc<DispatchPathMetrics>,
4123    lifecycle_probe: Option<&SubcTestLifecycleProbe>,
4124) -> Result<(), SubcError> {
4125    let route_id = completion.route;
4126    let Some(pending) = pending_binds.remove(&route_id) else {
4127        log::warn!(
4128            "subc attach: dropping RouteBind completion for non-pending route {}",
4129            completion.route
4130        );
4131        rollback_pending_bind_actor(
4132            executor,
4133            live_roots,
4134            pending_binds,
4135            &completion.bind_root_id,
4136            completion.inserted_new_actor,
4137        );
4138        let has_pending_bind = pending_binds
4139            .values()
4140            .any(|pending| pending.bind_root_id == completion.bind_root_id);
4141        if !root_channels
4142            .get(&completion.bind_root_id)
4143            .is_some_and(|channels| !channels.is_empty())
4144            && !has_pending_bind
4145        {
4146            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
4147        }
4148        remove_installed_route(installed_route_epochs, route_id);
4149        return Ok(());
4150    };
4151
4152    if pending.bind_root_id != completion.bind_root_id {
4153        log::warn!(
4154            "subc attach: pending RouteBind root mismatch for route {} (pending {} completion {})",
4155            completion.route,
4156            pending.bind_root_id.as_path().display(),
4157            completion.bind_root_id.as_path().display()
4158        );
4159    }
4160
4161    let inserted_new_actor = pending.inserted_new_actor || completion.inserted_new_actor;
4162    if pending.cancelled {
4163        rollback_pending_bind_actor(
4164            executor,
4165            live_roots,
4166            pending_binds,
4167            &completion.bind_root_id,
4168            inserted_new_actor,
4169        );
4170        let has_pending_bind = pending_binds
4171            .values()
4172            .any(|pending| pending.bind_root_id == completion.bind_root_id);
4173        if !root_channels
4174            .get(&completion.bind_root_id)
4175            .is_some_and(|channels| !channels.is_empty())
4176            && !has_pending_bind
4177        {
4178            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
4179        }
4180        log::debug!(
4181            "subc attach: discarded completed RouteBind for cancelled route {} root {}",
4182            completion.route,
4183            completion.bind_root_id.as_path().display()
4184        );
4185        remove_installed_route(installed_route_epochs, route_id);
4186        return Ok(());
4187    }
4188
4189    let failure = if !completion.configure_response.success {
4190        Some((
4191            &completion.configure_response,
4192            "configure failed during route bind",
4193        ))
4194    } else {
4195        None
4196    };
4197
4198    if let Some((response, fallback)) = failure {
4199        rollback_pending_bind_actor(
4200            executor,
4201            live_roots,
4202            pending_binds,
4203            &completion.bind_root_id,
4204            inserted_new_actor,
4205        );
4206        let has_pending_bind = pending_binds
4207            .values()
4208            .any(|pending| pending.bind_root_id == completion.bind_root_id);
4209        if !root_channels
4210            .get(&completion.bind_root_id)
4211            .is_some_and(|channels| !channels.is_empty())
4212            && !has_pending_bind
4213        {
4214            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
4215        }
4216        let message = response_message(response, fallback);
4217        let fatal = response_is_fatal_panic(response);
4218        let error_code = route_bind_error_code_for_configure_response(response);
4219        send_route_bind_error_parts(
4220            tx,
4221            completion.ver,
4222            completion.corr,
4223            completion.flags,
4224            error_code,
4225            &message,
4226            metrics,
4227        )
4228        .await?;
4229        remove_installed_route(installed_route_epochs, route_id);
4230        if fatal {
4231            signal_fatal_teardown(
4232                tx,
4233                Some(completion.route),
4234                completion.ver,
4235                completion.corr,
4236                shutdown,
4237                metrics,
4238            )
4239            .await;
4240        }
4241        return Ok(());
4242    }
4243
4244    remember_session_identity(session_identity, &completion.identity);
4245    let replay_key = push::ReplayKey::from_identity(&completion.identity);
4246    let bind_trust = completion.identity.trust;
4247    insert_route_channel(routes, root_channels, route_id, completion.identity);
4248    sync_bg_live_delivery_sessions(executor, routes, Some(&completion.bind_root_id));
4249    let restore_watcher = live_roots
4250        .get(&completion.bind_root_id)
4251        .is_some_and(|meta| meta.idle_artifacts_evicted || meta.unbound_quiesced);
4252    live_roots
4253        .entry(completion.bind_root_id.clone())
4254        .and_modify(|meta| {
4255            meta.reactivate_bound();
4256            meta.diagnostics_on_edit = completion.diagnostics_on_edit;
4257            meta.maintenance_poisoned = false;
4258        })
4259        .or_insert_with(|| RootMeta::new(Instant::now()));
4260    if let Some(meta) = live_roots.get_mut(&completion.bind_root_id) {
4261        meta.diagnostics_on_edit = completion.diagnostics_on_edit;
4262        meta.maintenance_poisoned = false;
4263    }
4264    if let Some(ctx) = executor.actor_context(&completion.bind_root_id) {
4265        // The bind transition revokes any matching unbound standing admission
4266        // before this session can select the shared artifact family.
4267        standing_actor.begin_session_bind(&ctx);
4268        ctx.mark_subc_bound();
4269        if restore_watcher {
4270            crate::commands::configure::ensure_project_watcher(&ctx);
4271        }
4272    }
4273
4274    let ack =
4275        serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).map_err(SubcError::Json)?;
4276    let response = Frame::build_with_version(
4277        completion.ver,
4278        FrameType::Response,
4279        control_flags(),
4280        0,
4281        0,
4282        completion.corr,
4283        ack,
4284    )
4285    .map_err(SubcError::FrameBuild)?;
4286    send_reliable_writer_frame(tx, metrics, response, "RouteBindAck").await?;
4287    queue_post_bind_configure_and_completion_maintenance(&completion.bind_root_id, live_roots);
4288    let replayed = push::replay_buffered_push_frames(
4289        tx,
4290        metrics,
4291        route_id,
4292        push_buffer,
4293        &replay_key,
4294        bind_trust,
4295        lifecycle_probe,
4296    );
4297    if replayed > 0 {
4298        log::debug!(
4299            "subc attach: replayed {} buffered Push frame(s) to route {} root {} harness {} session {}",
4300            replayed,
4301            completion.route,
4302            replay_key.root.as_path().display(),
4303            replay_key.harness,
4304            replay_key.session
4305        );
4306    }
4307    log::info!(
4308        "subc attach: route {} bound to root {}",
4309        completion.route,
4310        completion.bind_root_id.as_path().display()
4311    );
4312    Ok(())
4313}
4314
4315async fn expire_overdue_route_binds(
4316    tx: &WriterSender,
4317    executor: &Arc<Executor>,
4318    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
4319    installed_route_epochs: &mut HashMap<u16, u32>,
4320    metrics: &DispatchPathMetrics,
4321) -> Result<(), SubcError> {
4322    let now = Instant::now();
4323    let expired: Vec<_> = pending_binds
4324        .iter()
4325        .filter_map(|(route, pending)| {
4326            let age = now.saturating_duration_since(pending.started_at);
4327            (!pending.deadline_reported && age >= ROUTE_BIND_DEADLINE).then(|| {
4328                (
4329                    *route,
4330                    pending.corr,
4331                    pending.ver,
4332                    pending.flags,
4333                    pending.bind_root_id.clone(),
4334                    pending.configure_request_id.clone(),
4335                    age,
4336                )
4337            })
4338        })
4339        .collect();
4340
4341    for (route, corr, ver, flags, root_id, configure_request_id, age) in expired {
4342        if let Some(pending) = pending_binds.get_mut(&route) {
4343            pending.cancelled = true;
4344            pending.deadline_reported = true;
4345            let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
4346            log::debug!(
4347                "subc attach: cancelled overdue RouteBind configure for route {route} ({outcome:?})"
4348            );
4349        }
4350        remove_installed_route(installed_route_epochs, route);
4351        let age_ms = age.as_millis().min(u128::from(u64::MAX)) as u64;
4352        let deadline_ms = ROUTE_BIND_DEADLINE.as_millis();
4353        send_route_bind_error_parts(
4354            tx,
4355            ver,
4356            corr,
4357            flags,
4358            "actor_not_ready",
4359            &format!("route bind deadline exceeded after {age_ms}ms (deadline {deadline_ms}ms)"),
4360            metrics,
4361        )
4362        .await?;
4363        log::warn!(
4364            "subc attach: route {} bind for root {} exceeded {}ms deadline (configure_request_id={})",
4365            route,
4366            root_id.as_path().display(),
4367            deadline_ms,
4368            configure_request_id
4369        );
4370    }
4371
4372    Ok(())
4373}
4374
4375fn record_bg_runtime_from_snapshots(
4376    metrics: &DispatchPathMetrics,
4377    subscriptions: usize,
4378    wake_pending: usize,
4379    unacked_keys_by_root: &HashMap<ProjectRootId, HashSet<String>>,
4380) {
4381    let unacked_total = unacked_keys_by_root
4382        .values()
4383        .flatten()
4384        .collect::<HashSet<_>>()
4385        .len();
4386    metrics.record_bg_runtime(subscriptions, wake_pending, unacked_total);
4387}
4388
4389async fn send_cached_health_response(
4390    tx: &WriterSender,
4391    frame: &Frame,
4392    shared_app: &App,
4393    executor: &Executor,
4394    pending_binds: &HashMap<RouteChannel, PendingBind>,
4395    metrics: &DispatchPathMetrics,
4396    health_rollup_cache: &HealthRollupCache,
4397) -> Result<(), SubcError> {
4398    let report = build_health_report(
4399        health_rollup_cache,
4400        executor,
4401        pending_binds,
4402        metrics,
4403        shared_app,
4404    );
4405    let body = serde_json::to_vec(&ModuleControlResponse::from(report)).map_err(SubcError::Json)?;
4406    let response = Frame::build_with_version(
4407        frame.header.ver,
4408        FrameType::Response,
4409        frame.header.flags,
4410        0,
4411        0,
4412        frame.header.corr,
4413        body,
4414    )
4415    .map_err(SubcError::FrameBuild)?;
4416    send_frame(tx, metrics, response).await
4417}
4418
4419/// Channel-0 control requests: RouteBind plus the cached health probe.
4420/// Tool-provider binds reconcile RootConfig through the executor's Mutating lane;
4421/// management binds install immediately without creating project state.
4422#[allow(clippy::too_many_arguments)]
4423async fn handle_control_request(
4424    tx: &WriterSender,
4425    frame: &Frame,
4426    shared_app: &Arc<App>,
4427    executor: &Arc<Executor>,
4428    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4429    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
4430    installed_route_epochs: &mut HashMap<u16, u32>,
4431    routes: &mut HashMap<RouteChannel, RouteIdentity>,
4432    management_routes: &mut HashSet<RouteChannel>,
4433    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
4434    bg_subs: &mut HashMap<RouteChannel, BgSub>,
4435    bg_sub_by_session: &mut BgSubsBySession,
4436    bg_wake_pending: &mut BgWakePending,
4437    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
4438    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
4439    active_tool_calls: &ActiveToolCalls,
4440    pending_responses: &mut PendingSubcResponses,
4441    retry_buffer: &mut RetryBuffer,
4442    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
4443    shutdown: &Arc<Notify>,
4444    control_completion_tx: &mpsc::Sender<RouteBindCompletion>,
4445    metrics: &Arc<DispatchPathMetrics>,
4446    lifecycle_probe: Option<&SubcTestLifecycleProbe>,
4447    health_rollup_cache: &HealthRollupCache,
4448    push_senders: &PushSenders,
4449    dispatch: DispatchFn,
4450    user_config_path: Option<&Path>,
4451    tool_response_body_limit: usize,
4452) -> Result<(), SubcError> {
4453    let request =
4454        serde_json::from_slice::<ModuleControlRequest>(&frame.body).map_err(SubcError::Json)?;
4455    match request {
4456        ModuleControlRequest::RouteBind {
4457            route_channel,
4458            epoch,
4459            target,
4460            identity,
4461            principal,
4462            consumer_capabilities,
4463            admission_facts: _,
4464        } => {
4465            let route_id = route_key(route_channel, epoch);
4466            if epoch == 0 {
4467                return send_route_bind_error(
4468                    tx,
4469                    frame,
4470                    "config_divergence",
4471                    "route bind uses an invalid channel generation",
4472                    metrics,
4473                )
4474                .await;
4475            }
4476
4477            let bind_trust = trust_for_bind(&identity.harness, &principal);
4478            if let RouteTarget::ManagementSurface { module_id } = &target {
4479                if module_id != "aft" {
4480                    return send_route_bind_error(
4481                        tx,
4482                        frame,
4483                        "route_refused",
4484                        "management route target is not AFT",
4485                        metrics,
4486                    )
4487                    .await;
4488                }
4489                if !matches!(bind_trust, BindTrust::FirstParty) {
4490                    return send_route_bind_error(
4491                        tx,
4492                        frame,
4493                        "route_refused",
4494                        "AFT management routes require a first-party principal",
4495                        metrics,
4496                    )
4497                    .await;
4498                }
4499                if let Some(installed_epoch) = installed_route_epochs.get(&route_channel).copied() {
4500                    if installed_epoch >= epoch {
4501                        return send_route_bind_error(
4502                            tx,
4503                            frame,
4504                            "config_divergence",
4505                            "route bind generation is not newer than the installed generation",
4506                            metrics,
4507                        )
4508                        .await;
4509                    }
4510                    teardown_installed_route(
4511                        tx,
4512                        metrics,
4513                        executor,
4514                        route_key(route_channel, installed_epoch),
4515                        "higher-epoch RouteBind",
4516                        None,
4517                        installed_route_epochs,
4518                        routes,
4519                        management_routes,
4520                        root_channels,
4521                        bg_subs,
4522                        bg_sub_by_session,
4523                        bg_wake_pending,
4524                        pending_bash_asks,
4525                        live_roots,
4526                        route_bash_cancels,
4527                        active_tool_calls,
4528                        pending_responses,
4529                        pending_binds,
4530                        retry_buffer,
4531                        push_buffer,
4532                        shutdown,
4533                        tool_response_body_limit,
4534                        lifecycle_probe,
4535                    )
4536                    .await?;
4537                }
4538                if pending_binds.contains_key(&route_id) {
4539                    return send_route_bind_error(
4540                        tx,
4541                        frame,
4542                        "config_divergence",
4543                        "route bind is already pending for channel",
4544                        metrics,
4545                    )
4546                    .await;
4547                }
4548
4549                installed_route_epochs.insert(route_channel, epoch);
4550                management_routes.insert(route_id);
4551                return send_route_bind_ack(
4552                    tx,
4553                    frame.header.ver,
4554                    frame.header.corr,
4555                    frame.header.flags,
4556                    metrics,
4557                )
4558                .await;
4559            }
4560            if matches!(&target, RouteTarget::InternalService { .. }) {
4561                return send_route_bind_error(
4562                    tx,
4563                    frame,
4564                    "route_refused",
4565                    "AFT does not provide an internal-service route",
4566                    metrics,
4567                )
4568                .await;
4569            }
4570
4571            let mut bind_root_id = None;
4572            if let Some(installed_epoch) = installed_route_epochs.get(&route_channel).copied() {
4573                if installed_epoch >= epoch {
4574                    return send_route_bind_error(
4575                        tx,
4576                        frame,
4577                        "config_divergence",
4578                        "route bind generation is not newer than the installed generation",
4579                        metrics,
4580                    )
4581                    .await;
4582                }
4583
4584                let replacement_root = match ProjectRootId::from_path(&identity.project_root) {
4585                    Ok(root_id) => root_id,
4586                    Err(error) => {
4587                        return send_route_bind_error(
4588                            tx,
4589                            frame,
4590                            "config_divergence",
4591                            &format!("invalid route project root: {error}"),
4592                            metrics,
4593                        )
4594                        .await;
4595                    }
4596                };
4597                teardown_installed_route(
4598                    tx,
4599                    metrics,
4600                    executor,
4601                    route_key(route_channel, installed_epoch),
4602                    "higher-epoch RouteBind",
4603                    Some(&replacement_root),
4604                    installed_route_epochs,
4605                    routes,
4606                    management_routes,
4607                    root_channels,
4608                    bg_subs,
4609                    bg_sub_by_session,
4610                    bg_wake_pending,
4611                    pending_bash_asks,
4612                    live_roots,
4613                    route_bash_cancels,
4614                    active_tool_calls,
4615                    pending_responses,
4616                    pending_binds,
4617                    retry_buffer,
4618                    push_buffer,
4619                    shutdown,
4620                    tool_response_body_limit,
4621                    lifecycle_probe,
4622                )
4623                .await?;
4624                bind_root_id = Some(replacement_root);
4625            }
4626            if pending_binds.contains_key(&route_id) {
4627                return send_route_bind_error(
4628                    tx,
4629                    frame,
4630                    "config_divergence",
4631                    "route bind is already pending for channel",
4632                    metrics,
4633                )
4634                .await;
4635            }
4636            let bind_root_id = match bind_root_id {
4637                Some(root_id) => root_id,
4638                None => match ProjectRootId::from_path(&identity.project_root) {
4639                    Ok(root_id) => root_id,
4640                    Err(error) => {
4641                        return send_route_bind_error(
4642                            tx,
4643                            frame,
4644                            "config_divergence",
4645                            &format!("invalid route project root: {error}"),
4646                            metrics,
4647                        )
4648                        .await;
4649                    }
4650                },
4651            };
4652
4653            // Reconcile RootConfig: build a configure request from the bind
4654            // identity + forwarded config tiers and run it through the executor.
4655            let request_id = format!("subc-bind-{route_channel}");
4656            let bind_project_root = identity.project_root.clone();
4657            let bind_harness = identity.harness.clone();
4658            let bind_session = identity.session.clone();
4659            let bind_principal_id = principal_id(&principal);
4660            // Typed capability declaration from the consumer: the facade stamps it
4661            // from the MCP host's initialize-advertised capabilities. Absent
4662            // means no reverse-request capability — flat deny, fail-closed. A
4663            // consumer over-declaring only earns asks that TTL-deny.
4664            let consumer_elicitation_capable = consumer_capabilities
4665                .as_ref()
4666                .is_some_and(|capabilities| capabilities.iter().any(|c| c == "elicitation"));
4667            log::info!(
4668                "subc attach: route {} harness={} principal={} trust={} elicitation={}",
4669                route_channel,
4670                bind_harness,
4671                principal_label(&principal),
4672                bind_trust.label(),
4673                consumer_elicitation_capable
4674            );
4675
4676            // Config is read directly from the CortexKit user and project files;
4677            // wire-relayed tiers are ignored so a front cannot inject settings.
4678            // The resolver selects only this bind's harness override before it
4679            // applies the unchanged user/project trust boundary.
4680            let local_tiers = crate::subc_config::read_local_cortexkit_config_tiers(
4681                user_config_path,
4682                Path::new(&bind_project_root),
4683            );
4684            let config_tiers: Vec<Value> = local_tiers
4685                .iter()
4686                .map(|t| json!({ "tier": t.tier, "source": t.source, "doc": t.doc }))
4687                .collect();
4688            // Let configure return its structured invalid-harness error rather
4689            // than panicking while computing this optional registration setting.
4690            let active_harness = bind_harness.parse::<crate::harness::Harness>().ok();
4691            let diagnostics_on_edit = crate::config_resolve::resolve_config_for_harness(
4692                &local_tiers,
4693                active_harness.as_ref(),
4694            )
4695            .config
4696            .diagnostics_on_edit;
4697            let configure_json = json!({
4698                "id": request_id,
4699                "command": "configure",
4700                "project_root": bind_project_root,
4701                "harness": bind_harness,
4702                "session_id": bind_session.clone(),
4703                "config": config_tiers,
4704            });
4705            let configure_req = match serde_json::from_value::<RawRequest>(configure_json) {
4706                Ok(req) => req,
4707                Err(error) => {
4708                    return send_route_bind_error(
4709                        tx,
4710                        frame,
4711                        "config_divergence",
4712                        &format!("failed to build configure request: {error}"),
4713                        metrics,
4714                    )
4715                    .await;
4716                }
4717            };
4718
4719            let route_identity = RouteIdentity(Arc::new(RouteIdentityData {
4720                root: bind_root_id.clone(),
4721                project_root: PathBuf::from(&bind_project_root),
4722                harness: bind_harness.clone(),
4723                session: bind_session.clone(),
4724                trust: bind_trust,
4725                spawn_principal: AuthenticatedPrincipal::RouteBind {
4726                    trust: bind_trust.sandbox_trust(),
4727                    route_channel,
4728                    route_epoch: epoch,
4729                    project_root: PathBuf::from(&bind_project_root),
4730                    harness: bind_harness.clone(),
4731                    session_id: bind_session.clone(),
4732                    principal_id: bind_principal_id,
4733                },
4734                consumer_elicitation_capable,
4735            }));
4736            let configure_session = route_identity.session.clone();
4737            let root_was_live = live_roots.contains_key(&bind_root_id);
4738            let inserted_new_actor = register_actor_for_bind(
4739                shared_app,
4740                executor,
4741                push_senders,
4742                &bind_root_id,
4743                route_channel,
4744                root_was_live,
4745            );
4746
4747            sync_bg_live_delivery_sessions(executor, routes, Some(&bind_root_id));
4748            let configure_request_id = configure_req.id.clone();
4749            installed_route_epochs.insert(route_channel, epoch);
4750            if let Some(meta) = live_roots.get_mut(&bind_root_id) {
4751                meta.maintenance_queued_kinds.clear();
4752                meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
4753            }
4754            let (configure_rx, configure_cancellation) = executor.submit_cancellable_async(
4755                bind_root_id.clone(),
4756                Lane::Mutating,
4757                configure_request_id.clone(),
4758                Box::new(move |ctx| {
4759                    log_ctx::with_session(Some(configure_session.clone()), || {
4760                        dispatch(configure_req, ctx)
4761                    })
4762                }),
4763            );
4764            pending_binds.insert(
4765                route_id,
4766                PendingBind {
4767                    bind_root_id: bind_root_id.clone(),
4768                    inserted_new_actor,
4769                    cancelled: false,
4770                    configure_request_id: configure_request_id.clone(),
4771                    started_at: Instant::now(),
4772                    warned_half_deadline: false,
4773                    deadline_reported: false,
4774                    corr: frame.header.corr,
4775                    ver: frame.header.ver,
4776                    flags: frame.header.flags,
4777                    cancellation: configure_cancellation,
4778                },
4779            );
4780
4781            let completion_tx = control_completion_tx.clone();
4782            let completion_identity = route_identity;
4783            let completion_root = bind_root_id.clone();
4784            let completion_route_channel = route_channel;
4785            let completion_ver = frame.header.ver;
4786            let completion_corr = frame.header.corr;
4787            let completion_flags = frame.header.flags;
4788            let completion_metrics = Arc::clone(metrics);
4789            tokio::spawn(async move {
4790                let _response_task = ResponseTaskGuard::new(&completion_metrics);
4791                let configure_response =
4792                    await_executor_response(configure_rx, configure_request_id.clone()).await;
4793                // Send the route-bind acknowledgment as soon as configure succeeds.
4794                // Installing completed search or callgraph builds only refreshes cached
4795                // read data, so a later maintenance pass can do it without delaying the
4796                // daemon's confirmation that the route is usable.
4797                let completion = RouteBindCompletion {
4798                    route: route_key(completion_route_channel, epoch),
4799                    identity: completion_identity,
4800                    bind_root_id: completion_root,
4801                    inserted_new_actor,
4802                    configure_response,
4803                    diagnostics_on_edit,
4804                    ver: completion_ver,
4805                    corr: completion_corr,
4806                    flags: completion_flags,
4807                };
4808                if send_counted_channel(
4809                    &completion_tx,
4810                    &completion_metrics.control_completion_queued,
4811                    completion,
4812                )
4813                .await
4814                .is_err()
4815                {
4816                    log::debug!(
4817                        "subc attach: dropped RouteBind completion for route {} after loop exit",
4818                        completion_route_channel
4819                    );
4820                }
4821            });
4822
4823            // Bind completion wakes the health worker. A synchronous census here
4824            // scans every hosted root before the transport can accept another frame.
4825            Ok(())
4826        }
4827        ModuleControlRequest::HealthCheck {} => {
4828            send_cached_health_response(
4829                tx,
4830                frame,
4831                shared_app,
4832                executor,
4833                pending_binds,
4834                metrics,
4835                health_rollup_cache,
4836            )
4837            .await
4838        }
4839    }
4840}
4841
4842async fn handle_management_request(
4843    tx: &WriterSender,
4844    frame: &Frame,
4845    shared_app: &App,
4846    executor: &Executor,
4847    live_roots: &HashMap<ProjectRootId, RootMeta>,
4848    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
4849    health_rollup_cache: &HealthRollupCache,
4850    metrics: &DispatchPathMetrics,
4851) -> Result<(), SubcError> {
4852    let decoded = serde_json::from_slice::<Value>(&frame.body).ok();
4853    let operation = decoded
4854        .as_ref()
4855        .and_then(|value| value.get("op"))
4856        .and_then(Value::as_str);
4857    let Some(operation) = operation else {
4858        let error = build_error_frame(
4859            frame.header.ver,
4860            frame.header.channel,
4861            frame.header.epoch,
4862            frame.header.corr,
4863            frame.header.flags,
4864            "unknown_management_op",
4865            "management routes accept only declared operation envelopes",
4866        )?;
4867        return send_reliable_writer_frame(tx, metrics, error, "management refusal").await;
4868    };
4869
4870    let result = match operation {
4871        crate::commands::memory_census::MEMORY_CENSUS_OPERATION => Response::success(
4872            "management-memory-census",
4873            memory_census_with_lifecycle(
4874                health_rollup_cache,
4875                shared_app,
4876                executor,
4877                live_roots,
4878                root_channels,
4879            ),
4880        ),
4881        crate::commands::health_digest::HEALTH_DIGEST_OPERATION => {
4882            let params = decoded
4883                .as_ref()
4884                .and_then(|value| value.get("params"))
4885                .cloned()
4886                .unwrap_or_else(|| json!({}));
4887            let Some(params) = params.as_object() else {
4888                return send_management_response(
4889                    tx,
4890                    frame,
4891                    operation,
4892                    Response::error(
4893                        "management-health-digest",
4894                        "invalid_request",
4895                        "health.digest params must be an object",
4896                    ),
4897                    metrics,
4898                )
4899                .await;
4900            };
4901            let root = params
4902                .get("project_root")
4903                .or_else(|| params.get("root"))
4904                .and_then(Value::as_str);
4905            let Some(root) = root else {
4906                return send_management_response(
4907                    tx,
4908                    frame,
4909                    operation,
4910                    Response::error(
4911                        "management-health-digest",
4912                        "invalid_request",
4913                        "health.digest requires params.project_root",
4914                    ),
4915                    metrics,
4916                )
4917                .await;
4918            };
4919
4920            let mut request = params.clone();
4921            request.insert("id".to_string(), json!("management-health-digest"));
4922            request.insert(
4923                "command".to_string(),
4924                json!(crate::commands::health_digest::HEALTH_DIGEST_OPERATION),
4925            );
4926            let request = serde_json::from_value::<RawRequest>(Value::Object(request))
4927                .map_err(SubcError::Json)?;
4928            match ProjectRootId::from_path(Path::new(root))
4929                .ok()
4930                .and_then(|root_id| executor.actor_context(&root_id))
4931            {
4932                Some(ctx) => crate::commands::health_digest::handle_health_digest(&request, &ctx),
4933                None => crate::commands::health_digest::root_not_bound_response(&request, root),
4934            }
4935        }
4936        _ => {
4937            let error = build_error_frame(
4938                frame.header.ver,
4939                frame.header.channel,
4940                frame.header.epoch,
4941                frame.header.corr,
4942                frame.header.flags,
4943                "unknown_management_op",
4944                &format!("management operation {operation:?} is not declared by AFT"),
4945            )?;
4946            return send_reliable_writer_frame(tx, metrics, error, "management refusal").await;
4947        }
4948    };
4949
4950    send_management_response(tx, frame, operation, result, metrics).await
4951}
4952
4953fn memory_census_with_lifecycle(
4954    health_rollup_cache: &HealthRollupCache,
4955    shared_app: &App,
4956    executor: &Executor,
4957    live_roots: &HashMap<ProjectRootId, RootMeta>,
4958    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
4959) -> Value {
4960    let mut census = health_rollup_cache.memory_census();
4961    if let Some(rows) = census.get_mut("roots").and_then(Value::as_object_mut) {
4962        let lifecycle = shared_app.lifecycle_census_snapshot();
4963        for (root_id, meta) in live_roots {
4964            let root = root_id.as_path().display().to_string();
4965            let bound_routes = root_channels.get(root_id).map_or(0, HashSet::len);
4966            let age_ms = Instant::now()
4967                .saturating_duration_since(meta.last_touched)
4968                .as_millis()
4969                .min(u128::from(u64::MAX)) as u64;
4970            let ttl_ms = root_idle_ttl(executor, root_id)
4971                .as_millis()
4972                .min(u128::from(u64::MAX)) as u64;
4973            let lsp = lifecycle
4974                .lsp
4975                .children_by_root
4976                .iter()
4977                .find(|child| child.root == root);
4978            if let Some(row) = rows.get_mut(&root).and_then(Value::as_object_mut) {
4979                let evictable_bytes = row.get("evictable_bytes").cloned().unwrap_or(json!(0));
4980                row.insert("root_id".to_string(), json!(root));
4981                row.insert("bound_routes".to_string(), json!(bound_routes));
4982                row.insert("last_request_age_ms".to_string(), json!(age_ms));
4983                row.insert("idle_ttl_ms".to_string(), json!(ttl_ms));
4984                row.insert(
4985                    "lsp_idle_ttl_ms".to_string(),
4986                    json!(executor
4987                        .actor_context(root_id)
4988                        .map(|ctx| ctx
4989                            .config()
4990                            .idle
4991                            .lsp_ttl()
4992                            .as_millis()
4993                            .min(u128::from(u64::MAX)) as u64)
4994                        .unwrap_or(0)),
4995                );
4996                row.insert(
4997                    "evictable_in_ms".to_string(),
4998                    crate::commands::memory_census::evictable_in_ms(bound_routes, ttl_ms, age_ms)
4999                        .map_or(Value::Null, |value| json!(value)),
5000                );
5001                row.insert(
5002                    "evictable_bytes".to_string(),
5003                    if bound_routes == 0 {
5004                        evictable_bytes
5005                    } else {
5006                        json!(0)
5007                    },
5008                );
5009                row.insert(
5010                    "lsp_children".to_string(),
5011                    json!({
5012                        "count": lsp.map_or(0, |child| child.count),
5013                        "rss_bytes": lsp.map_or(0, |child| child.rss_bytes),
5014                    }),
5015                );
5016            }
5017        }
5018    }
5019    census
5020}
5021
5022async fn send_management_response(
5023    tx: &WriterSender,
5024    request: &Frame,
5025    operation: &str,
5026    result: Response,
5027    metrics: &DispatchPathMetrics,
5028) -> Result<(), SubcError> {
5029    let status = if result.success { "ok" } else { "error" };
5030    let body = json!({ "op": operation, "status": status, "data": result.data });
5031    let response = Frame::build_with_version(
5032        request.header.ver,
5033        FrameType::Response,
5034        request.header.flags,
5035        request.header.channel,
5036        request.header.epoch,
5037        request.header.corr,
5038        serde_json::to_vec(&body).map_err(SubcError::Json)?,
5039    )
5040    .map_err(SubcError::FrameBuild)?;
5041    send_reliable_writer_frame(tx, metrics, response, "management response").await
5042}
5043
5044fn install_bash_compressor(ctx: &AppContext) {
5045    // Mirrors main.rs per-actor compressor installation for subc-created actors.
5046    let filter_registry_handle = ctx.shared_filter_registry();
5047    let compress_flag = ctx.bash_compress_flag();
5048    ctx.bash_background().set_compressor_with_exit_code(
5049        move |command: &str, output: String, exit_code: Option<i32>| {
5050            if !compress_flag.load(std::sync::atomic::Ordering::Relaxed) {
5051                return crate::compress::CompressionResult::new(output);
5052            }
5053            let registry_guard = match filter_registry_handle.read() {
5054                Ok(g) => g,
5055                Err(poisoned) => poisoned.into_inner(),
5056            };
5057            crate::compress::compress_with_registry_exit_code(
5058                command,
5059                &output,
5060                exit_code,
5061                &registry_guard,
5062            )
5063        },
5064    );
5065}
5066
5067async fn send_route_bind_ack(
5068    tx: &WriterSender,
5069    ver: u8,
5070    corr: u64,
5071    flags: Flags,
5072    metrics: &DispatchPathMetrics,
5073) -> Result<(), SubcError> {
5074    let body =
5075        serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).map_err(SubcError::Json)?;
5076    let response = Frame::build_with_version(ver, FrameType::Response, flags, 0, 0, corr, body)
5077        .map_err(SubcError::FrameBuild)?;
5078    send_reliable_writer_frame(tx, metrics, response, "RouteBindAck").await
5079}
5080
5081async fn send_route_bind_error(
5082    tx: &WriterSender,
5083    frame: &Frame,
5084    code: &str,
5085    message: &str,
5086    metrics: &DispatchPathMetrics,
5087) -> Result<(), SubcError> {
5088    send_route_bind_error_parts(
5089        tx,
5090        frame.header.ver,
5091        frame.header.corr,
5092        frame.header.flags,
5093        code,
5094        message,
5095        metrics,
5096    )
5097    .await
5098}
5099
5100async fn send_route_bind_error_parts(
5101    tx: &WriterSender,
5102    ver: u8,
5103    corr: u64,
5104    flags: Flags,
5105    code: &str,
5106    message: &str,
5107    metrics: &DispatchPathMetrics,
5108) -> Result<(), SubcError> {
5109    let response = build_error_frame(ver, 0, 0, corr, flags, code, message)?;
5110    send_reliable_writer_frame(tx, metrics, response, "RouteBind error").await?;
5111    log_route_bind_rejection(code, message);
5112    Ok(())
5113}
5114
5115/// Per-message rate limit for the bind-rejection warn line. A caller that
5116/// re-attaches a dead root forever turns this line into the entire readable
5117/// tail of the SHARED daemon log (measured 2026-08-09: 2.08M copies, ~40/sec,
5118/// 936MB log — other modules' incident lines pushed out of the tail).
5119/// The line itself stays byte-identical so external counters keep matching;
5120/// repeats inside the window are summarized with a suppressed count on the
5121/// next emission (volume stays diagnosable, per the log-diet convention).
5122fn log_route_bind_rejection(code: &str, message: &str) {
5123    const WINDOW: Duration = Duration::from_secs(60);
5124    static SUPPRESSED: OnceLock<StdMutex<HashMap<String, (Instant, u64)>>> = OnceLock::new();
5125    let map = SUPPRESSED.get_or_init(|| StdMutex::new(HashMap::new()));
5126    let mut map = match map.try_lock() {
5127        Ok(map) => map,
5128        // Contended: log unsuppressed rather than blocking or dropping.
5129        Err(_) => {
5130            log::warn!("subc attach: route bind rejected ({code}): {message}");
5131            return;
5132        }
5133    };
5134    let now = Instant::now();
5135    // Bound the map: dead roots churn, and an unbounded suppression map is
5136    // its own leak. Sweep expired entries once it grows past a fleet-sized
5137    // number of distinct rejection messages.
5138    if map.len() > 512 {
5139        map.retain(|_, (start, _)| now.duration_since(*start) < WINDOW);
5140    }
5141    match map.get_mut(message) {
5142        Some((window_start, suppressed)) if now.duration_since(*window_start) < WINDOW => {
5143            *suppressed += 1;
5144        }
5145        Some((window_start, suppressed)) => {
5146            if *suppressed > 0 {
5147                log::warn!(
5148                    "subc attach: route bind rejected ({code}): {message} (repeated {}x in last 60s)",
5149                    *suppressed
5150                );
5151            } else {
5152                log::warn!("subc attach: route bind rejected ({code}): {message}");
5153            }
5154            *window_start = now;
5155            *suppressed = 0;
5156        }
5157        None => {
5158            log::warn!("subc attach: route bind rejected ({code}): {message}");
5159            map.insert(message.to_string(), (now, 0));
5160        }
5161    }
5162}
5163
5164/// Route-channel tool call: `{name, arguments}` → executor lane → dispatch to
5165/// the sync command core → wrap the structured Response in a CallToolResult
5166/// `{content, isError}`. Tool-result mapping: the whole `{success, ...}` Response
5167/// serialized into ONE text block; `isError` carries `success == false`.
5168async fn handle_tool_call(
5169    tx: &WriterSender,
5170    frame: &Frame,
5171    mut phase_trace: PhaseTrace,
5172    routes: &HashMap<RouteChannel, RouteIdentity>,
5173    pending_binds: &HashMap<RouteChannel, PendingBind>,
5174    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
5175    executor: &Arc<Executor>,
5176    active_tool_calls: &ActiveToolCalls,
5177    pending_deferred_setups: &Arc<AtomicUsize>,
5178    shutdown: &Arc<Notify>,
5179    connection_cancel: &PersistentCancelSignal,
5180    bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
5181    bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
5182    metrics: &Arc<DispatchPathMetrics>,
5183    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
5184    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
5185    next_bash_ask_corr: &mut u64,
5186    bg_subs: &mut HashMap<RouteChannel, BgSub>,
5187    bg_sub_by_session: &mut BgSubsBySession,
5188    bg_wake_pending: &mut BgWakePending,
5189    bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
5190    dispatch: DispatchFn,
5191    deferred_response_tx: &mpsc::UnboundedSender<PendingSubcResponse>,
5192    allow_native_passthrough: bool,
5193    tool_response_body_limit: usize,
5194) -> Result<(), SubcError> {
5195    let route_id = route_key(frame.header.channel, frame.header.epoch);
5196    if pending_binds.contains_key(&route_id) {
5197        let error = build_error_frame(
5198            frame.header.ver,
5199            frame.header.channel,
5200            frame.header.epoch,
5201            frame.header.corr,
5202            frame.header.flags,
5203            "route_not_bound",
5204            "route is not bound before tool call",
5205        )?;
5206        return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
5207    }
5208
5209    let Some(identity) = routes.get(&route_id).cloned() else {
5210        let error = build_error_frame(
5211            frame.header.ver,
5212            frame.header.channel,
5213            frame.header.epoch,
5214            frame.header.corr,
5215            frame.header.flags,
5216            "route_not_bound",
5217            "route is not bound before tool call",
5218        )?;
5219        return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
5220    };
5221    let restore_watcher = live_roots
5222        .get(&identity.root)
5223        .is_some_and(|meta| meta.idle_artifacts_evicted);
5224    if let Some(meta) = live_roots.get_mut(&identity.root) {
5225        meta.reactivate_bound();
5226    }
5227    if restore_watcher {
5228        if let Some(ctx) = executor.actor_context(&identity.root) {
5229            crate::commands::configure::ensure_project_watcher(&ctx);
5230        }
5231    }
5232
5233    let route_request = match serde_json::from_slice::<RouteRequest>(&frame.body) {
5234        Ok(request) => request,
5235        Err(error) => {
5236            let management_envelope = serde_json::from_slice::<Value>(&frame.body).ok();
5237            let Some(operation) = management_envelope
5238                .as_ref()
5239                .and_then(|value| value.get("op"))
5240                .and_then(Value::as_str)
5241            else {
5242                return Err(SubcError::Json(error));
5243            };
5244            RouteRequest::ToolCall(ToolCallRequest {
5245                name: operation.to_string(),
5246                arguments: management_envelope
5247                    .and_then(|value| value.get("params").cloned())
5248                    .unwrap_or_else(|| json!({})),
5249                edit_slot_survives: None,
5250                preview: false,
5251            })
5252        }
5253    };
5254    if matches!(
5255        route_request,
5256        RouteRequest::BgEvents(BgEventsRequest {
5257            op: BgEventsOp::BgEvents
5258        })
5259    ) {
5260        if let Some(old_sub) = bg_subs.get(&route_id).cloned() {
5261            metrics.record_bg_subscription_ended(
5262                &old_sub.root,
5263                &old_sub.session,
5264                route_id,
5265                "resubscribe",
5266            );
5267            push::send_reliable_bg_stream_end(tx, metrics, route_id, &old_sub).await?;
5268        }
5269        if !identity.trust.allows_bash_observation() {
5270            bg_subs.remove(&route_id);
5271            bg_wake_pending.remove(&route_id);
5272            remove_bg_subscription_index(bg_sub_by_session, route_id, Some(&identity));
5273            let denied_sub = BgSub {
5274                corr: frame.header.corr,
5275                ver: frame.header.ver,
5276                flags: frame.header.flags,
5277                root: identity.root.clone(),
5278                session: identity.session.clone(),
5279            };
5280            metrics.record_bg_subscription_ended(
5281                &identity.root,
5282                &identity.session,
5283                route_id,
5284                "subscribe-denied",
5285            );
5286            push::send_reliable_bg_stream_end(tx, metrics, route_id, &denied_sub).await?;
5287            return Ok(());
5288        }
5289        bg_subs.insert(
5290            route_id,
5291            BgSub {
5292                corr: frame.header.corr,
5293                ver: frame.header.ver,
5294                flags: frame.header.flags,
5295                root: identity.root.clone(),
5296                session: identity.session.clone(),
5297            },
5298        );
5299        insert_bg_subscription_index(
5300            bg_sub_by_session,
5301            identity.root.clone(),
5302            identity.session.clone(),
5303            route_id,
5304        );
5305        metrics.record_bg_subscription_installed(&identity.root, &identity.session, route_id);
5306        push::arm_bg_wake(
5307            identity.root.clone(),
5308            identity.session.clone(),
5309            route_id,
5310            bg_wake_pending,
5311            bg_wake_epoch,
5312            metrics,
5313        );
5314        return Ok(());
5315    }
5316
5317    let RouteRequest::ToolCall(call) = route_request else {
5318        unreachable!("background event subscription returned above")
5319    };
5320    let bare_name = call.name;
5321    let arguments = strip_agent_preview_arg_owned(call.arguments);
5322    let format_context = crate::subc_format::FormatContext::from_tool_call(
5323        &bare_name,
5324        &arguments,
5325        identity.project_root.as_path(),
5326    );
5327
5328    let request_id = format!("subc-{}-{}", frame.header.channel, frame.header.corr);
5329    let bind_trust = identity.trust;
5330    let diagnostics_on_edit = live_roots
5331        .get(&identity.root)
5332        .map(|meta| meta.diagnostics_on_edit)
5333        .unwrap_or(false);
5334
5335    let requests_host = matches!(bare_name.as_str(), "bash" | "powershell")
5336        && arguments
5337            .get("sandbox")
5338            .or_else(|| {
5339                arguments
5340                    .get("params")
5341                    .and_then(|params| params.get("sandbox"))
5342            })
5343            .and_then(Value::as_str)
5344            == Some("host");
5345    if matches!(bind_trust, BindTrust::Untrusted) && requests_host {
5346        let response = Response::error(
5347            request_id.clone(),
5348            "sandbox_escalation_denied",
5349            "sandbox host escalation is unavailable to untrusted principals",
5350        );
5351        let text = crate::subc_format::format_response_with_context(
5352            &bare_name,
5353            &response,
5354            &format_context,
5355        );
5356        let result = ToolCallResult { text, response };
5357        let response_frame = build_tool_response_frame_with_limit(
5358            frame.header.ver,
5359            route_id,
5360            frame.header.corr,
5361            frame.header.flags,
5362            &result,
5363            bind_trust,
5364            tool_response_body_limit,
5365        )?;
5366        return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
5367    }
5368
5369    if matches!(bind_trust, BindTrust::Untrusted)
5370        && is_bash_family_tool(&bare_name)
5371        && (!matches!(bare_name.as_str(), "bash" | "powershell")
5372            || !identity.consumer_elicitation_capable)
5373    {
5374        let response = bash::bash_denied_untrusted_response(request_id.clone());
5375        let text = crate::subc_format::format_response_with_context(
5376            &bare_name,
5377            &response,
5378            &format_context,
5379        );
5380        let result = ToolCallResult { text, response };
5381        let response_frame = build_tool_response_frame_with_limit(
5382            frame.header.ver,
5383            route_id,
5384            frame.header.corr,
5385            frame.header.flags,
5386            &result,
5387            bind_trust,
5388            tool_response_body_limit,
5389        )?;
5390        return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
5391    }
5392
5393    // A non-core name is NOT in the tool manifest. AFT fails closed and
5394    // does not trust subc to enforce the manifest: rejecting here is the
5395    // defense-in-depth backstop that prevents a forwarded native command
5396    // (e.g. `configure`, which would reach handle_configure and bypass
5397    // the RouteBind config-trust cap) from ever reaching dispatch. Only
5398    // the integration-test harness (run_subc_mode_for_test) opens this to
5399    // drive synthetic native commands through the executor.
5400    if !is_subc_agent_core_tool(&bare_name)
5401        && !is_subc_native_plumbing_tool(&bare_name)
5402        && !allow_native_passthrough
5403    {
5404        log::warn!(
5405            "subc tool call: rejecting non-manifest tool name {:?} on route {} (fail-closed)",
5406            bare_name,
5407            frame.header.channel
5408        );
5409        let response = Response::error(
5410            request_id.clone(),
5411            "unknown_tool",
5412            format!("tool {:?} is not in the AFT tool manifest", bare_name),
5413        );
5414        let text = crate::subc_format::format_response_with_context(
5415            &bare_name,
5416            &response,
5417            &format_context,
5418        );
5419        let result = ToolCallResult { text, response };
5420        let response_frame = build_tool_response_frame_with_limit(
5421            frame.header.ver,
5422            route_id,
5423            frame.header.corr,
5424            frame.header.flags,
5425            &result,
5426            bind_trust,
5427            tool_response_body_limit,
5428        )?;
5429        return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
5430    }
5431
5432    if matches!(bare_name.as_str(), "bash" | "powershell") {
5433        if matches!(bind_trust, BindTrust::Untrusted) {
5434            let plan = match bash::prepare_bash_elicitation_plan(
5435                &arguments,
5436                identity.project_root.as_path(),
5437            ) {
5438                Ok(plan) => plan,
5439                Err(error) => {
5440                    let response = Response::error(request_id.clone(), error.code, error.message);
5441                    let text = crate::subc_format::format_response_with_context(
5442                        &bare_name,
5443                        &response,
5444                        &format_context,
5445                    );
5446                    let result = ToolCallResult { text, response };
5447                    let response_frame = build_tool_response_frame_with_limit(
5448                        frame.header.ver,
5449                        route_id,
5450                        frame.header.corr,
5451                        frame.header.flags,
5452                        &result,
5453                        bind_trust,
5454                        tool_response_body_limit,
5455                    )?;
5456                    return send_reliable_writer_frame(
5457                        tx,
5458                        metrics,
5459                        response_frame,
5460                        "tool response",
5461                    )
5462                    .await;
5463                }
5464            };
5465
5466            let reverse_corr =
5467                allocate_reverse_corr(pending_bash_asks, route_id, next_bash_ask_corr);
5468            let ask_frame = build_bash_elicitation_request_frame(
5469                frame.header.ver,
5470                route_id,
5471                reverse_corr,
5472                frame.header.flags,
5473                &plan.command,
5474                &plan.asks,
5475            )?;
5476
5477            let meta = live_roots
5478                .entry(identity.root.clone())
5479                .or_insert_with(|| RootMeta::new(Instant::now()));
5480            meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
5481            meta.reactivate_bound();
5482
5483            let route_cancel =
5484                route_bash_cancels
5485                    .entry(route_id)
5486                    .or_insert_with(|| bash::RouteBashCancel {
5487                        token: PersistentCancelSignal::new(),
5488                        active_waits: 0,
5489                    });
5490            route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
5491            let cancel = bash::BashWaitCancel {
5492                connection: connection_cancel.clone(),
5493                route: route_cancel.token.clone(),
5494            };
5495            pending_bash_asks.insert(
5496                ReverseCorrKey {
5497                    route: route_id,
5498                    corr: reverse_corr,
5499                },
5500                PendingBashAsk {
5501                    route: route_id,
5502                    tool_corr: frame.header.corr,
5503                    tool_flags: frame.header.flags,
5504                    tool_ver: frame.header.ver,
5505                    root: identity.root.clone(),
5506                    project_root: identity.project_root.clone(),
5507                    session_id: identity.session.clone(),
5508                    spawn_principal: identity.spawn_principal.clone(),
5509                    edit_slot_survives: call.edit_slot_survives,
5510                    request_id,
5511                    arguments,
5512                    format_context,
5513                    cancel,
5514                    grants: plan.grants,
5515                    expires_at: Instant::now() + bash_elicitation_timeout(),
5516                },
5517            );
5518            return send_reliable_writer_frame(tx, metrics, ask_frame, "bash elicitation request")
5519                .await;
5520        }
5521
5522        let meta = live_roots
5523            .entry(identity.root.clone())
5524            .or_insert_with(|| RootMeta::new(Instant::now()));
5525        meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
5526        meta.reactivate_bound();
5527
5528        let route_cancel =
5529            route_bash_cancels
5530                .entry(route_id)
5531                .or_insert_with(|| bash::RouteBashCancel {
5532                    token: PersistentCancelSignal::new(),
5533                    active_waits: 0,
5534                });
5535        route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
5536        let cancel = bash::BashWaitCancel {
5537            connection: connection_cancel.clone(),
5538            route: route_cancel.token.clone(),
5539        };
5540
5541        bash::submit_deferred_bash(
5542            executor,
5543            bash_deferred_tx,
5544            bash_poll_touch_tx,
5545            metrics,
5546            dispatch,
5547            identity.root.clone(),
5548            identity.project_root.clone(),
5549            identity.session.clone(),
5550            request_id,
5551            route_id,
5552            frame.header.corr,
5553            frame.header.flags,
5554            frame.header.ver,
5555            arguments,
5556            format_context,
5557            cancel,
5558            bind_trust,
5559            identity.spawn_principal.clone(),
5560            call.edit_slot_survives,
5561            None,
5562        );
5563        return Ok(());
5564    }
5565
5566    let lane = command_lane(&bare_name);
5567    let tool_call_context = ToolCallContext {
5568        project_root: identity.project_root.clone(),
5569        session_id: Some(identity.session.clone()),
5570        request_id: request_id.clone(),
5571        diagnostics_on_edit,
5572        preview: call.preview,
5573        edit_slot_survives: call.edit_slot_survives,
5574        report_registration_downgrade: true,
5575    };
5576
5577    let uses_deferred_response_seam = bare_name == "inspect"
5578        || crate::commands::lsp_navigation::is_lsp_navigation_command(&bare_name);
5579    if uses_deferred_response_seam {
5580        let Some(deferred_ctx) = executor.actor_context(&identity.root) else {
5581            let response = Response::error(
5582                &request_id,
5583                "actor_not_registered",
5584                "executor actor is not registered",
5585            );
5586            let text = crate::subc_format::format_response_with_context(
5587                &bare_name,
5588                &response,
5589                &format_context,
5590            );
5591            let result = ToolCallResult { text, response };
5592            let response_frame = build_tool_response_frame_with_limit(
5593                frame.header.ver,
5594                route_id,
5595                frame.header.corr,
5596                frame.header.flags,
5597                &result,
5598                bind_trust,
5599                tool_response_body_limit,
5600            )?;
5601            return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
5602        };
5603        let identity_for_run = identity.clone();
5604        let request_id_for_force = request_id.clone();
5605        let format_context_for_run = format_context.clone();
5606        let bare_name_for_run = bare_name.clone();
5607        let (setup_tx, setup_rx) = oneshot::channel::<DeferredSetupOutcome>();
5608        phase_trace.mark_executor_submitted();
5609        let job: crate::executor::ExecutorJob = Box::new(move |ctx| {
5610            phase_trace.mark_job_admitted();
5611            log_ctx::with_session(Some(identity_for_run.session.clone()), || {
5612                let run = || match prepare_tool_call(
5613                    &bare_name_for_run,
5614                    arguments,
5615                    &format_context_for_run,
5616                    &tool_call_context,
5617                    ctx,
5618                    Some(&mut phase_trace),
5619                ) {
5620                    Err(result) => {
5621                        let response = result.response;
5622                        let _ = setup_tx.send(DeferredSetupOutcome::Immediate {
5623                            text: result.text,
5624                            phase_trace,
5625                        });
5626                        response
5627                    }
5628                    Ok(prepared) => {
5629                        let outcome = if bare_name_for_run == "inspect" {
5630                            crate::commands::inspect::handle_inspect_deferred_with_restriction(
5631                                &prepared.request,
5632                                Arc::clone(&deferred_ctx),
5633                                matches!(bind_trust, BindTrust::Untrusted),
5634                            )
5635                        } else {
5636                            crate::commands::lsp_navigation::handle_lsp_navigation_deferred_with_restriction(
5637                                &prepared.request,
5638                                Arc::clone(&deferred_ctx),
5639                                matches!(bind_trust, BindTrust::Untrusted),
5640                            )
5641                        };
5642                        match outcome {
5643                            DispatchOutcome::Deferred(pending) => {
5644                                let _ = setup_tx.send(DeferredSetupOutcome::Deferred {
5645                                    pending,
5646                                    surface_downgraded: prepared.surface_downgraded,
5647                                    phase_trace,
5648                                });
5649                                Response::success(
5650                                    request_id_for_force.clone(),
5651                                    json!({ "response_deferred": true }),
5652                                )
5653                            }
5654                            DispatchOutcome::Immediate(response) => {
5655                                phase_trace.mark_execute_done();
5656                                let finalizer = |response: &mut Response| {
5657                                    crate::response_finalize::finalize_response_with_bg_completions(
5658                                        response,
5659                                        ctx,
5660                                        &identity_for_run.session,
5661                                        &bare_name_for_run,
5662                                        bind_trust.allows_bash_observation(),
5663                                    );
5664                                };
5665                                let result = finish_tool_call_response(
5666                                    &bare_name_for_run,
5667                                    &format_context_for_run,
5668                                    response,
5669                                    prepared.surface_downgraded,
5670                                    Some(&finalizer),
5671                                    Some(&mut phase_trace),
5672                                );
5673                                let response = result.response;
5674                                let _ = setup_tx.send(DeferredSetupOutcome::Immediate {
5675                                    text: result.text,
5676                                    phase_trace,
5677                                });
5678                                response
5679                            }
5680                        }
5681                    }
5682                };
5683                if matches!(bind_trust, BindTrust::Untrusted) {
5684                    ctx.with_force_restrict(&request_id_for_force, run)
5685                } else {
5686                    run()
5687                }
5688            })
5689        });
5690        let deferred_setup_guard =
5691            PendingDeferredSetupGuard::new(Arc::clone(pending_deferred_setups));
5692        let rx = submit_active_tool_call(
5693            executor.as_ref(),
5694            active_tool_calls,
5695            route_id,
5696            frame.header.corr,
5697            identity.root.clone(),
5698            lane,
5699            request_id.clone(),
5700            RouteDetachPolicy::CancelOnDetach,
5701            job,
5702        );
5703
5704        let completion_tx = tx.clone();
5705        let completion_shutdown = Arc::clone(shutdown);
5706        let completion_metrics = Arc::clone(metrics);
5707        let active_tool_calls = Arc::clone(active_tool_calls);
5708        let deferred_response_tx = deferred_response_tx.clone();
5709        let route = route_id;
5710        let corr = frame.header.corr;
5711        let flags = frame.header.flags;
5712        let ver = frame.header.ver;
5713        let root = identity.root.clone();
5714        let session_id = identity.session.clone();
5715        tokio::spawn(async move {
5716            let _response_task = ResponseTaskGuard::new(&completion_metrics);
5717            let _deferred_setup = deferred_setup_guard;
5718            let response = await_executor_response(rx, request_id.clone()).await;
5719            match setup_rx.await {
5720                Ok(DeferredSetupOutcome::Deferred {
5721                    pending,
5722                    surface_downgraded,
5723                    phase_trace,
5724                }) => {
5725                    let pending = PendingSubcResponse {
5726                        route,
5727                        corr,
5728                        flags,
5729                        ver,
5730                        root,
5731                        session_id,
5732                        bare_name,
5733                        format_context,
5734                        bind_trust,
5735                        pending,
5736                        surface_downgraded,
5737                        phase_trace,
5738                    };
5739                    if let Err(error) = deferred_response_tx.send(pending) {
5740                        if let Some(cancellation) = &error.0.pending.cancellation {
5741                            cancellation.request_cancel();
5742                        }
5743                        finish_active_tool_call(&active_tool_calls, route, corr);
5744                    }
5745                }
5746                Ok(DeferredSetupOutcome::Immediate { text, phase_trace }) => {
5747                    if !finish_active_tool_call(&active_tool_calls, route, corr) {
5748                        return;
5749                    }
5750                    let result = ToolCallResult { text, response };
5751                    let fatal = note_fatal_panic_response(&result.response);
5752                    match build_tool_response_frame_with_limit(
5753                        ver,
5754                        route,
5755                        corr,
5756                        flags,
5757                        &result,
5758                        bind_trust,
5759                        tool_response_body_limit,
5760                    ) {
5761                        Ok(response_frame) => {
5762                            let trace = ToolResponseWriteTrace::new(
5763                                phase_trace,
5764                                bare_name.clone(),
5765                                identity.project_root.clone(),
5766                                identity.session.clone(),
5767                                route.channel,
5768                                corr,
5769                            );
5770                            if let Err(error) = send_traced_tool_response_frame(
5771                                &completion_tx,
5772                                &completion_metrics,
5773                                response_frame,
5774                                trace,
5775                            )
5776                            .await
5777                            {
5778                                log::warn!(
5779                                    "subc attach: failed to queue deferred-seam setup response: {error}"
5780                                );
5781                            }
5782                        }
5783                        Err(error) => {
5784                            log::error!(
5785                                "subc attach: failed to build deferred-seam setup response: {error}"
5786                            );
5787                        }
5788                    }
5789                    if fatal {
5790                        signal_fatal_teardown(
5791                            &completion_tx,
5792                            Some(route),
5793                            ver,
5794                            corr,
5795                            &completion_shutdown,
5796                            &completion_metrics,
5797                        )
5798                        .await;
5799                    }
5800                }
5801                Err(_) => {
5802                    if !finish_active_tool_call(&active_tool_calls, route, corr) {
5803                        return;
5804                    }
5805                    let text = crate::subc_format::format_response_with_context(
5806                        &bare_name,
5807                        &response,
5808                        &format_context,
5809                    );
5810                    let result = ToolCallResult { text, response };
5811                    if let Ok(response_frame) = build_tool_response_frame_with_limit(
5812                        ver,
5813                        route,
5814                        corr,
5815                        flags,
5816                        &result,
5817                        bind_trust,
5818                        tool_response_body_limit,
5819                    ) {
5820                        let _ = send_reliable_writer_frame(
5821                            &completion_tx,
5822                            &completion_metrics,
5823                            response_frame,
5824                            "deferred setup failure",
5825                        )
5826                        .await;
5827                    }
5828                }
5829            }
5830        });
5831        return Ok(());
5832    }
5833
5834    let bare_name_for_frame = bare_name.clone();
5835    let identity_for_run = identity.clone();
5836    let completion_session = identity.session.clone();
5837    let completion_root = identity.project_root.clone();
5838    let request_id_for_force = request_id.clone();
5839    let format_context_for_frame = format_context.clone();
5840    let (tool_call_tx, tool_call_rx) = oneshot::channel::<ToolCallCompletion>();
5841    phase_trace.mark_executor_submitted();
5842    let job: crate::executor::ExecutorJob = Box::new(move |ctx| {
5843        phase_trace.mark_job_admitted();
5844        log_ctx::with_session(Some(identity_for_run.session.clone()), || {
5845            let run = || {
5846                let finalizer = |response: &mut Response| {
5847                    crate::response_finalize::finalize_response_with_bg_completions(
5848                        response,
5849                        ctx,
5850                        &identity_for_run.session,
5851                        &bare_name,
5852                        bind_trust.allows_bash_observation(),
5853                    );
5854                };
5855                match run_tool_call(
5856                    &bare_name,
5857                    arguments,
5858                    &format_context,
5859                    &tool_call_context,
5860                    ctx,
5861                    &dispatch,
5862                    Some(&finalizer),
5863                    Some(&mut phase_trace),
5864                ) {
5865                    ToolCallOutcome::Unary(result) => {
5866                        let response = result.response;
5867                        let _ = tool_call_tx.send(ToolCallCompletion {
5868                            text: result.text,
5869                            phase_trace,
5870                        });
5871                        response
5872                    }
5873                }
5874            };
5875            if matches!(bind_trust, BindTrust::Untrusted) {
5876                ctx.with_force_restrict(&request_id_for_force, run)
5877            } else {
5878                run()
5879            }
5880        })
5881    });
5882    let rx = submit_active_tool_call(
5883        executor.as_ref(),
5884        active_tool_calls,
5885        route_id,
5886        frame.header.corr,
5887        identity.root.clone(),
5888        lane,
5889        request_id.clone(),
5890        RouteDetachPolicy::RetainForReplay,
5891        job,
5892    );
5893    let completion_tx = tx.clone();
5894    let completion_shutdown = Arc::clone(shutdown);
5895    let route = route_id;
5896    let corr = frame.header.corr;
5897    let flags = frame.header.flags;
5898    let ver = frame.header.ver;
5899    let completion_metrics = Arc::clone(metrics);
5900    let active_tool_calls = Arc::clone(active_tool_calls);
5901    tokio::spawn(async move {
5902        let _response_task = ResponseTaskGuard::new(&completion_metrics);
5903        let response = await_executor_response(rx, request_id.clone()).await;
5904        let (text, phase_trace) = match tool_call_rx.await {
5905            Ok(completion) => (completion.text, Some(completion.phase_trace)),
5906            Err(_) => (
5907                crate::subc_format::format_response_with_context(
5908                    &bare_name_for_frame,
5909                    &response,
5910                    &format_context_for_frame,
5911                ),
5912                None,
5913            ),
5914        };
5915        if !finish_active_tool_call(&active_tool_calls, route, corr) {
5916            return;
5917        }
5918        let result = ToolCallResult { text, response };
5919        let fatal = note_fatal_panic_response(&result.response);
5920        match build_tool_response_frame_with_limit(
5921            ver,
5922            route,
5923            corr,
5924            flags,
5925            &result,
5926            bind_trust,
5927            tool_response_body_limit,
5928        ) {
5929            Ok(response_frame) => {
5930                let send_result = if let Some(phase_trace) = phase_trace {
5931                    let trace = ToolResponseWriteTrace::new(
5932                        phase_trace,
5933                        bare_name_for_frame,
5934                        completion_root,
5935                        completion_session,
5936                        route.channel,
5937                        corr,
5938                    );
5939                    send_traced_tool_response_frame(
5940                        &completion_tx,
5941                        &completion_metrics,
5942                        response_frame,
5943                        trace,
5944                    )
5945                    .await
5946                } else {
5947                    send_reliable_writer_frame(
5948                        &completion_tx,
5949                        &completion_metrics,
5950                        response_frame,
5951                        "tool response",
5952                    )
5953                    .await
5954                };
5955                if let Err(error) = send_result {
5956                    log::warn!("subc attach: failed to queue tool response frame: {error}");
5957                }
5958            }
5959            Err(error) => {
5960                log::error!("subc attach: failed to build tool response frame: {error}");
5961            }
5962        }
5963        if fatal {
5964            signal_fatal_teardown(
5965                &completion_tx,
5966                Some(route),
5967                ver,
5968                corr,
5969                &completion_shutdown,
5970                &completion_metrics,
5971            )
5972            .await;
5973        }
5974    });
5975    Ok(())
5976}
5977
5978fn submit_maintenance_job(
5979    executor: &Arc<Executor>,
5980    root_id: ProjectRootId,
5981    kind: MaintenanceDrainKind,
5982    bg_sessions_to_check: Vec<(String, u64)>,
5983    completion_tx: &mpsc::Sender<MaintenanceCompletion>,
5984    metrics: &Arc<DispatchPathMetrics>,
5985) {
5986    let request_id = format!(
5987        "subc-maintenance-drain-{}-{}",
5988        kind.label(),
5989        root_id.as_path().to_string_lossy()
5990    );
5991    let response_id = request_id.clone();
5992    let completion_root_id = root_id.clone();
5993    let maintenance_generation = executor
5994        .actor_context(&root_id)
5995        .map(|ctx| ctx.configure_generation())
5996        .unwrap_or(0);
5997    let (outcome_tx, outcome_rx) = oneshot::channel::<MaintenanceJobOutcome>();
5998    // Deferred drains mutate subsystem state behind each subsystem's own lock.
5999    // Keeping every drain on MaintenanceCommit lets interactive reads and lazy
6000    // HeavyInit queries run while maintenance converges after the bind ack.
6001    let lane = Lane::MaintenanceCommit;
6002    let job: crate::executor::ExecutorJob = Box::new(move |ctx: &AppContext| {
6003        let outcome = match kind {
6004            MaintenanceDrainKind::Watcher => {
6005                let drained = runtime_drain::drain_watcher_events_bounded(
6006                    ctx,
6007                    runtime_drain::WATCHER_PATH_DRAIN_BATCH_CAP,
6008                );
6009                MaintenanceJobOutcome {
6010                    empty_bg_sessions: Vec::new(),
6011                    unacked_bg_keys: None,
6012                    requeue_kind: drained.has_more.then_some(kind),
6013                }
6014            }
6015            MaintenanceDrainKind::Lsp => {
6016                let drained = runtime_drain::drain_lsp_events_bounded(
6017                    ctx,
6018                    runtime_drain::LSP_EVENT_DRAIN_BATCH_CAP,
6019                );
6020                MaintenanceJobOutcome {
6021                    empty_bg_sessions: Vec::new(),
6022                    unacked_bg_keys: None,
6023                    requeue_kind: drained.has_more.then_some(kind),
6024                }
6025            }
6026            MaintenanceDrainKind::ConfigureTail => {
6027                runtime_drain::drain_deferred_configure_maintenance(ctx);
6028                runtime_drain::drain_configure_warning_events(ctx);
6029                MaintenanceJobOutcome::default()
6030            }
6031            MaintenanceDrainKind::CompletionDrains => {
6032                runtime_drain::drain_search_index_events(ctx);
6033                runtime_drain::drain_callgraph_store_events(ctx);
6034                runtime_drain::drain_semantic_index_events(ctx);
6035                runtime_drain::drain_semantic_refresh_events(ctx);
6036                runtime_drain::drain_inspect_events_for_generation(ctx, maintenance_generation);
6037                let empty_bg_sessions = bg_sessions_to_check
6038                    .into_iter()
6039                    .filter(|(session, _)| {
6040                        !ctx.bash_background().has_unacked_wakes_for_session(session)
6041                    })
6042                    .collect();
6043                MaintenanceJobOutcome {
6044                    empty_bg_sessions,
6045                    unacked_bg_keys: Some(ctx.bash_background().unacked_wake_keys()),
6046                    requeue_kind: None,
6047                }
6048            }
6049        };
6050        let requeued = outcome.requeue_kind.is_some();
6051        let _ = outcome_tx.send(outcome);
6052        Response::success(
6053            response_id,
6054            json!({ "drained": true, "kind": kind.label(), "requeued": requeued }),
6055        )
6056    });
6057    let rx = match kind {
6058        MaintenanceDrainKind::Watcher => executor.submit_coalescable_maintenance_async(
6059            root_id,
6060            lane,
6061            request_id.clone(),
6062            crate::executor::MaintenanceCoalesceKey::WatcherDrain,
6063            job,
6064        ),
6065        MaintenanceDrainKind::Lsp => executor.submit_coalescable_maintenance_async(
6066            root_id,
6067            lane,
6068            request_id.clone(),
6069            crate::executor::MaintenanceCoalesceKey::LspDrain,
6070            job,
6071        ),
6072        MaintenanceDrainKind::ConfigureTail | MaintenanceDrainKind::CompletionDrains => {
6073            executor.submit_maintenance_async(root_id, lane, request_id.clone(), job)
6074        }
6075    };
6076    let completion_tx = completion_tx.clone();
6077    let completion_metrics = Arc::clone(metrics);
6078    tokio::spawn(async move {
6079        let _response_task = ResponseTaskGuard::new(&completion_metrics);
6080        let response = await_executor_response(rx, request_id).await;
6081        let outcome = outcome_rx.await.unwrap_or_default();
6082        let _ = send_counted_channel(
6083            &completion_tx,
6084            &completion_metrics.maintenance_queued,
6085            MaintenanceCompletion {
6086                root_id: completion_root_id,
6087                kind,
6088                response,
6089                empty_bg_sessions: outcome.empty_bg_sessions,
6090                unacked_bg_keys: outcome.unacked_bg_keys,
6091                requeue_kind: outcome.requeue_kind,
6092            },
6093        )
6094        .await;
6095    });
6096}
6097
6098async fn await_executor_response(rx: oneshot::Receiver<Response>, request_id: String) -> Response {
6099    rx.await
6100        .unwrap_or_else(|_| Response::error(request_id, "internal_error", "executor dropped"))
6101}
6102
6103async fn deliver_resolved_subc_response(
6104    tx: &WriterSender,
6105    mut resolved: ResolvedSubcResponse,
6106    routes: &HashMap<RouteChannel, RouteIdentity>,
6107    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
6108    executor: &Executor,
6109    active_tool_calls: &ActiveToolCalls,
6110    shutdown: &Arc<Notify>,
6111    metrics: &DispatchPathMetrics,
6112    tool_response_body_limit: usize,
6113) -> Result<(), SubcError> {
6114    let entry = &mut resolved.entry;
6115    finish_active_tool_call(active_tool_calls, entry.route, entry.corr);
6116    if let Some(meta) = live_roots.get_mut(&entry.root) {
6117        meta.note_activity();
6118    }
6119
6120    let Some(identity) = routes.get(&entry.route) else {
6121        log::debug!(
6122            "subc attach: dropping deferred {} response {} for unbound route {}",
6123            entry.bare_name,
6124            entry.pending.request_id,
6125            entry.route
6126        );
6127        return Ok(());
6128    };
6129    let Some(ctx) = executor.actor_context(&entry.root) else {
6130        return Ok(());
6131    };
6132    entry.phase_trace.mark_execute_done();
6133    let finalizer = |response: &mut Response| {
6134        crate::response_finalize::finalize_response_with_bg_completions(
6135            response,
6136            &ctx,
6137            &entry.session_id,
6138            &entry.bare_name,
6139            entry.bind_trust.allows_bash_observation(),
6140        );
6141    };
6142    let result = finish_tool_call_response(
6143        &entry.bare_name,
6144        &entry.format_context,
6145        resolved.response,
6146        entry.surface_downgraded,
6147        Some(&finalizer),
6148        Some(&mut entry.phase_trace),
6149    );
6150    let fatal = note_fatal_panic_response(&result.response);
6151    let response_frame = build_tool_response_frame_with_limit(
6152        entry.ver,
6153        entry.route,
6154        entry.corr,
6155        entry.flags,
6156        &result,
6157        identity.trust,
6158        tool_response_body_limit,
6159    )?;
6160    let trace = ToolResponseWriteTrace::new(
6161        std::mem::replace(&mut entry.phase_trace, PhaseTrace::new(Instant::now())),
6162        entry.bare_name.clone(),
6163        identity.project_root.clone(),
6164        entry.session_id.clone(),
6165        entry.route.channel,
6166        entry.corr,
6167    );
6168    send_traced_tool_response_frame(tx, metrics, response_frame, trace).await?;
6169    if fatal {
6170        signal_fatal_teardown(
6171            tx,
6172            Some(entry.route),
6173            entry.ver,
6174            entry.corr,
6175            shutdown,
6176            metrics,
6177        )
6178        .await;
6179    }
6180    Ok(())
6181}
6182
6183async fn signal_fatal_teardown(
6184    tx: &WriterSender,
6185    route: Option<RouteChannel>,
6186    ver: u8,
6187    corr: u64,
6188    shutdown: &Arc<Notify>,
6189    metrics: &DispatchPathMetrics,
6190) {
6191    if let Some(route) = route {
6192        if let Ok(frame) = build_goodbye_frame(ver, route.channel, route.epoch, corr) {
6193            if let Err(error) = send_frame(tx, metrics, frame).await {
6194                log::warn!(
6195                    "subc attach: failed to queue fatal route Goodbye for route {route}: {error}"
6196                );
6197            }
6198        }
6199    }
6200    if let Ok(frame) = build_goodbye_frame(ver, 0, 0, 0) {
6201        if let Err(error) = send_frame(tx, metrics, frame).await {
6202            log::warn!("subc attach: failed to queue fatal channel-0 Goodbye: {error}");
6203        }
6204    }
6205    shutdown.notify_one();
6206}
6207#[derive(Debug, Deserialize)]
6208#[serde(untagged)]
6209enum RouteRequest {
6210    BgEvents(BgEventsRequest),
6211    ToolCall(ToolCallRequest),
6212}
6213
6214#[derive(Debug, Deserialize)]
6215struct BgEventsRequest {
6216    op: BgEventsOp,
6217}
6218
6219#[derive(Debug, Deserialize)]
6220#[serde(rename_all = "snake_case")]
6221enum BgEventsOp {
6222    BgEvents,
6223}
6224
6225#[derive(Debug, Deserialize)]
6226struct ToolCallRequest {
6227    name: String,
6228    #[serde(default)]
6229    arguments: Value,
6230    /// Host-computed registration fact; kept outside agent-controlled arguments.
6231    #[serde(default)]
6232    edit_slot_survives: Option<bool>,
6233    /// Server-owned preview control (B1c-0): the plugin's mutation flow is
6234    /// preview -> permission ask -> apply. Dropping this field made "preview"
6235    /// calls mutate disk before the permission prompt and the subsequent
6236    /// apply fail with not-found.
6237    #[serde(default)]
6238    preview: bool,
6239}
6240
6241#[cfg(test)]
6242pub(crate) mod test_support {
6243    use super::*;
6244    use crate::bash_background::BgTaskStatus;
6245    use crate::protocol::{
6246        BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, ConfigureWarningsFrame,
6247        ProgressFrame, StatusChangedFrame,
6248    };
6249    use serde_json::json;
6250
6251    pub(super) fn test_root(name: &str) -> (tempfile::TempDir, ProjectRootId) {
6252        let dir = tempfile::Builder::new()
6253            .prefix(name)
6254            .tempdir()
6255            .expect("temp root");
6256        let root = ProjectRootId::from_path(dir.path()).expect("project root id");
6257        (dir, root)
6258    }
6259
6260    pub(super) fn test_ctx() -> Arc<AppContext> {
6261        Arc::new(AppContext::new(
6262            Box::new(crate::parser::TreeSitterProvider::new()),
6263            crate::config::Config::default(),
6264        ))
6265    }
6266
6267    fn inspect_context(root: &Path) -> Arc<AppContext> {
6268        inspect_context_with_timeout(root, None)
6269    }
6270
6271    fn inspect_context_with_timeout(root: &Path, timeout_ms: Option<u64>) -> Arc<AppContext> {
6272        let mut config = crate::config::Config::default();
6273        config.project_root = Some(root.to_path_buf());
6274        if let Some(timeout_ms) = timeout_ms {
6275            config.inspect.diagnostics_timeout_ms = timeout_ms;
6276        }
6277        let ctx = Arc::new(AppContext::new(
6278            Box::new(crate::parser::TreeSitterProvider::new()),
6279            config,
6280        ));
6281        ctx.set_harness(crate::harness::Harness::Opencode);
6282        ctx
6283    }
6284
6285    fn inspect_request(id: &str) -> RawRequest {
6286        serde_json::from_value(json!({ "id": id, "command": "inspect" })).expect("inspect request")
6287    }
6288
6289    fn submit_deferred_inspect_setup(
6290        executor: &Arc<Executor>,
6291        root: &ProjectRootId,
6292        ctx: &Arc<AppContext>,
6293        request_id: &str,
6294    ) -> (PendingResponse, JobCancellation) {
6295        let (pending_tx, pending_rx) = std::sync::mpsc::sync_channel(1);
6296        let request = inspect_request(request_id);
6297        let inspect_ctx = Arc::clone(ctx);
6298        let (_rx, cancellation) = executor.submit_cancellable_async(
6299            root.clone(),
6300            Lane::SerialLspStatus,
6301            request_id.to_string(),
6302            Box::new(move |_| {
6303                let DispatchOutcome::Deferred(pending) =
6304                    crate::commands::inspect::handle_inspect_deferred_with_restriction(
6305                        &request,
6306                        inspect_ctx,
6307                        true,
6308                    )
6309                else {
6310                    panic!("inspect setup must defer")
6311                };
6312                pending_tx.send(pending).expect("send pending inspect");
6313                Response::success("inspect-setup", json!({}))
6314            }),
6315        );
6316        let pending = pending_rx
6317            .recv_timeout(Duration::from_secs(1))
6318            .expect("inspect setup leaves the executor lane");
6319        let deadline = Instant::now() + Duration::from_secs(1);
6320        while !executor.actor_is_idle(root) {
6321            assert!(
6322                Instant::now() < deadline,
6323                "inspect setup kept lane counters live"
6324            );
6325            std::thread::sleep(Duration::from_millis(5));
6326        }
6327        (pending, cancellation)
6328    }
6329
6330    fn wait_for_inspect_terminal(pending: &mut PendingResponse, ctx: &AppContext) -> Response {
6331        let deadline = Instant::now() + Duration::from_secs(60);
6332        loop {
6333            if let Some(response) = (pending.poll)(ctx) {
6334                return response;
6335            }
6336            assert!(Instant::now() < deadline, "inspect terminal timed out");
6337            std::thread::sleep(Duration::from_millis(5));
6338        }
6339    }
6340
6341    fn cold_navigation_context(root: &Path) -> (Arc<AppContext>, PathBuf) {
6342        let source_dir = root.join("src");
6343        std::fs::create_dir_all(&source_dir).expect("create source dir");
6344        std::fs::write(root.join("Cargo.toml"), "[package]\nname = \"fixture\"\n")
6345            .expect("write Cargo manifest");
6346        let source = source_dir.join("main.rs");
6347        std::fs::write(&source, "fn main() {}\n").expect("write source");
6348        let binary = root.join("cold-navigation-server");
6349        std::fs::write(&binary, b"fixture").expect("write server placeholder");
6350
6351        let ctx = inspect_context(root);
6352        ctx.lsp()
6353            .override_binary(crate::lsp::registry::ServerKind::Rust, binary);
6354        (ctx, source)
6355    }
6356
6357    fn navigation_request(id: &str, source: &Path) -> RawRequest {
6358        serde_json::from_value(json!({
6359            "id": id,
6360            "command": "lsp_hover",
6361            "file": source.display().to_string(),
6362            "line": 1,
6363            "character": 1,
6364        }))
6365        .expect("navigation request")
6366    }
6367
6368    fn submit_deferred_navigation_setup(
6369        executor: &Arc<Executor>,
6370        root: &ProjectRootId,
6371        ctx: &Arc<AppContext>,
6372        source: &Path,
6373        request_id: &str,
6374    ) -> (PendingResponse, JobCancellation) {
6375        let (pending_tx, pending_rx) = std::sync::mpsc::sync_channel(1);
6376        let request = navigation_request(request_id, source);
6377        let navigation_ctx = Arc::clone(ctx);
6378        let (_rx, cancellation) = executor.submit_cancellable_async(
6379            root.clone(),
6380            Lane::SerialLspStatus,
6381            request_id.to_string(),
6382            Box::new(move |_| {
6383                let DispatchOutcome::Deferred(pending) = crate::commands::lsp_navigation::
6384                    handle_lsp_navigation_deferred_with_restriction(
6385                        &request,
6386                        navigation_ctx,
6387                        true,
6388                    )
6389                else {
6390                    panic!("cold navigation setup must defer")
6391                };
6392                pending_tx.send(pending).expect("send pending navigation");
6393                Response::success("navigation-setup", json!({}))
6394            }),
6395        );
6396        let pending = pending_rx
6397            .recv_timeout(Duration::from_secs(1))
6398            .expect("navigation setup leaves the executor lane");
6399        let deadline = Instant::now() + Duration::from_secs(1);
6400        while !executor.actor_is_idle(root) {
6401            assert!(
6402                Instant::now() < deadline,
6403                "navigation setup kept lane counters live"
6404            );
6405            std::thread::sleep(Duration::from_millis(5));
6406        }
6407        (pending, cancellation)
6408    }
6409
6410    fn wait_for_navigation_terminal(pending: &mut PendingResponse, ctx: &AppContext) -> Response {
6411        let deadline = Instant::now() + Duration::from_secs(1);
6412        loop {
6413            if let Some(response) = (pending.poll)(ctx) {
6414                return response;
6415            }
6416            assert!(Instant::now() < deadline, "navigation terminal timed out");
6417            std::thread::sleep(Duration::from_millis(5));
6418        }
6419    }
6420
6421    #[test]
6422    fn deferred_inspect_releases_lane_for_bind_and_mutation() {
6423        let _serial = crate::commands::inspect::deferred_inspect_test_lock();
6424        let executor = Arc::new(Executor::new());
6425        let (dir, root) = test_root("deferred-inspect-storm");
6426        std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
6427        let ctx = inspect_context(dir.path());
6428        executor.register_actor(root.clone(), Arc::clone(&ctx));
6429        let (started_rx, release_tx) =
6430            crate::commands::inspect::install_deferred_inspect_stat_gate_for_test();
6431        let (mut pending, _cancellation) =
6432            submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-storm");
6433        started_rx
6434            .recv_timeout(Duration::from_secs(1))
6435            .expect("deferred inspect body starts");
6436
6437        for request_id in ["subc-bind-other-session", "subc-edit-other-session"] {
6438            let response = executor.submit(
6439                root.clone(),
6440                Lane::Mutating,
6441                request_id.to_string(),
6442                Box::new(move |_| Response::success(request_id, json!({ "admitted": true }))),
6443            );
6444            assert!(
6445                response
6446                    .recv_timeout(Duration::from_secs(1))
6447                    .expect("writer admits while inspect remains deferred")
6448                    .success
6449            );
6450        }
6451        assert_eq!(
6452            crate::commands::inspect::deferred_inspect_root_count_for_test(),
6453            1,
6454            "writer admissions must not finish the detached inspect"
6455        );
6456
6457        release_tx.send(()).expect("release inspect body");
6458        let terminal = wait_for_inspect_terminal(&mut pending, &ctx);
6459        assert!(
6460            terminal.data.get("inspect_terminal").is_some(),
6461            "inspect must still produce its terminal: {:?}",
6462            terminal.data
6463        );
6464    }
6465
6466    #[test]
6467    fn deferred_inspect_honors_the_shared_request_deadline() {
6468        let _serial = crate::commands::inspect::deferred_inspect_test_lock();
6469        let executor = Arc::new(Executor::new());
6470        let (dir, root) = test_root("deferred-inspect-deadline");
6471        std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
6472        let ctx = inspect_context_with_timeout(dir.path(), Some(10_000));
6473        executor.register_actor(root.clone(), Arc::clone(&ctx));
6474        let (started_rx, _release_tx) =
6475            crate::commands::inspect::install_deferred_inspect_body_gate_for_test();
6476        let started = Instant::now();
6477        let (mut pending, _cancellation) =
6478            submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-deadline");
6479        started_rx
6480            .recv_timeout(Duration::from_secs(1))
6481            .expect("deferred inspect body starts");
6482
6483        let terminal = wait_for_inspect_terminal(&mut pending, &ctx);
6484        assert_eq!(terminal.data["inspect_terminal"], "phase_failed");
6485        assert_eq!(terminal.data["failure_reason"], "inspect_request_timeout");
6486        assert_eq!(terminal.data["failed_phase"], "tier2_rescan");
6487        assert!(
6488            started.elapsed() < Duration::from_secs(8),
6489            "deferred inspect missed its terminal reserve: {:?}",
6490            started.elapsed()
6491        );
6492    }
6493
6494    #[test]
6495    fn deferred_cold_navigation_releases_lsp_lane_for_reads_and_mutation() {
6496        let _serial = crate::commands::lsp_navigation::deferred_navigation_test_lock();
6497        let executor = Arc::new(Executor::new());
6498        let (dir, root) = test_root("deferred-cold-navigation");
6499        let (ctx, source) = cold_navigation_context(dir.path());
6500        executor.register_actor(root.clone(), Arc::clone(&ctx));
6501        let (started_rx, _release_tx) =
6502            crate::commands::lsp_navigation::install_deferred_navigation_gate_for_test();
6503        let (mut pending, cancellation) = submit_deferred_navigation_setup(
6504            &executor,
6505            &root,
6506            &ctx,
6507            &source,
6508            "subc-cold-navigation",
6509        );
6510        started_rx
6511            .recv_timeout(Duration::from_secs(1))
6512            .expect("detached navigation reaches its cold-start gate");
6513
6514        for (lane, request_id) in [
6515            (Lane::PureRead, "subc-navigation-read"),
6516            (Lane::Mutating, "subc-navigation-write"),
6517        ] {
6518            let response = executor.submit(
6519                root.clone(),
6520                lane,
6521                request_id.to_string(),
6522                Box::new(move |_| Response::success(request_id, json!({ "admitted": true }))),
6523            );
6524            assert!(
6525                response
6526                    .recv_timeout(Duration::from_secs(1))
6527                    .expect("unrelated work admits while navigation remains deferred")
6528                    .success
6529            );
6530        }
6531        assert_eq!(
6532            crate::commands::lsp_navigation::deferred_navigation_worker_count_for_test(),
6533            1,
6534            "lane admissions must not finish the detached navigation"
6535        );
6536
6537        cancellation.request_cancel();
6538        let terminal = wait_for_navigation_terminal(&mut pending, &ctx);
6539        assert!(!terminal.success);
6540        let deadline = Instant::now() + Duration::from_secs(1);
6541        while crate::commands::lsp_navigation::deferred_navigation_worker_count_for_test() != 0 {
6542            assert!(
6543                Instant::now() < deadline,
6544                "cancelled navigation worker did not settle"
6545            );
6546            std::thread::sleep(Duration::from_millis(5));
6547        }
6548        assert!(executor.actor_is_idle(&root));
6549    }
6550
6551    #[test]
6552    fn cancelling_pending_navigation_removes_entry_without_reply() {
6553        let _serial = crate::commands::lsp_navigation::deferred_navigation_test_lock();
6554        let executor = Arc::new(Executor::new());
6555        let (dir, root) = test_root("cancelled-pending-navigation");
6556        let (ctx, source) = cold_navigation_context(dir.path());
6557        executor.register_actor(root.clone(), Arc::clone(&ctx));
6558        let (started_rx, _release_tx) =
6559            crate::commands::lsp_navigation::install_deferred_navigation_gate_for_test();
6560        let (pending, _cancellation) = submit_deferred_navigation_setup(
6561            &executor,
6562            &root,
6563            &ctx,
6564            &source,
6565            "subc-cancel-navigation",
6566        );
6567        started_rx
6568            .recv_timeout(Duration::from_secs(1))
6569            .expect("detached navigation reaches its cancellation gate");
6570
6571        let route = RouteChannel {
6572            channel: 17,
6573            epoch: 1,
6574        };
6575        let mut registry = PendingSubcResponses::default();
6576        registry.register(PendingSubcResponse {
6577            route,
6578            corr: 71,
6579            flags: Flags::new(false, Priority::Passive, false),
6580            ver: PROTOCOL_VERSION,
6581            root: root.clone(),
6582            session_id: "navigation-cancel-session".to_string(),
6583            bare_name: "lsp_hover".to_string(),
6584            format_context: crate::subc_format::FormatContext::from_tool_call(
6585                "lsp_hover",
6586                &json!({}),
6587                dir.path(),
6588            ),
6589            bind_trust: BindTrust::FirstParty,
6590            pending,
6591            surface_downgraded: false,
6592            phase_trace: PhaseTrace::new(Instant::now()),
6593        });
6594
6595        assert!(registry.cancel_request(route, 71));
6596        assert!(registry.is_empty());
6597        let deadline = Instant::now() + Duration::from_secs(1);
6598        while crate::commands::lsp_navigation::deferred_navigation_worker_count_for_test() != 0 {
6599            assert!(
6600                Instant::now() < deadline,
6601                "pending cancellation did not settle the detached worker"
6602            );
6603            std::thread::sleep(Duration::from_millis(5));
6604        }
6605        assert!(
6606            registry.poll_ready(executor.as_ref()).is_empty(),
6607            "a cancelled navigation must not leak a reply"
6608        );
6609    }
6610
6611    #[test]
6612    fn same_root_deferred_inspects_are_single_flight() {
6613        let _serial = crate::commands::inspect::deferred_inspect_test_lock();
6614        let executor = Arc::new(Executor::new());
6615        let (dir, root) = test_root("single-flight-deferred-inspect");
6616        std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
6617        let ctx = inspect_context(dir.path());
6618        executor.register_actor(root.clone(), Arc::clone(&ctx));
6619        let (started_rx, release_tx) =
6620            crate::commands::inspect::install_deferred_inspect_stat_gate_for_test();
6621        let (mut first, _first_cancellation) =
6622            submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-first");
6623        started_rx
6624            .recv_timeout(Duration::from_secs(1))
6625            .expect("first inspect owns the root flight");
6626        let (mut second, second_cancellation) =
6627            submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-second");
6628
6629        assert_eq!(
6630            crate::commands::inspect::deferred_inspect_root_count_for_test(),
6631            1,
6632            "only one detached body may run for a root"
6633        );
6634        assert!((second.poll)(&ctx).is_none(), "second inspect must queue");
6635        second_cancellation.request_cancel();
6636        let second_terminal = wait_for_inspect_terminal(&mut second, &ctx);
6637        assert_eq!(second_terminal.data["inspect_terminal"], "interrupted");
6638        assert_eq!(
6639            crate::commands::inspect::deferred_inspect_root_count_for_test(),
6640            1,
6641            "cancelling the queued request must not release the active flight"
6642        );
6643
6644        release_tx.send(()).expect("release first inspect");
6645        let first_terminal = wait_for_inspect_terminal(&mut first, &ctx);
6646        assert_eq!(first_terminal.data["inspect_terminal"], "fresh");
6647        assert_eq!(
6648            crate::commands::inspect::deferred_inspect_root_count_for_test(),
6649            0
6650        );
6651    }
6652
6653    #[test]
6654    fn route_abandonment_cancels_detached_inspect_thread() {
6655        let _serial = crate::commands::inspect::deferred_inspect_test_lock();
6656        let executor = Arc::new(Executor::new());
6657        let (dir, root) = test_root("cancelled-deferred-inspect");
6658        std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
6659        let ctx = inspect_context(dir.path());
6660        executor.register_actor(root.clone(), Arc::clone(&ctx));
6661        let active: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
6662        let route = RouteChannel {
6663            channel: 7,
6664            epoch: 1,
6665        };
6666        let (started_rx, _release_tx) =
6667            crate::commands::inspect::install_deferred_inspect_body_gate_for_test();
6668        let (mut pending, cancellation) =
6669            submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-abandoned");
6670        active.lock().expect("active tool call map").insert(
6671            (route, 41),
6672            ActiveToolCall {
6673                root_id: root.clone(),
6674                cancellation,
6675                detach_policy: RouteDetachPolicy::CancelOnDetach,
6676            },
6677        );
6678        started_rx
6679            .recv_timeout(Duration::from_secs(1))
6680            .expect("detached inspect reaches cancellation gate");
6681        assert!(ctx.request_force_restrict("subc-inspect-abandoned"));
6682
6683        assert!(cancel_active_tool_call(
6684            &active,
6685            executor.as_ref(),
6686            route,
6687            41,
6688            "test route abandonment"
6689        ));
6690        let terminal = wait_for_inspect_terminal(&mut pending, &ctx);
6691        assert_eq!(terminal.data["inspect_terminal"], "interrupted");
6692        assert_eq!(
6693            crate::commands::inspect::deferred_inspect_root_count_for_test(),
6694            0
6695        );
6696        assert!(executor.actor_is_idle(&root));
6697        assert!(active.lock().expect("active tool call map").is_empty());
6698        let restriction_deadline = Instant::now() + Duration::from_secs(1);
6699        while ctx.request_force_restrict("subc-inspect-abandoned") {
6700            assert!(
6701                Instant::now() < restriction_deadline,
6702                "detached force-restrict guard leaked"
6703            );
6704            std::thread::sleep(Duration::from_millis(5));
6705        }
6706    }
6707
6708    #[test]
6709    fn true_abandonment_cancels_but_route_detach_retains_interactive_search() {
6710        // This scenario requires the replayable and terminal calls to run at the
6711        // same time. Pin two read slots instead of deriving the topology from
6712        // the host, where a small runner can queue the terminal call forever
6713        // behind the intentionally retained call.
6714        let executor = Arc::new(Executor::with_config(crate::executor::ExecutorConfig {
6715            pool_size: 3,
6716            read_cap: 2,
6717            actor_cap: 2,
6718            heavy_permits: 1,
6719            drr_quantum: 1,
6720        }));
6721        let (_dir, root) = test_root("cancelled-interactive-search");
6722        executor.register_actor(root.clone(), test_ctx());
6723        let active: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
6724        let route = RouteChannel {
6725            channel: 9,
6726            epoch: 1,
6727        };
6728
6729        let disabled_iterations = Arc::new(AtomicUsize::new(0));
6730        let disabled_probe = Arc::clone(&disabled_iterations);
6731        let (disabled_started_tx, disabled_started_rx) = std::sync::mpsc::sync_channel(1);
6732        let (disabled_rx, disabled_cancellation) = executor.submit_cancellable_async(
6733            root.clone(),
6734            Lane::PureRead,
6735            "untracked-search".to_string(),
6736            Box::new(move |_| {
6737                disabled_started_tx
6738                    .send(())
6739                    .expect("signal untracked search");
6740                let deadline = Instant::now() + Duration::from_secs(5);
6741                while !crate::commands::semantic_search::search_cancellation_requested() {
6742                    if Instant::now() >= deadline {
6743                        return Response::error(
6744                            "untracked-search",
6745                            "test_timeout",
6746                            "untracked search did not receive cancellation",
6747                        );
6748                    }
6749                    disabled_probe.fetch_add(1, Ordering::Relaxed);
6750                    std::thread::yield_now();
6751                }
6752                Response::error(
6753                    "untracked-search",
6754                    "request_cancelled",
6755                    "cancelled at search checkpoint",
6756                )
6757            }),
6758        );
6759        disabled_started_rx
6760            .recv_timeout(Duration::from_secs(1))
6761            .expect("untracked search starts");
6762        assert_eq!(
6763            apply_route_work_disposition(
6764                &active,
6765                executor.as_ref(),
6766                route,
6767                RouteWorkDisposition::Abandon,
6768                "disabled cancellation wiring",
6769            ),
6770            0
6771        );
6772        let iterations_before = disabled_iterations.load(Ordering::Relaxed);
6773        std::thread::sleep(Duration::from_millis(10));
6774        assert!(
6775            disabled_iterations.load(Ordering::Relaxed) > iterations_before,
6776            "without route registration the search keeps computing"
6777        );
6778        disabled_cancellation.request_cancel();
6779        let disabled_response = disabled_rx
6780            .blocking_recv()
6781            .expect("untracked search settles after explicit cleanup");
6782        assert_eq!(disabled_response.data["code"], "request_cancelled");
6783
6784        let tracked_iterations = Arc::new(AtomicUsize::new(0));
6785        let tracked_probe = Arc::clone(&tracked_iterations);
6786        let (tracked_started_tx, tracked_started_rx) = std::sync::mpsc::sync_channel(1);
6787        let tracked_rx = submit_active_tool_call(
6788            executor.as_ref(),
6789            &active,
6790            route,
6791            42,
6792            root.clone(),
6793            Lane::PureRead,
6794            "tracked-search".to_string(),
6795            RouteDetachPolicy::RetainForReplay,
6796            Box::new(move |_| {
6797                tracked_started_tx.send(()).expect("signal tracked search");
6798                let deadline = Instant::now() + Duration::from_secs(5);
6799                while !crate::commands::semantic_search::search_cancellation_requested() {
6800                    if Instant::now() >= deadline {
6801                        return Response::error(
6802                            "tracked-search",
6803                            "test_timeout",
6804                            "tracked search did not receive cancellation",
6805                        );
6806                    }
6807                    tracked_probe.fetch_add(1, Ordering::Relaxed);
6808                    std::thread::yield_now();
6809                }
6810                Response::error(
6811                    "tracked-search",
6812                    "request_cancelled",
6813                    "cancelled at search checkpoint",
6814                )
6815            }),
6816        );
6817        tracked_started_rx
6818            .recv_timeout(Duration::from_secs(1))
6819            .expect("tracked search starts");
6820
6821        let (terminal_started_tx, terminal_started_rx) = std::sync::mpsc::sync_channel(1);
6822        let terminal_rx = submit_active_tool_call(
6823            executor.as_ref(),
6824            &active,
6825            route,
6826            43,
6827            root.clone(),
6828            Lane::PureRead,
6829            "teardown-terminal".to_string(),
6830            RouteDetachPolicy::CancelOnDetach,
6831            Box::new(move |_| {
6832                terminal_started_tx
6833                    .send(())
6834                    .expect("signal teardown-terminal call");
6835                let deadline = Instant::now() + Duration::from_secs(5);
6836                while !crate::executor::current_job_cancelled() {
6837                    if Instant::now() >= deadline {
6838                        return Response::error(
6839                            "teardown-terminal",
6840                            "test_timeout",
6841                            "terminal call did not receive cancellation",
6842                        );
6843                    }
6844                    std::thread::yield_now();
6845                }
6846                Response::error(
6847                    "teardown-terminal",
6848                    "request_cancelled",
6849                    "cancelled for terminal-emitting teardown",
6850                )
6851            }),
6852        );
6853        terminal_started_rx
6854            .recv_timeout(Duration::from_secs(1))
6855            .expect("teardown-terminal call starts");
6856        assert_eq!(
6857            apply_route_work_disposition(
6858                &active,
6859                executor.as_ref(),
6860                route,
6861                RouteWorkDisposition::RetainForReplay,
6862                "test route detach",
6863            ),
6864            1,
6865            "only the replayable search remains active after route detach"
6866        );
6867        let terminal_response = terminal_rx
6868            .blocking_recv()
6869            .expect("teardown-terminal call stops at cancellation checkpoint");
6870        assert_eq!(terminal_response.data["code"], "request_cancelled");
6871        let iterations_before_detach = tracked_iterations.load(Ordering::Relaxed);
6872        std::thread::sleep(Duration::from_millis(10));
6873        assert!(
6874            tracked_iterations.load(Ordering::Relaxed) > iterations_before_detach,
6875            "route detach must retain work whose response can replay after rebind"
6876        );
6877        assert_eq!(
6878            apply_route_work_disposition(
6879                &active,
6880                executor.as_ref(),
6881                route,
6882                RouteWorkDisposition::Abandon,
6883                "test session purge",
6884            ),
6885            1
6886        );
6887        let tracked_response = tracked_rx
6888            .blocking_recv()
6889            .expect("tracked search stops at cancellation checkpoint");
6890        assert_eq!(tracked_response.data["code"], "request_cancelled");
6891        assert!(active.lock().expect("active tool calls").is_empty());
6892
6893        let deadline = Instant::now() + Duration::from_secs(1);
6894        while !executor.actor_is_idle(&root) {
6895            assert!(
6896                Instant::now() < deadline,
6897                "cancelled search must release the PureRead lane"
6898            );
6899            std::thread::sleep(Duration::from_millis(2));
6900        }
6901    }
6902
6903    #[test]
6904    fn shutdown_drain_emits_terminal_and_clears_pending_inspect() {
6905        let _serial = crate::commands::inspect::deferred_inspect_test_lock();
6906        let executor = Arc::new(Executor::new());
6907        let (dir, root) = test_root("shutdown-deferred-inspect");
6908        std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
6909        let ctx = inspect_context(dir.path());
6910        executor.register_actor(root.clone(), Arc::clone(&ctx));
6911        let (started_rx, _release_tx) =
6912            crate::commands::inspect::install_deferred_inspect_body_gate_for_test();
6913        let (pending, cancellation) =
6914            submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-shutdown");
6915        let route = RouteChannel {
6916            channel: 8,
6917            epoch: 1,
6918        };
6919        started_rx
6920            .recv_timeout(Duration::from_secs(1))
6921            .expect("detached inspect reaches shutdown gate");
6922        let mut registry = PendingSubcResponses::default();
6923        registry.register(PendingSubcResponse {
6924            route,
6925            corr: 42,
6926            flags: Flags::new(false, Priority::Passive, false),
6927            ver: PROTOCOL_VERSION,
6928            root: root.clone(),
6929            session_id: "shutdown-session".to_string(),
6930            bare_name: "inspect".to_string(),
6931            format_context: crate::subc_format::FormatContext::from_tool_call(
6932                "inspect",
6933                &json!({}),
6934                dir.path(),
6935            ),
6936            bind_trust: BindTrust::FirstParty,
6937            pending,
6938            surface_downgraded: false,
6939            phase_trace: PhaseTrace::new(Instant::now()),
6940        });
6941        let active: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::from([(
6942            (route, 42),
6943            ActiveToolCall {
6944                root_id: root.clone(),
6945                cancellation,
6946                detach_policy: RouteDetachPolicy::CancelOnDetach,
6947            },
6948        )])));
6949
6950        let resolved = registry.drain_on_shutdown(executor.as_ref());
6951        assert!(registry.is_empty());
6952        assert_eq!(resolved.len(), 1);
6953        assert_eq!(
6954            resolved[0].response.data["failure_reason"],
6955            "daemon_shutdown"
6956        );
6957        finish_active_tool_call(&active, route, 42);
6958        let deadline = Instant::now() + Duration::from_secs(1);
6959        while crate::commands::inspect::deferred_inspect_root_count_for_test() != 0 {
6960            assert!(Instant::now() < deadline, "shutdown cancellation was inert");
6961            std::thread::sleep(Duration::from_millis(5));
6962        }
6963        assert!(active.lock().expect("active calls").is_empty());
6964        assert!(executor.actor_is_idle(&root));
6965    }
6966
6967    pub(super) fn wait_for_watcher_count(ctx: &AppContext, expected: usize) {
6968        let deadline = Instant::now() + Duration::from_secs(30);
6969        loop {
6970            let observed = ctx.watcher_registry_count();
6971            if observed == expected {
6972                return;
6973            }
6974            assert!(
6975                Instant::now() < deadline,
6976                "watcher count did not settle before deadline: expected={expected}, observed={observed}"
6977            );
6978            std::thread::sleep(Duration::from_millis(50));
6979        }
6980    }
6981
6982    /// Sweep until `root` is forgotten, mirroring how production reaps.
6983    ///
6984    /// `reap_idle_roots` probes actor idleness with a try-lock and retains the
6985    /// root when the scheduler holds that lock — correct behavior, since the
6986    /// real caller sweeps on a timer and simply catches the root next tick. A
6987    /// test that asserts a single sweep succeeds is therefore asserting it wins
6988    /// a lock race that nothing in production depends on: `register_actor` wakes
6989    /// the scheduler, which grabs the same lock, and on a loaded runner that
6990    /// window is wide enough to lose. Sweep to the outcome instead.
6991    pub(super) fn reap_until_forgotten(
6992        root: &ProjectRootId,
6993        live_roots: &mut HashMap<ProjectRootId, RootMeta>,
6994        pending_binds: &HashMap<RouteChannel, PendingBind>,
6995        root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
6996        executor: &Arc<Executor>,
6997        metrics: &DispatchPathMetrics,
6998    ) -> IdleReapOutcome {
6999        let deadline = Instant::now() + Duration::from_secs(30);
7000        loop {
7001            let outcome = reap_idle_roots(
7002                Instant::now(),
7003                live_roots,
7004                pending_binds,
7005                root_channels,
7006                executor,
7007                metrics,
7008            );
7009            if outcome.forgotten_deleted_roots.contains(root) {
7010                return outcome;
7011            }
7012            assert!(
7013                Instant::now() < deadline,
7014                "deleted root was never forgotten: {root:?}"
7015            );
7016            std::thread::sleep(Duration::from_millis(10));
7017        }
7018    }
7019
7020    pub(super) fn wait_for_actor_root_count(app: &App, expected: usize) {
7021        let deadline = Instant::now() + Duration::from_secs(30);
7022        loop {
7023            let observed = app.actor_root_count();
7024            if observed == expected {
7025                return;
7026            }
7027            assert!(
7028                Instant::now() < deadline,
7029                "actor root count did not settle before deadline: expected={expected}, observed={observed}"
7030            );
7031            std::thread::sleep(Duration::from_millis(50));
7032        }
7033    }
7034
7035    pub(super) fn status_frame(seq: u64) -> PushFrame {
7036        status_frame_with_session(seq, None)
7037    }
7038
7039    pub(super) fn status_frame_with_session(seq: u64, session_id: Option<&str>) -> PushFrame {
7040        PushFrame::StatusChanged(StatusChangedFrame {
7041            frame_type: "status_changed",
7042            session_id: session_id.map(str::to_string),
7043            snapshot: json!({ "seq": seq }),
7044        })
7045    }
7046
7047    pub(super) fn completion_frame(task_id: &str) -> PushFrame {
7048        completion_frame_with_session(task_id, "session-1")
7049    }
7050
7051    pub(super) fn completion_frame_with_session(task_id: &str, session_id: &str) -> PushFrame {
7052        PushFrame::BashCompleted(BashCompletedFrame {
7053            frame_type: "bash_completed",
7054            task_id: task_id.to_string(),
7055            session_id: session_id.to_string(),
7056            status: BgTaskStatus::Completed,
7057            exit_code: Some(0),
7058            command: format!("echo {task_id}"),
7059            output_preview: String::new(),
7060            bash_output_list_envelope: None,
7061            output_truncated: false,
7062            original_tokens: None,
7063            compressed_tokens: None,
7064            tokens_skipped: false,
7065            status_reason: None,
7066            live_descendants: Some(Vec::new()),
7067            live_descendants_omitted: 0,
7068            live_descendants_summary: None,
7069        })
7070    }
7071
7072    pub(super) fn long_running_frame(task_id: &str, elapsed_ms: u64) -> PushFrame {
7073        long_running_frame_with_session(task_id, "session-1", elapsed_ms)
7074    }
7075
7076    pub(super) fn long_running_frame_with_session(
7077        task_id: &str,
7078        session_id: &str,
7079        elapsed_ms: u64,
7080    ) -> PushFrame {
7081        PushFrame::BashLongRunning(BashLongRunningFrame {
7082            frame_type: "bash_long_running",
7083            task_id: task_id.to_string(),
7084            session_id: session_id.to_string(),
7085            command: format!("sleep {elapsed_ms}"),
7086            elapsed_ms,
7087        })
7088    }
7089
7090    pub(super) fn pattern_match_frame(session_id: &str) -> PushFrame {
7091        PushFrame::BashPatternMatch(BashPatternMatchFrame {
7092            frame_type: "bash_pattern_match",
7093            task_id: "task-pattern".to_string(),
7094            session_id: session_id.to_string(),
7095            watch_id: "watch-1".to_string(),
7096            match_text: "needle".to_string(),
7097            match_offset: 7,
7098            context: "haystack needle".to_string(),
7099            once: true,
7100            reason: "pattern_match",
7101        })
7102    }
7103
7104    pub(super) fn configure_warnings_frame(session_id: Option<&str>) -> PushFrame {
7105        PushFrame::ConfigureWarnings(ConfigureWarningsFrame {
7106            frame_type: "configure_warnings",
7107            session_id: session_id.map(str::to_string),
7108            project_root: "/tmp/subc-test".to_string(),
7109            warnings: Vec::new(),
7110        })
7111    }
7112
7113    pub(super) fn route_identity(root: &ProjectRootId, session_id: &str) -> RouteIdentity {
7114        route_identity_with_trust(root, session_id, BindTrust::FirstParty)
7115    }
7116
7117    pub(super) fn route_identity_with_trust(
7118        root: &ProjectRootId,
7119        session_id: &str,
7120        trust: BindTrust,
7121    ) -> RouteIdentity {
7122        RouteIdentity(Arc::new(RouteIdentityData {
7123            root: root.clone(),
7124            project_root: root.as_path().to_path_buf(),
7125            harness: "opencode".to_string(),
7126            session: session_id.to_string(),
7127            trust,
7128            spawn_principal: AuthenticatedPrincipal::RouteBind {
7129                trust: trust.sandbox_trust(),
7130                route_channel: 0,
7131                route_epoch: 0,
7132                project_root: root.as_path().to_path_buf(),
7133                harness: "opencode".to_string(),
7134                session_id: session_id.to_string(),
7135                principal_id: Some(match trust {
7136                    BindTrust::FirstParty => "direct".to_string(),
7137                    BindTrust::Untrusted => "unverified".to_string(),
7138                }),
7139            },
7140            consumer_elicitation_capable: false,
7141        }))
7142    }
7143
7144    pub(super) fn progress_frame(request_id: &str, kind: ProgressKind, chunk: &str) -> PushFrame {
7145        PushFrame::Progress(ProgressFrame::new(request_id, kind, chunk))
7146    }
7147
7148    pub(super) fn status_seq(frame: &PushFrame) -> Option<u64> {
7149        match frame {
7150            PushFrame::StatusChanged(status) => status.snapshot.get("seq").and_then(|v| v.as_u64()),
7151            _ => None,
7152        }
7153    }
7154
7155    pub(super) fn completion_task(frame: &PushFrame) -> Option<&str> {
7156        match frame {
7157            PushFrame::BashCompleted(completion) => Some(completion.task_id.as_str()),
7158            _ => None,
7159        }
7160    }
7161
7162    pub(super) fn push_frame_task_id(frame: &Frame) -> Option<String> {
7163        let body: serde_json::Value = serde_json::from_slice(&frame.body).expect("push body");
7164        body.get("task_id")
7165            .and_then(serde_json::Value::as_str)
7166            .map(str::to_string)
7167    }
7168}
7169
7170#[cfg(test)]
7171mod tests {
7172    use super::test_support::{
7173        completion_frame, reap_until_forgotten, route_identity, test_ctx, test_root,
7174        wait_for_actor_root_count, wait_for_watcher_count,
7175    };
7176    use super::*;
7177    use crate::bash_background::BgTaskStatus;
7178
7179    /// Only a daemon-requested stop may exit 0. The supervisor never respawns
7180    /// a clean exit, so a fatal-actor teardown that mapped to Ok(()) left the
7181    /// module down host-wide until a manual start (2026-09-11).
7182    #[test]
7183    fn only_a_graceful_goodbye_maps_to_a_clean_exit() {
7184        assert!(module_loop_exit_result(ModuleLoopExit::Graceful).is_ok());
7185        assert!(matches!(
7186            module_loop_exit_result(ModuleLoopExit::ConnectionLost),
7187            Err(SubcError::ConnectionLost)
7188        ));
7189        assert!(matches!(
7190            module_loop_exit_result(ModuleLoopExit::SkipSearchFlush),
7191            Err(SubcError::ActorFatal)
7192        ));
7193    }
7194
7195    #[test]
7196    fn fatal_panic_responses_are_detected_and_noted() {
7197        let panic = Response::error(
7198            "req-fatal",
7199            "actor_fatal",
7200            "start byte index 7 is not a char boundary",
7201        );
7202        assert!(note_fatal_panic_response(&panic));
7203        let ordinary = Response::error("req-ok", "invalid_request", "missing field");
7204        assert!(!note_fatal_panic_response(&ordinary));
7205    }
7206
7207    fn attach_error(kind: io::ErrorKind) -> SubcError {
7208        SubcError::Connect {
7209            endpoint: "127.0.0.1:1".to_string(),
7210            source: io::Error::new(kind, "constructed attach failure"),
7211        }
7212    }
7213
7214    fn auth_io_error(kind: io::ErrorKind) -> SubcError {
7215        SubcError::Auth {
7216            endpoint: "127.0.0.1:1".to_string(),
7217            source: subc_transport::AuthError::Io {
7218                stage: subc_transport::AuthStage::ServerProof,
7219                source: io::Error::new(kind, "constructed auth failure"),
7220            },
7221        }
7222    }
7223
7224    fn cpu_hunt_process_cpu_us() -> u64 {
7225        #[cfg(unix)]
7226        {
7227            let mut usage = std::mem::MaybeUninit::<libc::rusage>::uninit();
7228            // getrusage initializes the output on success; no pointer escapes.
7229            if unsafe { libc::getrusage(libc::RUSAGE_SELF, usage.as_mut_ptr()) } == 0 {
7230                let usage = unsafe { usage.assume_init() };
7231                return (usage.ru_utime.tv_sec + usage.ru_stime.tv_sec) as u64 * 1_000_000
7232                    + (usage.ru_utime.tv_usec + usage.ru_stime.tv_usec) as u64;
7233            }
7234        }
7235        0
7236    }
7237
7238    #[test]
7239    fn route_bind_does_not_recompute_fleet_health() {
7240        let runtime = tokio::runtime::Builder::new_current_thread()
7241            .enable_all()
7242            .build()
7243            .expect("bind runtime");
7244        runtime.block_on(async {
7245            let (dir, root) = test_root("bind-health-work-count");
7246            let app = App::default_shared();
7247            let executor = Arc::new(Executor::new());
7248            let ctx = Arc::new(AppContext::from_app(Arc::clone(&app), Config::default()));
7249            let mut fixture_dirs = Vec::new();
7250            // The opt-in probe accepts only an already-copied artifact below this
7251            // checkout's target directory; normal tests never open a live store.
7252            if let Some(copy) = std::env::var_os("AFT_CPU_HUNT_STORE_COPY") {
7253                let copy = std::fs::canonicalize(copy).expect("copied store exists");
7254                let project = Path::new(env!("CARGO_MANIFEST_DIR"))
7255                    .parent()
7256                    .expect("crates directory")
7257                    .parent()
7258                    .expect("checkout directory")
7259                    .canonicalize()
7260                    .expect("canonical checkout");
7261                assert!(copy.starts_with(project.join("target")));
7262                for index in 0..36 {
7263                    let actor = if index == 0 {
7264                        Arc::clone(&ctx)
7265                    } else {
7266                        Arc::new(AppContext::from_app(Arc::clone(&app), Config::default()))
7267                    };
7268                    actor.update_config(|config| config.project_root = Some(project.clone()));
7269                    *actor.callgraph_store().write().expect("store slot") = Some(Arc::new(
7270                        crate::callgraph_store::CallGraphStore::open_readonly(
7271                            copy.clone(),
7272                            project.clone(),
7273                        )
7274                        .expect("open copied graph")
7275                        .expect("copied graph ready"),
7276                    ));
7277                    if index > 0 {
7278                        let (fixture, id) = test_root(&format!("cpu-hunt-{index}"));
7279                        assert!(executor.register_actor(id, actor));
7280                        fixture_dirs.push(fixture);
7281                    }
7282                }
7283            }
7284            assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7285            let cache = HealthRollupCache::new();
7286            if !fixture_dirs.is_empty() {
7287                let started = Instant::now();
7288                let cpu = cpu_hunt_process_cpu_us();
7289                cache.refresh(&executor, &app);
7290                eprintln!(
7291                    "cpu_hunt health_rollup wall_us={} cpu_us={}",
7292                    started.elapsed().as_micros(),
7293                    cpu_hunt_process_cpu_us().saturating_sub(cpu)
7294                );
7295                let started = Instant::now();
7296                let cpu = cpu_hunt_process_cpu_us();
7297                std::hint::black_box(ctx.build_status_snapshot());
7298                eprintln!(
7299                    "cpu_hunt status wall_us={} cpu_us={}",
7300                    started.elapsed().as_micros(),
7301                    cpu_hunt_process_cpu_us().saturating_sub(cpu)
7302                );
7303            }
7304            let refreshes_before = cache.refresh_count_for_test();
7305            let metrics = Arc::new(DispatchPathMetrics::new());
7306            let (writer_tx, _writer_rx) = mpsc::channel(8);
7307            let (completion_tx, mut completion_rx) = mpsc::channel(8);
7308            let (lossy_tx, _lossy_rx) = mpsc::channel(8);
7309            let (reliable_tx, _reliable_rx) = mpsc::unbounded_channel();
7310            let senders = PushSenders {
7311                lossy_tx,
7312                reliable_tx,
7313                lossy_overflow: Arc::new(push::LossyOverflow::default()),
7314                lossy_seq: Arc::new(AtomicU64::new(0)),
7315                fleet_status_client: FleetStatusClient::channel(1).0,
7316            };
7317            let request = ModuleControlRequest::RouteBind {
7318                route_channel: 1,
7319                epoch: 1,
7320                target: RouteTarget::ToolProvider {
7321                    module_id: "aft".to_string(),
7322                },
7323                identity: subc_protocol::BindIdentity {
7324                    project_root: root.as_path().to_path_buf(),
7325                    harness: "opencode".to_string(),
7326                    session: "bind-health-work-count".to_string(),
7327                },
7328                principal: Some(subc_protocol::Principal::Direct),
7329                consumer_capabilities: None,
7330                admission_facts: Default::default(),
7331            };
7332            let frame = Frame::build_with_version(
7333                PROTOCOL_VERSION,
7334                FrameType::Request,
7335                control_flags(),
7336                0,
7337                0,
7338                1,
7339                serde_json::to_vec(&request).expect("bind body"),
7340            )
7341            .expect("bind frame");
7342            let mut pending_binds = HashMap::new();
7343            let started = Instant::now();
7344            let cpu = cpu_hunt_process_cpu_us();
7345            handle_control_request(
7346                &writer_tx,
7347                &frame,
7348                &app,
7349                &executor,
7350                &mut HashMap::new(),
7351                &mut pending_binds,
7352                &mut HashMap::new(),
7353                &mut HashMap::new(),
7354                &mut HashSet::new(),
7355                &mut HashMap::new(),
7356                &mut HashMap::new(),
7357                &mut HashMap::new(),
7358                &mut HashMap::new(),
7359                &mut HashMap::new(),
7360                &mut HashMap::new(),
7361                &Arc::new(StdMutex::new(HashMap::new())),
7362                &mut PendingSubcResponses::default(),
7363                &mut RetryBuffer::new(),
7364                &mut HashMap::new(),
7365                &Arc::new(Notify::new()),
7366                &completion_tx,
7367                &metrics,
7368                None,
7369                &cache,
7370                &senders,
7371                |request, _| Response::success(request.id, json!({})),
7372                Some(&dir.path().join("absent-user-config.json")),
7373                usize::MAX,
7374            )
7375            .await
7376            .expect("admit route bind");
7377            eprintln!(
7378                "route_bind_health admission_us={} cpu_us={} refreshes={}",
7379                started.elapsed().as_micros(),
7380                cpu_hunt_process_cpu_us().saturating_sub(cpu),
7381                cache.refresh_count_for_test() - refreshes_before
7382            );
7383            let completion = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
7384                .await
7385                .expect("configure completes")
7386                .expect("completion delivered");
7387            assert!(completion.configure_response.success);
7388            assert_eq!(pending_binds.len(), 1, "the bind must reach admission");
7389            assert_eq!(
7390                cache.refresh_count_for_test() - refreshes_before,
7391                0,
7392                "route admission must not perform a fleet-wide health census"
7393            );
7394        });
7395    }
7396
7397    #[test]
7398    fn channel_zero_health_response_does_not_wait_for_bash_background_db() {
7399        let (dir, root) = test_root("health-does-not-lock-bash-db");
7400        let executor = Arc::new(Executor::new());
7401        let ctx = test_ctx();
7402        ctx.set_harness(crate::harness::Harness::Opencode);
7403        let db = Arc::new(StdMutex::new(
7404            crate::db::open(&dir.path().join("health.db")).expect("open health test DB"),
7405        ));
7406        ctx.bash_background().set_db_pool(Arc::clone(&db));
7407        executor.register_actor(root, ctx);
7408
7409        let guard = db.lock().expect("hold bash-background DB mutex");
7410        let app = App::default_shared();
7411        let metrics = Arc::new(DispatchPathMetrics::new());
7412        let health_rollup_cache = Arc::new(HealthRollupCache::new());
7413        let frame = Frame::build_with_version(
7414            PROTOCOL_VERSION,
7415            FrameType::Request,
7416            control_flags(),
7417            0,
7418            0,
7419            77,
7420            Vec::new(),
7421        )
7422        .expect("health request frame");
7423        let (writer_tx, _writer_rx) = mpsc::channel::<WriterFrame>(1);
7424        let (done_tx, done_rx) = std::sync::mpsc::channel();
7425        let join = std::thread::spawn(move || {
7426            let runtime = tokio::runtime::Builder::new_current_thread()
7427                .enable_all()
7428                .build()
7429                .expect("health test runtime");
7430            let result = runtime.block_on(send_cached_health_response(
7431                &writer_tx,
7432                &frame,
7433                &app,
7434                &executor,
7435                &HashMap::new(),
7436                &metrics,
7437                &health_rollup_cache,
7438            ));
7439            done_tx.send(result).expect("report health result");
7440        });
7441
7442        let result = done_rx
7443            .recv_timeout(Duration::from_millis(500))
7444            .expect("channel-0 health blocked on bash-background DB mutex");
7445        result.expect("send cached health response");
7446        drop(guard);
7447        join.join().expect("health thread");
7448    }
7449
7450    #[test]
7451    fn maintenance_bg_runtime_refresh_deduplicates_shared_db_items() {
7452        let (_dir_a, root_a) = test_root("health-metric-root-a");
7453        let (_dir_b, root_b) = test_root("health-metric-root-b");
7454        let duplicate_key = "match\0session-1\0bash-0000000000000001\0watch-00000001";
7455        let snapshots = HashMap::from([
7456            (root_a, HashSet::from([duplicate_key.to_string()])),
7457            (root_b, HashSet::from([duplicate_key.to_string()])),
7458        ]);
7459        let metrics = DispatchPathMetrics::new();
7460
7461        record_bg_runtime_from_snapshots(&metrics, 2, 1, &snapshots);
7462
7463        assert_eq!(metrics.bg_runtime_for_test(), (2, 1, 1));
7464    }
7465
7466    #[test]
7467    fn initial_attach_error_classifier_distinguishes_transient_and_permanent_failures() {
7468        let transient_errors = vec![
7469            attach_error(io::ErrorKind::ConnectionRefused),
7470            attach_error(io::ErrorKind::TimedOut),
7471            attach_error(io::ErrorKind::ConnectionReset),
7472            auth_io_error(io::ErrorKind::ConnectionAborted),
7473            auth_io_error(io::ErrorKind::BrokenPipe),
7474            SubcError::Auth {
7475                endpoint: "127.0.0.1:1".to_string(),
7476                source: subc_transport::AuthError::UnexpectedEof {
7477                    stage: subc_transport::AuthStage::ServerProof,
7478                    expected: 4,
7479                    actual: 0,
7480                },
7481            },
7482            SubcError::Auth {
7483                endpoint: "127.0.0.1:1".to_string(),
7484                source: subc_transport::AuthError::Timeout {
7485                    stage: subc_transport::AuthStage::ServerProof,
7486                    deadline: AUTH_DEADLINE,
7487                },
7488            },
7489        ];
7490        for error in &transient_errors {
7491            assert_eq!(
7492                classify_attach_error(error),
7493                AttachErrorClass::Transient,
7494                "expected transient: {error}"
7495            );
7496        }
7497
7498        let permanent_errors = vec![
7499            attach_error(io::ErrorKind::PermissionDenied),
7500            auth_io_error(io::ErrorKind::InvalidData),
7501            SubcError::Auth {
7502                endpoint: "127.0.0.1:1".to_string(),
7503                source: subc_transport::AuthError::InvalidServerProof,
7504            },
7505            SubcError::Auth {
7506                endpoint: "127.0.0.1:1".to_string(),
7507                source: subc_transport::AuthError::DaemonIdMismatch,
7508            },
7509            SubcError::ConnectionFile {
7510                path: PathBuf::from("subc-connection.json"),
7511                source: subc_transport::ConnectionFileError::Invalid {
7512                    reason: "constructed invalid file".to_string(),
7513                },
7514            },
7515            SubcError::NoEndpoint {
7516                path: PathBuf::from("subc-connection.json"),
7517            },
7518            SubcError::InvalidEndpoint {
7519                path: PathBuf::from("subc-connection.json"),
7520                endpoint: "not-an-ip:1234".to_string(),
7521            },
7522        ];
7523        for error in &permanent_errors {
7524            assert_eq!(
7525                classify_attach_error(error),
7526                AttachErrorClass::Permanent,
7527                "expected permanent: {error}"
7528            );
7529        }
7530    }
7531
7532    #[test]
7533    fn incompatible_wire_version_is_rejected_before_tcp_connect() {
7534        let conn_dir = tempfile::tempdir().expect("connection tempdir");
7535        let conn_path = conn_dir.path().join("subc-connection.json");
7536        let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind listener");
7537        listener
7538            .set_nonblocking(true)
7539            .expect("set listener nonblocking");
7540        let port = listener.local_addr().expect("listener addr").port();
7541        connection_file::write_atomic(
7542            &conn_path,
7543            &connection_file::ConnectionInfo {
7544                schema: connection_file::SCHEMA_VERSION,
7545                wire_version: Some(PROTOCOL_VERSION.wrapping_add(1)),
7546                endpoints: vec![connection_file::Endpoint {
7547                    host: "127.0.0.1".to_string(),
7548                    port,
7549                }],
7550                key: vec![0x42; subc_transport::KEY_LEN],
7551                daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
7552                pid: std::process::id(),
7553                daemon_ver: "subc-test".to_string(),
7554            },
7555        )
7556        .expect("write connection file");
7557
7558        let runtime = tokio::runtime::Builder::new_current_thread()
7559            .enable_all()
7560            .build()
7561            .expect("test runtime");
7562        let result = runtime.block_on(connect_and_authenticate_with_policy(
7563            &conn_path,
7564            AttachRetryPolicy {
7565                budget: Duration::from_secs(1),
7566                initial_backoff: Duration::from_millis(5),
7567                max_backoff: Duration::from_millis(10),
7568                jitter_percent: 0,
7569            },
7570            None,
7571        ));
7572        assert!(matches!(
7573            result,
7574            Err(SubcError::ConnectionFile {
7575                source: connection_file::ConnectionFileError::WireVersionMismatch { .. },
7576                ..
7577            })
7578        ));
7579        assert!(matches!(
7580            listener.accept(),
7581            Err(error) if error.kind() == std::io::ErrorKind::WouldBlock
7582        ));
7583    }
7584
7585    #[test]
7586    fn initial_attach_unreachable_endpoint_retries_until_budget_then_fails_loud() {
7587        let conn_dir = tempfile::tempdir().expect("connection tempdir");
7588        let conn_path = conn_dir.path().join("subc-connection.json");
7589        let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve port");
7590        let port = listener.local_addr().expect("reserved addr").port();
7591        drop(listener);
7592        connection_file::write_atomic(
7593            &conn_path,
7594            &connection_file::ConnectionInfo {
7595                schema: connection_file::SCHEMA_VERSION,
7596                wire_version: Some(PROTOCOL_VERSION),
7597                endpoints: vec![connection_file::Endpoint {
7598                    host: "127.0.0.1".to_string(),
7599                    port,
7600                }],
7601                key: vec![0x42; subc_transport::KEY_LEN],
7602                daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
7603                pid: std::process::id(),
7604                daemon_ver: "subc-test".to_string(),
7605            },
7606        )
7607        .expect("write connection file");
7608
7609        let policy = AttachRetryPolicy {
7610            budget: Duration::from_millis(40),
7611            initial_backoff: Duration::from_millis(5),
7612            max_backoff: Duration::from_millis(10),
7613            jitter_percent: 0,
7614        };
7615        let runtime = tokio::runtime::Builder::new_current_thread()
7616            .enable_all()
7617            .build()
7618            .expect("test runtime");
7619        let started_at = Instant::now();
7620        let result = runtime.block_on(connect_and_authenticate_with_policy(
7621            &conn_path, policy, None,
7622        ));
7623        let elapsed = started_at.elapsed();
7624        let error = match result {
7625            Ok(_) => panic!("unreachable endpoint unexpectedly attached"),
7626            Err(error) => error,
7627        };
7628
7629        assert!(matches!(error, SubcError::Connect { .. }), "{error}");
7630        assert!(
7631            elapsed >= Duration::from_millis(35),
7632            "retry budget ended too early: {elapsed:?}"
7633        );
7634        assert!(
7635            elapsed < Duration::from_secs(1),
7636            "retry budget was not bounded: {elapsed:?}"
7637        );
7638    }
7639
7640    fn due_maintenance_jobs_without_actor_context(
7641        live_roots: &mut HashMap<ProjectRootId, RootMeta>,
7642        budget: usize,
7643        pending_bind_roots: &HashSet<ProjectRootId>,
7644    ) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
7645        due_maintenance_jobs(
7646            live_roots,
7647            None,
7648            &HashMap::new(),
7649            &BgWakePending::new(),
7650            budget,
7651            pending_bind_roots,
7652        )
7653    }
7654
7655    fn actor_ctx_with_dirty_search_index(
7656        root: &Path,
7657        storage: &Path,
7658        file_name: &str,
7659        old_contents: &str,
7660        new_contents: &str,
7661    ) -> (Arc<AppContext>, PathBuf, PathBuf) {
7662        let file = root.join(file_name);
7663        std::fs::write(&file, old_contents).expect("write source");
7664        let canonical_root = std::fs::canonicalize(root).expect("canonical root");
7665        let ctx = Arc::new(AppContext::new(
7666            Box::new(crate::parser::TreeSitterProvider::new()),
7667            Config {
7668                project_root: Some(root.to_path_buf()),
7669                storage_dir: Some(storage.to_path_buf()),
7670                ..Config::default()
7671            },
7672        ));
7673        ctx.set_canonical_cache_root(canonical_root.clone());
7674
7675        let cache_dir = crate::search_index::resolve_cache_dir(&canonical_root, Some(storage));
7676        let mut index = crate::search_index::SearchIndex::build(&canonical_root);
7677        let git_head = index.stored_git_head().map(str::to_owned);
7678        index.write_to_disk(&cache_dir, git_head.as_deref());
7679
7680        std::fs::write(&file, new_contents).expect("edit source");
7681        index.update_file(&file);
7682        *ctx.search_index()
7683            .write()
7684            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
7685        (ctx, canonical_root, cache_dir)
7686    }
7687
7688    #[test]
7689    fn graceful_shutdown_flushes_every_actor_search_index() {
7690        let storage = tempfile::tempdir().expect("storage tempdir");
7691        let (root1_dir, root1) = test_root("shutdown-flush-root-1");
7692        let (root2_dir, root2) = test_root("shutdown-flush-root-2");
7693        let (ctx1, canonical_root1, cache_dir1) = actor_ctx_with_dirty_search_index(
7694            root1_dir.path(),
7695            storage.path(),
7696            "alpha.txt",
7697            "old actor one token\n",
7698            "new actor one token\n",
7699        );
7700        let (ctx2, canonical_root2, cache_dir2) = actor_ctx_with_dirty_search_index(
7701            root2_dir.path(),
7702            storage.path(),
7703            "beta.txt",
7704            "old actor two token\n",
7705            "new actor two token\n",
7706        );
7707
7708        let executor = Executor::new();
7709        assert!(executor.register_actor(root1.clone(), Arc::clone(&ctx1)));
7710        assert!(executor.register_actor(root2.clone(), Arc::clone(&ctx2)));
7711
7712        flush_actor_indexes_on_graceful_shutdown(&executor.actor_contexts());
7713
7714        let mut restored1 =
7715            crate::search_index::SearchIndex::read_from_disk(&cache_dir1, &canonical_root1)
7716                .expect("load flushed root one index");
7717        restored1.ready = true;
7718        assert_eq!(
7719            restored1
7720                .grep("new actor one token", true, &[], &[], &canonical_root1, 10)
7721                .matches
7722                .len(),
7723            1,
7724            "graceful subc shutdown should flush the first root's trigram delta"
7725        );
7726
7727        let mut restored2 =
7728            crate::search_index::SearchIndex::read_from_disk(&cache_dir2, &canonical_root2)
7729                .expect("load flushed root two index");
7730        restored2.ready = true;
7731        assert_eq!(
7732            restored2
7733                .grep("new actor two token", true, &[], &[], &canonical_root2, 10)
7734                .matches
7735                .len(),
7736            1,
7737            "graceful subc shutdown should flush every registered root"
7738        );
7739    }
7740
7741    #[test]
7742    fn idle_root_reaper_closes_artifacts_and_stops_watcher() {
7743        let _ = env_logger::builder().is_test(true).try_init();
7744        let (root_dir, root) = test_root("idle-root-reaper");
7745        let storage = tempfile::tempdir().expect("storage tempdir");
7746        std::fs::write(
7747            root_dir.path().join("main.rs"),
7748            "fn entry() { leaf(); }\nfn leaf() {}\n",
7749        )
7750        .expect("source file");
7751        let canonical_root = std::fs::canonicalize(root_dir.path()).expect("canonical root");
7752        let app = App::default_shared();
7753        let ctx = Arc::new(AppContext::from_app(
7754            Arc::clone(&app),
7755            Config {
7756                project_root: Some(canonical_root.clone()),
7757                storage_dir: Some(storage.path().to_path_buf()),
7758                callgraph_store: true,
7759                search_index: true,
7760                ..Config::default()
7761            },
7762        ));
7763        ctx.set_canonical_cache_root(canonical_root.clone());
7764        let project_key = crate::search_index::artifact_cache_key(&canonical_root);
7765        crate::root_cache::configure_artifact_access(&canonical_root, &project_key, false);
7766        assert!(ctx
7767            .ensure_callgraph_store()
7768            .expect("build callgraph store")
7769            .is_some());
7770
7771        let cache_dir =
7772            crate::search_index::resolve_cache_dir(&canonical_root, Some(storage.path()));
7773        let mut index = crate::search_index::SearchIndex::build(&canonical_root);
7774        let git_head = index.stored_git_head().map(str::to_owned);
7775        index.write_to_disk(&cache_dir, git_head.as_deref());
7776        *ctx.search_index()
7777            .write()
7778            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
7779        // Seed a completed warm verification so the test can prove eviction
7780        // downgrades it to Strict rather than passing vacuously.
7781        let seeded_generation =
7782            crate::cache_freshness::artifact_generation(&cache_dir.join("cache.bin"))
7783                .expect("seeded artifact generation");
7784        crate::cache_freshness::record_verify_completed(
7785            &canonical_root,
7786            crate::cache_freshness::VerifyArtifact::Search,
7787            Some(seeded_generation),
7788        );
7789        assert!(
7790            matches!(
7791                crate::cache_freshness::warm_verify_plan(
7792                    canonical_root.as_path(),
7793                    crate::cache_freshness::VerifyArtifact::Search,
7794                    Some(seeded_generation),
7795                ),
7796                crate::cache_freshness::WarmVerifyPlan::Skip
7797            ),
7798            "memo must be warm before eviction for the downgrade assertion to bite"
7799        );
7800
7801        let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
7802        let _dispatch_tx = dispatch_tx;
7803        let shutdown = Arc::new(AtomicBool::new(false));
7804        let thread_shutdown = Arc::clone(&shutdown);
7805        let join = std::thread::spawn(move || {
7806            while !thread_shutdown.load(Ordering::SeqCst) {
7807                std::thread::yield_now();
7808            }
7809        });
7810        ctx.install_watcher_runtime(
7811            dispatch_rx,
7812            crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
7813        );
7814        wait_for_watcher_count(&ctx, 1);
7815
7816        let executor = Arc::new(Executor::new());
7817        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7818        ctx.mark_subc_unbound();
7819        let mut live_roots = HashMap::new();
7820        let mut meta = RootMeta::new(Instant::now());
7821        meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
7822        meta.unbound_quiesced = true;
7823        live_roots.insert(root.clone(), meta);
7824
7825        let message = idle_root_eviction_message(&root, &ctx.memory_root_snapshot(), None);
7826        assert!(message.contains("evicted idle root"));
7827        assert!(message.contains("freed ~"));
7828        assert!(message.contains("semantic"));
7829        assert!(!message.contains("semantic not estimated retained"));
7830        assert!(message.contains("trigram"));
7831        assert!(message.contains("retained: bash"));
7832        assert!(message.contains("parser_pool"));
7833
7834        assert_eq!(
7835            reap_idle_roots(
7836                Instant::now(),
7837                &mut live_roots,
7838                &HashMap::new(),
7839                &HashMap::new(),
7840                &executor,
7841                &DispatchPathMetrics::new(),
7842            )
7843            .evicted,
7844            1
7845        );
7846        assert!(ctx.search_index().read().unwrap().is_none());
7847        wait_for_watcher_count(&ctx, 0);
7848        // The watcher stopped with the eviction, so the idle interval is
7849        // unobserved: the pre-seeded warm-verify memo (Skip) must fall back to
7850        // strict content verification (stat-first would miss same-size,
7851        // preserved-mtime edits made while nobody was watching).
7852        assert!(
7853            matches!(
7854                crate::cache_freshness::warm_verify_plan(
7855                    canonical_root.as_path(),
7856                    crate::cache_freshness::VerifyArtifact::Search,
7857                    Some(seeded_generation),
7858                ),
7859                crate::cache_freshness::WarmVerifyPlan::Strict
7860            ),
7861            "idle eviction must force strict re-verification"
7862        );
7863        assert!(
7864            crate::search_index::SearchIndex::read_from_disk(&cache_dir, &canonical_root).is_some()
7865        );
7866        ctx.mark_subc_bound();
7867        assert!(ctx
7868            .ensure_callgraph_store()
7869            .expect("reopen callgraph store")
7870            .is_some());
7871        assert!(live_roots[&root].idle_artifacts_evicted);
7872    }
7873
7874    #[test]
7875    fn idle_root_reaper_applies_ttl_to_unbound_roots() {
7876        let (_root_dir, root) = test_root("idle-root-ttl-gate");
7877        let ctx = test_ctx();
7878        let executor = Arc::new(Executor::new());
7879        assert!(executor.register_actor(root.clone(), ctx));
7880        let ctx = executor.actor_context(&root).expect("actor context");
7881        ctx.mark_subc_unbound();
7882        let now = Instant::now();
7883        let mut meta = RootMeta::new(now);
7884        meta.unbound_quiesced = true;
7885        let mut live_roots = HashMap::from([(root.clone(), meta)]);
7886
7887        // A recently-unbound root stays warm: a transient unbind (host
7888        // restart) must not pay the strict-verify + forced-rebuild teardown
7889        // on the next maintenance sweep.
7890        assert_eq!(
7891            reap_idle_roots(
7892                now,
7893                &mut live_roots,
7894                &HashMap::new(),
7895                &HashMap::new(),
7896                &executor,
7897                &DispatchPathMetrics::new(),
7898            )
7899            .evicted,
7900            0
7901        );
7902        assert!(!live_roots[&root].idle_artifacts_evicted);
7903
7904        // Past the TTL the same unbound root pays the full teardown. The
7905        // pending reconciliation paths retained across the transient-unbind
7906        // window would block eviction forever through
7907        // `artifact_eviction_blocked`; the reaper disposes them because the
7908        // strict gap invalidation subsumes their purpose.
7909        ctx.add_pending_search_index_paths([root.as_path().join("retained.rs")]);
7910        assert_eq!(
7911            reap_idle_roots(
7912                now + IDLE_ROOT_TTL,
7913                &mut live_roots,
7914                &HashMap::new(),
7915                &HashMap::new(),
7916                &executor,
7917                &DispatchPathMetrics::new(),
7918            )
7919            .evicted,
7920            1
7921        );
7922        assert!(live_roots[&root].idle_artifacts_evicted);
7923        assert!(
7924            ctx.take_pending_search_index_paths().is_empty(),
7925            "TTL eviction must dispose retained pending reconciliation paths"
7926        );
7927    }
7928
7929    #[test]
7930    fn blocked_ttl_eviction_restores_taken_pending_reconciliation_state() {
7931        let (_root_dir, root) = test_root("ttl-eviction-blocked-restore");
7932        let ctx = test_ctx();
7933        let executor = Arc::new(Executor::new());
7934        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7935        ctx.mark_subc_unbound();
7936
7937        // Retained pending path from the transient-unbind window plus a
7938        // SECONDARY eviction blocker (a non-ready resident search index, the
7939        // dirty-index blocker in artifact_eviction_blocked). Disposal must be
7940        // transactional: the blocked eviction may be followed by a rebind, and
7941        // the path is the only repair record for its consumed watcher event.
7942        let pending = root.as_path().join("edited-while-unbound.rs");
7943        ctx.add_pending_search_index_paths([pending.clone()]);
7944        let dirty_source = root.as_path().join("dirty.rs");
7945        std::fs::write(&dirty_source, "fn dirty() {}\n").expect("dirty source");
7946        let mut dirty = crate::search_index::SearchIndex::new();
7947        dirty.ready = true;
7948        dirty.update_file(&dirty_source);
7949        *ctx.search_index()
7950            .write()
7951            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(dirty);
7952        assert!(ctx.artifact_eviction_blocked());
7953
7954        let mut live_roots = HashMap::new();
7955        let mut meta = RootMeta::new(Instant::now());
7956        meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
7957        meta.unbound_quiesced = true;
7958        live_roots.insert(root.clone(), meta);
7959
7960        assert_eq!(
7961            reap_idle_roots(
7962                Instant::now(),
7963                &mut live_roots,
7964                &HashMap::new(),
7965                &HashMap::new(),
7966                &executor,
7967                &DispatchPathMetrics::new(),
7968            )
7969            .evicted,
7970            0,
7971            "the dirty index must still block this eviction"
7972        );
7973        assert_eq!(
7974            ctx.take_pending_search_index_paths(),
7975            vec![pending],
7976            "a blocked eviction must restore the taken pending paths"
7977        );
7978    }
7979
7980    #[test]
7981    fn idle_reap_with_bound_route_keeps_watcher_running() {
7982        let (_root_dir, root) = test_root("bound-root-reap-gate");
7983        let ctx = test_ctx();
7984        let executor = Arc::new(Executor::new());
7985        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7986
7987        let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
7988        let _dispatch_tx = dispatch_tx;
7989        let shutdown = Arc::new(AtomicBool::new(false));
7990        let thread_shutdown = Arc::clone(&shutdown);
7991        let join = std::thread::spawn(move || {
7992            while !thread_shutdown.load(Ordering::SeqCst) {
7993                std::thread::yield_now();
7994            }
7995        });
7996        ctx.install_watcher_runtime(
7997            dispatch_rx,
7998            crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
7999        );
8000
8001        let mut meta = RootMeta::new(Instant::now());
8002        meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
8003        let mut live_roots = HashMap::from([(root.clone(), meta)]);
8004        let bound = HashMap::from([(root, HashSet::from([route_key(7, 1)]))]);
8005        assert_eq!(
8006            reap_idle_roots(
8007                Instant::now(),
8008                &mut live_roots,
8009                &HashMap::new(),
8010                &bound,
8011                &executor,
8012                &DispatchPathMetrics::new(),
8013            )
8014            .evicted,
8015            0
8016        );
8017        wait_for_watcher_count(&ctx, 1);
8018        ctx.stop_watcher_runtime_in_background();
8019        wait_for_watcher_count(&ctx, 0);
8020    }
8021
8022    #[test]
8023    fn deleted_root_with_bound_route_is_reclaimed_after_confirmation_and_routes_are_purged() {
8024        let (root_dir, root) = test_root("deleted-bound-root-reap");
8025        let executor = Arc::new(Executor::new());
8026        assert!(executor.register_actor(root.clone(), test_ctx()));
8027        root_dir.close().expect("delete project root");
8028
8029        let route = route_key(19, 3);
8030        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
8031        let cancel_signal = PersistentCancelSignal::new();
8032        let mut routes = HashMap::from([(route, route_identity(&root, "deleted-route"))]);
8033        let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
8034        let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
8035        let mut route_bash_cancels = HashMap::from([(
8036            route,
8037            bash::RouteBashCancel {
8038                token: cancel_signal.clone(),
8039                active_waits: 0,
8040            },
8041        )]);
8042        let metrics = DispatchPathMetrics::new();
8043
8044        let first = reap_idle_roots(
8045            Instant::now(),
8046            &mut live_roots,
8047            &HashMap::new(),
8048            &root_channels,
8049            &executor,
8050            &metrics,
8051        );
8052        assert!(first.forgotten_deleted_roots.is_empty());
8053        assert!(executor.actor_registered(&root));
8054
8055        let mut forgotten = Vec::new();
8056        for _ in 0..100 {
8057            let outcome = reap_idle_roots(
8058                Instant::now(),
8059                &mut live_roots,
8060                &HashMap::new(),
8061                &root_channels,
8062                &executor,
8063                &metrics,
8064            );
8065            if !outcome.forgotten_deleted_roots.is_empty() {
8066                forgotten = outcome.forgotten_deleted_roots;
8067                break;
8068            }
8069            std::thread::sleep(Duration::from_millis(10));
8070        }
8071        assert_eq!(forgotten, vec![root.clone()]);
8072        assert!(!executor.actor_registered(&root));
8073
8074        let mut retry_buffer = HashMap::new();
8075        let mut reclaimed_routes = ReclaimedRoutes::default();
8076        let mut session_identity = HashMap::new();
8077        let mut push_buffer = HashMap::new();
8078        let mut bg_subs = HashMap::from([(
8079            route,
8080            BgSub {
8081                corr: 77,
8082                ver: PROTOCOL_VERSION,
8083                flags: control_flags(),
8084                root: root.clone(),
8085                session: "deleted-route".to_string(),
8086            },
8087        )]);
8088        let mut bg_sub_by_session = HashMap::from([(
8089            (root.clone(), "deleted-route".to_string()),
8090            HashSet::from([route]),
8091        )]);
8092        let mut bg_wake_pending =
8093            BgWakePending::from([(route, BgWakeState::armed(Instant::now()))]);
8094        let mut bg_wake_epoch = HashMap::new();
8095        let mut pending_bash_asks = HashMap::new();
8096        let active_tool_calls: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
8097        health::take_bg_observability_logs_for_test();
8098        purge_deleted_root_residents(
8099            &root,
8100            &mut routes,
8101            &mut root_channels,
8102            &mut installed_route_epochs,
8103            &mut route_bash_cancels,
8104            &active_tool_calls,
8105            executor.as_ref(),
8106            &mut retry_buffer,
8107            &mut reclaimed_routes,
8108            &mut session_identity,
8109            &mut push_buffer,
8110            &mut bg_subs,
8111            &mut bg_sub_by_session,
8112            &mut bg_wake_pending,
8113            &mut bg_wake_epoch,
8114            &mut pending_bash_asks,
8115            &metrics,
8116        );
8117
8118        assert!(routes.is_empty());
8119        assert!(root_channels.is_empty());
8120        assert!(installed_route_epochs.is_empty());
8121        assert!(route_bash_cancels.is_empty());
8122        assert!(reclaimed_routes.contains(route));
8123        assert!(cancel_signal.is_cancelled());
8124        assert_eq!(
8125            health::take_bg_observability_logs_for_test(),
8126            vec![format!(
8127                "subc bg subscription: ended root={} session=deleted-route channel=19@3 cause=root-reclaim suppressed=0",
8128                root.as_path().display()
8129            )]
8130        );
8131    }
8132
8133    /// The control for the deleted-root reclamation above: a root whose
8134    /// directory still EXISTS must stay retained while it holds a bound route,
8135    /// even with every other reap precondition satisfied. Without this, the
8136    /// suite cannot tell "reclaim roots that are provably gone" apart from
8137    /// "reap any root that looks idle" — the second would tear down live
8138    /// sessions, and both satisfy the deleted-root tests.
8139    #[test]
8140    fn live_root_with_bound_route_is_never_reclaimed() {
8141        let (_root_dir, root) = test_root("live-bound-root-retained");
8142        let ctx = test_ctx();
8143        ctx.mark_subc_unbound();
8144        let executor = Arc::new(Executor::new());
8145        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
8146
8147        let route = RouteChannel {
8148            channel: 7,
8149            epoch: 1,
8150        };
8151        let mut meta = RootMeta::new(Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1));
8152        meta.unbound_quiesced = true;
8153        let mut live_roots = HashMap::from([(root.clone(), meta)]);
8154        let root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
8155
8156        // Sweep three times because reclamation requires the root path to be
8157        // observed missing on two CONSECUTIVE sweeps. A single sweep would pass
8158        // here even if reclamation were wrongly unconditional, since the first
8159        // absence never reclaims on its own.
8160        for _ in 0..3 {
8161            let outcome = reap_idle_roots(
8162                Instant::now(),
8163                &mut live_roots,
8164                &HashMap::new(),
8165                &root_channels,
8166                &executor,
8167                &DispatchPathMetrics::new(),
8168            );
8169            assert!(
8170                outcome.forgotten_deleted_roots.is_empty(),
8171                "a root whose directory exists must never be forgotten"
8172            );
8173        }
8174
8175        assert!(live_roots.contains_key(&root), "live root must be retained");
8176        assert!(
8177            executor.actor_registered(&root),
8178            "live root's actor must survive"
8179        );
8180        assert!(
8181            root.as_path().exists(),
8182            "test vehicle must keep the directory alive; otherwise this control proves nothing"
8183        );
8184    }
8185
8186    #[test]
8187    fn deleted_root_is_not_reclaimed_on_first_absence_observation() {
8188        let (root_dir, root) = test_root("deleted-root-first-observation");
8189        let ctx = test_ctx();
8190        ctx.mark_subc_unbound();
8191        let executor = Arc::new(Executor::new());
8192        assert!(executor.register_actor(root.clone(), ctx));
8193        root_dir.close().expect("delete project root");
8194
8195        let mut meta = RootMeta::new(Instant::now());
8196        meta.unbound_quiesced = true;
8197        let mut live_roots = HashMap::from([(root.clone(), meta)]);
8198        let outcome = reap_idle_roots(
8199            Instant::now(),
8200            &mut live_roots,
8201            &HashMap::new(),
8202            &HashMap::new(),
8203            &executor,
8204            &DispatchPathMetrics::new(),
8205        );
8206
8207        assert!(outcome.forgotten_deleted_roots.is_empty());
8208        assert!(live_roots.contains_key(&root));
8209        assert!(executor.actor_registered(&root));
8210    }
8211
8212    fn spawn_background_for_root(
8213        ctx: &AppContext,
8214        root: &ProjectRootId,
8215        storage: &tempfile::TempDir,
8216        session_id: &str,
8217    ) -> (String, u32) {
8218        // Windows refuses to delete a directory that is a running process's
8219        // cwd (ERROR_SHARING_VIOLATION), so the task must not live inside the
8220        // project root these tests delete. The kill path matches on the task's
8221        // registered project_root, not its cwd, so pointing the cwd at task
8222        // storage keeps the association under test intact.
8223        let command = if cfg!(windows) {
8224            // timeout.exe requires a console; ping is the standard sleep shim.
8225            "ping -n 31 127.0.0.1 > nul"
8226        } else {
8227            "sleep 30"
8228        };
8229        let task_id = ctx
8230            .bash_background()
8231            .spawn(
8232                crate::sandbox_spawn::SpawnPlan::Unsandboxed,
8233                command,
8234                session_id.to_string(),
8235                storage.path().to_path_buf(),
8236                HashMap::new(),
8237                Some(Duration::from_secs(60)),
8238                storage.path().to_path_buf(),
8239                8,
8240                true,
8241                false,
8242                Some(root.as_path().to_path_buf()),
8243            )
8244            .expect("spawn background task");
8245        let snapshot = ctx
8246            .bash_background()
8247            .status(
8248                &task_id,
8249                session_id,
8250                Some(root.as_path()),
8251                Some(storage.path()),
8252                0,
8253            )
8254            .expect("background task status");
8255        (task_id, snapshot.child_pid.expect("background child pid"))
8256    }
8257
8258    fn wait_for_background_exit(pid: u32) {
8259        let deadline = Instant::now() + Duration::from_secs(5);
8260        while crate::bash_background::process::is_process_alive(pid) {
8261            assert!(
8262                Instant::now() < deadline,
8263                "background task process survived kill"
8264            );
8265            std::thread::sleep(Duration::from_millis(20));
8266        }
8267    }
8268
8269    #[test]
8270    fn deleted_root_reclaims_background_task_after_two_absence_sweeps() {
8271        let (root_dir, root) = test_root("deleted-root-background-task");
8272        let storage = tempfile::tempdir().expect("task storage");
8273        let ctx = test_ctx();
8274        let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "reclaim-session");
8275        assert!(crate::bash_background::process::is_process_alive(pid));
8276
8277        let executor = Arc::new(Executor::new());
8278        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
8279        assert!(executor.actor_is_idle(&root));
8280        root_dir.close().expect("delete project root");
8281        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
8282        let pending_binds = HashMap::new();
8283        let root_channels = HashMap::new();
8284        let metrics = DispatchPathMetrics::new();
8285
8286        let first = reap_idle_roots(
8287            Instant::now(),
8288            &mut live_roots,
8289            &pending_binds,
8290            &root_channels,
8291            &executor,
8292            &metrics,
8293        );
8294        assert!(first.forgotten_deleted_roots.is_empty());
8295        assert!(crate::bash_background::process::is_process_alive(pid));
8296
8297        let outcome = reap_until_forgotten(
8298            &root,
8299            &mut live_roots,
8300            &pending_binds,
8301            &root_channels,
8302            &executor,
8303            &metrics,
8304        );
8305        assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
8306        wait_for_background_exit(pid);
8307
8308        let snapshot = ctx
8309            .bash_background()
8310            .status(
8311                &task_id,
8312                "reclaim-session",
8313                Some(root.as_path()),
8314                Some(storage.path()),
8315                0,
8316            )
8317            .expect("reclaimed task status");
8318        assert_eq!(snapshot.info.status, BgTaskStatus::Killed);
8319        assert_eq!(
8320            snapshot.info.status_reason.as_deref(),
8321            Some(crate::bash_background::registry::ROOT_RECLAIMED_REASON)
8322        );
8323        assert_eq!(
8324            serde_json::to_value(&snapshot).expect("serialize bash status")["status_reason"],
8325            crate::bash_background::registry::ROOT_RECLAIMED_REASON
8326        );
8327        let completion = ctx
8328            .bash_background()
8329            .drain_completions_for_session(Some("reclaim-session"))
8330            .pop()
8331            .expect("reclaimed task completion");
8332        assert_eq!(
8333            completion.status_reason.as_deref(),
8334            Some(crate::bash_background::registry::ROOT_RECLAIMED_REASON)
8335        );
8336    }
8337
8338    #[test]
8339    fn existing_unbound_root_keeps_background_task_alive_across_sweeps() {
8340        let (root_dir, root) = test_root("existing-root-background-task");
8341        let storage = tempfile::tempdir().expect("task storage");
8342        let ctx = test_ctx();
8343        ctx.mark_subc_unbound();
8344        let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "existing-session");
8345
8346        let executor = Arc::new(Executor::new());
8347        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
8348        let mut meta = RootMeta::new(
8349            Instant::now()
8350                .checked_sub(IDLE_ROOT_TTL + Duration::from_secs(1))
8351                .expect("old root timestamp"),
8352        );
8353        meta.unbound_quiesced = true;
8354        let mut live_roots = HashMap::from([(root.clone(), meta)]);
8355        let pending_binds = HashMap::new();
8356        let root_channels = HashMap::new();
8357        let metrics = DispatchPathMetrics::new();
8358
8359        for _ in 0..8 {
8360            reap_idle_roots(
8361                Instant::now(),
8362                &mut live_roots,
8363                &pending_binds,
8364                &root_channels,
8365                &executor,
8366                &metrics,
8367            );
8368            std::thread::sleep(Duration::from_millis(10));
8369        }
8370        assert!(root_dir.path().exists());
8371        assert!(crate::bash_background::process::is_process_alive(pid));
8372        let snapshot = ctx
8373            .bash_background()
8374            .status(
8375                &task_id,
8376                "existing-session",
8377                Some(root.as_path()),
8378                Some(storage.path()),
8379                0,
8380            )
8381            .expect("existing task status");
8382        assert_eq!(snapshot.info.status, BgTaskStatus::Running);
8383        let _ = ctx.bash_background().kill(&task_id, "existing-session");
8384        wait_for_background_exit(pid);
8385    }
8386
8387    #[test]
8388    fn restored_root_between_absence_sweeps_keeps_background_task_alive() {
8389        let (root_dir, root) = test_root("restored-root-background-task");
8390        let storage = tempfile::tempdir().expect("task storage");
8391        let ctx = test_ctx();
8392        let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "restored-session");
8393
8394        let executor = Arc::new(Executor::new());
8395        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
8396        root_dir.close().expect("delete project root");
8397        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
8398        let pending_binds = HashMap::new();
8399        let root_channels = HashMap::new();
8400        let metrics = DispatchPathMetrics::new();
8401
8402        let first = reap_idle_roots(
8403            Instant::now(),
8404            &mut live_roots,
8405            &pending_binds,
8406            &root_channels,
8407            &executor,
8408            &metrics,
8409        );
8410        assert!(first.forgotten_deleted_roots.is_empty());
8411        std::fs::create_dir_all(root.as_path()).expect("restore project root");
8412        let second = reap_idle_roots(
8413            Instant::now(),
8414            &mut live_roots,
8415            &pending_binds,
8416            &root_channels,
8417            &executor,
8418            &metrics,
8419        );
8420        assert!(second.forgotten_deleted_roots.is_empty());
8421        assert!(crate::bash_background::process::is_process_alive(pid));
8422        let _ = ctx.bash_background().kill(&task_id, "restored-session");
8423        wait_for_background_exit(pid);
8424    }
8425
8426    #[test]
8427    fn observing_root_again_resets_deleted_sweep_confirmation() {
8428        let (root_dir, root) = test_root("deleted-root-observation-reset");
8429        let ctx = test_ctx();
8430        ctx.mark_subc_unbound();
8431        let executor = Arc::new(Executor::new());
8432        assert!(executor.register_actor(root.clone(), ctx));
8433        root_dir.close().expect("delete project root");
8434
8435        let mut meta = RootMeta::new(Instant::now());
8436        meta.unbound_quiesced = true;
8437        let mut live_roots = HashMap::from([(root.clone(), meta)]);
8438        let pending_binds = HashMap::new();
8439        let root_channels = HashMap::new();
8440        let metrics = DispatchPathMetrics::new();
8441
8442        let first = reap_idle_roots(
8443            Instant::now(),
8444            &mut live_roots,
8445            &pending_binds,
8446            &root_channels,
8447            &executor,
8448            &metrics,
8449        );
8450        assert!(first.forgotten_deleted_roots.is_empty());
8451
8452        std::fs::create_dir_all(root.as_path()).expect("restore project root");
8453        reap_idle_roots(
8454            Instant::now(),
8455            &mut live_roots,
8456            &pending_binds,
8457            &root_channels,
8458            &executor,
8459            &metrics,
8460        );
8461        std::fs::remove_dir_all(root.as_path()).expect("delete project root again");
8462
8463        let after_reset = reap_idle_roots(
8464            Instant::now(),
8465            &mut live_roots,
8466            &pending_binds,
8467            &root_channels,
8468            &executor,
8469            &metrics,
8470        );
8471        assert!(after_reset.forgotten_deleted_roots.is_empty());
8472        assert!(live_roots.contains_key(&root));
8473        assert!(executor.actor_registered(&root));
8474    }
8475
8476    #[test]
8477    fn deleted_idle_root_is_fully_forgotten_and_status_counts_drop() {
8478        let (root_dir, root) = test_root("deleted-root-reap");
8479        let app = App::default_shared();
8480        let ctx = Arc::new(AppContext::from_app(
8481            Arc::clone(&app),
8482            Config {
8483                project_root: Some(root.as_path().to_path_buf()),
8484                ..Config::default()
8485            },
8486        ));
8487        ctx.set_canonical_cache_root(root.as_path().to_path_buf());
8488        ctx.mark_subc_unbound();
8489        let executor = Arc::new(Executor::new());
8490        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
8491        assert_eq!(app.actor_root_count(), 1);
8492        drop(ctx);
8493        root_dir.close().expect("delete project root");
8494
8495        let mut meta = RootMeta::new(Instant::now());
8496        meta.unbound_quiesced = true;
8497        let mut live_roots = HashMap::from([(root.clone(), meta)]);
8498        let outcome = reap_until_forgotten(
8499            &root,
8500            &mut live_roots,
8501            &HashMap::new(),
8502            &HashMap::new(),
8503            &executor,
8504            &DispatchPathMetrics::new(),
8505        );
8506        assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
8507        assert!(!executor.actor_registered(&root));
8508        assert!(!live_roots.contains_key(&root));
8509        wait_for_actor_root_count(&app, 0);
8510
8511        let status_ctx = AppContext::from_app(app, Config::default());
8512        let status = status_ctx.build_status_snapshot();
8513        assert_eq!(status["runtime"]["live_actor_roots"], 0);
8514        assert_eq!(status["runtime"]["open_routes"], 0);
8515    }
8516
8517    #[test]
8518    fn deleted_root_reap_blocker_census_is_exposed_in_health_metrics() {
8519        let (root_dir, root) = test_root("deleted-root-reap-census");
8520        let executor = Arc::new(Executor::new());
8521        assert!(executor.register_actor(root.clone(), test_ctx()));
8522        root_dir.close().expect("delete project root");
8523
8524        let mut live_roots = HashMap::from([(root, RootMeta::new(Instant::now()))]);
8525        let metrics = DispatchPathMetrics::new();
8526        let outcome = reap_idle_roots(
8527            Instant::now(),
8528            &mut live_roots,
8529            &HashMap::new(),
8530            &HashMap::new(),
8531            &executor,
8532            &metrics,
8533        );
8534        assert_eq!(outcome.evicted, 0);
8535
8536        let app = crate::context::App::default_shared();
8537        let health_rollup_cache = HealthRollupCache::new();
8538        health_rollup_cache.refresh(&executor, &app);
8539        let report = build_health_report(
8540            &health_rollup_cache,
8541            &executor,
8542            &HashMap::new(),
8543            &metrics,
8544            &app,
8545        );
8546        let reap = report
8547            .metrics
8548            .as_ref()
8549            .and_then(|metrics| metrics.get("reap"))
8550            .expect("reap health metrics");
8551        assert_eq!(reap["deleted_retained"].as_u64(), Some(1));
8552        assert_eq!(reap["blockers"]["absence_unconfirmed"].as_u64(), Some(1));
8553        assert_eq!(reap["blockers"]["unbound_quiesced"].as_u64(), Some(0));
8554        assert_eq!(reap["blockers"]["actor_busy"].as_u64(), Some(0));
8555    }
8556
8557    #[test]
8558    fn connection_exit_quiesces_queued_maintenance_and_deleted_root_is_purged() {
8559        let (root_dir, root) = test_root("connection-exit-deleted-root");
8560        let executor = Arc::new(Executor::new());
8561        assert!(executor.register_actor(root.clone(), test_ctx()));
8562
8563        let route = route_key(11, 1);
8564        let mut meta = RootMeta::new(Instant::now());
8565        meta.maintenance_pending = true;
8566        meta.maintenance_queued_kinds
8567            .push_back(MaintenanceDrainKind::CompletionDrains);
8568        let mut live_roots = HashMap::from([(root.clone(), meta)]);
8569        let mut pending_binds = HashMap::new();
8570        let mut routes = HashMap::from([(route, route_identity(&root, "abandoned"))]);
8571        let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
8572        let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
8573        let mut route_bash_cancels = HashMap::new();
8574        let active_tool_calls: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
8575
8576        quiesce_connection_roots(
8577            &mut live_roots,
8578            &mut pending_binds,
8579            &mut routes,
8580            &mut root_channels,
8581            &mut installed_route_epochs,
8582            &mut route_bash_cancels,
8583            &active_tool_calls,
8584            &executor,
8585        );
8586        assert!(live_roots[&root].unbound_quiesced);
8587        assert!(!live_roots[&root].maintenance_pending);
8588        assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
8589        assert!(routes.is_empty());
8590        assert!(root_channels.is_empty());
8591
8592        root_dir.close().expect("delete project root");
8593        let metrics = DispatchPathMetrics::new();
8594        let outcome = reap_until_forgotten(
8595            &root,
8596            &mut live_roots,
8597            &pending_binds,
8598            &root_channels,
8599            &executor,
8600            &metrics,
8601        );
8602        let mut session_identity = HashMap::new();
8603        let mut push_buffer = HashMap::new();
8604        let mut bg_subs = HashMap::new();
8605        let mut bg_sub_by_session = HashMap::new();
8606        let mut bg_wake_pending = BgWakePending::new();
8607        let mut bg_wake_epoch = HashMap::new();
8608        let mut pending_bash_asks = HashMap::new();
8609        let mut retry_buffer = HashMap::new();
8610        let mut reclaimed_routes = ReclaimedRoutes::default();
8611        for forgotten in &outcome.forgotten_deleted_roots {
8612            purge_deleted_root_residents(
8613                forgotten,
8614                &mut routes,
8615                &mut root_channels,
8616                &mut installed_route_epochs,
8617                &mut route_bash_cancels,
8618                &active_tool_calls,
8619                executor.as_ref(),
8620                &mut retry_buffer,
8621                &mut reclaimed_routes,
8622                &mut session_identity,
8623                &mut push_buffer,
8624                &mut bg_subs,
8625                &mut bg_sub_by_session,
8626                &mut bg_wake_pending,
8627                &mut bg_wake_epoch,
8628                &mut pending_bash_asks,
8629                &metrics,
8630            );
8631        }
8632
8633        assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
8634        assert!(!executor.actor_registered(&root));
8635        assert!(!live_roots.contains_key(&root));
8636    }
8637
8638    #[test]
8639    fn unbound_root_quiesces_maintenance_without_removing_actor() {
8640        let (_root_dir, root) = test_root("unbound-root-quiesce");
8641        let ctx = test_ctx();
8642        let executor = Arc::new(Executor::new());
8643        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
8644        let mut meta = RootMeta::new(Instant::now());
8645        meta.maintenance_pending = true;
8646        meta.maintenance_jobs_in_flight = 1;
8647        meta.maintenance_queued_kinds
8648            .push_back(MaintenanceDrainKind::ConfigureTail);
8649        let mut live_roots = HashMap::from([(root.clone(), meta)]);
8650        // Warm state planted before the unbind: quiesce must keep it.
8651        *ctx.search_index()
8652            .write()
8653            .unwrap_or_else(std::sync::PoisonError::into_inner) =
8654            Some(crate::search_index::SearchIndex::new());
8655        ctx.set_cache_writer_capabilities(true, true);
8656        let pending = root.as_path().join("pending.rs");
8657        ctx.add_pending_search_index_paths([pending.clone()]);
8658        // A warm verify memo must survive the transient unbind: the watcher
8659        // keeps running, so no unobserved window exists and the next warm
8660        // reload must not pay a strict full-corpus re-hash.
8661        let canonical_root = root.as_path().to_path_buf();
8662        let artifact = canonical_root.join("cache.bin");
8663        std::fs::write(&artifact, b"warm-artifact").expect("write artifact");
8664        let seeded_generation = crate::cache_freshness::artifact_generation(&artifact);
8665        crate::cache_freshness::record_verify_completed(
8666            &canonical_root,
8667            crate::cache_freshness::VerifyArtifact::Search,
8668            seeded_generation,
8669        );
8670        assert!(matches!(
8671            crate::cache_freshness::warm_verify_plan(
8672                &canonical_root,
8673                crate::cache_freshness::VerifyArtifact::Search,
8674                seeded_generation,
8675            ),
8676            crate::cache_freshness::WarmVerifyPlan::Skip
8677        ));
8678        // A live watcher runtime must survive quiesce (its events accumulate
8679        // for the rebind replay).
8680        let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
8681        let _dispatch_tx = dispatch_tx;
8682        let shutdown = Arc::new(AtomicBool::new(false));
8683        let thread_shutdown = Arc::clone(&shutdown);
8684        let join = std::thread::spawn(move || {
8685            while !thread_shutdown.load(Ordering::SeqCst) {
8686                std::thread::yield_now();
8687            }
8688        });
8689        ctx.install_watcher_runtime(
8690            dispatch_rx,
8691            crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
8692        );
8693        assert!(ctx.watcher_runtime_active());
8694
8695        quiesce_unbound_root(&root, &mut live_roots, &executor);
8696        let meta = &live_roots[&root];
8697        assert!(meta.unbound_quiesced);
8698        assert!(ctx.subc_unbound_quiesced());
8699        assert!(meta.maintenance_pending);
8700        assert!(meta.maintenance_queued_kinds.is_empty());
8701        assert!(executor.actor_registered(&root));
8702        // Transient unbind keeps the root warm: resident artifacts stay
8703        // resident, no forced callgraph rebuild is planted, and pending
8704        // reconciliation paths survive for the rebind replay.
8705        assert!(
8706            ctx.search_index()
8707                .read()
8708                .unwrap_or_else(std::sync::PoisonError::into_inner)
8709                .is_some(),
8710            "quiesce must not evict resident artifacts"
8711        );
8712        assert_eq!(
8713            ctx.pending_callgraph_store_force_token(),
8714            None,
8715            "quiesce must not force a callgraph rebuild"
8716        );
8717        assert_eq!(
8718            ctx.take_pending_search_index_paths(),
8719            vec![pending],
8720            "quiesce must retain pending watcher-derived paths"
8721        );
8722        assert!(
8723            matches!(
8724                crate::cache_freshness::warm_verify_plan(
8725                    &canonical_root,
8726                    crate::cache_freshness::VerifyArtifact::Search,
8727                    seeded_generation,
8728                ),
8729                crate::cache_freshness::WarmVerifyPlan::Skip
8730            ),
8731            "quiesce must not invalidate the warm verify memo"
8732        );
8733        assert!(
8734            ctx.watcher_runtime_active(),
8735            "quiesce must not stop a running watcher"
8736        );
8737        ctx.stop_watcher_runtime();
8738
8739        let meta = live_roots.get_mut(&root).expect("root metadata");
8740        note_maintenance_completion(
8741            meta,
8742            Some(MaintenanceDrainKind::ConfigureTail),
8743            false,
8744            meta.unbound_quiesced,
8745        );
8746        assert!(!meta.maintenance_pending);
8747        assert!(meta.maintenance_queued_kinds.is_empty());
8748    }
8749
8750    #[test]
8751    fn same_root_higher_epoch_replacement_does_not_quiesce_between_generations() {
8752        let (_dir, root) = test_root("same-root-replacement");
8753        let route = route_key(7, 1);
8754        let installed_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
8755        let root_channels = HashMap::new();
8756
8757        assert!(!route_removal_will_quiesce_root(
8758            &root,
8759            route,
8760            &installed_channels,
8761            false,
8762            Some(&root),
8763        ));
8764        assert!(route_removal_will_quiesce_root(
8765            &root,
8766            route,
8767            &installed_channels,
8768            false,
8769            None,
8770        ));
8771        assert!(!should_quiesce_removed_root(
8772            &root,
8773            &root_channels,
8774            false,
8775            Some(&root),
8776        ));
8777        assert!(should_quiesce_removed_root(
8778            &root,
8779            &root_channels,
8780            false,
8781            None,
8782        ));
8783        assert!(!should_quiesce_removed_root(
8784            &root,
8785            &root_channels,
8786            true,
8787            None,
8788        ));
8789    }
8790
8791    #[test]
8792    fn root_quiesces_only_after_its_last_route_is_removed_and_reactivates_on_bind() {
8793        let (_root_dir, root) = test_root("unbound-root-route-count");
8794        let executor = Arc::new(Executor::new());
8795        assert!(executor.register_actor(root.clone(), test_ctx()));
8796        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
8797        let mut root_channels = HashMap::from([(
8798            root.clone(),
8799            HashSet::from([route_key(7, 1), route_key(8, 1)]),
8800        )]);
8801
8802        remove_root_channel(&mut root_channels, &root, route_key(7, 1));
8803        if !root_channels.contains_key(&root) {
8804            quiesce_unbound_root(&root, &mut live_roots, &executor);
8805        }
8806        assert!(!live_roots[&root].unbound_quiesced);
8807
8808        remove_root_channel(&mut root_channels, &root, route_key(8, 1));
8809        if !root_channels.contains_key(&root) {
8810            quiesce_unbound_root(&root, &mut live_roots, &executor);
8811        }
8812        assert!(live_roots[&root].unbound_quiesced);
8813
8814        live_roots
8815            .get_mut(&root)
8816            .expect("root metadata")
8817            .note_activity();
8818        assert!(
8819            live_roots[&root].unbound_quiesced,
8820            "late asynchronous activity must not reactivate an unbound root"
8821        );
8822
8823        live_roots
8824            .get_mut(&root)
8825            .expect("root metadata")
8826            .reactivate_bound();
8827        assert!(!live_roots[&root].unbound_quiesced);
8828    }
8829
8830    #[test]
8831    fn allocator_pressure_relief_requires_every_root_to_be_idle() {
8832        let (_idle_dir, idle_root) = test_root("allocator-relief-idle");
8833        let (_active_dir, active_root) = test_root("allocator-relief-active");
8834        let now = Instant::now();
8835        let mut live_roots = HashMap::new();
8836        let mut idle = RootMeta::new(now);
8837        idle.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
8838        live_roots.insert(idle_root, idle);
8839        let executor = Executor::new();
8840        assert!(process_has_been_idle(now, &live_roots, &executor));
8841
8842        live_roots.insert(active_root.clone(), RootMeta::new(now));
8843        assert!(!process_has_been_idle(now, &live_roots, &executor));
8844
8845        let active = live_roots
8846            .get_mut(&active_root)
8847            .expect("active root metadata");
8848        active.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
8849        active.active_bash_waits = 1;
8850        assert!(!process_has_been_idle(now, &live_roots, &executor));
8851    }
8852
8853    #[test]
8854    fn pressure_relief_log_reports_before_and_after_measurements() {
8855        let allocator = crate::memory::AllocatorMemorySnapshot {
8856            status: "measured",
8857            bytes_in_use: Some(8 * 1024 * 1024),
8858            size_allocated: Some(12 * 1024 * 1024),
8859            retained_slack_bytes: Some(4 * 1024 * 1024),
8860            not_estimated: None,
8861        };
8862        let relief = crate::memory::AllocatorPressureRelief {
8863            bytes_released: 3 * 1024 * 1024,
8864            rss_before_bytes: Some(20 * 1024 * 1024),
8865            rss_after_bytes: Some(17 * 1024 * 1024),
8866            allocator_before: allocator.clone(),
8867            allocator_after: crate::memory::AllocatorMemorySnapshot {
8868                size_allocated: Some(9 * 1024 * 1024),
8869                retained_slack_bytes: Some(1024 * 1024),
8870                ..allocator
8871            },
8872        };
8873        let message = pressure_relief_label(&relief);
8874        assert!(message.contains("RSS 20.0 MB -> 17.0 MB"));
8875        assert!(message.contains("allocated 12.0 MB -> 9.0 MB"));
8876        assert!(message.contains("slack 4.0 MB -> 1.0 MB"));
8877        assert!(message.contains("reported 3.0 MB released"));
8878    }
8879
8880    #[test]
8881    fn due_maintenance_jobs_skip_poisoned_roots() {
8882        let (_healthy_dir, healthy_root) = test_root("maintenance-healthy");
8883        let (_poisoned_dir, poisoned_root) = test_root("maintenance-poisoned");
8884        let mut live_roots = HashMap::new();
8885        live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
8886        let mut poisoned_meta = RootMeta::new(Instant::now());
8887        poisoned_meta.maintenance_poisoned = true;
8888        live_roots.insert(poisoned_root.clone(), poisoned_meta);
8889
8890        let (due, deferred) = due_maintenance_jobs_without_actor_context(
8891            &mut live_roots,
8892            MAINTENANCE_SUBMIT_BUDGET,
8893            &HashSet::new(),
8894        );
8895
8896        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
8897        assert!(due.iter().all(|(root, _)| root == &healthy_root));
8898        assert!(!deferred);
8899        assert!(live_roots[&healthy_root].maintenance_pending);
8900        assert_eq!(
8901            live_roots[&healthy_root].maintenance_jobs_in_flight,
8902            INITIAL_MAINTENANCE_JOB_COUNT
8903        );
8904        assert!(!live_roots[&poisoned_root].maintenance_pending);
8905    }
8906
8907    #[test]
8908    fn due_maintenance_jobs_do_not_restart_quiesced_root_work() {
8909        let (_dir, root) = test_root("maintenance-unbound");
8910        let mut meta = RootMeta::new(Instant::now());
8911        meta.unbound_quiesced = true;
8912        let mut live_roots = HashMap::from([(root.clone(), meta)]);
8913
8914        let (due, deferred) = due_maintenance_jobs_without_actor_context(
8915            &mut live_roots,
8916            MAINTENANCE_SUBMIT_BUDGET,
8917            &HashSet::new(),
8918        );
8919
8920        assert!(due.is_empty());
8921        assert!(!deferred);
8922        assert!(!live_roots[&root].maintenance_pending);
8923    }
8924
8925    #[test]
8926    fn idle_bg_subscription_queues_no_jobs_until_a_wake_arrives() {
8927        let (_dir, root) = test_root("maintenance-idle-bg-subscription");
8928        let ctx = test_ctx();
8929        assert!(!ctx.completion_drains_have_work());
8930
8931        let executor = Executor::new();
8932        assert!(executor.register_actor(root.clone(), ctx));
8933        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
8934        let session = "idle-session".to_string();
8935        let channel = route_key(17, 1);
8936        let metrics = DispatchPathMetrics::new();
8937        let bg_sub_by_session =
8938            HashMap::from([((root.clone(), session.clone()), HashSet::from([channel]))]);
8939        let mut bg_wake_pending = BgWakePending::new();
8940
8941        let (idle_tick_jobs, deferred) = due_maintenance_jobs(
8942            &mut live_roots,
8943            Some(&executor),
8944            &bg_sub_by_session,
8945            &bg_wake_pending,
8946            MAINTENANCE_SUBMIT_BUDGET,
8947            &HashSet::new(),
8948        );
8949        assert!(idle_tick_jobs.is_empty());
8950        assert!(!deferred);
8951        assert!(!live_roots[&root].maintenance_pending);
8952
8953        // A completion can arm its wake after the idle tick's probes. The wake
8954        // remains loop-owned state, so the following tick must observe it.
8955        let mut bg_wake_epoch = HashMap::new();
8956        push::arm_bg_wake(
8957            root.clone(),
8958            session,
8959            channel,
8960            &mut bg_wake_pending,
8961            &mut bg_wake_epoch,
8962            &metrics,
8963        );
8964        let (next_tick_jobs, deferred) = due_maintenance_jobs(
8965            &mut live_roots,
8966            Some(&executor),
8967            &bg_sub_by_session,
8968            &bg_wake_pending,
8969            MAINTENANCE_SUBMIT_BUDGET,
8970            &HashSet::new(),
8971        );
8972        assert_eq!(
8973            next_tick_jobs,
8974            vec![(root, MaintenanceDrainKind::CompletionDrains)]
8975        );
8976        assert!(!deferred);
8977    }
8978
8979    async fn assert_slow_configure_tail_admission(
8980        config: crate::executor::ExecutorConfig,
8981        shape: &'static str,
8982    ) {
8983        let root_dir = tempfile::tempdir().unwrap();
8984        let root_path = std::fs::canonicalize(root_dir.path()).unwrap();
8985        let root = ProjectRootId::from_path(&root_path).unwrap();
8986        let ctx = test_ctx();
8987        ctx.mark_subc_bound();
8988        let storage_root = ctx.storage_dir();
8989        ctx.enqueue_configure_maintenance(crate::context::ConfigureMaintenanceJob {
8990            generation: ctx.configure_generation(),
8991            root_path: root_path.clone(),
8992            canonical_cache_root: root_path.clone(),
8993            harness: crate::harness::Harness::Opencode,
8994            storage_root: storage_root.clone(),
8995            harness_dir: storage_root.join("opencode"),
8996            session_id: "first-search-admission".to_string(),
8997            home_match: false,
8998            format_tool_cache_clear_needed: false,
8999            run_bash_replay: false,
9000            refresh_project_runtime: false,
9001            sync_bash_compress_flag: false,
9002            reset_filter_registry: false,
9003            clear_failed_spawns: false,
9004            warm_callgraph_store: false,
9005            supersede_search_artifact_persistence: false,
9006            supersede_callgraph_artifact_persistence: false,
9007            supersede_semantic_artifact_persistence: false,
9008            search_artifact_load_start: None,
9009            semantic_artifact_load_start: None,
9010        })
9011        .expect("queue configure maintenance");
9012        let (_gate, maintenance_reached, release_maintenance) =
9013            crate::commands::configure::gate_configure_deferred_maintenance_for_test(
9014                root_path.clone(),
9015            );
9016
9017        let expected_pool_size = config.pool_size;
9018        let expected_actor_cap = config.actor_cap;
9019        let executor = Arc::new(Executor::with_config(config));
9020        assert_eq!(executor.pool_size(), expected_pool_size);
9021        assert_eq!(executor.actor_cap(), expected_actor_cap);
9022        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
9023        let metrics = Arc::new(DispatchPathMetrics::new());
9024        let (completion_tx, mut completion_rx) = mpsc::channel(2);
9025        submit_maintenance_job(
9026            &executor,
9027            root.clone(),
9028            MaintenanceDrainKind::ConfigureTail,
9029            Vec::new(),
9030            &completion_tx,
9031            &metrics,
9032        );
9033        maintenance_reached
9034            .recv_timeout(Duration::from_secs(2))
9035            .expect("configure tail reached gate");
9036
9037        let admission_started = std::time::Instant::now();
9038        let (search_admitted_tx, search_admitted_rx) = crossbeam_channel::bounded(1);
9039        let search = executor.submit_async(
9040            root.clone(),
9041            Lane::HeavyInit,
9042            "first-search".to_string(),
9043            Box::new(move |_ctx| {
9044                search_admitted_tx
9045                    .send(())
9046                    .expect("signal search admission");
9047                Response::success("first-search", json!({}))
9048            }),
9049        );
9050        let (mutation_started_tx, mutation_started_rx) = crossbeam_channel::bounded(1);
9051        let mutation = executor.submit_async(
9052            root,
9053            Lane::Mutating,
9054            "queued-mutation".to_string(),
9055            Box::new(move |_ctx| {
9056                mutation_started_tx.send(()).expect("signal mutation start");
9057                Response::success("queued-mutation", json!({}))
9058            }),
9059        );
9060
9061        // The decision under test is ORDERING, not latency: this recv happens
9062        // while the maintenance gate is still held, so a search that could only
9063        // admit after maintenance completes can never satisfy it. The budget is
9064        // a hang catch - a tight bound here just measures runner scheduling
9065        // (the census S-class shape) and flaked on loaded macOS CI at 100ms.
9066        let search_admission = search_admitted_rx.recv_timeout(Duration::from_secs(10));
9067        let admission_elapsed = admission_started.elapsed();
9068        let mutation_waited = mutation_started_rx.try_recv().is_err();
9069        release_maintenance
9070            .send(())
9071            .expect("release configure maintenance");
9072        search_admission
9073            .expect("first search must admit while configure maintenance remains gated");
9074        eprintln!(
9075            "first-search admission while configure maintenance is gated ({shape}): {}ms",
9076            admission_elapsed.as_millis()
9077        );
9078        assert!(
9079            mutation_waited,
9080            "mutating work must wait for configure maintenance to release its read epoch"
9081        );
9082        tokio::time::timeout(Duration::from_secs(5), search)
9083            .await
9084            .expect("first search completion timed out")
9085            .expect("first search completion channel closed");
9086        tokio::time::timeout(Duration::from_secs(5), mutation)
9087            .await
9088            .expect("mutation completion timed out")
9089            .expect("mutation completion channel closed");
9090        tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
9091            .await
9092            .expect("configure-tail completion timed out")
9093            .expect("configure-tail completion channel closed");
9094    }
9095
9096    #[tokio::test]
9097    async fn slow_configure_tail_admits_first_search_but_not_mutating_work() {
9098        assert_slow_configure_tail_admission(
9099            crate::executor::ExecutorConfig {
9100                pool_size: 2,
9101                read_cap: 1,
9102                actor_cap: 1,
9103                heavy_permits: 1,
9104                drr_quantum: 1,
9105            },
9106            "pool=2 actor_cap=1",
9107        )
9108        .await;
9109        assert_slow_configure_tail_admission(
9110            crate::executor::ExecutorConfig {
9111                pool_size: 4,
9112                read_cap: 3,
9113                actor_cap: 3,
9114                heavy_permits: 3,
9115                drr_quantum: 1,
9116            },
9117            "pool=4 actor_cap=3",
9118        )
9119        .await;
9120    }
9121
9122    #[tokio::test]
9123    async fn subc_configure_tail_precedes_completed_search_install() {
9124        let root_dir = tempfile::tempdir().unwrap();
9125        let storage = tempfile::tempdir().unwrap();
9126        let root = ProjectRootId::from_path(root_dir.path()).unwrap();
9127        let (ctx, ignored_path) =
9128            runtime_drain::configure_search_order_context_for_test(root_dir.path(), storage.path());
9129        let ctx = Arc::new(ctx);
9130        assert!(!runtime_drain::watcher_path_is_ignored_by_current_matcher(
9131            &ctx,
9132            &ignored_path
9133        ));
9134
9135        let executor = Arc::new(Executor::new());
9136        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
9137        let metrics = Arc::new(DispatchPathMetrics::new());
9138        let (completion_tx, mut completion_rx) = mpsc::channel(4);
9139        submit_maintenance_job(
9140            &executor,
9141            root.clone(),
9142            MaintenanceDrainKind::ConfigureTail,
9143            Vec::new(),
9144            &completion_tx,
9145            &metrics,
9146        );
9147        submit_maintenance_job(
9148            &executor,
9149            root,
9150            MaintenanceDrainKind::CompletionDrains,
9151            Vec::new(),
9152            &completion_tx,
9153            &metrics,
9154        );
9155
9156        let first = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
9157            .await
9158            .expect("configure-tail completion timed out")
9159            .expect("configure-tail completion channel closed");
9160        let second = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
9161            .await
9162            .expect("completion-drains completion timed out")
9163            .expect("completion-drains completion channel closed");
9164        assert!(first.response.id.contains("configure-tail"));
9165        assert!(second.response.id.contains("completion-drains"));
9166        assert!(runtime_drain::watcher_path_is_ignored_by_current_matcher(
9167            &ctx,
9168            &ignored_path
9169        ));
9170        assert_eq!(
9171            ctx.search_index()
9172                .read()
9173                .unwrap_or_else(std::sync::PoisonError::into_inner)
9174                .as_ref()
9175                .expect("completed search index installed")
9176                .file_count(),
9177            0,
9178            "configure must install the ignore matcher before pending paths replay"
9179        );
9180        ctx.stop_watcher_runtime();
9181    }
9182
9183    #[test]
9184    fn post_bind_configure_and_completion_jobs_are_queued_in_order() {
9185        let (_dir, root) = test_root("maintenance-post-bind");
9186        let mut live_roots = HashMap::new();
9187        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
9188
9189        queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
9190        queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
9191
9192        let meta = live_roots.get(&root).expect("root metadata");
9193        assert!(meta.maintenance_pending);
9194        assert_eq!(meta.maintenance_jobs_in_flight, 0);
9195        assert_eq!(
9196            meta.maintenance_queued_kinds
9197                .iter()
9198                .copied()
9199                .collect::<Vec<_>>(),
9200            vec![
9201                MaintenanceDrainKind::ConfigureTail,
9202                MaintenanceDrainKind::CompletionDrains,
9203            ]
9204        );
9205
9206        let (due, deferred) = due_maintenance_jobs_without_actor_context(
9207            &mut live_roots,
9208            MAINTENANCE_SUBMIT_BUDGET,
9209            &HashSet::new(),
9210        );
9211
9212        assert_eq!(
9213            due,
9214            vec![
9215                (root.clone(), MaintenanceDrainKind::ConfigureTail),
9216                (root.clone(), MaintenanceDrainKind::CompletionDrains),
9217            ]
9218        );
9219        assert!(!deferred);
9220        assert_eq!(live_roots[&root].maintenance_jobs_in_flight, 2);
9221        assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
9222    }
9223
9224    #[test]
9225    fn due_maintenance_jobs_defers_unsubmitted_roots_without_marking_pending() {
9226        let mut live_roots = HashMap::new();
9227        let mut root_ids = Vec::new();
9228        let mut _dirs = Vec::new();
9229        for index in 0..4 {
9230            let (dir, root_id) = test_root(&format!("maintenance-budget-{index}"));
9231            live_roots.insert(root_id.clone(), RootMeta::new(Instant::now()));
9232            root_ids.push(root_id);
9233            _dirs.push(dir);
9234        }
9235
9236        let small_budget = INITIAL_MAINTENANCE_JOB_COUNT + 1;
9237        let (first_due, first_deferred) = due_maintenance_jobs_without_actor_context(
9238            &mut live_roots,
9239            small_budget,
9240            &HashSet::new(),
9241        );
9242
9243        assert_eq!(first_due.len(), small_budget);
9244        assert!(first_deferred);
9245        let first_due_set: HashSet<_> = first_due.into_iter().map(|(root, _)| root).collect();
9246        assert!(first_due_set
9247            .iter()
9248            .all(|root| live_roots[root].maintenance_pending));
9249        assert!(first_due_set
9250            .iter()
9251            .any(|root| !live_roots[root].maintenance_queued_kinds.is_empty()));
9252
9253        let all_roots: HashSet<_> = root_ids.into_iter().collect();
9254        let deferred_roots: HashSet<_> = all_roots.difference(&first_due_set).cloned().collect();
9255        assert!(deferred_roots
9256            .iter()
9257            .all(|root| !live_roots[root].maintenance_pending));
9258    }
9259
9260    #[test]
9261    fn due_maintenance_jobs_defers_pending_bind_roots() {
9262        let (_bind_dir, bind_root) = test_root("maintenance-pending-bind");
9263        let (_healthy_dir, healthy_root) = test_root("maintenance-no-bind");
9264        let mut live_roots = HashMap::new();
9265        live_roots.insert(bind_root.clone(), RootMeta::new(Instant::now()));
9266        live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
9267        let pending_bind_roots = HashSet::from([bind_root.clone()]);
9268
9269        let (due, deferred) = due_maintenance_jobs_without_actor_context(
9270            &mut live_roots,
9271            usize::MAX,
9272            &pending_bind_roots,
9273        );
9274
9275        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
9276        assert!(due.iter().all(|(root, _)| root == &healthy_root));
9277        assert!(!deferred);
9278        assert!(!live_roots[&bind_root].maintenance_pending);
9279        assert!(live_roots[&bind_root].maintenance_queued_kinds.is_empty());
9280    }
9281
9282    #[test]
9283    fn maintenance_pending_survives_requeue_and_clears_after_final_batch() {
9284        let (_dir, root) = test_root("maintenance-requeue");
9285        let mut live_roots = HashMap::new();
9286        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
9287        let (due, deferred) = due_maintenance_jobs_without_actor_context(
9288            &mut live_roots,
9289            usize::MAX,
9290            &HashSet::new(),
9291        );
9292        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
9293        assert!(due.iter().all(|(due_root, _)| due_root == &root));
9294        assert!(!deferred);
9295
9296        let meta = live_roots.get_mut(&root).unwrap();
9297        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, false);
9298        assert!(meta.maintenance_pending);
9299        assert_eq!(
9300            meta.maintenance_jobs_in_flight,
9301            INITIAL_MAINTENANCE_JOB_COUNT - 1
9302        );
9303        assert_eq!(meta.maintenance_queued_kinds.len(), 1);
9304
9305        let (requeued, deferred) =
9306            due_maintenance_jobs_without_actor_context(&mut live_roots, 1, &HashSet::new());
9307        assert_eq!(
9308            requeued,
9309            vec![(root.clone(), MaintenanceDrainKind::Watcher)]
9310        );
9311        assert!(!deferred);
9312        let meta = live_roots.get_mut(&root).unwrap();
9313        assert_eq!(
9314            meta.maintenance_jobs_in_flight,
9315            INITIAL_MAINTENANCE_JOB_COUNT
9316        );
9317        assert!(meta.maintenance_queued_kinds.is_empty());
9318
9319        for _ in 0..INITIAL_MAINTENANCE_JOB_COUNT {
9320            note_maintenance_completion(meta, None, false, false);
9321        }
9322        assert!(!meta.maintenance_pending);
9323        assert_eq!(meta.maintenance_jobs_in_flight, 0);
9324    }
9325
9326    #[test]
9327    fn maintenance_requeue_drops_while_bind_is_pending() {
9328        let (_dir, root) = test_root("maintenance-bind-requeue");
9329        let mut live_roots = HashMap::new();
9330        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
9331        let (due, _) = due_maintenance_jobs_without_actor_context(
9332            &mut live_roots,
9333            usize::MAX,
9334            &HashSet::new(),
9335        );
9336        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
9337
9338        let meta = live_roots.get_mut(&root).unwrap();
9339        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, true);
9340
9341        assert_eq!(
9342            meta.maintenance_jobs_in_flight,
9343            INITIAL_MAINTENANCE_JOB_COUNT - 1
9344        );
9345        assert!(meta.maintenance_queued_kinds.is_empty());
9346        assert!(meta.maintenance_pending);
9347    }
9348
9349    #[test]
9350    fn parked_lsp_completion_never_requiesces_or_cancels_a_pending_bind() {
9351        let mut meta = RootMeta::new(Instant::now());
9352        meta.unbound_quiesced = true;
9353
9354        assert!(!should_requiesce_after_maintenance(
9355            &meta,
9356            MaintenanceDrainKind::Lsp,
9357            false,
9358        ));
9359        assert!(!should_requiesce_after_maintenance(
9360            &meta,
9361            MaintenanceDrainKind::ConfigureTail,
9362            true,
9363        ));
9364        assert!(should_requiesce_after_maintenance(
9365            &meta,
9366            MaintenanceDrainKind::ConfigureTail,
9367            false,
9368        ));
9369    }
9370
9371    #[test]
9372    fn maintenance_pending_clears_and_poison_stops_requeue_after_fatal() {
9373        let (_dir, root) = test_root("maintenance-fatal");
9374        let mut live_roots = HashMap::new();
9375        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
9376        let (due, _) = due_maintenance_jobs_without_actor_context(
9377            &mut live_roots,
9378            usize::MAX,
9379            &HashSet::new(),
9380        );
9381        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
9382
9383        let meta = live_roots.get_mut(&root).unwrap();
9384        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), true, false);
9385        assert!(meta.maintenance_poisoned);
9386        assert!(meta.maintenance_queued_kinds.is_empty());
9387
9388        for _ in 1..INITIAL_MAINTENANCE_JOB_COUNT {
9389            note_maintenance_completion(meta, None, false, false);
9390        }
9391        assert!(!meta.maintenance_pending);
9392        assert_eq!(meta.maintenance_jobs_in_flight, 0);
9393    }
9394
9395    #[test]
9396    fn trust_for_principal_matrix() {
9397        assert_eq!(
9398            trust_for_principal(&Some(Principal::Direct)),
9399            BindTrust::FirstParty
9400        );
9401        // Every first-party reserved id is asserted BY NAME, in one loop over
9402        // the full set, so two failure classes stay distinguishable: an empty
9403        // or broken allowlist reddens every name at once, while a dropped
9404        // single entry (the rename hazard) reddens exactly the missing name.
9405        // Both halves of each transitional rename pair stay listed until the
9406        // flip settles (see the allowlist comment).
9407        for module_id in [
9408            "llm-runner",
9409            "aft",
9410            "broca",
9411            "alfonso-core",
9412            "prefrontal",
9413            "prefrontal-core",
9414        ] {
9415            assert_eq!(
9416                trust_for_principal(&Some(Principal::Reserved {
9417                    module_id: module_id.to_string(),
9418                })),
9419                BindTrust::FirstParty,
9420                "reserved module id '{module_id}' must resolve to first-party trust"
9421            );
9422        }
9423        assert_eq!(
9424            trust_for_principal(&Some(Principal::Reserved {
9425                module_id: "subc-mcp".to_string(),
9426            })),
9427            BindTrust::Untrusted
9428        );
9429        assert_eq!(
9430            trust_for_principal(&Some(Principal::Reserved {
9431                module_id: "anything-unknown".to_string(),
9432            })),
9433            BindTrust::Untrusted
9434        );
9435        assert_eq!(
9436            trust_for_principal(&Some(Principal::Unverified)),
9437            BindTrust::Untrusted
9438        );
9439        assert_eq!(trust_for_principal(&None), BindTrust::Untrusted);
9440    }
9441
9442    #[test]
9443    fn fed_harness_class_maps_to_untrusted_regardless_of_fingerprint_value() {
9444        let principal = Some(Principal::Direct);
9445        let fingerprint_a = "fed:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
9446        let fingerprint_b = "fed:0123456789abcdef111111111111111111111111111111111111111111111111";
9447
9448        assert_eq!(
9449            trust_for_bind(fingerprint_a, &principal),
9450            BindTrust::Untrusted
9451        );
9452        assert_eq!(
9453            trust_for_bind(fingerprint_b, &principal),
9454            BindTrust::Untrusted
9455        );
9456    }
9457
9458    /// The table above proves `trust_for_principal` maps correctly, and the
9459    /// test above proves the `fed:` harness override wins — but neither
9460    /// exercises the ordinary path, so an implementation that ignored the
9461    /// principal entirely for non-fed harnesses would satisfy both. Pin the
9462    /// delegation itself: on a normal harness the verdict must still come from
9463    /// the principal, in both directions.
9464    #[test]
9465    fn trust_for_bind_delegates_to_the_principal_on_ordinary_harnesses() {
9466        for harness in ["opencode", "pi", "runner", "mcp:claude"] {
9467            assert_eq!(
9468                trust_for_bind(harness, &Some(Principal::Direct)),
9469                BindTrust::FirstParty,
9470                "a direct principal must stay first-party on {harness}"
9471            );
9472            assert_eq!(
9473                trust_for_bind(harness, &Some(Principal::Unverified)),
9474                BindTrust::Untrusted,
9475                "an unverified principal must stay untrusted on {harness}"
9476            );
9477            assert_eq!(
9478                trust_for_bind(harness, &None),
9479                BindTrust::Untrusted,
9480                "an absent principal must fail closed on {harness}"
9481            );
9482            assert_eq!(
9483                trust_for_bind(
9484                    harness,
9485                    &Some(Principal::Reserved {
9486                        module_id: "subc-mcp".to_string(),
9487                    })
9488                ),
9489                BindTrust::Untrusted,
9490                "a non-allowlisted reserved module must stay untrusted on {harness}"
9491            );
9492        }
9493    }
9494
9495    #[tokio::test]
9496    async fn persistent_cancel_resolves_when_fired_before_await() {
9497        // The lost-wakeup guard: cancel() fires exactly once via notify_waiters()
9498        // (no stored permit). A waiter that registers AFTER the cancel must still
9499        // observe it via the flag; a waiter racing the cancel must still be woken.
9500        let signal = PersistentCancelSignal::new();
9501        signal.cancel();
9502        // Fired before we ever call cancelled() — must return immediately, not park.
9503        tokio::time::timeout(Duration::from_secs(1), signal.cancelled())
9504            .await
9505            .expect("cancelled() must resolve when cancel fired beforehand");
9506
9507        // A fresh signal cancelled concurrently with an in-flight cancelled().
9508        let racing = PersistentCancelSignal::new();
9509        let racing_for_task = racing.clone();
9510        let waiter = tokio::spawn(async move { racing_for_task.cancelled().await });
9511        racing.cancel();
9512        tokio::time::timeout(Duration::from_secs(1), waiter)
9513            .await
9514            .expect("cancelled() must resolve when cancel races the await")
9515            .expect("waiter task panicked");
9516    }
9517
9518    #[test]
9519    fn ingress_epoch_validation_rejects_reclaimed_requests_and_drops_other_stale_epochs() {
9520        let installed = HashMap::from([(7, 9)]);
9521        let mut reclaimed = ReclaimedRoutes::default();
9522        reclaimed.insert(route_key(8, 1));
9523        for ty in [
9524            FrameType::Request,
9525            FrameType::Response,
9526            FrameType::Error,
9527            FrameType::Push,
9528            FrameType::Cancel,
9529            FrameType::Goodbye,
9530        ] {
9531            let body = if ty.is_pure_header() {
9532                Vec::new()
9533            } else {
9534                br#"{}"#.to_vec()
9535            };
9536            let stale = Frame::build(ty, control_flags(), 7, 8, 41, body).unwrap();
9537            assert!(
9538                !ingress_route_should_be_processed(&installed, &reclaimed, &stale),
9539                "{ty:?}"
9540            );
9541        }
9542
9543        let reclaimed_request = Frame::build(
9544            FrameType::Request,
9545            control_flags(),
9546            8,
9547            1,
9548            42,
9549            br#"{}"#.to_vec(),
9550        )
9551        .unwrap();
9552        assert!(ingress_route_should_be_processed(
9553            &installed,
9554            &reclaimed,
9555            &reclaimed_request
9556        ));
9557
9558        let never_installed = Frame::build(
9559            FrameType::Request,
9560            control_flags(),
9561            9,
9562            1,
9563            43,
9564            br#"{}"#.to_vec(),
9565        )
9566        .unwrap();
9567        assert!(!ingress_route_should_be_processed(
9568            &installed,
9569            &reclaimed,
9570            &never_installed
9571        ));
9572
9573        let current = Frame::build(
9574            FrameType::Request,
9575            control_flags(),
9576            7,
9577            9,
9578            43,
9579            br#"{}"#.to_vec(),
9580        )
9581        .unwrap();
9582        let control = Frame::build(FrameType::Ping, control_flags(), 0, 0, 44, Vec::new()).unwrap();
9583        assert!(ingress_route_should_be_processed(
9584            &installed, &reclaimed, &current
9585        ));
9586        assert!(ingress_route_should_be_processed(
9587            &installed, &reclaimed, &control
9588        ));
9589        assert_eq!(installed, HashMap::from([(7, 9)]));
9590    }
9591
9592    #[tokio::test]
9593    async fn route_bind_ack_precedes_route_egress_in_writer_queue() {
9594        let (_dir, root) = test_root("route-bind-b2-ordering");
9595        let route = route_key(7, 3);
9596        let identity = RouteIdentity(Arc::new(RouteIdentityData {
9597            root: root.clone(),
9598            project_root: root.as_path().to_path_buf(),
9599            harness: "opencode".to_string(),
9600            session: "b2-session".to_string(),
9601            trust: BindTrust::FirstParty,
9602            spawn_principal: AuthenticatedPrincipal::FirstParty,
9603            consumer_elicitation_capable: false,
9604        }));
9605        let replay_key = push::ReplayKey::from_identity(&identity);
9606        let completion = RouteBindCompletion {
9607            route,
9608            identity,
9609            bind_root_id: root.clone(),
9610            inserted_new_actor: false,
9611            configure_response: Response::success("subc-bind-7", json!({})),
9612            diagnostics_on_edit: false,
9613            ver: PROTOCOL_VERSION,
9614            corr: 91,
9615            flags: control_flags(),
9616        };
9617        let mut pending_binds = HashMap::from([(
9618            route,
9619            PendingBind {
9620                bind_root_id: root,
9621                inserted_new_actor: false,
9622                cancelled: false,
9623                configure_request_id: "subc-bind-7".to_string(),
9624                started_at: Instant::now(),
9625                warned_half_deadline: false,
9626                deadline_reported: false,
9627                corr: 91,
9628                ver: PROTOCOL_VERSION,
9629                flags: control_flags(),
9630                cancellation: crate::executor::JobCancellation::new(),
9631            },
9632        )]);
9633        let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
9634        let mut push_buffer =
9635            HashMap::from([(replay_key, VecDeque::from([completion_frame("b2-replay")]))]);
9636        let (writer_tx, mut writer_rx) = mpsc::channel(8);
9637        let metrics = Arc::new(DispatchPathMetrics::new());
9638        let executor = Arc::new(Executor::new());
9639        let standing_actor =
9640            standing::StandingActor::new(App::default_shared(), Arc::clone(&executor));
9641
9642        handle_route_bind_completion(
9643            &writer_tx,
9644            completion,
9645            &mut HashMap::new(),
9646            &mut HashMap::new(),
9647            &mut HashMap::new(),
9648            &mut push_buffer,
9649            &mut HashMap::new(),
9650            &mut pending_binds,
9651            &mut installed_route_epochs,
9652            &executor,
9653            &standing_actor,
9654            &Arc::new(Notify::new()),
9655            &metrics,
9656            None,
9657        )
9658        .await
9659        .unwrap();
9660
9661        let ack = writer_rx.try_recv().expect("RouteBindAck");
9662        assert_eq!(ack.header.ty, FrameType::Response);
9663        assert_eq!((ack.header.channel, ack.header.epoch), (0, 0));
9664        let route_frame = writer_rx.try_recv().expect("post-ack route frame");
9665        assert_eq!(route_frame.header.ty, FrameType::Push);
9666        assert_eq!(
9667            (route_frame.header.channel, route_frame.header.epoch),
9668            (route.channel, route.epoch)
9669        );
9670    }
9671}