Skip to main content

aft/subc/
mod.rs

1//! subc daemon attach — transport edge.
2//!
3//! When AFT is launched as `aft --subc <connection-file>`, it does NOT run the
4//! standalone NDJSON-over-stdin loop. Instead it connects to a running subc
5//! daemon over loopback TCP, authenticates with the pre-envelope HMAC handshake
6//! (`subc-transport`), then speaks the subc frame protocol (`subc-protocol`):
7//! ModuleHello → HelloAck (register provider capabilities), then a channel-0
8//! control loop (Ping/Pong, RouteBind) plus tool and management route calls.
9//!
10//! Concurrency: subc routes tool calls through the executor. The tokio
11//! edge never dispatches against `AppContext` inline; per-actor executor lanes
12//! own the reader/mutator epoch, while a writer task serializes outbound frames.
13
14pub mod blob_store;
15
16use std::collections::{HashMap, HashSet, VecDeque};
17use std::fmt;
18use std::io;
19use std::net::{IpAddr, SocketAddr};
20use std::ops::Deref;
21use std::path::{Path, PathBuf};
22use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
23use std::sync::{Arc, LazyLock, Mutex as StdMutex, OnceLock};
24use std::time::{Duration, Instant};
25
26use serde::Deserialize;
27use serde_json::{json, Value};
28
29use crate::config::Config;
30use crate::context::{App, AppContext, ProgressSender, RootHealthSnapshot};
31use crate::executor::{Executor, JobCancellation, Lane, PreExecutionCancelOutcome};
32use crate::fleet_status::{spawn_fleet_status_dial, FleetStatusClient};
33use crate::log_ctx;
34use crate::path_identity::ProjectRootId;
35use crate::protocol::{ProgressKind, PushFrame, RawRequest, Response};
36use crate::response_finalize::{DispatchOutcome, PendingResponse};
37use crate::run_tool_call::{
38    finish_tool_call_response, prepare_tool_call, run_tool_call, strip_agent_preview_arg_owned,
39    PhaseTrace, ToolCallContext, ToolCallOutcome, ToolCallResult,
40};
41use crate::runtime_drain;
42use crate::sandbox_spawn::{AuthenticatedPrincipal, PrincipalTrust};
43
44use subc_protocol::manifest::{
45    Bindings, Concurrency, ExecutionMode, IdentityBinding, IdentityScope, ManagementOperation,
46    ManagementOperationKind, ModuleManifest, ProviderRole, StorageBinding, StorageKind,
47    StorageScope, Tool, TrustTier,
48};
49use subc_protocol::session::{
50    HealthReport, HealthStatus, ModuleControlRequest, ModuleControlResponse,
51    MODULE_CONTROL_OP_HEALTH_CHECK,
52};
53use subc_protocol::{
54    ErrorBody, Flags, Frame, FrameType, ModuleHelloBody, Principal, Priority, RouteTarget,
55    MAX_FRAME_BODY_LEN, PROTOCOL_VERSION,
56};
57use subc_transport::{authenticate_client, connection_file, read_frame, write_frame};
58use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
59use tokio::net::TcpStream;
60use tokio::sync::{mpsc, oneshot, Notify};
61use tokio::task::JoinHandle;
62
63/// Per-attempt handshake deadline. The initial attach loop has a separate total
64/// budget so a stalled peer cannot consume an unbounded supervisor launch window.
65const AUTH_DEADLINE: Duration = Duration::from_secs(5);
66const ATTACH_RETRY_BUDGET: Duration = Duration::from_secs(60);
67const ATTACH_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(250);
68const ATTACH_RETRY_MAX_BACKOFF: Duration = Duration::from_secs(5);
69const ATTACH_RETRY_JITTER_PERCENT: u64 = 20;
70
71/// Correlation id for the initial ModuleHello (channel 0).
72const HELLO_CORR: u64 = 1;
73
74/// Per-session in-memory replay cap for must-deliver Push frames. This covers
75/// detach/re-attach while AFT stays alive; cross-restart replay is phased later.
76const PUSH_BUFFER_MAX_PER_KEY: usize = 256;
77
78/// Bounded guard for control-frame sends. If the daemon stops reading and the
79/// writer queue stays full, tear the subc edge down instead of stalling the
80/// route loop indefinitely.
81const CONTROL_SEND_TIMEOUT: Duration = Duration::from_millis(250);
82
83/// Cadence for the loop's deadline-driven drain work (retry-buffer flush,
84/// bg-wake emission, maintenance submission). Checked at the top of every
85/// loop turn so busy select arms cannot starve it.
86const DRAIN_TICK_PERIOD: Duration = Duration::from_millis(250);
87
88/// Fallback unbound-root artifact TTL when a root has no actor config yet.
89const IDLE_ROOT_TTL: Duration = Duration::from_secs(30 * 60);
90
91const WRITER_QUEUE_CAPACITY: usize = 256;
92
93/// Keep reliable Push bursts from monopolizing the current-thread subc loop;
94/// any remaining must-deliver frames stay queued for the next loop turn.
95const RELIABLE_PUSH_DRAIN_BUDGET: usize = 32;
96
97/// Limit maintenance submissions per tick so background drains cannot delay
98/// control-plane work such as completed RouteBind acknowledgements.
99///
100/// The decomposed maintenance pass charges this budget by Mutating job, not by
101/// root. Size the default burst for one maintenance pass over eight live roots,
102/// while follow-up batches still re-enter the capped queue instead of bypassing
103/// the budget.
104const MAINTENANCE_SUBMIT_BUDGET: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len() * 8;
105const INITIAL_MAINTENANCE_DRAIN_KINDS: [MaintenanceDrainKind; 4] = [
106    MaintenanceDrainKind::Watcher,
107    MaintenanceDrainKind::Lsp,
108    MaintenanceDrainKind::ConfigureTail,
109    MaintenanceDrainKind::CompletionDrains,
110];
111#[cfg(test)]
112const INITIAL_MAINTENANCE_JOB_COUNT: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len();
113
114const RELIABLE_WRITER_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(10);
115const RELIABLE_WRITER_RETRY_MAX_BACKOFF: Duration = Duration::from_millis(250);
116
117const DISPATCH_PATH_BIND_WARN_AFTER: Duration = Duration::from_secs(6);
118const ROUTE_BIND_DEADLINE: Duration = Duration::from_secs(12);
119
120/// Small bounded memory of completed task ids used to suppress stale lossy
121/// long-running reminders that arrive after their reliable completion event.
122const COMPLETED_TASK_SUPPRESSION_MAX: usize = 4096;
123
124/// Bash foreground orchestration polls detached tasks with short read-lane jobs.
125/// The sleep between polls is outside the executor so no read or write worker is
126/// pinned while a foreground command is still running.
127const PENDING_POLL_INTERVAL: Duration = Duration::from_millis(100);
128
129/// Host elicitation asks fail closed if the MCP facade does not answer promptly.
130const BASH_ELICITATION_TIMEOUT: Duration = Duration::from_secs(60);
131const BASH_ELICITATION_CREATE_METHOD: &str = "elicitation/create";
132
133#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
134struct RouteChannel {
135    channel: u16,
136    epoch: u32,
137}
138
139impl fmt::Display for RouteChannel {
140    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
141        write!(f, "{}@{}", self.channel, self.epoch)
142    }
143}
144
145type PushEnvelope = (ProjectRootId, PushFrame);
146type LossyPushEnvelope = (u64, ProjectRootId, PushFrame);
147type RetryBuffer = HashMap<RouteChannel, VecDeque<(push::ReplayKey, PushFrame)>>;
148mod bash;
149mod health;
150mod manifest;
151mod push;
152mod standing;
153mod wire;
154
155use self::health::{
156    build_health_report, warn_slow_pending_binds, warn_slow_running_interactive_jobs,
157    DeferredBashWaitGuard, DispatchPathMetrics, HealthRollupCache, HealthRollupWorker,
158    ReapBlockerCensus, ResponseTaskGuard,
159};
160pub(crate) use self::manifest::is_subc_native_plumbing_tool;
161use self::manifest::{
162    build_manifest, command_lane, control_flags, control_ops, is_bash_family_tool,
163    is_subc_agent_core_tool,
164};
165pub use self::wire::SubcError;
166
167/// Lifecycle milestones emitted only by the dedicated subc integration-test runner.
168///
169/// Production entry points never install a probe, so these notifications cannot
170/// affect routing or delivery behavior.
171#[doc(hidden)]
172#[derive(Clone, Debug, PartialEq, Eq)]
173pub enum SubcLifecycleEvent {
174    AttachDecision {
175        attempt: u32,
176        will_retry: bool,
177    },
178    RouteDetached {
179        route_channel: u16,
180        route_epoch: u32,
181        session_id: String,
182    },
183    ReliableCompletionRetained {
184        task_id: String,
185        session_id: String,
186    },
187    ReliableCompletionReplayed {
188        route_channel: u16,
189        route_epoch: u32,
190        task_id: String,
191        session_id: String,
192    },
193}
194
195/// Test-only observer for the detach/rebind lifecycle.
196#[doc(hidden)]
197#[derive(Clone)]
198pub struct SubcTestLifecycleProbe {
199    events_tx: mpsc::UnboundedSender<SubcLifecycleEvent>,
200}
201
202impl SubcTestLifecycleProbe {
203    #[doc(hidden)]
204    pub fn new(events_tx: mpsc::UnboundedSender<SubcLifecycleEvent>) -> Self {
205        Self { events_tx }
206    }
207
208    fn attach_decision(&self, attempt: u32, will_retry: bool) {
209        let _ = self.events_tx.send(SubcLifecycleEvent::AttachDecision {
210            attempt,
211            will_retry,
212        });
213    }
214
215    fn route_detached(&self, route: RouteChannel, session_id: &str) {
216        let _ = self.events_tx.send(SubcLifecycleEvent::RouteDetached {
217            route_channel: route.channel,
218            route_epoch: route.epoch,
219            session_id: session_id.to_string(),
220        });
221    }
222
223    fn reliable_completion_retained(&self, task_id: &str, session_id: &str) {
224        let _ = self
225            .events_tx
226            .send(SubcLifecycleEvent::ReliableCompletionRetained {
227                task_id: task_id.to_string(),
228                session_id: session_id.to_string(),
229            });
230    }
231
232    fn reliable_completion_replayed(&self, route: RouteChannel, task_id: &str, session_id: &str) {
233        let _ = self
234            .events_tx
235            .send(SubcLifecycleEvent::ReliableCompletionReplayed {
236                route_channel: route.channel,
237                route_epoch: route.epoch,
238                task_id: task_id.to_string(),
239                session_id: session_id.to_string(),
240            });
241    }
242}
243
244/// Test-only view of the fail-closed tool-call gate: would `name` be admitted
245/// on a bound route (as an agent tool or native plumbing)? Used by the
246/// plugin-send drift guard in `subc_plumbing_drift_test.rs`.
247pub fn is_tool_call_admitted_for_test(name: &str) -> bool {
248    manifest::is_subc_agent_core_tool(name) || manifest::is_subc_native_plumbing_tool(name)
249}
250use self::wire::{
251    build_error_frame, build_goodbye_frame, build_tool_response_frame,
252    build_tool_response_frame_with_limit, decrement_counted_channel, response_is_fatal_panic,
253    response_message, send_counted_channel, send_frame, send_reliable_writer_frame,
254    send_traced_tool_response_frame, ToolResponseWriteTrace, WriterFrame, WriterSender,
255};
256
257struct DecodedFrame {
258    frame: Frame,
259    phase_trace: PhaseTrace,
260}
261
262struct ToolCallCompletion {
263    text: String,
264    phase_trace: PhaseTrace,
265}
266
267#[derive(Clone, Copy, Debug, PartialEq, Eq)]
268enum RouteDetachPolicy {
269    RetainForReplay,
270    CancelOnDetach,
271}
272
273#[derive(Clone)]
274struct ActiveToolCall {
275    root_id: ProjectRootId,
276    cancellation: JobCancellation,
277    detach_policy: RouteDetachPolicy,
278}
279
280type ActiveToolCalls = Arc<StdMutex<HashMap<(RouteChannel, u64), ActiveToolCall>>>;
281
282struct PendingDeferredSetupGuard(Arc<AtomicUsize>);
283
284impl PendingDeferredSetupGuard {
285    fn new(count: Arc<AtomicUsize>) -> Self {
286        count.fetch_add(1, Ordering::SeqCst);
287        Self(count)
288    }
289}
290
291impl Drop for PendingDeferredSetupGuard {
292    fn drop(&mut self) {
293        self.0.fetch_sub(1, Ordering::SeqCst);
294    }
295}
296
297enum DeferredSetupOutcome {
298    Immediate {
299        text: String,
300        phase_trace: PhaseTrace,
301    },
302    Deferred {
303        pending: PendingResponse,
304        surface_downgraded: bool,
305        phase_trace: PhaseTrace,
306    },
307}
308
309struct PendingSubcResponse {
310    route: RouteChannel,
311    corr: u64,
312    flags: Flags,
313    ver: u8,
314    root: ProjectRootId,
315    session_id: String,
316    bare_name: String,
317    format_context: crate::subc_format::FormatContext,
318    bind_trust: BindTrust,
319    pending: PendingResponse,
320    surface_downgraded: bool,
321    phase_trace: PhaseTrace,
322}
323
324struct ResolvedSubcResponse {
325    entry: PendingSubcResponse,
326    response: Response,
327}
328
329#[derive(Default)]
330struct PendingSubcResponses {
331    entries: Vec<PendingSubcResponse>,
332}
333
334impl PendingSubcResponses {
335    fn register(&mut self, pending: PendingSubcResponse) {
336        self.entries.retain(|entry| {
337            let keep = entry.route != pending.route || entry.corr != pending.corr;
338            if !keep {
339                if let Some(cancellation) = &entry.pending.cancellation {
340                    cancellation.request_cancel();
341                }
342            }
343            keep
344        });
345        self.entries.push(pending);
346    }
347
348    fn poll_ready(&mut self, executor: &Executor) -> Vec<ResolvedSubcResponse> {
349        let mut ready = Vec::new();
350        let mut waiting = Vec::with_capacity(self.entries.len());
351        for mut entry in self.entries.drain(..) {
352            let response = executor
353                .actor_context(&entry.root)
354                .and_then(|ctx| (entry.pending.poll)(&ctx));
355            if let Some(response) = response {
356                ready.push(ResolvedSubcResponse { entry, response });
357            } else {
358                waiting.push(entry);
359            }
360        }
361        self.entries = waiting;
362        ready
363    }
364
365    fn cancel_request(&mut self, route: RouteChannel, corr: u64) -> bool {
366        let mut cancelled = false;
367        self.entries.retain(|entry| {
368            let keep = entry.route != route || entry.corr != corr;
369            if !keep {
370                cancelled = true;
371                if let Some(cancellation) = &entry.pending.cancellation {
372                    cancellation.request_cancel();
373                }
374            }
375            keep
376        });
377        cancelled
378    }
379
380    fn drain_route(
381        &mut self,
382        route: RouteChannel,
383        executor: &Executor,
384    ) -> Vec<ResolvedSubcResponse> {
385        self.drain_matching(executor, |entry| entry.route == route)
386    }
387
388    fn drain_on_shutdown(&mut self, executor: &Executor) -> Vec<ResolvedSubcResponse> {
389        self.drain_matching(executor, |_| true)
390    }
391
392    fn drain_matching(
393        &mut self,
394        executor: &Executor,
395        matches: impl Fn(&PendingSubcResponse) -> bool,
396    ) -> Vec<ResolvedSubcResponse> {
397        let mut resolved = Vec::new();
398        let mut waiting = Vec::with_capacity(self.entries.len());
399        for mut entry in self.entries.drain(..) {
400            if !matches(&entry) {
401                waiting.push(entry);
402                continue;
403            }
404            if let Some(cancellation) = &entry.pending.cancellation {
405                cancellation.request_cancel();
406            }
407            if let Some(ctx) = executor.actor_context(&entry.root) {
408                if let Some(on_shutdown) = entry.pending.on_shutdown.as_mut() {
409                    let response = on_shutdown(&ctx);
410                    resolved.push(ResolvedSubcResponse { entry, response });
411                }
412            }
413        }
414        self.entries = waiting;
415        resolved
416    }
417
418    fn is_empty(&self) -> bool {
419        self.entries.is_empty()
420    }
421}
422
423#[derive(Clone)]
424struct PushSenders {
425    lossy_tx: mpsc::Sender<LossyPushEnvelope>,
426    reliable_tx: mpsc::UnboundedSender<PushEnvelope>,
427    lossy_overflow: Arc<push::LossyOverflow>,
428    lossy_seq: Arc<AtomicU64>,
429    fleet_status_client: FleetStatusClient,
430}
431
432#[derive(Clone)]
433struct PersistentCancelSignal {
434    inner: Arc<PersistentCancelInner>,
435}
436
437struct PersistentCancelInner {
438    cancelled: AtomicBool,
439    notify: Notify,
440}
441
442impl PersistentCancelSignal {
443    fn new() -> Self {
444        Self {
445            inner: Arc::new(PersistentCancelInner {
446                cancelled: AtomicBool::new(false),
447                notify: Notify::new(),
448            }),
449        }
450    }
451
452    fn cancel(&self) {
453        if !self.inner.cancelled.swap(true, Ordering::SeqCst) {
454            self.inner.notify.notify_waiters();
455        }
456    }
457
458    fn is_cancelled(&self) -> bool {
459        self.inner.cancelled.load(Ordering::SeqCst)
460    }
461
462    async fn cancelled(&self) {
463        // `enable()` REGISTERS this waiter before we read the flag, closing the
464        // lost-wakeup window: `notify_waiters()` only wakes already-registered
465        // waiters and stores no permit, so without enable() a `cancel()` firing
466        // between the flag read and `.await` would be missed and the future
467        // would park forever (cancel() fires only once). With enable(), a cancel
468        // racing the flag read still wakes the registered waiter. The loop is a
469        // belt-and-suspenders re-check on spurious wakeups.
470        loop {
471            let notified = self.inner.notify.notified();
472            tokio::pin!(notified);
473            notified.as_mut().enable();
474            if self.is_cancelled() {
475                return;
476            }
477            notified.await;
478        }
479    }
480}
481
482fn submit_active_tool_call(
483    executor: &Executor,
484    active: &ActiveToolCalls,
485    route: RouteChannel,
486    corr: u64,
487    root_id: ProjectRootId,
488    lane: Lane,
489    request_id: String,
490    detach_policy: RouteDetachPolicy,
491    job: crate::executor::ExecutorJob,
492) -> oneshot::Receiver<Response> {
493    let (rx, cancellation) =
494        executor.submit_cancellable_async(root_id.clone(), lane, request_id, job);
495    active
496        .lock()
497        .unwrap_or_else(std::sync::PoisonError::into_inner)
498        .insert(
499            (route, corr),
500            ActiveToolCall {
501                root_id,
502                cancellation,
503                detach_policy,
504            },
505        );
506    rx
507}
508
509fn finish_active_tool_call(active: &ActiveToolCalls, route: RouteChannel, corr: u64) -> bool {
510    active
511        .lock()
512        .unwrap_or_else(std::sync::PoisonError::into_inner)
513        .remove(&(route, corr))
514        .is_some()
515}
516
517fn active_tool_call_is_registered(
518    active: &ActiveToolCalls,
519    route: RouteChannel,
520    corr: u64,
521) -> bool {
522    active
523        .lock()
524        .unwrap_or_else(std::sync::PoisonError::into_inner)
525        .contains_key(&(route, corr))
526}
527
528fn cancel_active_tool_call(
529    active: &ActiveToolCalls,
530    executor: &Executor,
531    route: RouteChannel,
532    corr: u64,
533    reason: &str,
534) -> bool {
535    let call = active
536        .lock()
537        .unwrap_or_else(std::sync::PoisonError::into_inner)
538        .remove(&(route, corr));
539    let Some(call) = call else {
540        return false;
541    };
542    let outcome = executor.cancel_job(&call.root_id, &call.cancellation);
543    log::debug!(
544        "subc attach: cancelled active tool call route={route} corr={corr} reason={reason} outcome={outcome:?}"
545    );
546    true
547}
548
549#[derive(Clone, Copy, Debug, PartialEq, Eq)]
550enum RouteWorkDisposition {
551    RetainForReplay,
552    RetainStartedForReplay,
553    Abandon,
554}
555
556fn apply_route_work_disposition(
557    active: &ActiveToolCalls,
558    executor: &Executor,
559    route: RouteChannel,
560    disposition: RouteWorkDisposition,
561    reason: &str,
562) -> usize {
563    if disposition != RouteWorkDisposition::Abandon {
564        let route_calls = active
565            .lock()
566            .unwrap_or_else(std::sync::PoisonError::into_inner)
567            .iter()
568            .filter(|((call_route, _), _)| *call_route == route)
569            .map(|(key, call)| (*key, call.clone()))
570            .collect::<Vec<_>>();
571        let mut retained = 0usize;
572        let mut cancelled_before_execution = 0usize;
573        let mut cancelled_terminal = 0usize;
574
575        for (key, call) in route_calls {
576            let remove = match (call.detach_policy, disposition) {
577                (RouteDetachPolicy::RetainForReplay, RouteWorkDisposition::RetainForReplay) => {
578                    retained += 1;
579                    false
580                }
581                (
582                    RouteDetachPolicy::RetainForReplay,
583                    RouteWorkDisposition::RetainStartedForReplay,
584                ) => {
585                    match executor.cancel_job_before_execution(&call.root_id, &call.cancellation) {
586                        PreExecutionCancelOutcome::AlreadyStarted => {
587                            retained += 1;
588                            false
589                        }
590                        PreExecutionCancelOutcome::QueuedRemoved
591                        | PreExecutionCancelOutcome::DispatchedCancelled => {
592                            cancelled_before_execution += 1;
593                            true
594                        }
595                    }
596                }
597                (RouteDetachPolicy::CancelOnDetach, _) => {
598                    executor.cancel_job(&call.root_id, &call.cancellation);
599                    cancelled_terminal += 1;
600                    true
601                }
602                (_, RouteWorkDisposition::Abandon) => unreachable!("handled below"),
603            };
604            if remove {
605                active
606                    .lock()
607                    .unwrap_or_else(std::sync::PoisonError::into_inner)
608                    .remove(&key);
609            }
610        }
611        log::debug!(
612            "subc attach: retained {retained} replayable tool call(s), cancelled {cancelled_before_execution} replayable call(s) before execution, and cancelled {cancelled_terminal} teardown-terminal call(s) route={route} reason={reason}"
613        );
614        return retained;
615    }
616
617    let cancelled = {
618        let mut calls = active
619            .lock()
620            .unwrap_or_else(std::sync::PoisonError::into_inner);
621        let mut cancelled = Vec::new();
622        calls.retain(|(call_route, _), call| {
623            if *call_route == route {
624                cancelled.push(call.clone());
625                false
626            } else {
627                true
628            }
629        });
630        cancelled
631    };
632    for call in &cancelled {
633        let outcome = executor.cancel_job(&call.root_id, &call.cancellation);
634        log::debug!(
635            "subc attach: cancelled active tool call route={route} reason={reason} outcome={outcome:?}"
636        );
637    }
638    cancelled.len()
639}
640
641fn cancel_all_active_tool_calls(
642    active: &ActiveToolCalls,
643    executor: &Executor,
644    reason: &str,
645) -> usize {
646    let cancelled = {
647        let mut calls = active
648            .lock()
649            .unwrap_or_else(std::sync::PoisonError::into_inner);
650        calls.drain().map(|(_, call)| call).collect::<Vec<_>>()
651    };
652    for call in &cancelled {
653        let outcome = executor.cancel_job(&call.root_id, &call.cancellation);
654        log::debug!("subc attach: cancelled active tool call reason={reason} outcome={outcome:?}");
655    }
656    cancelled.len()
657}
658
659#[derive(Debug, Clone, Copy, PartialEq, Eq)]
660pub enum BindTrust {
661    FirstParty,
662    Untrusted,
663}
664
665impl BindTrust {
666    fn allows_bash_observation(self) -> bool {
667        matches!(self, Self::FirstParty)
668    }
669
670    fn label(self) -> &'static str {
671        match self {
672            Self::FirstParty => "first_party",
673            Self::Untrusted => "untrusted",
674        }
675    }
676
677    fn sandbox_trust(self) -> PrincipalTrust {
678        match self {
679            Self::FirstParty => PrincipalTrust::FirstParty,
680            Self::Untrusted => PrincipalTrust::Untrusted,
681        }
682    }
683}
684
685pub(super) fn trust_for_principal(principal: &Option<Principal>) -> BindTrust {
686    match principal {
687        Some(Principal::Direct) => BindTrust::FirstParty,
688        // Module renames are flag-days: the daemon registry refuses duplicate
689        // active ids, so a renaming module cannot advertise both names during
690        // its transition. This allowlist is DIALLED, not dialling — it must
691        // accept a module's NEW name in a released binary before the module
692        // starts using it, and the old name stays until the flip has settled.
693        // That is why transitional pairs appear here: llm-runner/broca was the
694        // previous rename, alfonso-core/prefrontal is the current one. When
695        // retiring an old name, confirm the fleet no longer spawns it — a
696        // stale entry here is inert, but a missing one silently downgrades a
697        // first-party module to Untrusted and revokes its bash access.
698        Some(Principal::Reserved { module_id })
699            if module_id == "llm-runner"
700                || module_id == "aft"
701                || module_id == "broca"
702                || module_id == "alfonso-core"
703                || module_id == "prefrontal"
704                || module_id == "prefrontal-core" =>
705        {
706            BindTrust::FirstParty
707        }
708        Some(Principal::Reserved { .. }) | Some(Principal::Unverified) | None => {
709            BindTrust::Untrusted
710        }
711    }
712}
713
714fn harness_forces_untrusted(harness: &str) -> bool {
715    harness.starts_with("fed:")
716}
717
718pub(super) fn trust_for_bind(harness: &str, principal: &Option<Principal>) -> BindTrust {
719    if harness_forces_untrusted(harness) {
720        BindTrust::Untrusted
721    } else {
722        trust_for_principal(principal)
723    }
724}
725
726fn principal_id(principal: &Option<Principal>) -> Option<String> {
727    match principal {
728        Some(Principal::Direct) => Some("direct".to_string()),
729        Some(Principal::Reserved { module_id }) => Some(format!("reserved:{module_id}")),
730        Some(Principal::Unverified) => Some("unverified".to_string()),
731        None => None,
732    }
733}
734
735fn principal_label(principal: &Option<Principal>) -> String {
736    principal_id(principal).unwrap_or_else(|| "absent".to_string())
737}
738
739#[derive(Debug)]
740/// Per-root route metadata owned by the subc loop. The `active_bash_waits` field
741/// counts detached bash processes that are still being observed for this root.
742/// Any future logic that evicts roots based on idle time must not evict a root
743/// while this count is greater than zero, because a foreground bash response may
744/// still arrive later.
745struct RootMeta {
746    maintenance_pending: bool,
747    maintenance_jobs_in_flight: usize,
748    maintenance_queued_kinds: VecDeque<MaintenanceDrainKind>,
749    maintenance_last_submitted: Option<Instant>,
750    maintenance_poisoned: bool,
751    last_touched: Instant,
752    diagnostics_on_edit: bool,
753    active_bash_waits: usize,
754    idle_artifacts_evicted: bool,
755    unbound_quiesced: bool,
756    consecutive_missing_sweeps: u8,
757}
758
759#[derive(Debug)]
760struct PendingBind {
761    bind_root_id: ProjectRootId,
762    inserted_new_actor: bool,
763    cancelled: bool,
764    configure_request_id: String,
765    started_at: Instant,
766    warned_half_deadline: bool,
767    deadline_reported: bool,
768    corr: u64,
769    ver: u8,
770    flags: Flags,
771    /// Exact-job cancellation for the submitted configure: Goodbye and
772    /// deadline expiry cancel the executor job operationally (queued jobs are
773    /// removed, running configures return at their next checkpoint) instead of
774    /// only marking bookkeeping.
775    cancellation: crate::executor::JobCancellation,
776}
777
778struct RouteBindCompletion {
779    route: RouteChannel,
780    identity: RouteIdentity,
781    bind_root_id: ProjectRootId,
782    inserted_new_actor: bool,
783    configure_response: Response,
784    diagnostics_on_edit: bool,
785    ver: u8,
786    corr: u64,
787    flags: Flags,
788}
789
790#[derive(Debug, Clone)]
791struct RouteIdentity(Arc<RouteIdentityData>);
792
793#[derive(Debug)]
794struct RouteIdentityData {
795    root: ProjectRootId,
796    project_root: PathBuf,
797    harness: String,
798    session: String,
799    trust: BindTrust,
800    spawn_principal: AuthenticatedPrincipal,
801    consumer_elicitation_capable: bool,
802}
803
804impl Deref for RouteIdentity {
805    type Target = RouteIdentityData;
806
807    fn deref(&self) -> &Self::Target {
808        &self.0
809    }
810}
811
812#[derive(Debug, Clone)]
813struct RetainedSessionIdentity {
814    harness: String,
815    trust: BindTrust,
816}
817
818#[derive(Clone)]
819struct BgSub {
820    corr: u64,
821    ver: u8,
822    flags: Flags,
823    root: ProjectRootId,
824    session: String,
825}
826
827#[derive(Clone, Copy, Debug)]
828struct BgWakeState {
829    next_nudge_at: Instant,
830    nudges_sent: u32,
831}
832
833impl BgWakeState {
834    fn armed(now: Instant) -> Self {
835        Self {
836            next_nudge_at: now,
837            nudges_sent: 0,
838        }
839    }
840}
841
842type BgWakePending = HashMap<RouteChannel, BgWakeState>;
843
844// A session can be observed by multiple long-lived consumer records. Retain
845// every route so each wake uses the correlation captured by that route's BgSub.
846type BgSubsBySession = HashMap<(ProjectRootId, String), HashSet<RouteChannel>>;
847
848struct MaintenanceCompletion {
849    root_id: ProjectRootId,
850    kind: MaintenanceDrainKind,
851    response: Response,
852    empty_bg_sessions: Vec<(String, u64)>,
853    unacked_bg_keys: Option<HashSet<String>>,
854    requeue_kind: Option<MaintenanceDrainKind>,
855}
856
857#[derive(Clone, Copy, Debug, PartialEq, Eq)]
858enum MaintenanceDrainKind {
859    Watcher,
860    Lsp,
861    ConfigureTail,
862    CompletionDrains,
863}
864
865impl MaintenanceDrainKind {
866    fn label(self) -> &'static str {
867        match self {
868            Self::Watcher => "watcher",
869            Self::Lsp => "lsp",
870            Self::ConfigureTail => "configure-tail",
871            Self::CompletionDrains => "completion-drains",
872        }
873    }
874}
875
876#[derive(Debug, Default)]
877struct MaintenanceJobOutcome {
878    empty_bg_sessions: Vec<(String, u64)>,
879    unacked_bg_keys: Option<HashSet<String>>,
880    requeue_kind: Option<MaintenanceDrainKind>,
881}
882
883#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
884struct ReverseCorrKey {
885    route: RouteChannel,
886    corr: u64,
887}
888
889struct PendingBashAsk {
890    route: RouteChannel,
891    tool_corr: u64,
892    tool_flags: Flags,
893    tool_ver: u8,
894    root: ProjectRootId,
895    project_root: PathBuf,
896    session_id: String,
897    spawn_principal: AuthenticatedPrincipal,
898    edit_slot_survives: Option<bool>,
899    request_id: String,
900    arguments: Value,
901    format_context: crate::subc_format::FormatContext,
902    cancel: bash::BashWaitCancel,
903    grants: Vec<String>,
904    expires_at: Instant,
905}
906
907impl RootMeta {
908    fn new(now: Instant) -> Self {
909        Self {
910            maintenance_pending: false,
911            maintenance_jobs_in_flight: 0,
912            maintenance_queued_kinds: VecDeque::new(),
913            maintenance_last_submitted: None,
914            maintenance_poisoned: false,
915            last_touched: now,
916            diagnostics_on_edit: false,
917            active_bash_waits: 0,
918            idle_artifacts_evicted: false,
919            unbound_quiesced: false,
920            consecutive_missing_sweeps: 0,
921        }
922    }
923
924    fn note_activity(&mut self) {
925        self.last_touched = Instant::now();
926    }
927
928    fn reactivate_bound(&mut self) {
929        self.note_activity();
930        self.idle_artifacts_evicted = false;
931        self.unbound_quiesced = false;
932    }
933}
934
935fn due_maintenance_jobs(
936    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
937    executor: Option<&Executor>,
938    bg_sub_by_session: &BgSubsBySession,
939    bg_wake_pending: &BgWakePending,
940    budget: usize,
941    pending_bind_roots: &HashSet<ProjectRootId>,
942) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
943    let mut jobs = Vec::new();
944    let mut deferred = false;
945    let mut roots = live_roots.keys().cloned().collect::<Vec<_>>();
946    roots.sort_by(|left, right| {
947        let left_last = live_roots
948            .get(left)
949            .and_then(|meta| meta.maintenance_last_submitted);
950        let right_last = live_roots
951            .get(right)
952            .and_then(|meta| meta.maintenance_last_submitted);
953        left_last
954            .cmp(&right_last)
955            .then_with(|| left.as_path().cmp(right.as_path()))
956    });
957
958    for root_id in roots {
959        let Some(meta) = live_roots.get_mut(&root_id) else {
960            continue;
961        };
962        if meta.maintenance_poisoned {
963            continue;
964        }
965
966        if pending_bind_roots.contains(&root_id) {
967            if meta.maintenance_pending || !meta.maintenance_queued_kinds.is_empty() {
968                deferred = true;
969            }
970            continue;
971        }
972
973        if !meta.maintenance_pending {
974            if jobs.len() >= budget {
975                deferred = true;
976                continue;
977            }
978            // Only enqueue kinds with pending work. Probes are cheap and
979            // fail-open (contended sources count as pending), so an idle root
980            // costs four probes per tick instead of four dispatched jobs.
981            let executor_actor_context =
982                executor.and_then(|executor| executor.actor_context(&root_id));
983            let root_has_pending_bg_wake =
984                bg_sub_by_session.iter().any(|((sub_root, _), channels)| {
985                    sub_root == &root_id
986                        && channels
987                            .iter()
988                            .any(|channel| bg_wake_pending.contains_key(channel))
989                });
990            let kinds_with_work: Vec<MaintenanceDrainKind> = match executor_actor_context {
991                Some(ctx) => INITIAL_MAINTENANCE_DRAIN_KINDS
992                    .into_iter()
993                    .filter(|kind| {
994                        if meta.unbound_quiesced && !matches!(kind, MaintenanceDrainKind::Lsp) {
995                            return false;
996                        }
997                        match kind {
998                            MaintenanceDrainKind::Watcher => ctx.watcher_drain_has_work(),
999                            MaintenanceDrainKind::Lsp => ctx.lsp_drain_has_work(),
1000                            MaintenanceDrainKind::ConfigureTail => ctx.configure_tail_has_work(),
1001                            // Every CompletionDrains source is visible at this enqueue site:
1002                            // AppContext probes completion queues, this loop owns bg wakes,
1003                            // and queued continuations bypass probing via maintenance_pending.
1004                            // New drain sources must expose a probe here rather than making
1005                            // every subscribed root fail open again.
1006                            MaintenanceDrainKind::CompletionDrains => {
1007                                root_has_pending_bg_wake || ctx.completion_drains_have_work()
1008                            }
1009                        }
1010                    })
1011                    .collect(),
1012                None if meta.unbound_quiesced => Vec::new(),
1013                // No context handle (actor gone mid-tick): enqueue everything.
1014                None => INITIAL_MAINTENANCE_DRAIN_KINDS.to_vec(),
1015            };
1016            if kinds_with_work.is_empty() {
1017                continue;
1018            }
1019            meta.maintenance_pending = true;
1020            meta.maintenance_queued_kinds.extend(kinds_with_work);
1021        }
1022
1023        while let Some(kind) = meta.maintenance_queued_kinds.pop_front() {
1024            if jobs.len() >= budget {
1025                meta.maintenance_queued_kinds.push_front(kind);
1026                deferred = true;
1027                break;
1028            }
1029            meta.maintenance_jobs_in_flight += 1;
1030            meta.maintenance_last_submitted = Some(Instant::now());
1031            jobs.push((root_id.clone(), kind));
1032        }
1033
1034        meta.maintenance_pending =
1035            meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
1036    }
1037
1038    (jobs, deferred)
1039}
1040
1041fn eviction_estimate_label(estimate: &crate::memory::MemoryEstimate) -> String {
1042    match estimate.estimated_bytes {
1043        Some(bytes) => format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
1044        None if estimate.status == "busy" => "busy".to_string(),
1045        None => "not estimated".to_string(),
1046    }
1047}
1048
1049fn optional_memory_label(bytes: Option<u64>) -> String {
1050    bytes.map_or_else(
1051        || "not estimated".to_string(),
1052        |bytes| format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
1053    )
1054}
1055
1056fn pressure_relief_label(relief: &crate::memory::AllocatorPressureRelief) -> String {
1057    format!(
1058        "; allocator pressure relief: RSS {} -> {}, in-use {} -> {}, allocated {} -> {}, slack {} -> {}, allocator reported {:.1} MB released",
1059        optional_memory_label(relief.rss_before_bytes),
1060        optional_memory_label(relief.rss_after_bytes),
1061        optional_memory_label(relief.allocator_before.bytes_in_use),
1062        optional_memory_label(relief.allocator_after.bytes_in_use),
1063        optional_memory_label(relief.allocator_before.size_allocated),
1064        optional_memory_label(relief.allocator_after.size_allocated),
1065        optional_memory_label(relief.allocator_before.retained_slack_bytes),
1066        optional_memory_label(relief.allocator_after.retained_slack_bytes),
1067        relief.bytes_released as f64 / (1024.0 * 1024.0),
1068    )
1069}
1070
1071fn idle_root_eviction_message(
1072    root_id: &ProjectRootId,
1073    memory: &crate::memory::RootMemorySnapshot,
1074    pressure_relief: Option<&crate::memory::AllocatorPressureRelief>,
1075) -> String {
1076    // Bash, LSP, and parser state remain resident. The freed total is deliberately
1077    // only the known-byte portion of handles eviction actually drops.
1078    let freed_bytes = [
1079        &memory.semantic,
1080        &memory.trigram,
1081        &memory.symbols,
1082        &memory.callgraph,
1083        &memory.inspect,
1084    ]
1085    .iter()
1086    .filter_map(|estimate| estimate.estimated_bytes)
1087    .fold(0u64, u64::saturating_add);
1088    let mut message = format!(
1089        "evicted idle root {}: freed ~{:.1} MB (semantic {}, trigram {}, symbols {}, callgraph {}, inspect {}; retained: bash {}, lsp {}, parser_pool {})",
1090        root_id.as_path().display(),
1091        freed_bytes as f64 / (1024.0 * 1024.0),
1092        eviction_estimate_label(&memory.semantic),
1093        eviction_estimate_label(&memory.trigram),
1094        eviction_estimate_label(&memory.symbols),
1095        eviction_estimate_label(&memory.callgraph),
1096        eviction_estimate_label(&memory.inspect),
1097        eviction_estimate_label(&memory.bash),
1098        eviction_estimate_label(&memory.lsp),
1099        eviction_estimate_label(&memory.parser_pool),
1100    );
1101    if let Some(pressure_relief) = pressure_relief {
1102        message.push_str(&pressure_relief_label(pressure_relief));
1103    }
1104    message
1105}
1106
1107fn root_idle_ttl(executor: &Executor, root_id: &ProjectRootId) -> Duration {
1108    executor
1109        .actor_context(root_id)
1110        .map(|ctx| ctx.config().idle.root_ttl())
1111        .unwrap_or(IDLE_ROOT_TTL)
1112}
1113
1114fn process_has_been_idle(
1115    now: Instant,
1116    live_roots: &HashMap<ProjectRootId, RootMeta>,
1117    executor: &Executor,
1118) -> bool {
1119    !live_roots.is_empty()
1120        && live_roots.iter().all(|(root_id, meta)| {
1121            now.saturating_duration_since(meta.last_touched) >= root_idle_ttl(executor, root_id)
1122                && meta.active_bash_waits == 0
1123                && !meta.maintenance_pending
1124                && meta.maintenance_queued_kinds.is_empty()
1125        })
1126}
1127
1128fn allocator_pressure_relief_after_idle_sweep(
1129    now: Instant,
1130    live_roots: &HashMap<ProjectRootId, RootMeta>,
1131    executor: &Executor,
1132) -> Option<crate::memory::AllocatorPressureRelief> {
1133    if !process_has_been_idle(now, live_roots, executor)
1134        || live_roots.keys().any(|root_id| {
1135            executor
1136                .actor_context(root_id)
1137                .is_some_and(|ctx| ctx.artifact_eviction_blocked())
1138        })
1139    {
1140        return None;
1141    }
1142
1143    #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu")))]
1144    {
1145        Some(crate::memory::relieve_allocator_pressure())
1146    }
1147    #[cfg(not(any(target_os = "macos", all(target_os = "linux", target_env = "gnu"))))]
1148    {
1149        None
1150    }
1151}
1152
1153fn quiesce_unbound_root(
1154    root_id: &ProjectRootId,
1155    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1156    executor: &Arc<Executor>,
1157) {
1158    let Some(meta) = live_roots.get_mut(root_id) else {
1159        return;
1160    };
1161
1162    let ctx = executor.actor_context(root_id);
1163    if let Some(ctx) = ctx.as_ref() {
1164        // Close lifecycle admission before touching scheduler queues. A running
1165        // ConfigureTail cannot release gates, install a watcher, or reserve a
1166        // callgraph build after this transition becomes visible.
1167        ctx.mark_subc_unbound();
1168        ctx.bash_background()
1169            .replace_live_delivery_sessions(HashSet::new());
1170    }
1171    let cancelled = executor.cancel_queued_maintenance(root_id);
1172    // Transient unbind keeps the root WARM: the watcher stays running (its
1173    // events accumulate and replay on rebind, so no unobserved gap exists) and
1174    // resident artifacts stay resident. Host restarts unbind every root and
1175    // rebind seconds later; stopping the watcher here would force strict
1176    // re-verification plus a full callgraph rebuild on every restart. The
1177    // expensive teardown (watcher stop + gap invalidation) belongs to the
1178    // idle-TTL reaper and the root-deleted path.
1179    let discarded = ctx
1180        .map(|ctx| crate::commands::configure::cancel_deferred_configure_maintenance(&ctx))
1181        .unwrap_or(0);
1182    meta.unbound_quiesced = true;
1183    meta.maintenance_queued_kinds.clear();
1184    meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
1185    log::info!(
1186        "subc attach: quiesced unbound root {} (cancelled {} queued maintenance job(s), cancelled {} configure maintenance job(s)); cause=goodbye_unbound",
1187        root_id.as_path().display(),
1188        cancelled,
1189        discarded
1190    );
1191}
1192
1193#[allow(clippy::too_many_arguments)]
1194fn quiesce_connection_roots(
1195    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1196    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
1197    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1198    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1199    installed_route_epochs: &mut HashMap<u16, u32>,
1200    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1201    active_tool_calls: &ActiveToolCalls,
1202    executor: &Arc<Executor>,
1203) {
1204    cancel_all_active_tool_calls(active_tool_calls, executor, "connection teardown");
1205    for cancel in route_bash_cancels.values() {
1206        cancel.token.cancel();
1207    }
1208    route_bash_cancels.clear();
1209
1210    let mut roots = live_roots.keys().cloned().collect::<HashSet<_>>();
1211    for pending in pending_binds.values_mut() {
1212        pending.cancelled = true;
1213        roots.insert(pending.bind_root_id.clone());
1214        let _ = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
1215    }
1216
1217    // A connection exit abandons every installed route at once. Close lifecycle
1218    // admission before cancelling maintenance so no deferred worker can restore
1219    // root activity after the loop-owned route tables disappear.
1220    for root_id in roots {
1221        if live_roots.contains_key(&root_id) {
1222            quiesce_unbound_root(&root_id, live_roots, executor);
1223        } else if let Some(ctx) = executor.actor_context(&root_id) {
1224            ctx.mark_subc_unbound();
1225            executor.cancel_queued_maintenance(&root_id);
1226            crate::commands::configure::cancel_deferred_configure_maintenance(&ctx);
1227        }
1228    }
1229
1230    routes.clear();
1231    root_channels.clear();
1232    installed_route_epochs.clear();
1233}
1234
1235/// Per-channel epoch watermarks for roots reclaimed without a client Goodbye.
1236/// The 16-bit channel space bounds this map, and no root identity or resource
1237/// handle is retained. It exists only so late requests receive a typed error.
1238#[derive(Debug, Default)]
1239struct ReclaimedRoutes {
1240    highest_epoch_by_channel: HashMap<u16, u32>,
1241}
1242
1243impl ReclaimedRoutes {
1244    fn insert(&mut self, route: RouteChannel) {
1245        self.highest_epoch_by_channel
1246            .entry(route.channel)
1247            .and_modify(|epoch| *epoch = (*epoch).max(route.epoch))
1248            .or_insert(route.epoch);
1249    }
1250
1251    fn contains(&self, route: RouteChannel) -> bool {
1252        self.highest_epoch_by_channel
1253            .get(&route.channel)
1254            .is_some_and(|epoch| route.epoch <= *epoch)
1255    }
1256}
1257
1258#[derive(Debug, Default)]
1259struct IdleReapOutcome {
1260    evicted: usize,
1261    forgotten_deleted_roots: Vec<ProjectRootId>,
1262}
1263
1264fn reap_idle_roots(
1265    now: Instant,
1266    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1267    pending_binds: &HashMap<RouteChannel, PendingBind>,
1268    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1269    executor: &Arc<Executor>,
1270    metrics: &DispatchPathMetrics,
1271) -> IdleReapOutcome {
1272    let pending_bind_roots = pending_binds
1273        .values()
1274        .map(|pending| pending.bind_root_id.clone())
1275        .collect::<HashSet<_>>();
1276    let mut census = ReapBlockerCensus::default();
1277    let mut candidates = Vec::new();
1278
1279    for (root_id, meta) in live_roots.iter_mut() {
1280        let deleted = !root_id.as_path().exists();
1281        if deleted {
1282            // A missing directory makes a bound route obsolete, but one failed
1283            // lookup is not enough evidence to tear down a client-visible actor.
1284            // Requiring two maintenance sweeps protects atomic replacement and
1285            // transient filesystem failures; observing the path resets the proof.
1286            // Absence also covers renames: a task's cwd handle can follow the
1287            // moved directory while the registered path disappears. The old
1288            // path is deliberately treated as a retired root identity, so the
1289            // reaper accepts killing such tasks; rename a project only with no
1290            // live tasks rather than relying on cwd-resolution heuristics.
1291            meta.consecutive_missing_sweeps = meta.consecutive_missing_sweeps.saturating_add(1);
1292        } else {
1293            meta.consecutive_missing_sweeps = 0;
1294        }
1295        let deletion_confirmed = meta.consecutive_missing_sweeps >= 2;
1296        let has_bound_route = root_channels
1297            .get(root_id)
1298            .is_some_and(|channels| !channels.is_empty());
1299        let has_pending_bind = pending_bind_roots.contains(root_id);
1300
1301        if deleted {
1302            let mut retained = false;
1303            if !deletion_confirmed {
1304                census.absence_unconfirmed += 1;
1305                retained = true;
1306            }
1307            // Once absence is confirmed, the directory cannot serve this route
1308            // again. Neither a stale route nor the lack of normal unbind cleanup
1309            // justifies retaining the root; purge removes the route after retirement.
1310            if meta.active_bash_waits > 0 {
1311                census.bash_waits += 1;
1312                retained = true;
1313            }
1314            if meta.maintenance_pending {
1315                census.maintenance_pending += 1;
1316                retained = true;
1317            }
1318            if !meta.maintenance_queued_kinds.is_empty() {
1319                census.maintenance_queued += 1;
1320                retained = true;
1321            }
1322            if has_pending_bind {
1323                census.pending_binds += 1;
1324                retained = true;
1325            }
1326            match executor.try_actor_is_idle(root_id) {
1327                Some(true) => {}
1328                Some(false) => {
1329                    census.actor_busy += 1;
1330                    retained = true;
1331                }
1332                None => {
1333                    census.actor_state_busy += 1;
1334                    retained = true;
1335                }
1336            }
1337            if retained {
1338                census.deleted_retained += 1;
1339                continue;
1340            }
1341        } else {
1342            // Route teardown marks the lifecycle admission gate before the last
1343            // channel disappears. Requiring zero bound channels and a quiesced
1344            // lifecycle prevents a still-bound root from losing its watcher.
1345            if has_bound_route
1346                || !meta.unbound_quiesced
1347                || meta.idle_artifacts_evicted
1348                || now.saturating_duration_since(meta.last_touched)
1349                    < root_idle_ttl(executor, root_id)
1350                || meta.active_bash_waits > 0
1351                || meta.maintenance_pending
1352                || !meta.maintenance_queued_kinds.is_empty()
1353                || has_pending_bind
1354                || !executor.actor_is_idle(root_id)
1355            {
1356                continue;
1357            }
1358        }
1359        candidates.push((root_id.clone(), deleted));
1360    }
1361
1362    let mut reaped = Vec::new();
1363    let mut forgotten_deleted_roots = Vec::new();
1364    for (root_id, deleted) in candidates {
1365        let Some(ctx) = executor.actor_context(&root_id) else {
1366            if deleted {
1367                census.deleted_retained += 1;
1368                census.actor_busy += 1;
1369            }
1370            continue;
1371        };
1372        // A TTL-aged unbound root retained its watcher-derived pending paths
1373        // across the transient-unbind window. Strict gap invalidation subsumes
1374        // them, but every abort path must restore them because a rebind can
1375        // still happen until eviction commits.
1376        //
1377        // After two consecutive directory-absence scans confirm that the
1378        // root is gone, terminate its background task before checking the
1379        // artifact-eviction gate. The task can otherwise keep the root's
1380        // artifacts in use; cleanup first lets confirmed reclamation finish
1381        // without weakening the gate for unrelated active work.
1382        if deleted {
1383            ctx.bash_background()
1384                .kill_running_tasks_for_root(root_id.as_path());
1385        }
1386        let taken_pending = Some(ctx.take_pending_reconciliation_state());
1387        if ctx.artifact_eviction_blocked() {
1388            if let Some(pending) = taken_pending {
1389                ctx.restore_pending_reconciliation_state(pending);
1390            }
1391            if deleted {
1392                census.deleted_retained += 1;
1393                census.artifact_eviction_blocked += 1;
1394            }
1395            continue;
1396        }
1397        let memory_before = ctx.memory_root_snapshot();
1398        if !ctx.evict_idle_artifacts() {
1399            if let Some(pending) = taken_pending {
1400                ctx.restore_pending_reconciliation_state(pending);
1401            }
1402            if deleted {
1403                census.deleted_retained += 1;
1404                census.artifact_eviction_failed += 1;
1405            }
1406            continue;
1407        }
1408        drop(taken_pending);
1409        ctx.stop_watcher_runtime_in_background();
1410        // Edits during watcher downtime are unobserved. Advance publication
1411        // epochs and force strict verification before any later warm reload.
1412        ctx.invalidate_artifacts_after_watcher_gap();
1413
1414        if deleted {
1415            if executor.retire_idle_actor_in_background(&root_id) {
1416                live_roots.remove(&root_id);
1417                forgotten_deleted_roots.push(root_id.clone());
1418            } else {
1419                census.deleted_retained += 1;
1420                census.actor_busy += 1;
1421            }
1422        } else {
1423            if let Some(meta) = live_roots.get_mut(&root_id) {
1424                meta.idle_artifacts_evicted = true;
1425            }
1426            ctx.release_idle_reopenable_resources_in_background();
1427        }
1428        reaped.push((root_id, memory_before));
1429    }
1430
1431    // Emit-on-change: the census is on the health surface every sweep; the log
1432    // line is for transitions, including the one back to zero retained.
1433    let census_changed = metrics.record_reap(census);
1434    if census_changed {
1435        log::info!(
1436            "subc attach: retained {} deleted root(s) during idle reap; blockers={}",
1437            census.deleted_retained,
1438            census.blocker_histogram()
1439        );
1440    }
1441
1442    let pressure_relief = (!reaped.is_empty())
1443        .then(|| allocator_pressure_relief_after_idle_sweep(now, live_roots, executor))
1444        .flatten();
1445    for (root_id, memory_before) in &reaped {
1446        log::info!(
1447            "{}",
1448            idle_root_eviction_message(root_id, memory_before, pressure_relief.as_ref())
1449        );
1450    }
1451    IdleReapOutcome {
1452        evicted: reaped.len(),
1453        forgotten_deleted_roots,
1454    }
1455}
1456
1457/// Shut down language servers for roots that have had no request for the
1458/// configured LSP idle window. This is independent of artifact eviction and
1459/// runs even while the root is still bound.
1460///
1461/// `meta.last_touched` moves on every inbound tool call for a bound root
1462/// (`reactivate_bound` in the route-request handler) and on response delivery,
1463/// so an active harness does not look idle.
1464fn reap_idle_lsp_servers(
1465    now: Instant,
1466    live_roots: &HashMap<ProjectRootId, RootMeta>,
1467    executor: &Executor,
1468) {
1469    for (root_id, meta) in live_roots {
1470        let Some(ctx) = executor.actor_context(root_id) else {
1471            continue;
1472        };
1473        crate::runtime_drain::shutdown_idle_lsp_at(&ctx, now, meta.last_touched);
1474    }
1475}
1476
1477#[allow(clippy::too_many_arguments)]
1478fn purge_deleted_root_residents(
1479    root_id: &ProjectRootId,
1480    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1481    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1482    installed_route_epochs: &mut HashMap<u16, u32>,
1483    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1484    active_tool_calls: &ActiveToolCalls,
1485    executor: &Executor,
1486    retry_buffer: &mut RetryBuffer,
1487    reclaimed_routes: &mut ReclaimedRoutes,
1488    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1489    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1490    bg_subs: &mut HashMap<RouteChannel, BgSub>,
1491    bg_sub_by_session: &mut BgSubsBySession,
1492    bg_wake_pending: &mut BgWakePending,
1493    bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
1494    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1495    metrics: &DispatchPathMetrics,
1496) {
1497    let mut stale_routes = root_channels.get(root_id).cloned().unwrap_or_default();
1498    stale_routes.extend(
1499        routes
1500            .iter()
1501            .filter_map(|(route, identity)| (&identity.root == root_id).then_some(*route)),
1502    );
1503    stale_routes.extend(
1504        bg_sub_by_session
1505            .iter()
1506            .filter(|((root, _), _)| root == root_id)
1507            .flat_map(|(_, routes)| routes.iter().copied()),
1508    );
1509    stale_routes.extend(
1510        pending_bash_asks
1511            .values()
1512            .filter_map(|ask| (&ask.root == root_id).then_some(ask.route)),
1513    );
1514
1515    for route in stale_routes {
1516        reclaimed_routes.insert(route);
1517        remove_installed_route(installed_route_epochs, route);
1518        remove_route_channel(routes, root_channels, route);
1519        if let Some(cancel) = route_bash_cancels.remove(&route) {
1520            cancel.token.cancel();
1521        }
1522        apply_route_work_disposition(
1523            active_tool_calls,
1524            executor,
1525            route,
1526            RouteWorkDisposition::Abandon,
1527            "root reclaim",
1528        );
1529        retry_buffer.remove(&route);
1530        if let Some(sub) = bg_subs.remove(&route) {
1531            metrics.record_bg_subscription_ended(&sub.root, &sub.session, route, "root-reclaim");
1532        }
1533        bg_wake_pending.remove(&route);
1534    }
1535    root_channels.remove(root_id);
1536    session_identity.retain(|(root, _), _| root != root_id);
1537    push_buffer.retain(|key, _| &key.root != root_id);
1538    bg_wake_epoch.retain(|(root, _), _| root != root_id);
1539    pending_bash_asks.retain(|_, ask| &ask.root != root_id);
1540    bg_sub_by_session.retain(|(root, _), _| root != root_id);
1541    sync_bg_live_delivery_sessions(executor, routes, Some(root_id));
1542
1543    log::info!(
1544        "subc attach: fully forgot deleted root {}; cause=absence_reclaim",
1545        root_id.as_path().display()
1546    );
1547}
1548
1549#[allow(clippy::too_many_arguments)]
1550fn submit_due_maintenance_jobs(
1551    executor: &Arc<Executor>,
1552    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1553    pending_binds: &HashMap<RouteChannel, PendingBind>,
1554    bg_sub_by_session: &BgSubsBySession,
1555    bg_wake_pending: &BgWakePending,
1556    bg_wake_epoch: &HashMap<(ProjectRootId, String), u64>,
1557    maintenance_tx: &mpsc::Sender<MaintenanceCompletion>,
1558    metrics: &Arc<DispatchPathMetrics>,
1559) {
1560    let pending_bind_roots = pending_binds
1561        .values()
1562        .map(|pending| pending.bind_root_id.clone())
1563        .collect::<HashSet<_>>();
1564    let (due_jobs, deferred_jobs) = due_maintenance_jobs(
1565        live_roots,
1566        Some(executor),
1567        bg_sub_by_session,
1568        bg_wake_pending,
1569        MAINTENANCE_SUBMIT_BUDGET,
1570        &pending_bind_roots,
1571    );
1572    if deferred_jobs {
1573        metrics
1574            .maintenance_budget_deferrals
1575            .fetch_add(1, Ordering::Relaxed);
1576    }
1577    for (root_id, kind) in due_jobs {
1578        let bg_sessions_to_check = if kind == MaintenanceDrainKind::CompletionDrains {
1579            bg_sub_by_session
1580                .iter()
1581                .filter_map(|((root, session), _)| {
1582                    if root == &root_id {
1583                        Some((
1584                            session.clone(),
1585                            bg_wake_epoch
1586                                .get(&(root_id.clone(), session.clone()))
1587                                .copied()
1588                                .unwrap_or(0),
1589                        ))
1590                    } else {
1591                        None
1592                    }
1593                })
1594                .collect()
1595        } else {
1596            Vec::new()
1597        };
1598        submit_maintenance_job(
1599            executor,
1600            root_id,
1601            kind,
1602            bg_sessions_to_check,
1603            maintenance_tx,
1604            metrics,
1605        );
1606    }
1607}
1608
1609fn should_requiesce_after_maintenance(
1610    meta: &RootMeta,
1611    completed_kind: MaintenanceDrainKind,
1612    bind_pending: bool,
1613) -> bool {
1614    meta.unbound_quiesced && completed_kind != MaintenanceDrainKind::Lsp && !bind_pending
1615}
1616
1617fn note_maintenance_completion(
1618    meta: &mut RootMeta,
1619    requeue_kind: Option<MaintenanceDrainKind>,
1620    fatal: bool,
1621    defer_requeue: bool,
1622) {
1623    if fatal {
1624        meta.maintenance_poisoned = true;
1625    }
1626
1627    if let Some(kind) = requeue_kind.filter(|_| !meta.maintenance_poisoned && !defer_requeue) {
1628        meta.maintenance_queued_kinds.push_back(kind);
1629    }
1630
1631    meta.maintenance_jobs_in_flight = meta.maintenance_jobs_in_flight.saturating_sub(1);
1632    meta.maintenance_pending =
1633        meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
1634}
1635
1636fn route_key(channel: u16, epoch: u32) -> RouteChannel {
1637    RouteChannel { channel, epoch }
1638}
1639
1640fn remove_installed_route(installed_epochs: &mut HashMap<u16, u32>, route: RouteChannel) {
1641    if installed_epochs.get(&route.channel).copied() == Some(route.epoch) {
1642        installed_epochs.remove(&route.channel);
1643    }
1644}
1645
1646fn ingress_route_should_be_processed(
1647    installed_epochs: &HashMap<u16, u32>,
1648    reclaimed_routes: &ReclaimedRoutes,
1649    frame: &Frame,
1650) -> bool {
1651    if frame.header.channel == 0
1652        || installed_epochs.get(&frame.header.channel).copied() == Some(frame.header.epoch)
1653    {
1654        return true;
1655    }
1656
1657    // A late request for a reclaimed root reaches the normal unknown-route
1658    // handler, which returns the typed `route_not_bound` error. Other stale or
1659    // never-installed generations remain silent so they cannot affect a newer
1660    // route or change the protocol's rejected-bind behavior.
1661    frame.header.ty == FrameType::Request
1662        && reclaimed_routes.contains(route_key(frame.header.channel, frame.header.epoch))
1663}
1664
1665fn bash_elicitation_timeout() -> Duration {
1666    if cfg!(debug_assertions) {
1667        if let Ok(raw) = std::env::var("AFT_TEST_SUBC_BASH_ELICITATION_TTL_MS") {
1668            if let Ok(ms) = raw.parse::<u64>() {
1669                if ms > 0 {
1670                    return Duration::from_millis(ms);
1671                }
1672            }
1673        }
1674    }
1675    BASH_ELICITATION_TIMEOUT
1676}
1677
1678fn allocate_reverse_corr(
1679    pending_bash_asks: &HashMap<ReverseCorrKey, PendingBashAsk>,
1680    route: RouteChannel,
1681    next_corr: &mut u64,
1682) -> u64 {
1683    loop {
1684        let corr = *next_corr;
1685        *next_corr = (*next_corr).wrapping_add(1).max(1);
1686        if !pending_bash_asks.contains_key(&ReverseCorrKey { route, corr }) {
1687            return corr;
1688        }
1689    }
1690}
1691
1692fn bash_permission_kind_label(kind: &crate::bash_permissions::PermissionKind) -> &'static str {
1693    match kind {
1694        crate::bash_permissions::PermissionKind::ExternalDirectory => "external directory",
1695        crate::bash_permissions::PermissionKind::Bash => "bash",
1696    }
1697}
1698
1699fn bash_elicitation_patterns(asks: &[crate::bash_permissions::PermissionAsk]) -> Vec<String> {
1700    let mut patterns = Vec::new();
1701    let mut seen = HashSet::new();
1702    for ask in asks {
1703        for pattern in ask.patterns.iter().chain(ask.always.iter()) {
1704            if seen.insert(pattern.clone()) {
1705                patterns.push(pattern.clone());
1706            }
1707        }
1708    }
1709    patterns
1710}
1711
1712fn bash_elicitation_message(
1713    command: &str,
1714    asks: &[crate::bash_permissions::PermissionAsk],
1715) -> String {
1716    let command = command.split_whitespace().collect::<Vec<_>>().join(" ");
1717    let patterns = bash_elicitation_patterns(asks);
1718    let pattern_text = if patterns.is_empty() {
1719        "no matched permission patterns".to_string()
1720    } else {
1721        patterns.join(", ")
1722    };
1723    let ask_kinds = asks
1724        .iter()
1725        .map(|ask| bash_permission_kind_label(&ask.kind))
1726        .collect::<HashSet<_>>()
1727        .into_iter()
1728        .collect::<Vec<_>>()
1729        .join(", ");
1730    if ask_kinds.is_empty() {
1731        format!("Allow bash command `{command}`? Matched patterns: {pattern_text}")
1732    } else {
1733        format!("Allow bash command `{command}`? Matched {ask_kinds} patterns: {pattern_text}")
1734    }
1735}
1736
1737fn bash_elicitation_request_body(
1738    command: &str,
1739    asks: &[crate::bash_permissions::PermissionAsk],
1740) -> Value {
1741    json!({
1742        "method": BASH_ELICITATION_CREATE_METHOD,
1743        "params": {
1744            "mode": "form",
1745            "message": bash_elicitation_message(command, asks),
1746            "requestedSchema": {
1747                "type": "object",
1748                "properties": {
1749                    "decision": {
1750                        "type": "string",
1751                        "enum": ["allow", "deny"],
1752                        "description": "Choose allow to run this bash command once, or deny to block it."
1753                    }
1754                },
1755                "required": ["decision"],
1756                "additionalProperties": false
1757            },
1758            "_meta": {
1759                "aft": {
1760                    "tool": "bash",
1761                    "command": command,
1762                    "asks": asks
1763                }
1764            }
1765        }
1766    })
1767}
1768
1769fn build_bash_elicitation_request_frame(
1770    ver: u8,
1771    route: RouteChannel,
1772    corr: u64,
1773    flags: Flags,
1774    command: &str,
1775    asks: &[crate::bash_permissions::PermissionAsk],
1776) -> Result<Frame, SubcError> {
1777    let body = bash_elicitation_request_body(command, asks);
1778    Frame::build_with_version(
1779        ver,
1780        FrameType::Request,
1781        flags,
1782        route.channel,
1783        route.epoch,
1784        corr,
1785        serde_json::to_vec(&body).map_err(SubcError::Json)?,
1786    )
1787    .map_err(SubcError::FrameBuild)
1788}
1789
1790fn bash_elicitation_reply_is_allow(body: &[u8]) -> bool {
1791    let Ok(value) = serde_json::from_slice::<Value>(body) else {
1792        return false;
1793    };
1794    flat_bash_elicitation_reply_is_allow(&value) || mcp_bash_elicitation_reply_is_allow(&value)
1795}
1796
1797fn flat_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1798    let Some(object) = value.as_object() else {
1799        return false;
1800    };
1801    object.len() == 1 && object.get("decision").and_then(Value::as_str) == Some("allow")
1802}
1803
1804fn mcp_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1805    let Some(object) = value.as_object() else {
1806        return false;
1807    };
1808    if object.len() != 2 || object.get("action").and_then(Value::as_str) != Some("accept") {
1809        return false;
1810    }
1811    let Some(content) = object.get("content").and_then(Value::as_object) else {
1812        return false;
1813    };
1814    content.len() == 1 && content.get("decision").and_then(Value::as_str) == Some("allow")
1815}
1816
1817#[allow(clippy::too_many_arguments)]
1818async fn settle_pending_bash_ask_denied(
1819    tx: &WriterSender,
1820    pending: PendingBashAsk,
1821    routes: &HashMap<RouteChannel, RouteIdentity>,
1822    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1823    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1824    shutdown: &Arc<Notify>,
1825    metrics: &DispatchPathMetrics,
1826) -> Result<(), SubcError> {
1827    let completion = bash::bash_denied_untrusted_completion(
1828        pending.route,
1829        pending.tool_corr,
1830        pending.tool_flags,
1831        pending.tool_ver,
1832        pending.root,
1833        pending.request_id,
1834        pending.format_context,
1835    );
1836    bash::handle_bash_deferred_completion(
1837        tx,
1838        completion,
1839        routes,
1840        live_roots,
1841        route_bash_cancels,
1842        shutdown,
1843        metrics,
1844    )
1845    .await
1846}
1847
1848fn take_pending_bash_asks_for_route(
1849    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1850    route: RouteChannel,
1851) -> Vec<PendingBashAsk> {
1852    let keys = pending_bash_asks
1853        .keys()
1854        .copied()
1855        .filter(|key| key.route == route)
1856        .collect::<Vec<_>>();
1857    keys.into_iter()
1858        .filter_map(|key| pending_bash_asks.remove(&key))
1859        .collect()
1860}
1861
1862#[allow(clippy::too_many_arguments)]
1863async fn settle_pending_bash_asks_for_route(
1864    tx: &WriterSender,
1865    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1866    route: RouteChannel,
1867    routes: &HashMap<RouteChannel, RouteIdentity>,
1868    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1869    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1870    shutdown: &Arc<Notify>,
1871    metrics: &DispatchPathMetrics,
1872) -> Result<(), SubcError> {
1873    for pending in take_pending_bash_asks_for_route(pending_bash_asks, route) {
1874        settle_pending_bash_ask_denied(
1875            tx,
1876            pending,
1877            routes,
1878            live_roots,
1879            route_bash_cancels,
1880            shutdown,
1881            metrics,
1882        )
1883        .await?;
1884    }
1885    Ok(())
1886}
1887
1888#[allow(clippy::too_many_arguments)]
1889async fn settle_all_pending_bash_asks(
1890    tx: &WriterSender,
1891    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1892    routes: &HashMap<RouteChannel, RouteIdentity>,
1893    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1894    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1895    shutdown: &Arc<Notify>,
1896    metrics: &DispatchPathMetrics,
1897) -> Result<(), SubcError> {
1898    let pending = pending_bash_asks
1899        .drain()
1900        .map(|(_, pending)| pending)
1901        .collect::<Vec<_>>();
1902    for pending in pending {
1903        settle_pending_bash_ask_denied(
1904            tx,
1905            pending,
1906            routes,
1907            live_roots,
1908            route_bash_cancels,
1909            shutdown,
1910            metrics,
1911        )
1912        .await?;
1913    }
1914    Ok(())
1915}
1916
1917#[allow(clippy::too_many_arguments)]
1918async fn expire_pending_bash_asks(
1919    tx: &WriterSender,
1920    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1921    routes: &HashMap<RouteChannel, RouteIdentity>,
1922    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1923    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1924    shutdown: &Arc<Notify>,
1925    metrics: &DispatchPathMetrics,
1926) -> Result<(), SubcError> {
1927    let now = Instant::now();
1928    let expired = pending_bash_asks
1929        .iter()
1930        .filter_map(|(key, pending)| (pending.expires_at <= now).then_some(*key))
1931        .collect::<Vec<_>>();
1932    for key in expired {
1933        if let Some(pending) = pending_bash_asks.remove(&key) {
1934            log::debug!(
1935                "subc attach: bash elicitation request {} on route {} expired fail-closed",
1936                key.corr,
1937                pending.route
1938            );
1939            settle_pending_bash_ask_denied(
1940                tx,
1941                pending,
1942                routes,
1943                live_roots,
1944                route_bash_cancels,
1945                shutdown,
1946                metrics,
1947            )
1948            .await?;
1949        }
1950    }
1951    Ok(())
1952}
1953
1954#[allow(clippy::too_many_arguments)]
1955async fn handle_bash_elicitation_reply(
1956    tx: &WriterSender,
1957    frame: &Frame,
1958    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1959    routes: &HashMap<RouteChannel, RouteIdentity>,
1960    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1961    executor: &Arc<Executor>,
1962    shutdown: &Arc<Notify>,
1963    bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
1964    bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
1965    metrics: &Arc<DispatchPathMetrics>,
1966    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1967    dispatch: DispatchFn,
1968) -> Result<(), SubcError> {
1969    let key = ReverseCorrKey {
1970        route: route_key(frame.header.channel, frame.header.epoch),
1971        corr: frame.header.corr,
1972    };
1973    let Some(pending) = pending_bash_asks.remove(&key) else {
1974        return Ok(());
1975    };
1976
1977    if frame.header.ty == FrameType::Response && bash_elicitation_reply_is_allow(&frame.body) {
1978        if routes.contains_key(&key.route) {
1979            bash::submit_deferred_bash(
1980                executor,
1981                bash_deferred_tx,
1982                bash_poll_touch_tx,
1983                metrics,
1984                dispatch,
1985                pending.root,
1986                pending.project_root,
1987                pending.session_id,
1988                pending.request_id,
1989                pending.route,
1990                pending.tool_corr,
1991                pending.tool_flags,
1992                pending.tool_ver,
1993                pending.arguments,
1994                pending.format_context,
1995                pending.cancel,
1996                BindTrust::Untrusted,
1997                pending.spawn_principal,
1998                pending.edit_slot_survives,
1999                Some(pending.grants),
2000            );
2001            return Ok(());
2002        }
2003        log::debug!(
2004            "subc attach: dropping allowed bash elicitation reply {} for unbound route {}",
2005            key.corr,
2006            pending.route
2007        );
2008    }
2009
2010    settle_pending_bash_ask_denied(
2011        tx,
2012        pending,
2013        routes,
2014        live_roots,
2015        route_bash_cancels,
2016        shutdown,
2017        metrics,
2018    )
2019    .await
2020}
2021
2022#[allow(clippy::too_many_arguments)]
2023async fn cancel_pending_bash_ask_for_tool_call(
2024    tx: &WriterSender,
2025    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
2026    route: RouteChannel,
2027    tool_corr: u64,
2028    routes: &HashMap<RouteChannel, RouteIdentity>,
2029    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2030    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
2031    shutdown: &Arc<Notify>,
2032    metrics: &DispatchPathMetrics,
2033) -> Result<(), SubcError> {
2034    let keys = pending_bash_asks
2035        .iter()
2036        .filter_map(|(key, pending)| {
2037            (key.route == route && pending.tool_corr == tool_corr).then_some(*key)
2038        })
2039        .collect::<Vec<_>>();
2040    for key in keys {
2041        if let Some(pending) = pending_bash_asks.remove(&key) {
2042            settle_pending_bash_ask_denied(
2043                tx,
2044                pending,
2045                routes,
2046                live_roots,
2047                route_bash_cancels,
2048                shutdown,
2049                metrics,
2050            )
2051            .await?;
2052        }
2053    }
2054    Ok(())
2055}
2056
2057fn remove_root_channel(
2058    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2059    root: &ProjectRootId,
2060    channel: RouteChannel,
2061) {
2062    let remove_root = if let Some(channels) = root_channels.get_mut(root) {
2063        channels.remove(&channel);
2064        channels.is_empty()
2065    } else {
2066        false
2067    };
2068    if remove_root {
2069        root_channels.remove(root);
2070    }
2071}
2072
2073fn remove_route_channel(
2074    routes: &mut HashMap<RouteChannel, RouteIdentity>,
2075    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2076    channel: RouteChannel,
2077) -> Option<RouteIdentity> {
2078    let removed = routes.remove(&channel);
2079    if let Some(identity) = &removed {
2080        remove_root_channel(root_channels, &identity.root, channel);
2081    }
2082    removed
2083}
2084
2085fn insert_route_channel(
2086    routes: &mut HashMap<RouteChannel, RouteIdentity>,
2087    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2088    channel: RouteChannel,
2089    identity: RouteIdentity,
2090) {
2091    if let Some(previous) = routes.insert(channel, identity.clone()) {
2092        remove_root_channel(root_channels, &previous.root, channel);
2093    }
2094    root_channels
2095        .entry(identity.root.clone())
2096        .or_default()
2097        .insert(channel);
2098}
2099
2100fn sync_bg_live_delivery_sessions(
2101    executor: &Executor,
2102    routes: &HashMap<RouteChannel, RouteIdentity>,
2103    additional_root: Option<&ProjectRootId>,
2104) {
2105    // The loop-owned installed-route table is the lifecycle source of truth:
2106    // an originating session is live exactly while the daemon has an installed,
2107    // bash-observation-capable route whose identity names that session.
2108    let sessions = routes
2109        .values()
2110        .filter(|identity| identity.trust.allows_bash_observation())
2111        .map(|identity| identity.session.clone())
2112        .collect::<HashSet<_>>();
2113    let mut roots = routes
2114        .values()
2115        .map(|identity| identity.root.clone())
2116        .collect::<HashSet<_>>();
2117    roots.extend(additional_root.cloned());
2118    for root in roots {
2119        if let Some(ctx) = executor.actor_context(&root) {
2120            ctx.bash_background()
2121                .replace_live_delivery_sessions(sessions.clone());
2122        }
2123    }
2124}
2125
2126fn insert_bg_subscription_index(
2127    bg_sub_by_session: &mut BgSubsBySession,
2128    root: ProjectRootId,
2129    session: String,
2130    channel: RouteChannel,
2131) {
2132    bg_sub_by_session
2133        .entry((root, session))
2134        .or_default()
2135        .insert(channel);
2136}
2137
2138fn remove_bg_subscription_index(
2139    bg_sub_by_session: &mut BgSubsBySession,
2140    channel: RouteChannel,
2141    identity: Option<&RouteIdentity>,
2142) {
2143    if let Some(identity) = identity {
2144        let key = (identity.root.clone(), identity.session.clone());
2145        let remove_key = bg_sub_by_session.get_mut(&key).is_some_and(|channels| {
2146            channels.remove(&channel);
2147            channels.is_empty()
2148        });
2149        if remove_key {
2150            bg_sub_by_session.remove(&key);
2151        }
2152    } else {
2153        bg_sub_by_session.retain(|_, channels| {
2154            channels.remove(&channel);
2155            !channels.is_empty()
2156        });
2157    }
2158}
2159
2160fn route_removal_will_quiesce_root(
2161    root: &ProjectRootId,
2162    route: RouteChannel,
2163    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
2164    has_pending_bind: bool,
2165    replacement_root: Option<&ProjectRootId>,
2166) -> bool {
2167    let removes_last_route = root_channels
2168        .get(root)
2169        .is_some_and(|channels| channels.len() == 1 && channels.contains(&route));
2170    removes_last_route && !has_pending_bind && replacement_root != Some(root)
2171}
2172
2173fn should_quiesce_removed_root(
2174    root: &ProjectRootId,
2175    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
2176    has_pending_bind: bool,
2177    replacement_root: Option<&ProjectRootId>,
2178) -> bool {
2179    !root_channels.contains_key(root) && !has_pending_bind && replacement_root != Some(root)
2180}
2181
2182async fn end_bg_subscription(
2183    writer_tx: &WriterSender,
2184    metrics: &DispatchPathMetrics,
2185    bg_subs: &mut HashMap<RouteChannel, BgSub>,
2186    bg_sub_by_session: &mut BgSubsBySession,
2187    bg_wake_pending: &mut BgWakePending,
2188    channel: RouteChannel,
2189    identity: Option<&RouteIdentity>,
2190    cause: &str,
2191) -> Result<(), SubcError> {
2192    if let Some(sub) = bg_subs.remove(&channel) {
2193        bg_wake_pending.remove(&channel);
2194        remove_bg_subscription_index(bg_sub_by_session, channel, identity);
2195        metrics.record_bg_subscription_ended(&sub.root, &sub.session, channel, cause);
2196        push::send_reliable_bg_stream_end(writer_tx, metrics, channel, &sub).await?;
2197    }
2198    Ok(())
2199}
2200
2201#[allow(clippy::too_many_arguments)]
2202async fn teardown_installed_route(
2203    tx: &WriterSender,
2204    metrics: &DispatchPathMetrics,
2205    executor: &Arc<Executor>,
2206    channel: RouteChannel,
2207    cancellation_reason: &str,
2208    replacement_root: Option<&ProjectRootId>,
2209    installed_route_epochs: &mut HashMap<u16, u32>,
2210    routes: &mut HashMap<RouteChannel, RouteIdentity>,
2211    management_routes: &mut HashSet<RouteChannel>,
2212    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2213    bg_subs: &mut HashMap<RouteChannel, BgSub>,
2214    bg_sub_by_session: &mut BgSubsBySession,
2215    bg_wake_pending: &mut BgWakePending,
2216    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
2217    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2218    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
2219    active_tool_calls: &ActiveToolCalls,
2220    pending_responses: &mut PendingSubcResponses,
2221    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2222    retry_buffer: &mut RetryBuffer,
2223    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2224    shutdown: &Arc<Notify>,
2225    tool_response_body_limit: usize,
2226    lifecycle_probe: Option<&SubcTestLifecycleProbe>,
2227) -> Result<(), SubcError> {
2228    remove_installed_route(installed_route_epochs, channel);
2229    management_routes.remove(&channel);
2230    let bg_end_cause = match cancellation_reason {
2231        "Goodbye" => "goodbye",
2232        "higher-epoch RouteBind" => "higher-epoch",
2233        other => other,
2234    };
2235    end_bg_subscription(
2236        tx,
2237        metrics,
2238        bg_subs,
2239        bg_sub_by_session,
2240        bg_wake_pending,
2241        channel,
2242        routes.get(&channel),
2243        bg_end_cause,
2244    )
2245    .await?;
2246    settle_pending_bash_asks_for_route(
2247        tx,
2248        pending_bash_asks,
2249        channel,
2250        routes,
2251        live_roots,
2252        route_bash_cancels,
2253        shutdown,
2254        metrics,
2255    )
2256    .await?;
2257    if let Some(cancel) = route_bash_cancels.remove(&channel) {
2258        cancel.token.cancel();
2259    }
2260    for resolved in pending_responses.drain_route(channel, executor) {
2261        deliver_resolved_subc_response(
2262            tx,
2263            resolved,
2264            routes,
2265            live_roots,
2266            executor.as_ref(),
2267            active_tool_calls,
2268            shutdown,
2269            metrics,
2270            tool_response_body_limit,
2271        )
2272        .await?;
2273    }
2274    // A higher-epoch replacement keeps replayable work because the logical
2275    // route remains live. A route that actually closes keeps only calls that
2276    // already started; pre-execution work has no response to replay and must
2277    // not retain scheduler capacity or an epoch-reader admission.
2278    let work_disposition = if replacement_root.is_some() {
2279        RouteWorkDisposition::RetainForReplay
2280    } else {
2281        RouteWorkDisposition::RetainStartedForReplay
2282    };
2283    apply_route_work_disposition(
2284        active_tool_calls,
2285        executor,
2286        channel,
2287        work_disposition,
2288        cancellation_reason,
2289    );
2290    if let Some(pending) = pending_binds.get_mut(&channel) {
2291        pending.cancelled = true;
2292        let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
2293        log::debug!(
2294            "subc attach: cancelled pending RouteBind for route {} on {cancellation_reason} (configure job: {outcome:?})",
2295            channel.channel
2296        );
2297    }
2298    let migrated = push::migrate_retry_buffer_to_push_buffer(retry_buffer, channel, push_buffer);
2299    if let Some(identity) = routes.get(&channel) {
2300        let has_pending_bind = pending_binds
2301            .values()
2302            .any(|pending| pending.bind_root_id == identity.root);
2303        if route_removal_will_quiesce_root(
2304            &identity.root,
2305            channel,
2306            root_channels,
2307            has_pending_bind,
2308            replacement_root,
2309        ) {
2310            if let Some(ctx) = executor.actor_context(&identity.root) {
2311                // Fence deferred admissions before the final route disappears
2312                // from the loop-owned routing tables.
2313                ctx.mark_subc_unbound();
2314            }
2315        }
2316    }
2317    // This test-only delay lets the lifecycle probe verify that a queued rebind
2318    // cannot run before the route is removed and a completion is recorded for replay.
2319    delay_route_detach_for_test(lifecycle_probe).await;
2320    if let Some(identity) = remove_route_channel(routes, root_channels, channel) {
2321        sync_bg_live_delivery_sessions(executor, routes, Some(&identity.root));
2322        if let Some(probe) = lifecycle_probe {
2323            probe.route_detached(channel, &identity.session);
2324        }
2325        let session_still_routed = routes
2326            .values()
2327            .any(|route| route.root == identity.root && route.session == identity.session);
2328        if !session_still_routed {
2329            if let Some(ctx) = executor.actor_context(&identity.root) {
2330                ctx.hashline_bindings()
2331                    .teardown(identity.root.as_path(), &identity.session);
2332            }
2333        }
2334        if migrated > 0 {
2335            log::debug!(
2336                "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from route {} into detach replay",
2337                channel.channel
2338            );
2339        }
2340        if let Some(meta) = live_roots.get_mut(&identity.root) {
2341            let idle_for = meta.last_touched.elapsed();
2342            meta.note_activity();
2343            log::debug!(
2344                "subc attach: route {} torn down for root {} harness {} session {} (last touched {:?} ago)",
2345                channel.channel,
2346                identity.root.as_path().display(),
2347                identity.harness,
2348                identity.session,
2349                idle_for
2350            );
2351        } else {
2352            log::debug!(
2353                "subc attach: route {} torn down for root {} harness {} session {}",
2354                channel.channel,
2355                identity.root.as_path().display(),
2356                identity.harness,
2357                identity.session
2358            );
2359        }
2360        let has_pending_bind = pending_binds
2361            .values()
2362            .any(|pending| pending.bind_root_id == identity.root);
2363        if should_quiesce_removed_root(
2364            &identity.root,
2365            root_channels,
2366            has_pending_bind,
2367            replacement_root,
2368        ) {
2369            quiesce_unbound_root(&identity.root, live_roots, executor);
2370        }
2371    } else {
2372        if migrated > 0 {
2373            log::debug!(
2374                "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from unbound route {} into detach replay",
2375                channel.channel
2376            );
2377        }
2378        log::debug!("subc attach: unbound route {} torn down", channel.channel);
2379    }
2380    Ok(())
2381}
2382
2383async fn delay_route_detach_for_test(lifecycle_probe: Option<&SubcTestLifecycleProbe>) {
2384    if lifecycle_probe.is_none() {
2385        return;
2386    }
2387    let Some(delay) = std::env::var("AFT_TEST_SUBC_ROUTE_DETACH_DELAY_MS")
2388        .ok()
2389        .and_then(|raw| raw.parse::<u64>().ok())
2390    else {
2391        return;
2392    };
2393    tokio::time::sleep(Duration::from_millis(delay)).await;
2394}
2395
2396fn remember_session_identity(
2397    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2398    identity: &RouteIdentity,
2399) {
2400    let key = (identity.root.clone(), identity.session.clone());
2401    if matches!(identity.trust, BindTrust::Untrusted)
2402        && session_identity
2403            .get(&key)
2404            .is_some_and(|retained| matches!(retained.trust, BindTrust::FirstParty))
2405    {
2406        return;
2407    }
2408
2409    // Retained after route Goodbye so reliable session-scoped frames emitted while
2410    // the session is detached can still be keyed by the full (root,harness,session)
2411    // replay triple. Untrusted binds never overwrite a retained first-party
2412    // session identity, because bash completion replay is an observation channel.
2413    session_identity.insert(
2414        key,
2415        RetainedSessionIdentity {
2416            harness: identity.harness.clone(),
2417            trust: identity.trust,
2418        },
2419    );
2420}
2421
2422fn replay_key_for_session(
2423    session_identity: &HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2424    root: &ProjectRootId,
2425    session: &str,
2426) -> Option<(push::ReplayKey, BindTrust)> {
2427    let retained = session_identity.get(&(root.clone(), session.to_string()))?;
2428    Some((
2429        push::ReplayKey {
2430            root: root.clone(),
2431            harness: retained.harness.clone(),
2432            session: session.to_string(),
2433        },
2434        retained.trust,
2435    ))
2436}
2437/// Sync command dispatch, passed in from `main` (the binary owns the command
2438/// table). Invoked only inside executor jobs in subc mode.
2439pub type DispatchFn = fn(RawRequest, &AppContext) -> Response;
2440
2441#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2442enum ModuleLoopExit {
2443    /// The daemon asked us to stop (channel-0 Goodbye). Exit 0 is correct:
2444    /// the supervisor treats a clean exit as "stopped on request".
2445    Graceful,
2446    /// The connection ended without a Goodbye. Indexes still flush, but the
2447    /// process must exit non-zero so the supervisor restarts it.
2448    ConnectionLost,
2449    /// An actor went fatal (worker panic on a mutating job) and the loop tore
2450    /// the module down. Index flushes are skipped because the state that
2451    /// panicked cannot be trusted, and the process exits non-zero for the
2452    /// same reason as `ConnectionLost`.
2453    SkipSearchFlush,
2454}
2455
2456/// Entry point for `aft --subc <connection-file>`. Synchronous on the outside;
2457/// owns an isolated current-thread tokio runtime for the async transport.
2458/// Returns `Err` (fail-loud) on any connect/auth/protocol failure — we never
2459/// fall back to the standalone loop, to avoid split-brain index state.
2460pub fn run_subc_mode(
2461    connection_file_path: &Path,
2462    ctx: Arc<AppContext>,
2463    executor: Arc<Executor>,
2464    dispatch: DispatchFn,
2465    user_config_path: Option<PathBuf>,
2466) -> Result<(), SubcError> {
2467    // Production NEVER allows non-manifest tool names on route channels: AFT
2468    // fails closed and does not trust subc to enforce the manifest. The
2469    // test-only harness sets this through `run_subc_mode_for_test`.
2470    run_subc_mode_inner(
2471        connection_file_path,
2472        ctx,
2473        executor,
2474        dispatch,
2475        user_config_path,
2476        false,
2477        MAX_FRAME_BODY_LEN as usize,
2478        None,
2479    )
2480}
2481
2482fn run_subc_mode_inner(
2483    connection_file_path: &Path,
2484    ctx: Arc<AppContext>,
2485    executor: Arc<Executor>,
2486    dispatch: DispatchFn,
2487    user_config_path: Option<PathBuf>,
2488    allow_native_passthrough: bool,
2489    tool_response_body_limit: usize,
2490    lifecycle_probe: Option<SubcTestLifecycleProbe>,
2491) -> Result<(), SubcError> {
2492    let runtime = tokio::runtime::Builder::new_current_thread()
2493        .enable_all()
2494        .build()
2495        .map_err(SubcError::Runtime)?;
2496
2497    let executor_for_loop = Arc::clone(&executor);
2498    let loop_result = runtime.block_on(async move {
2499        let shared_app = ctx.app();
2500        drop(ctx);
2501        let stream =
2502            connect_and_authenticate(connection_file_path, lifecycle_probe.as_ref()).await?;
2503        log::info!(
2504            "subc attach: authenticated to daemon via {}",
2505            connection_file_path.display()
2506        );
2507        let (read_half, write_half) = tokio::io::split(stream);
2508        run_module_loop(
2509            read_half,
2510            write_half,
2511            connection_file_path,
2512            shared_app,
2513            executor_for_loop,
2514            dispatch,
2515            user_config_path,
2516            allow_native_passthrough,
2517            tool_response_body_limit,
2518            lifecycle_probe,
2519        )
2520        .await
2521    });
2522
2523    let actor_contexts = executor.actor_contexts();
2524    if matches!(
2525        loop_result,
2526        Ok(ModuleLoopExit::Graceful | ModuleLoopExit::ConnectionLost)
2527    ) {
2528        // EOF/Goodbye teardown flushes each root's index deltas and queued
2529        // callgraph refreshes. Fatal/panic teardown skips this best-effort work.
2530        flush_actor_indexes_on_graceful_shutdown(&actor_contexts);
2531    }
2532    for actor_ctx in &actor_contexts {
2533        actor_ctx.lsp().shutdown_all();
2534        actor_ctx.bash_background().detach();
2535    }
2536
2537    match loop_result {
2538        Ok(exit) => module_loop_exit_result(exit),
2539        Err(error) => Err(error),
2540    }
2541}
2542
2543/// Maps how the module loop ended onto the process outcome the supervisor
2544/// reads. Only a daemon-requested stop may exit 0: the supervisor never
2545/// respawns a clean exit, so every other ending must surface as an error.
2546fn module_loop_exit_result(exit: ModuleLoopExit) -> Result<(), SubcError> {
2547    match exit {
2548        ModuleLoopExit::Graceful => Ok(()),
2549        ModuleLoopExit::ConnectionLost => Err(SubcError::ConnectionLost),
2550        ModuleLoopExit::SkipSearchFlush => Err(SubcError::ActorFatal),
2551    }
2552}
2553
2554/// Records a fatal panic response in the module log before the teardown it
2555/// triggers. The panic text otherwise lives only in the error frame sent to
2556/// the caller, which leaves the crash undiagnosable from the host afterwards.
2557fn note_fatal_panic_response(response: &Response) -> bool {
2558    let fatal = response_is_fatal_panic(response);
2559    if fatal {
2560        log::error!(
2561            "subc attach: request {} returned a fatal panic response; tearing the module down: {}",
2562            response.id,
2563            response
2564                .data
2565                .get("message")
2566                .and_then(Value::as_str)
2567                .unwrap_or("(no message)")
2568        );
2569    }
2570    fatal
2571}
2572
2573fn flush_actor_indexes_on_graceful_shutdown(actor_contexts: &[Arc<AppContext>]) {
2574    for actor_ctx in actor_contexts {
2575        let _ = actor_ctx.flush_search_index_on_graceful_shutdown();
2576    }
2577    let _ = crate::callgraph_store::flush_callgraph_store_refreshes_on_graceful_shutdown();
2578}
2579
2580/// Test-only entry that enables the non-manifest native-command passthrough on
2581/// route channels. Integration tests drive synthetic native commands (`glob`,
2582/// `callers`, `subc_test_echo_session`, …) through the executor to exercise
2583/// mechanics; production callers use [`run_subc_mode`], which fails closed.
2584#[doc(hidden)]
2585pub fn run_subc_mode_for_test(
2586    connection_file_path: &Path,
2587    ctx: Arc<AppContext>,
2588    executor: Arc<Executor>,
2589    dispatch: DispatchFn,
2590    user_config_path: Option<PathBuf>,
2591) -> Result<(), SubcError> {
2592    run_subc_mode_inner(
2593        connection_file_path,
2594        ctx,
2595        executor,
2596        dispatch,
2597        user_config_path,
2598        true,
2599        MAX_FRAME_BODY_LEN as usize,
2600        None,
2601    )
2602}
2603
2604/// Test-only entry that observes detach/rebind lifecycle milestones.
2605#[doc(hidden)]
2606pub fn run_subc_mode_for_test_with_lifecycle_probe(
2607    connection_file_path: &Path,
2608    ctx: Arc<AppContext>,
2609    executor: Arc<Executor>,
2610    dispatch: DispatchFn,
2611    user_config_path: Option<PathBuf>,
2612    lifecycle_probe: SubcTestLifecycleProbe,
2613) -> Result<(), SubcError> {
2614    run_subc_mode_inner(
2615        connection_file_path,
2616        ctx,
2617        executor,
2618        dispatch,
2619        user_config_path,
2620        true,
2621        MAX_FRAME_BODY_LEN as usize,
2622        Some(lifecycle_probe),
2623    )
2624}
2625
2626/// Test-only entry that lowers the effective tool-response body limit without
2627/// allocating a 64 MiB fixture. The fixed fallback envelope needs 4 KiB of room.
2628#[doc(hidden)]
2629pub fn run_subc_mode_for_test_with_response_body_limit(
2630    connection_file_path: &Path,
2631    ctx: Arc<AppContext>,
2632    executor: Arc<Executor>,
2633    dispatch: DispatchFn,
2634    user_config_path: Option<PathBuf>,
2635    tool_response_body_limit: usize,
2636) -> Result<(), SubcError> {
2637    assert!((4 * 1_024..=MAX_FRAME_BODY_LEN as usize).contains(&tool_response_body_limit));
2638    run_subc_mode_inner(
2639        connection_file_path,
2640        ctx,
2641        executor,
2642        dispatch,
2643        user_config_path,
2644        true,
2645        tool_response_body_limit,
2646        None,
2647    )
2648}
2649
2650#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2651enum AttachErrorClass {
2652    Transient,
2653    Permanent,
2654}
2655
2656impl fmt::Display for AttachErrorClass {
2657    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2658        match self {
2659            Self::Transient => f.write_str("transient"),
2660            Self::Permanent => f.write_str("permanent"),
2661        }
2662    }
2663}
2664
2665#[derive(Clone, Copy)]
2666struct AttachRetryPolicy {
2667    budget: Duration,
2668    initial_backoff: Duration,
2669    max_backoff: Duration,
2670    jitter_percent: u64,
2671}
2672
2673const ATTACH_RETRY_POLICY: AttachRetryPolicy = AttachRetryPolicy {
2674    budget: ATTACH_RETRY_BUDGET,
2675    initial_backoff: ATTACH_RETRY_INITIAL_BACKOFF,
2676    max_backoff: ATTACH_RETRY_MAX_BACKOFF,
2677    jitter_percent: ATTACH_RETRY_JITTER_PERCENT,
2678};
2679
2680/// Retry only failures that can be caused by a daemon bounce or an interrupted
2681/// handshake. Protocol and credential failures are permanent for this process.
2682fn classify_attach_error(error: &SubcError) -> AttachErrorClass {
2683    let transient = match error {
2684        SubcError::Connect { source, .. } => is_transient_attach_io(source.kind()),
2685        SubcError::Auth { source, .. } => match source {
2686            subc_transport::AuthError::Timeout { .. }
2687            | subc_transport::AuthError::UnexpectedEof { .. } => true,
2688            subc_transport::AuthError::Io { source, .. } => is_transient_attach_io(source.kind()),
2689            _ => false,
2690        },
2691        _ => false,
2692    };
2693    if transient {
2694        AttachErrorClass::Transient
2695    } else {
2696        AttachErrorClass::Permanent
2697    }
2698}
2699
2700fn is_transient_attach_io(kind: io::ErrorKind) -> bool {
2701    matches!(
2702        kind,
2703        io::ErrorKind::ConnectionRefused
2704            | io::ErrorKind::TimedOut
2705            | io::ErrorKind::ConnectionReset
2706            | io::ErrorKind::ConnectionAborted
2707            | io::ErrorKind::BrokenPipe
2708            | io::ErrorKind::UnexpectedEof
2709    )
2710}
2711
2712/// Read the connection file → resolve the first endpoint → TCP connect → HMAC
2713/// handshake. Transient initial-attach failures retry on fresh sockets and reread
2714/// the file so a daemon bounce can publish a new endpoint or authentication key.
2715async fn connect_and_authenticate(
2716    connection_file_path: &Path,
2717    lifecycle_probe: Option<&SubcTestLifecycleProbe>,
2718) -> Result<TcpStream, SubcError> {
2719    connect_and_authenticate_with_policy(connection_file_path, ATTACH_RETRY_POLICY, lifecycle_probe)
2720        .await
2721}
2722
2723async fn connect_and_authenticate_with_policy(
2724    connection_file_path: &Path,
2725    policy: AttachRetryPolicy,
2726    lifecycle_probe: Option<&SubcTestLifecycleProbe>,
2727) -> Result<TcpStream, SubcError> {
2728    let started_at = Instant::now();
2729    let deadline = started_at + policy.budget;
2730    let mut attempt = 0_u32;
2731    let mut backoff = policy.initial_backoff;
2732    let mut history = Vec::new();
2733
2734    loop {
2735        attempt = attempt.saturating_add(1);
2736        let error = match connect_and_authenticate_once(connection_file_path, deadline).await {
2737            Ok(stream) => return Ok(stream),
2738            Err(error) => error,
2739        };
2740        let class = classify_attach_error(&error);
2741        let will_retry = class != AttachErrorClass::Permanent;
2742        if let Some(probe) = lifecycle_probe {
2743            probe.attach_decision(attempt, will_retry);
2744        }
2745        let error_text = error.to_string().lines().collect::<Vec<_>>().join(" ");
2746        history.push(format!("attempt {attempt} [{class}]: {error_text}"));
2747
2748        if !will_retry {
2749            log_attach_final_failure(started_at.elapsed(), &history);
2750            return Err(error);
2751        }
2752
2753        let remaining = deadline.saturating_duration_since(Instant::now());
2754        if remaining.is_zero() {
2755            log_attach_final_failure(started_at.elapsed(), &history);
2756            return Err(error);
2757        }
2758
2759        let delay = jittered_attach_delay(backoff, policy.jitter_percent, attempt).min(remaining);
2760        log::info!(
2761            "subc attach retry: attempt {attempt} failed; error_class={class}; error={error_text}; next_delay={delay:?}"
2762        );
2763        tokio::time::sleep(delay).await;
2764
2765        if Instant::now() >= deadline {
2766            log_attach_final_failure(started_at.elapsed(), &history);
2767            return Err(error);
2768        }
2769        backoff = backoff.saturating_mul(2).min(policy.max_backoff);
2770    }
2771}
2772
2773fn jittered_attach_delay(base: Duration, jitter_percent: u64, attempt: u32) -> Duration {
2774    let jitter_percent = jitter_percent.min(100);
2775    if jitter_percent == 0 {
2776        return base;
2777    }
2778
2779    let mut random_bytes = [0_u8; 8];
2780    let random = if getrandom::fill(&mut random_bytes).is_ok() {
2781        u64::from_le_bytes(random_bytes)
2782    } else {
2783        let timestamp = std::time::SystemTime::now()
2784            .duration_since(std::time::UNIX_EPOCH)
2785            .unwrap_or_default()
2786            .subsec_nanos();
2787        u64::from(timestamp) ^ u64::from(attempt)
2788    };
2789    let span = jitter_percent.saturating_mul(2).saturating_add(1);
2790    let multiplier_percent = 100 - jitter_percent + random % span;
2791    let base_millis = u64::try_from(base.as_millis()).unwrap_or(u64::MAX);
2792    Duration::from_millis(base_millis.saturating_mul(multiplier_percent) / 100)
2793}
2794
2795fn log_attach_final_failure(elapsed: Duration, history: &[String]) {
2796    log::error!(
2797        "subc initial attach failed after {} attempt(s) in {elapsed:?}; attempt history: {}",
2798        history.len(),
2799        history.join(" | ")
2800    );
2801}
2802
2803async fn connect_and_authenticate_once(
2804    connection_file_path: &Path,
2805    deadline: Instant,
2806) -> Result<TcpStream, SubcError> {
2807    // This read intentionally lives inside the per-attempt function. The daemon
2808    // publishes connection files atomically and may change both port and key.
2809    let conn = connection_file::read_for_client(connection_file_path).map_err(|source| {
2810        SubcError::ConnectionFile {
2811            path: connection_file_path.to_path_buf(),
2812            source,
2813        }
2814    })?;
2815
2816    let endpoint = conn
2817        .endpoints
2818        .first()
2819        .ok_or_else(|| SubcError::NoEndpoint {
2820            path: connection_file_path.to_path_buf(),
2821        })?;
2822    let endpoint_label = format!("{}:{}", endpoint.host, endpoint.port);
2823    let ip = endpoint
2824        .host
2825        .parse::<IpAddr>()
2826        .map_err(|_| SubcError::InvalidEndpoint {
2827            path: connection_file_path.to_path_buf(),
2828            endpoint: endpoint_label.clone(),
2829        })?;
2830    let addr = SocketAddr::new(ip, endpoint.port);
2831
2832    let connect_budget = deadline.saturating_duration_since(Instant::now());
2833    let mut stream = tokio::time::timeout(connect_budget, TcpStream::connect(addr))
2834        .await
2835        .map_err(|_| SubcError::Connect {
2836            endpoint: endpoint_label.clone(),
2837            source: io::Error::new(
2838                io::ErrorKind::TimedOut,
2839                "initial subc attach retry budget elapsed during TCP connect",
2840            ),
2841        })?
2842        .map_err(|source| SubcError::Connect {
2843            endpoint: endpoint_label.clone(),
2844            source,
2845        })?;
2846    stream
2847        .set_nodelay(true)
2848        .map_err(|source| SubcError::Connect {
2849            endpoint: endpoint_label.clone(),
2850            source,
2851        })?;
2852
2853    let auth_budget = AUTH_DEADLINE.min(deadline.saturating_duration_since(Instant::now()));
2854    authenticate_client(&mut stream, &conn, auth_budget)
2855        .await
2856        .map_err(|source| SubcError::Auth {
2857            endpoint: endpoint_label,
2858            source,
2859        })?;
2860
2861    Ok(stream)
2862}
2863
2864#[allow(clippy::too_many_arguments)]
2865async fn process_route_bind_completion(
2866    writer_tx: &WriterSender,
2867    completion: RouteBindCompletion,
2868    routes: &mut HashMap<RouteChannel, RouteIdentity>,
2869    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2870    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2871    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2872    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2873    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2874    installed_route_epochs: &mut HashMap<u16, u32>,
2875    executor: &Arc<Executor>,
2876    standing_actor: &standing::StandingActor,
2877    shutdown: &Arc<Notify>,
2878    metrics: &Arc<DispatchPathMetrics>,
2879    lifecycle_probe: Option<&SubcTestLifecycleProbe>,
2880) -> Result<(), SubcError> {
2881    decrement_counted_channel(&metrics.control_completion_queued);
2882    handle_route_bind_completion(
2883        writer_tx,
2884        completion,
2885        routes,
2886        root_channels,
2887        session_identity,
2888        push_buffer,
2889        live_roots,
2890        pending_binds,
2891        installed_route_epochs,
2892        executor,
2893        standing_actor,
2894        shutdown,
2895        metrics,
2896        lifecycle_probe,
2897    )
2898    .await
2899}
2900
2901#[allow(clippy::too_many_arguments)]
2902async fn drain_pending_route_bind_completions(
2903    control_completion_rx: &mut mpsc::Receiver<RouteBindCompletion>,
2904    writer_tx: &WriterSender,
2905    routes: &mut HashMap<RouteChannel, RouteIdentity>,
2906    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2907    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2908    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2909    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2910    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2911    installed_route_epochs: &mut HashMap<u16, u32>,
2912    executor: &Arc<Executor>,
2913    standing_actor: &standing::StandingActor,
2914    shutdown: &Arc<Notify>,
2915    metrics: &Arc<DispatchPathMetrics>,
2916    lifecycle_probe: Option<&SubcTestLifecycleProbe>,
2917) -> Result<usize, SubcError> {
2918    let mut drained = 0;
2919    while let Ok(completion) = control_completion_rx.try_recv() {
2920        process_route_bind_completion(
2921            writer_tx,
2922            completion,
2923            routes,
2924            root_channels,
2925            session_identity,
2926            push_buffer,
2927            live_roots,
2928            pending_binds,
2929            installed_route_epochs,
2930            executor,
2931            standing_actor,
2932            shutdown,
2933            metrics,
2934            lifecycle_probe,
2935        )
2936        .await?;
2937        drained += 1;
2938    }
2939    Ok(drained)
2940}
2941
2942/// ModuleHello → HelloAck → control/route loop. Runs until the daemon closes
2943/// the connection (EOF), sends channel-0 Goodbye, or a fatal mutating executor
2944/// response requests whole-connection teardown.
2945async fn run_module_loop<R, W>(
2946    mut read: R,
2947    mut write: W,
2948    connection_file_path: &Path,
2949    shared_app: Arc<App>,
2950    executor: Arc<Executor>,
2951    dispatch: DispatchFn,
2952    user_config_path: Option<PathBuf>,
2953    allow_native_passthrough: bool,
2954    tool_response_body_limit: usize,
2955    lifecycle_probe: Option<SubcTestLifecycleProbe>,
2956) -> Result<ModuleLoopExit, SubcError>
2957where
2958    R: AsyncRead + Unpin + Send + 'static,
2959    W: AsyncWrite + Unpin + Send + 'static,
2960{
2961    // ModuleHello registers the tool and management providers and advertises
2962    // the separate channel-0 control operations.
2963    // Echo the one-time launch nonce the daemon injected via SUBC_LAUNCH_NONCE so a
2964    // reserved module_id's HELLO is accepted; absent for non-reserved/self-connect.
2965    let hello = ModuleHelloBody {
2966        manifest: build_manifest(),
2967        protocol_ver: PROTOCOL_VERSION,
2968        control_ops: control_ops(),
2969        launch_nonce: std::env::var("SUBC_LAUNCH_NONCE").ok(),
2970    };
2971    let hello_frame = Frame::build(
2972        FrameType::Hello,
2973        control_flags(),
2974        0,
2975        0,
2976        HELLO_CORR,
2977        serde_json::to_vec(&hello).map_err(SubcError::Json)?,
2978    )
2979    .map_err(SubcError::FrameBuild)?;
2980    write_frame(&mut write, &hello_frame)
2981        .await
2982        .map_err(SubcError::FrameIo)?;
2983
2984    // Expect HelloAck (registered) or a channel-0 Error (manifest/version reject).
2985    match read_frame(&mut read).await.map_err(SubcError::FrameIo)? {
2986        None => return Err(SubcError::ClosedBeforeHelloAck),
2987        Some(frame) => match frame.header.ty {
2988            FrameType::HelloAck => {
2989                log::info!("subc attach: registered (HelloAck received)");
2990            }
2991            FrameType::Error => {
2992                let body = serde_json::from_slice::<ErrorBody>(&frame.body).ok();
2993                return Err(SubcError::HelloRejected { body });
2994            }
2995            other => return Err(SubcError::UnexpectedFrame { ty: other }),
2996        },
2997    }
2998
2999    let dispatch_path_metrics = Arc::new(DispatchPathMetrics::new());
3000    let (writer_tx, writer_rx) = mpsc::channel::<WriterFrame>(WRITER_QUEUE_CAPACITY);
3001    let writer_task = spawn_writer_task(write, writer_rx, Arc::clone(&dispatch_path_metrics));
3002    // `read_frame` is NOT cancellation-safe, so it must never sit directly inside
3003    // the `select!` below: a drain-interval tick (or shutdown) firing while a
3004    // frame is mid-transit would drop the partially-consumed bytes and desync the
3005    // stream (the next read would parse a body byte as a frame header). A
3006    // dedicated reader task owns the socket, reads whole frames sequentially, and
3007    // forwards them over a channel; the loop selects on the cancel-safe `recv()`.
3008    let (reader_tx, mut reader_rx) = mpsc::channel::<Result<DecodedFrame, SubcError>>(256);
3009    let reader_task = spawn_reader_task(read, reader_tx);
3010    let shutdown = Arc::new(Notify::new());
3011    // Drain-tick deadline is tracked manually and checked at the TOP of every
3012    // loop turn rather than as an Interval select arm: the select below is
3013    // `biased` (bind completions first), and biased polling means a saturated
3014    // higher arm (sustained lossy push traffic keeps lossy_rx always-ready)
3015    // would starve every arm below it, including a timer arm — leaving
3016    // backpressured reliable frames parked in the retry buffer past their
3017    // delivery deadline. The pre-turn check cannot be starved by arm order;
3018    // the sleep_until arm below only exists to wake an otherwise-idle loop.
3019    let mut next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
3020    let mut next_maintenance_at = next_drain_at;
3021    let standing_actor =
3022        standing::StandingActor::new(Arc::clone(&shared_app), Arc::clone(&executor));
3023    // Startup reconciliation is intentionally direct; subsequent passes use
3024    // this existing maintenance timer arm and never create a standing timer.
3025    standing_actor.reconcile_at_startup();
3026    let mut next_standing_pass_at = tokio::time::Instant::now();
3027    let (maintenance_tx, mut maintenance_rx) = mpsc::channel::<MaintenanceCompletion>(256);
3028    let (bash_deferred_tx, mut bash_deferred_rx) =
3029        mpsc::channel::<bash::BashDeferredCompletion>(256);
3030    let (deferred_response_tx, mut deferred_response_rx) =
3031        mpsc::unbounded_channel::<PendingSubcResponse>();
3032    let (bash_poll_touch_tx, mut bash_poll_touch_rx) = mpsc::channel::<ProjectRootId>(256);
3033    let (control_completion_tx, mut control_completion_rx) =
3034        mpsc::channel::<RouteBindCompletion>(256);
3035    let (lossy_tx, mut lossy_rx) = mpsc::channel::<LossyPushEnvelope>(1024);
3036    let lossy_overflow = Arc::new(push::LossyOverflow::default());
3037    let lossy_seq = Arc::new(AtomicU64::new(0));
3038    let (reliable_tx, mut reliable_rx) = mpsc::unbounded_channel::<PushEnvelope>();
3039    let (fleet_status_client, fleet_status_task) =
3040        spawn_fleet_status_dial(connection_file_path, 64);
3041    let push_senders = PushSenders {
3042        lossy_tx,
3043        reliable_tx,
3044        lossy_overflow: Arc::clone(&lossy_overflow),
3045        lossy_seq,
3046        fleet_status_client: fleet_status_client.clone(),
3047    };
3048    let connection_cancel = PersistentCancelSignal::new();
3049    let mut installed_route_epochs: HashMap<u16, u32> = HashMap::new();
3050    let mut routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
3051    let mut management_routes: HashSet<RouteChannel> = HashSet::new();
3052    let mut bg_subs: HashMap<RouteChannel, BgSub> = HashMap::new();
3053    let mut bg_sub_by_session: BgSubsBySession = HashMap::new();
3054    let mut bg_wake_pending = BgWakePending::new();
3055    let mut bg_wake_epoch: HashMap<(ProjectRootId, String), u64> = HashMap::new();
3056    let mut bg_unacked_keys_by_root: HashMap<ProjectRootId, HashSet<String>> = HashMap::new();
3057    let mut root_channels: HashMap<ProjectRootId, HashSet<RouteChannel>> = HashMap::new();
3058    let mut session_identity: HashMap<(ProjectRootId, String), RetainedSessionIdentity> =
3059        HashMap::new();
3060    let mut push_buffer: HashMap<push::ReplayKey, VecDeque<PushFrame>> = HashMap::new();
3061    let mut retry_buffer: RetryBuffer = HashMap::new();
3062    let mut reclaimed_routes = ReclaimedRoutes::default();
3063    let mut completed_tasks = push::CompletedTaskIds::default();
3064    let mut live_roots: HashMap<ProjectRootId, RootMeta> = HashMap::new();
3065    let mut pending_binds: HashMap<RouteChannel, PendingBind> = HashMap::new();
3066    let mut pending_bash_asks: HashMap<ReverseCorrKey, PendingBashAsk> = HashMap::new();
3067    let mut next_bash_ask_corr: u64 = 1;
3068    let mut route_bash_cancels: HashMap<RouteChannel, bash::RouteBashCancel> = HashMap::new();
3069    let active_tool_calls: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
3070    let pending_deferred_setups = Arc::new(AtomicUsize::new(0));
3071    let mut pending_responses = PendingSubcResponses::default();
3072    let health_rollup_cache = Arc::new(HealthRollupCache::new());
3073    let health_rollup_worker = HealthRollupWorker::start(
3074        Arc::clone(&health_rollup_cache),
3075        Arc::clone(&executor),
3076        Arc::clone(&shared_app),
3077    );
3078
3079    let loop_result: Result<ModuleLoopExit, SubcError> = 'module_loop: loop {
3080        shared_app.set_open_route_count(routes.len() + management_routes.len());
3081        crate::logging::perf_tick(Some(&executor));
3082        dispatch_path_metrics.mark_frame_loop_tick();
3083        let ready_inspects = pending_responses.poll_ready(executor.as_ref());
3084        for resolved in ready_inspects {
3085            if let Err(error) = deliver_resolved_subc_response(
3086                &writer_tx,
3087                resolved,
3088                &routes,
3089                &mut live_roots,
3090                executor.as_ref(),
3091                &active_tool_calls,
3092                &shutdown,
3093                &dispatch_path_metrics,
3094                tool_response_body_limit,
3095            )
3096            .await
3097            {
3098                break 'module_loop Err(error);
3099            }
3100        }
3101        if let Err(error) = expire_pending_bash_asks(
3102            &writer_tx,
3103            &mut pending_bash_asks,
3104            &routes,
3105            &mut live_roots,
3106            &mut route_bash_cancels,
3107            &shutdown,
3108            &dispatch_path_metrics,
3109        )
3110        .await
3111        {
3112            break Err(error);
3113        }
3114
3115        // RouteBind completions are control-plane unblockers. Drain any completed
3116        // binds before entering other branch work so Push and maintenance bursts
3117        // can only add one loop-turn of latency.
3118        match drain_pending_route_bind_completions(
3119            &mut control_completion_rx,
3120            &writer_tx,
3121            &mut routes,
3122            &mut root_channels,
3123            &mut session_identity,
3124            &mut push_buffer,
3125            &mut live_roots,
3126            &mut pending_binds,
3127            &mut installed_route_epochs,
3128            &executor,
3129            &standing_actor,
3130            &shutdown,
3131            &dispatch_path_metrics,
3132            lifecycle_probe.as_ref(),
3133        )
3134        .await
3135        {
3136            Ok(drained) => {
3137                if drained > 0 {
3138                    next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
3139                    health_rollup_worker.request_refresh();
3140                }
3141            }
3142            Err(error) => break Err(error),
3143        }
3144
3145        if tokio::time::Instant::now() >= next_drain_at {
3146            push::emit_bg_event_wakes(
3147                &writer_tx,
3148                &dispatch_path_metrics,
3149                &bg_subs,
3150                &mut bg_wake_pending,
3151            );
3152            dispatch_path_metrics.warn_stuck_pending_watches(&executor, &bg_sub_by_session);
3153            warn_slow_pending_binds(&mut pending_binds, &executor);
3154            warn_slow_running_interactive_jobs(&executor);
3155            if let Err(error) = expire_overdue_route_binds(
3156                &writer_tx,
3157                &executor,
3158                &mut pending_binds,
3159                &mut installed_route_epochs,
3160                &dispatch_path_metrics,
3161            )
3162            .await
3163            {
3164                break Err(error);
3165            }
3166
3167            let retried = push::drain_retry_buffers_for_bound_routes(
3168                &writer_tx,
3169                &dispatch_path_metrics,
3170                &routes,
3171                &mut retry_buffer,
3172            );
3173            if retried > 0 {
3174                log::debug!(
3175                    "subc attach: retried {retried} reliable Push frame(s) after writer backpressure"
3176                );
3177            }
3178
3179            next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
3180        }
3181
3182        // A lossy emitter may place its newest update in the overflow buffer
3183        // when the bounded channel is full, while this receive loop is draining
3184        // the channel. Drain overflow before selecting again so that raced
3185        // update is delivered on the next timer tick instead of waiting for
3186        // another lossy enqueue.
3187        let overflow_batch = lossy_overflow.drain();
3188        if !overflow_batch.is_empty() {
3189            let (_, deferred) = push::drain_reliable_push_turn(
3190                &writer_tx,
3191                &dispatch_path_metrics,
3192                &routes,
3193                &root_channels,
3194                &session_identity,
3195                &mut retry_buffer,
3196                &mut push_buffer,
3197                &mut completed_tasks,
3198                &bg_sub_by_session,
3199                &mut bg_wake_pending,
3200                &mut bg_wake_epoch,
3201                &mut reliable_rx,
3202                None,
3203                lifecycle_probe.as_ref(),
3204            );
3205            if deferred {
3206                tokio::task::yield_now().await;
3207            }
3208
3209            let mut batch = Vec::new();
3210            while let Ok(item) = lossy_rx.try_recv() {
3211                batch.push(item);
3212            }
3213            batch.extend(overflow_batch);
3214            push::process_lossy_push_envelope_batch(
3215                &writer_tx,
3216                &dispatch_path_metrics,
3217                &routes,
3218                &root_channels,
3219                &completed_tasks,
3220                batch,
3221            );
3222        }
3223
3224        tokio::select! {
3225            biased;
3226            Some(completion) = control_completion_rx.recv() => {
3227                if let Err(error) = process_route_bind_completion(
3228                    &writer_tx,
3229                    completion,
3230                    &mut routes,
3231                    &mut root_channels,
3232                    &mut session_identity,
3233                    &mut push_buffer,
3234                    &mut live_roots,
3235                    &mut pending_binds,
3236                    &mut installed_route_epochs,
3237                    &executor,
3238                    &standing_actor,
3239                    &shutdown,
3240                    &dispatch_path_metrics,
3241                    lifecycle_probe.as_ref(),
3242                )
3243                .await
3244                {
3245                    break Err(error);
3246                }
3247                next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
3248                health_rollup_worker.request_refresh();
3249            }
3250            _ = shutdown.notified() => {
3251                log::warn!("subc attach: fatal executor response requested teardown");
3252                break Ok(ModuleLoopExit::SkipSearchFlush);
3253            }
3254            maybe_frame = reader_rx.recv() => {
3255                let frame = match maybe_frame {
3256                    None => {
3257                        log::warn!(
3258                            "subc attach: daemon connection ended without Goodbye; exiting for restart"
3259                        );
3260                        break Ok(ModuleLoopExit::ConnectionLost);
3261                    }
3262                    Some(Err(error)) => break Err(error),
3263                    Some(Ok(frame)) => frame,
3264                };
3265                let phase_trace = frame.phase_trace;
3266                let frame = frame.frame;
3267
3268                if !ingress_route_should_be_processed(
3269                    &installed_route_epochs,
3270                    &reclaimed_routes,
3271                    &frame,
3272                ) {
3273                    log::debug!(
3274                        "subc attach: silently dropping {:?} for uninstalled route {}@{}",
3275                        frame.header.ty,
3276                        frame.header.channel,
3277                        frame.header.epoch
3278                    );
3279                    continue;
3280                }
3281
3282                match frame.header.ty {
3283                    FrameType::Ping if frame.header.channel == 0 => {
3284                        let pong = match Frame::build_with_version(
3285                            frame.header.ver,
3286                            FrameType::Pong,
3287                            frame.header.flags,
3288                            0,
3289                            0,
3290                            frame.header.corr,
3291                            Vec::new(),
3292                        ) {
3293                            Ok(pong) => pong,
3294                            Err(error) => break Err(SubcError::FrameBuild(error)),
3295                        };
3296                        if let Err(error) = send_frame(&writer_tx, &dispatch_path_metrics, pong).await {
3297                            break Err(error);
3298                        }
3299                    }
3300                    FrameType::Goodbye if frame.header.channel == 0 => {
3301                        log::info!("subc attach: received channel-0 Goodbye");
3302                        break Ok(ModuleLoopExit::Graceful);
3303                    }
3304                    FrameType::Goodbye => {
3305                        let channel = route_key(frame.header.channel, frame.header.epoch);
3306                        if let Err(error) = teardown_installed_route(
3307                            &writer_tx,
3308                            &dispatch_path_metrics,
3309                            &executor,
3310                            channel,
3311                            "Goodbye",
3312                            None,
3313                            &mut installed_route_epochs,
3314                            &mut routes,
3315                            &mut management_routes,
3316                            &mut root_channels,
3317                            &mut bg_subs,
3318                            &mut bg_sub_by_session,
3319                            &mut bg_wake_pending,
3320                            &mut pending_bash_asks,
3321                            &mut live_roots,
3322                            &mut route_bash_cancels,
3323                            &active_tool_calls,
3324                            &mut pending_responses,
3325                            &mut pending_binds,
3326                            &mut retry_buffer,
3327                            &mut push_buffer,
3328                            &shutdown,
3329                            tool_response_body_limit,
3330                            lifecycle_probe.as_ref(),
3331                        )
3332                        .await
3333                        {
3334                            break Err(error);
3335                        }
3336                    }
3337                    FrameType::Response | FrameType::Error if frame.header.channel != 0 => {
3338                        if let Err(error) = handle_bash_elicitation_reply(
3339                            &writer_tx,
3340                            &frame,
3341                            &mut pending_bash_asks,
3342                            &routes,
3343                            &mut live_roots,
3344                            &executor,
3345                            &shutdown,
3346                            &bash_deferred_tx,
3347                            &bash_poll_touch_tx,
3348                            &dispatch_path_metrics,
3349                            &mut route_bash_cancels,
3350                            dispatch,
3351                        )
3352                        .await
3353                        {
3354                            break Err(error);
3355                        }
3356                    }
3357                    FrameType::Request if frame.header.channel == 0 => {
3358                        if let Err(error) = handle_control_request(
3359                            &writer_tx,
3360                            &frame,
3361                            &shared_app,
3362                            &executor,
3363                            &mut live_roots,
3364                            &mut pending_binds,
3365                            &mut installed_route_epochs,
3366                            &mut routes,
3367                            &mut management_routes,
3368                            &mut root_channels,
3369                            &mut bg_subs,
3370                            &mut bg_sub_by_session,
3371                            &mut bg_wake_pending,
3372                            &mut pending_bash_asks,
3373                            &mut route_bash_cancels,
3374                            &active_tool_calls,
3375                            &mut pending_responses,
3376                            &mut retry_buffer,
3377                            &mut push_buffer,
3378                            &shutdown,
3379                            &control_completion_tx,
3380                            &dispatch_path_metrics,
3381                            lifecycle_probe.as_ref(),
3382                            &health_rollup_cache,
3383                            &push_senders,
3384                            dispatch,
3385                            user_config_path.as_deref(),
3386                            tool_response_body_limit,
3387                        )
3388                        .await
3389                        {
3390                            break Err(error);
3391                        }
3392                    }
3393                    FrameType::Request => {
3394                        let route = route_key(frame.header.channel, frame.header.epoch);
3395                        let result = if management_routes.contains(&route) {
3396                            handle_management_request(
3397                                &writer_tx,
3398                                &frame,
3399                                &shared_app,
3400                                &executor,
3401                                &live_roots,
3402                                &root_channels,
3403                                &health_rollup_cache,
3404                                &dispatch_path_metrics,
3405                            )
3406                            .await
3407                        } else {
3408                            handle_tool_call(
3409                                &writer_tx,
3410                                &frame,
3411                                phase_trace,
3412                                &routes,
3413                                &pending_binds,
3414                                &mut live_roots,
3415                                &executor,
3416                                &active_tool_calls,
3417                                &pending_deferred_setups,
3418                                &shutdown,
3419                                &connection_cancel,
3420                                &bash_deferred_tx,
3421                                &bash_poll_touch_tx,
3422                                &dispatch_path_metrics,
3423                                &mut route_bash_cancels,
3424                                &mut pending_bash_asks,
3425                                &mut next_bash_ask_corr,
3426                                &mut bg_subs,
3427                                &mut bg_sub_by_session,
3428                                &mut bg_wake_pending,
3429                                &mut bg_wake_epoch,
3430                                dispatch,
3431                                &deferred_response_tx,
3432                                allow_native_passthrough,
3433                                tool_response_body_limit,
3434                            )
3435                            .await
3436                        };
3437                        if let Err(error) = result {
3438                            break Err(error);
3439                        }
3440                    }
3441                    FrameType::Cancel => {
3442                        let channel = route_key(frame.header.channel, frame.header.epoch);
3443                        cancel_active_tool_call(
3444                            &active_tool_calls,
3445                            executor.as_ref(),
3446                            channel,
3447                            frame.header.corr,
3448                            "Cancel frame",
3449                        );
3450                        pending_responses.cancel_request(channel, frame.header.corr);
3451                        if bg_subs.contains_key(&channel) {
3452                            if let Err(error) = end_bg_subscription(
3453                                &writer_tx,
3454                                &dispatch_path_metrics,
3455                                &mut bg_subs,
3456                                &mut bg_sub_by_session,
3457                                &mut bg_wake_pending,
3458                                channel,
3459                                routes.get(&channel),
3460                                "cancel",
3461                            )
3462                            .await
3463                            {
3464                                break Err(error);
3465                            }
3466                        }
3467                        if let Err(error) = cancel_pending_bash_ask_for_tool_call(
3468                            &writer_tx,
3469                            &mut pending_bash_asks,
3470                            channel,
3471                            frame.header.corr,
3472                            &routes,
3473                            &mut live_roots,
3474                            &mut route_bash_cancels,
3475                            &shutdown,
3476                            &dispatch_path_metrics,
3477                        )
3478                        .await
3479                        {
3480                            break Err(error);
3481                        }
3482                    }
3483                    // Incoming push messages are ignored here. Cancel frames are
3484                    // handled above for active and deferred tool calls plus pending
3485                    // bash elicitation requests.
3486                    _ => {}
3487                }
3488            }
3489            Some(pending) = deferred_response_rx.recv() => {
3490                if routes.contains_key(&pending.route)
3491                    && active_tool_call_is_registered(
3492                        &active_tool_calls,
3493                        pending.route,
3494                        pending.corr,
3495                    )
3496                {
3497                    pending_responses.register(pending);
3498                } else {
3499                    if let Some(cancellation) = &pending.pending.cancellation {
3500                        cancellation.request_cancel();
3501                    }
3502                    finish_active_tool_call(&active_tool_calls, pending.route, pending.corr);
3503                }
3504            }
3505            Some((root_id, frame)) = reliable_rx.recv() => {
3506                // Reliable Push frames are FIFO and must-deliver, but draining an
3507                // unbounded burst in one current-thread turn can starve RouteBind
3508                // completions. The budget defers excess frames, never drops them.
3509                let (_, deferred) = push::drain_reliable_push_turn(
3510                    &writer_tx,
3511                    &dispatch_path_metrics,
3512                    &routes,
3513                    &root_channels,
3514                    &session_identity,
3515                    &mut retry_buffer,
3516                    &mut push_buffer,
3517                    &mut completed_tasks,
3518                    &bg_sub_by_session,
3519                    &mut bg_wake_pending,
3520                    &mut bg_wake_epoch,
3521                    &mut reliable_rx,
3522                    Some((root_id, frame)),
3523                    lifecycle_probe.as_ref(),
3524                );
3525                if deferred {
3526                    tokio::task::yield_now().await;
3527                }
3528            }
3529            Some((order, root_id, frame)) = lossy_rx.recv() => {
3530                // When both push lanes have work, handle a small reliable slice before lossy work.
3531                // That ordering lets completed task ids suppress stale BashLongRunning frames.
3532                // The slice stays bounded so reliable bursts cannot monopolize this loop turn.
3533                let (_, deferred) = push::drain_reliable_push_turn(
3534                    &writer_tx,
3535                    &dispatch_path_metrics,
3536                    &routes,
3537                    &root_channels,
3538                    &session_identity,
3539                    &mut retry_buffer,
3540                    &mut push_buffer,
3541                    &mut completed_tasks,
3542                    &bg_sub_by_session,
3543                    &mut bg_wake_pending,
3544                    &mut bg_wake_epoch,
3545                    &mut reliable_rx,
3546                    None,
3547                    lifecycle_probe.as_ref(),
3548                );
3549                if deferred {
3550                    tokio::task::yield_now().await;
3551                }
3552
3553                // Drain the currently queued burst in one loop turn so lossy
3554                // status/progress updates can be merged before reaching subc's
3555                // shared egress queue. Each lossy frame gets a sequence number
3556                // before it goes to the channel or overflow buffer, so the
3557                // combined batch is sorted back into producer order before
3558                // coalescing drops stale updates for the same key.
3559                let mut batch = vec![(order, root_id, frame)];
3560                while let Ok(item) = lossy_rx.try_recv() {
3561                    batch.push(item);
3562                }
3563                batch.extend(lossy_overflow.drain());
3564                push::process_lossy_push_envelope_batch(
3565                    &writer_tx,
3566                    &dispatch_path_metrics,
3567                    &routes,
3568                    &root_channels,
3569                    &completed_tasks,
3570                    batch,
3571                );
3572            }
3573            Some(done) = bash_deferred_rx.recv() => {
3574                decrement_counted_channel(&dispatch_path_metrics.bash_deferred_queued);
3575                if let Err(error) = bash::handle_bash_deferred_completion(
3576                    &writer_tx,
3577                    done,
3578                    &routes,
3579                    &mut live_roots,
3580                    &mut route_bash_cancels,
3581                    &shutdown,
3582                    &dispatch_path_metrics,
3583                )
3584                .await
3585                {
3586                    break Err(error);
3587                }
3588            }
3589            Some(root_id) = bash_poll_touch_rx.recv() => {
3590                decrement_counted_channel(&dispatch_path_metrics.bash_poll_touch_queued);
3591                if let Some(meta) = live_roots.get_mut(&root_id) {
3592                    meta.note_activity();
3593                }
3594            }
3595            Some(completion) = maintenance_rx.recv() => {
3596                decrement_counted_channel(&dispatch_path_metrics.maintenance_queued);
3597                let root_id = completion.root_id.clone();
3598                let response = completion.response;
3599                let response_is_fatal = response_is_fatal_panic(&response);
3600                let bind_pending = pending_binds
3601                    .values()
3602                    .any(|pending| pending.bind_root_id == root_id);
3603                let requiesce = if let Some(meta) = live_roots.get_mut(&root_id) {
3604                    let defer_requeue = meta.unbound_quiesced || bind_pending;
3605                    note_maintenance_completion(
3606                        meta,
3607                        completion.requeue_kind,
3608                        response_is_fatal,
3609                        defer_requeue,
3610                    );
3611                    should_requiesce_after_maintenance(meta, completion.kind, bind_pending)
3612                } else {
3613                    false
3614                };
3615                if requiesce {
3616                    quiesce_unbound_root(&root_id, &mut live_roots, &executor);
3617                }
3618                push::clear_stale_bg_wakes_for_empty_sessions(
3619                    &root_id,
3620                    &completion.empty_bg_sessions,
3621                    &bg_sub_by_session,
3622                    &mut bg_wake_pending,
3623                    &bg_wake_epoch,
3624                );
3625                if let Some(keys) = completion.unacked_bg_keys {
3626                    bg_unacked_keys_by_root.insert(root_id.clone(), keys);
3627                }
3628                record_bg_runtime_from_snapshots(
3629                    &dispatch_path_metrics,
3630                    bg_subs.len(),
3631                    bg_wake_pending.len(),
3632                    &bg_unacked_keys_by_root,
3633                );
3634                if response_is_fatal {
3635                    if let Some(meta) = live_roots.get_mut(&root_id) {
3636                        meta.maintenance_poisoned = true;
3637                    }
3638                    log::warn!(
3639                        "subc attach: maintenance drain observed a fatal actor; deferring teardown until a route request can receive actor_fatal"
3640                    );
3641                }
3642            }
3643            _ = tokio::time::sleep(PENDING_POLL_INTERVAL), if !pending_responses.is_empty() => {
3644                // The next loop turn polls detached inspect completions. Keeping
3645                // the timer here lets already-ready control frames run first.
3646            }
3647            _ = tokio::time::sleep_until(next_drain_at) => {
3648                // Wakes an otherwise-idle loop so the pre-turn drain check
3649                // above runs on schedule; the drain work itself lives there.
3650            }
3651            _ = tokio::time::sleep_until(next_maintenance_at) => {
3652                // Delay cache-draining maintenance until any already-ready
3653                // inbound route/control messages and push completions have run,
3654                // so maintenance does not block the actor from handling the
3655                // first request that arrives after a route bind is acknowledged.
3656                crate::logging::maybe_sweep_logs();
3657                crate::db::compression_events::maybe_spawn_retention(shared_app.db());
3658                let reaped_lsp_children = shared_app
3659                    .lsp_child_registry()
3660                    .reap_children_with_gone_cwd_or_reclaimed_root();
3661                if reaped_lsp_children > 0 {
3662                    log::warn!(
3663                        "subc attach: reaped {reaped_lsp_children} orphaned LSP child process group(s)"
3664                    );
3665                }
3666                let now = Instant::now();
3667                reap_idle_lsp_servers(now, &live_roots, &executor);
3668                let reap = reap_idle_roots(
3669                    now,
3670                    &mut live_roots,
3671                    &pending_binds,
3672                    &root_channels,
3673                    &executor,
3674                    &dispatch_path_metrics,
3675                );
3676                for root_id in &reap.forgotten_deleted_roots {
3677                    bg_unacked_keys_by_root.remove(root_id);
3678                    purge_deleted_root_residents(
3679                        root_id,
3680                        &mut routes,
3681                        &mut root_channels,
3682                        &mut installed_route_epochs,
3683                        &mut route_bash_cancels,
3684                        &active_tool_calls,
3685                        executor.as_ref(),
3686                        &mut retry_buffer,
3687                        &mut reclaimed_routes,
3688                        &mut session_identity,
3689                        &mut push_buffer,
3690                        &mut bg_subs,
3691                        &mut bg_sub_by_session,
3692                        &mut bg_wake_pending,
3693                        &mut bg_wake_epoch,
3694                        &mut pending_bash_asks,
3695                        &dispatch_path_metrics,
3696                    );
3697                }
3698                if reap.evicted > 0 {
3699                    log::debug!("subc attach: reaped {} idle root(s)", reap.evicted);
3700                }
3701                record_bg_runtime_from_snapshots(
3702                    &dispatch_path_metrics,
3703                    bg_subs.len(),
3704                    bg_wake_pending.len(),
3705                    &bg_unacked_keys_by_root,
3706                );
3707                submit_due_maintenance_jobs(
3708                    &executor,
3709                    &mut live_roots,
3710                    &pending_binds,
3711                    &bg_sub_by_session,
3712                    &bg_wake_pending,
3713                    &bg_wake_epoch,
3714                    &maintenance_tx,
3715                    &dispatch_path_metrics,
3716                );
3717                if tokio::time::Instant::now() >= next_standing_pass_at {
3718                    standing_actor.tick();
3719                    next_standing_pass_at = tokio::time::Instant::now()
3720                        + standing::STANDING_MAINTENANCE_INTERVAL;
3721                }
3722                // Opportunistic allocator relief, independent of the idle
3723                // sweep: the sweep's whole-process idle gate never opens while
3724                // any session stays active, which let freed warm-up arenas sit
3725                // resident for the process lifetime (5.1 GB RSS over ~600 MB
3726                // live). Slack threshold + spacing live in memory.rs; the pass
3727                // itself runs on a detached thread.
3728                #[cfg(any(target_os = "macos", target_os = "linux"))]
3729                {
3730                    let now_std = std::time::Instant::now();
3731                    let _ = crate::memory::spawn_allocator_slack_relief_if_due(now_std);
3732                }
3733                next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
3734            }
3735        }
3736    };
3737
3738    shared_app.set_open_route_count(0);
3739    health_rollup_worker.shutdown();
3740
3741    connection_cancel.cancel();
3742    cancel_all_active_tool_calls(&active_tool_calls, executor.as_ref(), "connection teardown");
3743    let setup_drain_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
3744    while pending_deferred_setups.load(Ordering::SeqCst) != 0
3745        && tokio::time::Instant::now() < setup_drain_deadline
3746    {
3747        tokio::select! {
3748            biased;
3749            Some(pending) = deferred_response_rx.recv() => pending_responses.register(pending),
3750            _ = tokio::time::sleep(Duration::from_millis(5)) => {}
3751        }
3752    }
3753    if pending_deferred_setups.load(Ordering::SeqCst) != 0 {
3754        log::warn!(
3755            "subc attach: timed out waiting for deferred response setup registration during shutdown"
3756        );
3757    }
3758    while let Ok(pending) = deferred_response_rx.try_recv() {
3759        pending_responses.register(pending);
3760    }
3761    for resolved in pending_responses.drain_on_shutdown(executor.as_ref()) {
3762        if let Err(error) = deliver_resolved_subc_response(
3763            &writer_tx,
3764            resolved,
3765            &routes,
3766            &mut live_roots,
3767            executor.as_ref(),
3768            &active_tool_calls,
3769            &shutdown,
3770            &dispatch_path_metrics,
3771            tool_response_body_limit,
3772        )
3773        .await
3774        {
3775            log::warn!("subc attach: failed to emit deferred shutdown terminal: {error}");
3776        }
3777    }
3778    // Channel-0 Goodbye, EOF, and fatal exits bypass per-route Goodbye. Settle
3779    // their root lifecycle state before loop-owned routing metadata is dropped.
3780    quiesce_connection_roots(
3781        &mut live_roots,
3782        &mut pending_binds,
3783        &mut routes,
3784        &mut root_channels,
3785        &mut installed_route_epochs,
3786        &mut route_bash_cancels,
3787        &active_tool_calls,
3788        &executor,
3789    );
3790
3791    fleet_status_client.set_route_live(false);
3792    fleet_status_task.abort();
3793    let _ = fleet_status_task.await;
3794
3795    let mut loop_result = loop_result;
3796    if !pending_bash_asks.is_empty() {
3797        let no_routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
3798        if let Err(error) = settle_all_pending_bash_asks(
3799            &writer_tx,
3800            &mut pending_bash_asks,
3801            &no_routes,
3802            &mut live_roots,
3803            &mut route_bash_cancels,
3804            &shutdown,
3805            &dispatch_path_metrics,
3806        )
3807        .await
3808        {
3809            loop_result = loop_result.and(Err(error));
3810        }
3811    }
3812
3813    // The reader task may be parked on `read_frame`; abort it (we are done with
3814    // the connection) and flush the writer.
3815    reader_task.abort();
3816    drop(writer_tx);
3817    let writer_result = finish_writer_task(writer_task).await;
3818    loop_result.and_then(|exit| writer_result.map(|_| exit))
3819}
3820
3821fn spawn_writer_task<W>(
3822    mut write: W,
3823    mut rx: mpsc::Receiver<WriterFrame>,
3824    metrics: Arc<DispatchPathMetrics>,
3825) -> JoinHandle<Result<(), subc_transport::FrameIoError>>
3826where
3827    W: AsyncWrite + Unpin + Send + 'static,
3828{
3829    tokio::spawn(async move {
3830        let mut write_buffer = Vec::new();
3831        while let Some(mut queued) = rx.recv().await {
3832            let measure = queued.tool_response_trace.is_some();
3833            let dequeued = measure.then(Instant::now);
3834            metrics.writer_active.store(true, Ordering::Relaxed);
3835            decrement_counted_channel(&metrics.writer_queued);
3836            let write_timing = write_frame_contiguous(
3837                &mut write,
3838                queued.frame(),
3839                queued.body(),
3840                &mut write_buffer,
3841                measure,
3842            )
3843            .await;
3844            metrics.writer_active.store(false, Ordering::Relaxed);
3845            let write_timing = write_timing?;
3846
3847            if let (Some(trace), Some(dequeued), Some(write_timing)) =
3848                (queued.tool_response_trace.take(), dequeued, write_timing)
3849            {
3850                if let Some(completed) = trace.finish(
3851                    dequeued,
3852                    write_timing.write_started,
3853                    write_timing.write_finished,
3854                    write_timing.frame_bytes,
3855                ) {
3856                    log_ctx::with_session(Some(completed.session), || {
3857                        crate::logging::note_tool_call_trace(
3858                            &completed.name,
3859                            &completed.root,
3860                            completed.channel,
3861                            completed.corr,
3862                            completed.phases,
3863                        );
3864                    });
3865                }
3866            }
3867        }
3868        Ok(())
3869    })
3870}
3871
3872struct FrameWriteTiming {
3873    write_started: Instant,
3874    write_finished: Instant,
3875    frame_bytes: usize,
3876}
3877
3878/// Encode one complete frame into the existing reusable buffer and write it
3879/// without interleaving bytes from another channel. Timing is collected only
3880/// for tool responses, so Push and control frames add no clock reads.
3881async fn write_frame_contiguous<W>(
3882    writer: &mut W,
3883    frame: &Frame,
3884    body: &[u8],
3885    buffer: &mut Vec<u8>,
3886    measure: bool,
3887) -> Result<Option<FrameWriteTiming>, subc_transport::FrameIoError>
3888where
3889    W: AsyncWrite + Unpin,
3890{
3891    if frame.header.len as usize != body.len() {
3892        return Err(subc_transport::FrameIoError::BodyLengthMismatch {
3893            header_len: frame.header.len,
3894            body_len: body.len(),
3895        });
3896    }
3897
3898    let header = frame.header.encode();
3899    buffer.clear();
3900    buffer.reserve(header.len() + body.len());
3901    buffer.extend_from_slice(&header);
3902    buffer.extend_from_slice(body);
3903    let write_started = measure.then(Instant::now);
3904    writer
3905        .write_all(buffer)
3906        .await
3907        .map_err(subc_transport::FrameIoError::Io)?;
3908    Ok(write_started.map(|write_started| FrameWriteTiming {
3909        write_started,
3910        write_finished: Instant::now(),
3911        frame_bytes: buffer.len(),
3912    }))
3913}
3914
3915fn spawn_reader_task<R>(
3916    mut read: R,
3917    tx: mpsc::Sender<Result<DecodedFrame, SubcError>>,
3918) -> JoinHandle<()>
3919where
3920    R: AsyncRead + Unpin + Send + 'static,
3921{
3922    tokio::spawn(async move {
3923        loop {
3924            match read_frame(&mut read).await {
3925                Ok(Some(frame)) => {
3926                    let decoded = DecodedFrame {
3927                        frame,
3928                        phase_trace: PhaseTrace::new(Instant::now()),
3929                    };
3930                    if tx.send(Ok(decoded)).await.is_err() {
3931                        return;
3932                    }
3933                }
3934                Ok(None) => {
3935                    // EOF: let the loop observe channel close as "daemon closed".
3936                    return;
3937                }
3938                Err(error) => {
3939                    // A killed daemon surfaces as ConnectionReset (RST) on
3940                    // Windows where Unix delivers a clean EOF (FIN); a
3941                    // mid-teardown daemon can also abort the socket. Both mean
3942                    // "daemon went away", not a wire fault — normalize them to
3943                    // the clean-close path so module exit behavior matches
3944                    // across platforms (same class subc-core fixed in d33d9a71).
3945                    if let subc_transport::FrameIoError::Io(io_error) = &error {
3946                        if matches!(
3947                            io_error.kind(),
3948                            std::io::ErrorKind::ConnectionReset
3949                                | std::io::ErrorKind::ConnectionAborted
3950                        ) {
3951                            log::info!(
3952                                "subc attach: connection reset by daemon; treating as close"
3953                            );
3954                            return;
3955                        }
3956                    }
3957                    let _ = tx.send(Err(SubcError::FrameIo(error))).await;
3958                    return;
3959                }
3960            }
3961        }
3962    })
3963}
3964
3965async fn finish_writer_task(
3966    mut writer_task: JoinHandle<Result<(), subc_transport::FrameIoError>>,
3967) -> Result<(), SubcError> {
3968    match tokio::time::timeout(Duration::from_millis(100), &mut writer_task).await {
3969        Ok(Ok(Ok(()))) => Ok(()),
3970        Ok(Ok(Err(error))) => Err(SubcError::FrameIo(error)),
3971        Ok(Err(error)) => Err(SubcError::WriterJoin(error)),
3972        Err(_) => {
3973            writer_task.abort();
3974            Ok(())
3975        }
3976    }
3977}
3978
3979fn register_actor_for_bind(
3980    shared_app: &Arc<App>,
3981    executor: &Arc<Executor>,
3982    push_senders: &PushSenders,
3983    bind_root_id: &ProjectRootId,
3984    route_channel: u16,
3985    root_was_live: bool,
3986) -> bool {
3987    if executor.actor_registered(bind_root_id) {
3988        log::debug!(
3989            "subc attach: reusing actor for route {} root {}",
3990            route_channel,
3991            bind_root_id.as_path().display()
3992        );
3993        return false;
3994    }
3995
3996    if root_was_live {
3997        log::warn!(
3998            "subc attach: recreating missing actor for live root {} on route {}",
3999            bind_root_id.as_path().display(),
4000            route_channel
4001        );
4002    }
4003
4004    let actor_ctx = Arc::new(AppContext::from_app(
4005        Arc::clone(shared_app),
4006        Config::default(),
4007    ));
4008    install_bash_compressor(&actor_ctx);
4009    actor_ctx.install_fleet_status_client(Some(push_senders.fleet_status_client.clone()));
4010    actor_ctx.set_progress_sender(Some(push::progress_sender_for_root(
4011        push_senders.clone(),
4012        bind_root_id.clone(),
4013    )));
4014    let inserted = executor.register_actor(bind_root_id.clone(), Arc::clone(&actor_ctx));
4015    drop(actor_ctx);
4016    if inserted {
4017        // Do not insert into live_roots until configure succeeds: live_roots
4018        // drives maintenance, and a half-configured new actor must not be
4019        // maintenance-eligible before its route/session identity exists.
4020        log::debug!(
4021            "subc attach: registered actor for route {} root {}",
4022            route_channel,
4023            bind_root_id.as_path().display()
4024        );
4025    } else {
4026        log::debug!(
4027            "subc attach: actor appeared while binding route {} root {}; reusing it",
4028            route_channel,
4029            bind_root_id.as_path().display()
4030        );
4031    }
4032    inserted
4033}
4034
4035fn rollback_pending_bind_actor(
4036    executor: &Arc<Executor>,
4037    live_roots: &HashMap<ProjectRootId, RootMeta>,
4038    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
4039    root_id: &ProjectRootId,
4040    inserted_new_actor: bool,
4041) {
4042    if !inserted_new_actor || live_roots.contains_key(root_id) {
4043        return;
4044    }
4045
4046    if let Some((route, pending)) = pending_binds
4047        .iter_mut()
4048        .find(|(_, pending)| &pending.bind_root_id == root_id)
4049    {
4050        pending.inserted_new_actor = true;
4051        log::debug!(
4052            "subc attach: transferred rollback ownership for root {} to pending route {}",
4053            root_id.as_path().display(),
4054            route
4055        );
4056        return;
4057    }
4058
4059    executor.remove_actor(root_id);
4060}
4061
4062fn route_bind_error_code_for_configure_response(response: &Response) -> &'static str {
4063    match response.data.get("code").and_then(|code| code.as_str()) {
4064        // Preserve typed configure rejections across the bind boundary: a
4065        // malformed fed fingerprint means a federation-module bug or
4066        // fingerprint-format drift, and the fed side matches on the code rather
4067        // than parsing prose.
4068        Some("bad_harness_fingerprint") => "bad_harness_fingerprint",
4069        // Cache-key probe failures are transient (fd pressure, git spawn
4070        // contention); the client retries the bind rather than treating the
4071        // root as permanently divergent.
4072        Some("cache_key_probe_failed") => "cache_key_probe_failed",
4073        // Actor lifecycle gaps are transient from the daemon/client viewpoint:
4074        // a fresh bind can create or join a healthy actor, so do not classify
4075        // them as permanent config divergence.
4076        Some("actor_not_registered" | "actor_fatal") => "actor_not_ready",
4077        _ => "config_divergence",
4078    }
4079}
4080
4081fn queue_post_bind_configure_and_completion_maintenance(
4082    root_id: &ProjectRootId,
4083    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4084) {
4085    let Some(meta) = live_roots.get_mut(root_id) else {
4086        return;
4087    };
4088    if meta.maintenance_poisoned || meta.maintenance_pending {
4089        return;
4090    }
4091
4092    meta.maintenance_pending = true;
4093    meta.maintenance_queued_kinds
4094        .push_back(MaintenanceDrainKind::ConfigureTail);
4095    meta.maintenance_queued_kinds
4096        .push_back(MaintenanceDrainKind::CompletionDrains);
4097}
4098
4099#[allow(clippy::too_many_arguments)]
4100async fn handle_route_bind_completion(
4101    tx: &WriterSender,
4102    completion: RouteBindCompletion,
4103    routes: &mut HashMap<RouteChannel, RouteIdentity>,
4104    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
4105    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
4106    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
4107    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4108    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
4109    installed_route_epochs: &mut HashMap<u16, u32>,
4110    executor: &Arc<Executor>,
4111    standing_actor: &standing::StandingActor,
4112    shutdown: &Arc<Notify>,
4113    metrics: &Arc<DispatchPathMetrics>,
4114    lifecycle_probe: Option<&SubcTestLifecycleProbe>,
4115) -> Result<(), SubcError> {
4116    let route_id = completion.route;
4117    let Some(pending) = pending_binds.remove(&route_id) else {
4118        log::warn!(
4119            "subc attach: dropping RouteBind completion for non-pending route {}",
4120            completion.route
4121        );
4122        rollback_pending_bind_actor(
4123            executor,
4124            live_roots,
4125            pending_binds,
4126            &completion.bind_root_id,
4127            completion.inserted_new_actor,
4128        );
4129        let has_pending_bind = pending_binds
4130            .values()
4131            .any(|pending| pending.bind_root_id == completion.bind_root_id);
4132        if !root_channels
4133            .get(&completion.bind_root_id)
4134            .is_some_and(|channels| !channels.is_empty())
4135            && !has_pending_bind
4136        {
4137            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
4138        }
4139        remove_installed_route(installed_route_epochs, route_id);
4140        return Ok(());
4141    };
4142
4143    if pending.bind_root_id != completion.bind_root_id {
4144        log::warn!(
4145            "subc attach: pending RouteBind root mismatch for route {} (pending {} completion {})",
4146            completion.route,
4147            pending.bind_root_id.as_path().display(),
4148            completion.bind_root_id.as_path().display()
4149        );
4150    }
4151
4152    let inserted_new_actor = pending.inserted_new_actor || completion.inserted_new_actor;
4153    if pending.cancelled {
4154        rollback_pending_bind_actor(
4155            executor,
4156            live_roots,
4157            pending_binds,
4158            &completion.bind_root_id,
4159            inserted_new_actor,
4160        );
4161        let has_pending_bind = pending_binds
4162            .values()
4163            .any(|pending| pending.bind_root_id == completion.bind_root_id);
4164        if !root_channels
4165            .get(&completion.bind_root_id)
4166            .is_some_and(|channels| !channels.is_empty())
4167            && !has_pending_bind
4168        {
4169            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
4170        }
4171        log::debug!(
4172            "subc attach: discarded completed RouteBind for cancelled route {} root {}",
4173            completion.route,
4174            completion.bind_root_id.as_path().display()
4175        );
4176        remove_installed_route(installed_route_epochs, route_id);
4177        return Ok(());
4178    }
4179
4180    let failure = if !completion.configure_response.success {
4181        Some((
4182            &completion.configure_response,
4183            "configure failed during route bind",
4184        ))
4185    } else {
4186        None
4187    };
4188
4189    if let Some((response, fallback)) = failure {
4190        rollback_pending_bind_actor(
4191            executor,
4192            live_roots,
4193            pending_binds,
4194            &completion.bind_root_id,
4195            inserted_new_actor,
4196        );
4197        let has_pending_bind = pending_binds
4198            .values()
4199            .any(|pending| pending.bind_root_id == completion.bind_root_id);
4200        if !root_channels
4201            .get(&completion.bind_root_id)
4202            .is_some_and(|channels| !channels.is_empty())
4203            && !has_pending_bind
4204        {
4205            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
4206        }
4207        let message = response_message(response, fallback);
4208        let fatal = response_is_fatal_panic(response);
4209        let error_code = route_bind_error_code_for_configure_response(response);
4210        send_route_bind_error_parts(
4211            tx,
4212            completion.ver,
4213            completion.corr,
4214            completion.flags,
4215            error_code,
4216            &message,
4217            metrics,
4218        )
4219        .await?;
4220        remove_installed_route(installed_route_epochs, route_id);
4221        if fatal {
4222            signal_fatal_teardown(
4223                tx,
4224                Some(completion.route),
4225                completion.ver,
4226                completion.corr,
4227                shutdown,
4228                metrics,
4229            )
4230            .await;
4231        }
4232        return Ok(());
4233    }
4234
4235    remember_session_identity(session_identity, &completion.identity);
4236    let replay_key = push::ReplayKey::from_identity(&completion.identity);
4237    let bind_trust = completion.identity.trust;
4238    insert_route_channel(routes, root_channels, route_id, completion.identity);
4239    sync_bg_live_delivery_sessions(executor, routes, Some(&completion.bind_root_id));
4240    let restore_watcher = live_roots
4241        .get(&completion.bind_root_id)
4242        .is_some_and(|meta| meta.idle_artifacts_evicted || meta.unbound_quiesced);
4243    live_roots
4244        .entry(completion.bind_root_id.clone())
4245        .and_modify(|meta| {
4246            meta.reactivate_bound();
4247            meta.diagnostics_on_edit = completion.diagnostics_on_edit;
4248            meta.maintenance_poisoned = false;
4249        })
4250        .or_insert_with(|| RootMeta::new(Instant::now()));
4251    if let Some(meta) = live_roots.get_mut(&completion.bind_root_id) {
4252        meta.diagnostics_on_edit = completion.diagnostics_on_edit;
4253        meta.maintenance_poisoned = false;
4254    }
4255    if let Some(ctx) = executor.actor_context(&completion.bind_root_id) {
4256        // The bind transition revokes any matching unbound standing admission
4257        // before this session can select the shared artifact family.
4258        standing_actor.begin_session_bind(&ctx);
4259        ctx.mark_subc_bound();
4260        if restore_watcher {
4261            crate::commands::configure::ensure_project_watcher(&ctx);
4262        }
4263    }
4264
4265    let ack =
4266        serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).map_err(SubcError::Json)?;
4267    let response = Frame::build_with_version(
4268        completion.ver,
4269        FrameType::Response,
4270        control_flags(),
4271        0,
4272        0,
4273        completion.corr,
4274        ack,
4275    )
4276    .map_err(SubcError::FrameBuild)?;
4277    send_reliable_writer_frame(tx, metrics, response, "RouteBindAck").await?;
4278    queue_post_bind_configure_and_completion_maintenance(&completion.bind_root_id, live_roots);
4279    let replayed = push::replay_buffered_push_frames(
4280        tx,
4281        metrics,
4282        route_id,
4283        push_buffer,
4284        &replay_key,
4285        bind_trust,
4286        lifecycle_probe,
4287    );
4288    if replayed > 0 {
4289        log::debug!(
4290            "subc attach: replayed {} buffered Push frame(s) to route {} root {} harness {} session {}",
4291            replayed,
4292            completion.route,
4293            replay_key.root.as_path().display(),
4294            replay_key.harness,
4295            replay_key.session
4296        );
4297    }
4298    log::info!(
4299        "subc attach: route {} bound to root {}",
4300        completion.route,
4301        completion.bind_root_id.as_path().display()
4302    );
4303    Ok(())
4304}
4305
4306async fn expire_overdue_route_binds(
4307    tx: &WriterSender,
4308    executor: &Arc<Executor>,
4309    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
4310    installed_route_epochs: &mut HashMap<u16, u32>,
4311    metrics: &DispatchPathMetrics,
4312) -> Result<(), SubcError> {
4313    let now = Instant::now();
4314    let expired: Vec<_> = pending_binds
4315        .iter()
4316        .filter_map(|(route, pending)| {
4317            let age = now.saturating_duration_since(pending.started_at);
4318            (!pending.deadline_reported && age >= ROUTE_BIND_DEADLINE).then(|| {
4319                (
4320                    *route,
4321                    pending.corr,
4322                    pending.ver,
4323                    pending.flags,
4324                    pending.bind_root_id.clone(),
4325                    pending.configure_request_id.clone(),
4326                    age,
4327                )
4328            })
4329        })
4330        .collect();
4331
4332    for (route, corr, ver, flags, root_id, configure_request_id, age) in expired {
4333        if let Some(pending) = pending_binds.get_mut(&route) {
4334            pending.cancelled = true;
4335            pending.deadline_reported = true;
4336            let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
4337            log::debug!(
4338                "subc attach: cancelled overdue RouteBind configure for route {route} ({outcome:?})"
4339            );
4340        }
4341        remove_installed_route(installed_route_epochs, route);
4342        let age_ms = age.as_millis().min(u128::from(u64::MAX)) as u64;
4343        let deadline_ms = ROUTE_BIND_DEADLINE.as_millis();
4344        send_route_bind_error_parts(
4345            tx,
4346            ver,
4347            corr,
4348            flags,
4349            "actor_not_ready",
4350            &format!("route bind deadline exceeded after {age_ms}ms (deadline {deadline_ms}ms)"),
4351            metrics,
4352        )
4353        .await?;
4354        log::warn!(
4355            "subc attach: route {} bind for root {} exceeded {}ms deadline (configure_request_id={})",
4356            route,
4357            root_id.as_path().display(),
4358            deadline_ms,
4359            configure_request_id
4360        );
4361    }
4362
4363    Ok(())
4364}
4365
4366fn record_bg_runtime_from_snapshots(
4367    metrics: &DispatchPathMetrics,
4368    subscriptions: usize,
4369    wake_pending: usize,
4370    unacked_keys_by_root: &HashMap<ProjectRootId, HashSet<String>>,
4371) {
4372    let unacked_total = unacked_keys_by_root
4373        .values()
4374        .flatten()
4375        .collect::<HashSet<_>>()
4376        .len();
4377    metrics.record_bg_runtime(subscriptions, wake_pending, unacked_total);
4378}
4379
4380async fn send_cached_health_response(
4381    tx: &WriterSender,
4382    frame: &Frame,
4383    shared_app: &App,
4384    executor: &Executor,
4385    pending_binds: &HashMap<RouteChannel, PendingBind>,
4386    metrics: &DispatchPathMetrics,
4387    health_rollup_cache: &HealthRollupCache,
4388) -> Result<(), SubcError> {
4389    let report = build_health_report(
4390        health_rollup_cache,
4391        executor,
4392        pending_binds,
4393        metrics,
4394        shared_app,
4395    );
4396    let body = serde_json::to_vec(&ModuleControlResponse::from(report)).map_err(SubcError::Json)?;
4397    let response = Frame::build_with_version(
4398        frame.header.ver,
4399        FrameType::Response,
4400        frame.header.flags,
4401        0,
4402        0,
4403        frame.header.corr,
4404        body,
4405    )
4406    .map_err(SubcError::FrameBuild)?;
4407    send_frame(tx, metrics, response).await
4408}
4409
4410/// Channel-0 control requests: RouteBind plus the cached health probe.
4411/// Tool-provider binds reconcile RootConfig through the executor's Mutating lane;
4412/// management binds install immediately without creating project state.
4413#[allow(clippy::too_many_arguments)]
4414async fn handle_control_request(
4415    tx: &WriterSender,
4416    frame: &Frame,
4417    shared_app: &Arc<App>,
4418    executor: &Arc<Executor>,
4419    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4420    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
4421    installed_route_epochs: &mut HashMap<u16, u32>,
4422    routes: &mut HashMap<RouteChannel, RouteIdentity>,
4423    management_routes: &mut HashSet<RouteChannel>,
4424    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
4425    bg_subs: &mut HashMap<RouteChannel, BgSub>,
4426    bg_sub_by_session: &mut BgSubsBySession,
4427    bg_wake_pending: &mut BgWakePending,
4428    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
4429    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
4430    active_tool_calls: &ActiveToolCalls,
4431    pending_responses: &mut PendingSubcResponses,
4432    retry_buffer: &mut RetryBuffer,
4433    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
4434    shutdown: &Arc<Notify>,
4435    control_completion_tx: &mpsc::Sender<RouteBindCompletion>,
4436    metrics: &Arc<DispatchPathMetrics>,
4437    lifecycle_probe: Option<&SubcTestLifecycleProbe>,
4438    health_rollup_cache: &HealthRollupCache,
4439    push_senders: &PushSenders,
4440    dispatch: DispatchFn,
4441    user_config_path: Option<&Path>,
4442    tool_response_body_limit: usize,
4443) -> Result<(), SubcError> {
4444    let request =
4445        serde_json::from_slice::<ModuleControlRequest>(&frame.body).map_err(SubcError::Json)?;
4446    match request {
4447        ModuleControlRequest::RouteBind {
4448            route_channel,
4449            epoch,
4450            target,
4451            identity,
4452            principal,
4453            consumer_capabilities,
4454            admission_facts: _,
4455        } => {
4456            let route_id = route_key(route_channel, epoch);
4457            if epoch == 0 {
4458                return send_route_bind_error(
4459                    tx,
4460                    frame,
4461                    "config_divergence",
4462                    "route bind uses an invalid channel generation",
4463                    metrics,
4464                )
4465                .await;
4466            }
4467
4468            let bind_trust = trust_for_bind(&identity.harness, &principal);
4469            if let RouteTarget::ManagementSurface { module_id } = &target {
4470                if module_id != "aft" {
4471                    return send_route_bind_error(
4472                        tx,
4473                        frame,
4474                        "route_refused",
4475                        "management route target is not AFT",
4476                        metrics,
4477                    )
4478                    .await;
4479                }
4480                if !matches!(bind_trust, BindTrust::FirstParty) {
4481                    return send_route_bind_error(
4482                        tx,
4483                        frame,
4484                        "route_refused",
4485                        "AFT management routes require a first-party principal",
4486                        metrics,
4487                    )
4488                    .await;
4489                }
4490                if let Some(installed_epoch) = installed_route_epochs.get(&route_channel).copied() {
4491                    if installed_epoch >= epoch {
4492                        return send_route_bind_error(
4493                            tx,
4494                            frame,
4495                            "config_divergence",
4496                            "route bind generation is not newer than the installed generation",
4497                            metrics,
4498                        )
4499                        .await;
4500                    }
4501                    teardown_installed_route(
4502                        tx,
4503                        metrics,
4504                        executor,
4505                        route_key(route_channel, installed_epoch),
4506                        "higher-epoch RouteBind",
4507                        None,
4508                        installed_route_epochs,
4509                        routes,
4510                        management_routes,
4511                        root_channels,
4512                        bg_subs,
4513                        bg_sub_by_session,
4514                        bg_wake_pending,
4515                        pending_bash_asks,
4516                        live_roots,
4517                        route_bash_cancels,
4518                        active_tool_calls,
4519                        pending_responses,
4520                        pending_binds,
4521                        retry_buffer,
4522                        push_buffer,
4523                        shutdown,
4524                        tool_response_body_limit,
4525                        lifecycle_probe,
4526                    )
4527                    .await?;
4528                }
4529                if pending_binds.contains_key(&route_id) {
4530                    return send_route_bind_error(
4531                        tx,
4532                        frame,
4533                        "config_divergence",
4534                        "route bind is already pending for channel",
4535                        metrics,
4536                    )
4537                    .await;
4538                }
4539
4540                installed_route_epochs.insert(route_channel, epoch);
4541                management_routes.insert(route_id);
4542                return send_route_bind_ack(
4543                    tx,
4544                    frame.header.ver,
4545                    frame.header.corr,
4546                    frame.header.flags,
4547                    metrics,
4548                )
4549                .await;
4550            }
4551            if matches!(&target, RouteTarget::InternalService { .. }) {
4552                return send_route_bind_error(
4553                    tx,
4554                    frame,
4555                    "route_refused",
4556                    "AFT does not provide an internal-service route",
4557                    metrics,
4558                )
4559                .await;
4560            }
4561
4562            let mut bind_root_id = None;
4563            if let Some(installed_epoch) = installed_route_epochs.get(&route_channel).copied() {
4564                if installed_epoch >= epoch {
4565                    return send_route_bind_error(
4566                        tx,
4567                        frame,
4568                        "config_divergence",
4569                        "route bind generation is not newer than the installed generation",
4570                        metrics,
4571                    )
4572                    .await;
4573                }
4574
4575                let replacement_root = match ProjectRootId::from_path(&identity.project_root) {
4576                    Ok(root_id) => root_id,
4577                    Err(error) => {
4578                        return send_route_bind_error(
4579                            tx,
4580                            frame,
4581                            "config_divergence",
4582                            &format!("invalid route project root: {error}"),
4583                            metrics,
4584                        )
4585                        .await;
4586                    }
4587                };
4588                teardown_installed_route(
4589                    tx,
4590                    metrics,
4591                    executor,
4592                    route_key(route_channel, installed_epoch),
4593                    "higher-epoch RouteBind",
4594                    Some(&replacement_root),
4595                    installed_route_epochs,
4596                    routes,
4597                    management_routes,
4598                    root_channels,
4599                    bg_subs,
4600                    bg_sub_by_session,
4601                    bg_wake_pending,
4602                    pending_bash_asks,
4603                    live_roots,
4604                    route_bash_cancels,
4605                    active_tool_calls,
4606                    pending_responses,
4607                    pending_binds,
4608                    retry_buffer,
4609                    push_buffer,
4610                    shutdown,
4611                    tool_response_body_limit,
4612                    lifecycle_probe,
4613                )
4614                .await?;
4615                bind_root_id = Some(replacement_root);
4616            }
4617            if pending_binds.contains_key(&route_id) {
4618                return send_route_bind_error(
4619                    tx,
4620                    frame,
4621                    "config_divergence",
4622                    "route bind is already pending for channel",
4623                    metrics,
4624                )
4625                .await;
4626            }
4627            let bind_root_id = match bind_root_id {
4628                Some(root_id) => root_id,
4629                None => match ProjectRootId::from_path(&identity.project_root) {
4630                    Ok(root_id) => root_id,
4631                    Err(error) => {
4632                        return send_route_bind_error(
4633                            tx,
4634                            frame,
4635                            "config_divergence",
4636                            &format!("invalid route project root: {error}"),
4637                            metrics,
4638                        )
4639                        .await;
4640                    }
4641                },
4642            };
4643
4644            // Reconcile RootConfig: build a configure request from the bind
4645            // identity + forwarded config tiers and run it through the executor.
4646            let request_id = format!("subc-bind-{route_channel}");
4647            let bind_project_root = identity.project_root.clone();
4648            let bind_harness = identity.harness.clone();
4649            let bind_session = identity.session.clone();
4650            let bind_principal_id = principal_id(&principal);
4651            // Typed capability declaration from the consumer: the facade stamps it
4652            // from the MCP host's initialize-advertised capabilities. Absent
4653            // means no reverse-request capability — flat deny, fail-closed. A
4654            // consumer over-declaring only earns asks that TTL-deny.
4655            let consumer_elicitation_capable = consumer_capabilities
4656                .as_ref()
4657                .is_some_and(|capabilities| capabilities.iter().any(|c| c == "elicitation"));
4658            log::info!(
4659                "subc attach: route {} harness={} principal={} trust={} elicitation={}",
4660                route_channel,
4661                bind_harness,
4662                principal_label(&principal),
4663                bind_trust.label(),
4664                consumer_elicitation_capable
4665            );
4666
4667            // Config is read directly from the CortexKit user and project files;
4668            // wire-relayed tiers are ignored so a front cannot inject settings.
4669            // The resolver selects only this bind's harness override before it
4670            // applies the unchanged user/project trust boundary.
4671            let local_tiers = crate::subc_config::read_local_cortexkit_config_tiers(
4672                user_config_path,
4673                Path::new(&bind_project_root),
4674            );
4675            let config_tiers: Vec<Value> = local_tiers
4676                .iter()
4677                .map(|t| json!({ "tier": t.tier, "source": t.source, "doc": t.doc }))
4678                .collect();
4679            // Let configure return its structured invalid-harness error rather
4680            // than panicking while computing this optional registration setting.
4681            let active_harness = bind_harness.parse::<crate::harness::Harness>().ok();
4682            let diagnostics_on_edit = crate::config_resolve::resolve_config_for_harness(
4683                &local_tiers,
4684                active_harness.as_ref(),
4685            )
4686            .config
4687            .diagnostics_on_edit;
4688            let configure_json = json!({
4689                "id": request_id,
4690                "command": "configure",
4691                "project_root": bind_project_root,
4692                "harness": bind_harness,
4693                "session_id": bind_session.clone(),
4694                "config": config_tiers,
4695            });
4696            let configure_req = match serde_json::from_value::<RawRequest>(configure_json) {
4697                Ok(req) => req,
4698                Err(error) => {
4699                    return send_route_bind_error(
4700                        tx,
4701                        frame,
4702                        "config_divergence",
4703                        &format!("failed to build configure request: {error}"),
4704                        metrics,
4705                    )
4706                    .await;
4707                }
4708            };
4709
4710            let route_identity = RouteIdentity(Arc::new(RouteIdentityData {
4711                root: bind_root_id.clone(),
4712                project_root: PathBuf::from(&bind_project_root),
4713                harness: bind_harness.clone(),
4714                session: bind_session.clone(),
4715                trust: bind_trust,
4716                spawn_principal: AuthenticatedPrincipal::RouteBind {
4717                    trust: bind_trust.sandbox_trust(),
4718                    route_channel,
4719                    route_epoch: epoch,
4720                    project_root: PathBuf::from(&bind_project_root),
4721                    harness: bind_harness.clone(),
4722                    session_id: bind_session.clone(),
4723                    principal_id: bind_principal_id,
4724                },
4725                consumer_elicitation_capable,
4726            }));
4727            let configure_session = route_identity.session.clone();
4728            let root_was_live = live_roots.contains_key(&bind_root_id);
4729            let inserted_new_actor = register_actor_for_bind(
4730                shared_app,
4731                executor,
4732                push_senders,
4733                &bind_root_id,
4734                route_channel,
4735                root_was_live,
4736            );
4737
4738            sync_bg_live_delivery_sessions(executor, routes, Some(&bind_root_id));
4739            let configure_request_id = configure_req.id.clone();
4740            installed_route_epochs.insert(route_channel, epoch);
4741            if let Some(meta) = live_roots.get_mut(&bind_root_id) {
4742                meta.maintenance_queued_kinds.clear();
4743                meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
4744            }
4745            let (configure_rx, configure_cancellation) = executor.submit_cancellable_async(
4746                bind_root_id.clone(),
4747                Lane::Mutating,
4748                configure_request_id.clone(),
4749                Box::new(move |ctx| {
4750                    log_ctx::with_session(Some(configure_session.clone()), || {
4751                        dispatch(configure_req, ctx)
4752                    })
4753                }),
4754            );
4755            pending_binds.insert(
4756                route_id,
4757                PendingBind {
4758                    bind_root_id: bind_root_id.clone(),
4759                    inserted_new_actor,
4760                    cancelled: false,
4761                    configure_request_id: configure_request_id.clone(),
4762                    started_at: Instant::now(),
4763                    warned_half_deadline: false,
4764                    deadline_reported: false,
4765                    corr: frame.header.corr,
4766                    ver: frame.header.ver,
4767                    flags: frame.header.flags,
4768                    cancellation: configure_cancellation,
4769                },
4770            );
4771
4772            let completion_tx = control_completion_tx.clone();
4773            let completion_identity = route_identity;
4774            let completion_root = bind_root_id.clone();
4775            let completion_route_channel = route_channel;
4776            let completion_ver = frame.header.ver;
4777            let completion_corr = frame.header.corr;
4778            let completion_flags = frame.header.flags;
4779            let completion_metrics = Arc::clone(metrics);
4780            tokio::spawn(async move {
4781                let _response_task = ResponseTaskGuard::new(&completion_metrics);
4782                let configure_response =
4783                    await_executor_response(configure_rx, configure_request_id.clone()).await;
4784                // Send the route-bind acknowledgment as soon as configure succeeds.
4785                // Installing completed search or callgraph builds only refreshes cached
4786                // read data, so a later maintenance pass can do it without delaying the
4787                // daemon's confirmation that the route is usable.
4788                let completion = RouteBindCompletion {
4789                    route: route_key(completion_route_channel, epoch),
4790                    identity: completion_identity,
4791                    bind_root_id: completion_root,
4792                    inserted_new_actor,
4793                    configure_response,
4794                    diagnostics_on_edit,
4795                    ver: completion_ver,
4796                    corr: completion_corr,
4797                    flags: completion_flags,
4798                };
4799                if send_counted_channel(
4800                    &completion_tx,
4801                    &completion_metrics.control_completion_queued,
4802                    completion,
4803                )
4804                .await
4805                .is_err()
4806                {
4807                    log::debug!(
4808                        "subc attach: dropped RouteBind completion for route {} after loop exit",
4809                        completion_route_channel
4810                    );
4811                }
4812            });
4813
4814            // Bind completion wakes the health worker. A synchronous census here
4815            // scans every hosted root before the transport can accept another frame.
4816            Ok(())
4817        }
4818        ModuleControlRequest::HealthCheck {} => {
4819            send_cached_health_response(
4820                tx,
4821                frame,
4822                shared_app,
4823                executor,
4824                pending_binds,
4825                metrics,
4826                health_rollup_cache,
4827            )
4828            .await
4829        }
4830    }
4831}
4832
4833async fn handle_management_request(
4834    tx: &WriterSender,
4835    frame: &Frame,
4836    shared_app: &App,
4837    executor: &Executor,
4838    live_roots: &HashMap<ProjectRootId, RootMeta>,
4839    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
4840    health_rollup_cache: &HealthRollupCache,
4841    metrics: &DispatchPathMetrics,
4842) -> Result<(), SubcError> {
4843    let decoded = serde_json::from_slice::<Value>(&frame.body).ok();
4844    let operation = decoded
4845        .as_ref()
4846        .and_then(|value| value.get("op"))
4847        .and_then(Value::as_str);
4848    let Some(operation) = operation else {
4849        let error = build_error_frame(
4850            frame.header.ver,
4851            frame.header.channel,
4852            frame.header.epoch,
4853            frame.header.corr,
4854            frame.header.flags,
4855            "unknown_management_op",
4856            "management routes accept only declared operation envelopes",
4857        )?;
4858        return send_reliable_writer_frame(tx, metrics, error, "management refusal").await;
4859    };
4860
4861    let result = match operation {
4862        crate::commands::memory_census::MEMORY_CENSUS_OPERATION => Response::success(
4863            "management-memory-census",
4864            memory_census_with_lifecycle(
4865                health_rollup_cache,
4866                shared_app,
4867                executor,
4868                live_roots,
4869                root_channels,
4870            ),
4871        ),
4872        crate::commands::health_digest::HEALTH_DIGEST_OPERATION => {
4873            let params = decoded
4874                .as_ref()
4875                .and_then(|value| value.get("params"))
4876                .cloned()
4877                .unwrap_or_else(|| json!({}));
4878            let Some(params) = params.as_object() else {
4879                return send_management_response(
4880                    tx,
4881                    frame,
4882                    operation,
4883                    Response::error(
4884                        "management-health-digest",
4885                        "invalid_request",
4886                        "health.digest params must be an object",
4887                    ),
4888                    metrics,
4889                )
4890                .await;
4891            };
4892            let root = params
4893                .get("project_root")
4894                .or_else(|| params.get("root"))
4895                .and_then(Value::as_str);
4896            let Some(root) = root else {
4897                return send_management_response(
4898                    tx,
4899                    frame,
4900                    operation,
4901                    Response::error(
4902                        "management-health-digest",
4903                        "invalid_request",
4904                        "health.digest requires params.project_root",
4905                    ),
4906                    metrics,
4907                )
4908                .await;
4909            };
4910
4911            let mut request = params.clone();
4912            request.insert("id".to_string(), json!("management-health-digest"));
4913            request.insert(
4914                "command".to_string(),
4915                json!(crate::commands::health_digest::HEALTH_DIGEST_OPERATION),
4916            );
4917            let request = serde_json::from_value::<RawRequest>(Value::Object(request))
4918                .map_err(SubcError::Json)?;
4919            match ProjectRootId::from_path(Path::new(root))
4920                .ok()
4921                .and_then(|root_id| executor.actor_context(&root_id))
4922            {
4923                Some(ctx) => crate::commands::health_digest::handle_health_digest(&request, &ctx),
4924                None => crate::commands::health_digest::root_not_bound_response(&request, root),
4925            }
4926        }
4927        _ => {
4928            let error = build_error_frame(
4929                frame.header.ver,
4930                frame.header.channel,
4931                frame.header.epoch,
4932                frame.header.corr,
4933                frame.header.flags,
4934                "unknown_management_op",
4935                &format!("management operation {operation:?} is not declared by AFT"),
4936            )?;
4937            return send_reliable_writer_frame(tx, metrics, error, "management refusal").await;
4938        }
4939    };
4940
4941    send_management_response(tx, frame, operation, result, metrics).await
4942}
4943
4944fn memory_census_with_lifecycle(
4945    health_rollup_cache: &HealthRollupCache,
4946    shared_app: &App,
4947    executor: &Executor,
4948    live_roots: &HashMap<ProjectRootId, RootMeta>,
4949    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
4950) -> Value {
4951    let mut census = health_rollup_cache.memory_census();
4952    if let Some(rows) = census.get_mut("roots").and_then(Value::as_object_mut) {
4953        let lifecycle = shared_app.lifecycle_census_snapshot();
4954        for (root_id, meta) in live_roots {
4955            let root = root_id.as_path().display().to_string();
4956            let bound_routes = root_channels.get(root_id).map_or(0, HashSet::len);
4957            let age_ms = Instant::now()
4958                .saturating_duration_since(meta.last_touched)
4959                .as_millis()
4960                .min(u128::from(u64::MAX)) as u64;
4961            let ttl_ms = root_idle_ttl(executor, root_id)
4962                .as_millis()
4963                .min(u128::from(u64::MAX)) as u64;
4964            let lsp = lifecycle
4965                .lsp
4966                .children_by_root
4967                .iter()
4968                .find(|child| child.root == root);
4969            if let Some(row) = rows.get_mut(&root).and_then(Value::as_object_mut) {
4970                let evictable_bytes = row.get("evictable_bytes").cloned().unwrap_or(json!(0));
4971                row.insert("root_id".to_string(), json!(root));
4972                row.insert("bound_routes".to_string(), json!(bound_routes));
4973                row.insert("last_request_age_ms".to_string(), json!(age_ms));
4974                row.insert("idle_ttl_ms".to_string(), json!(ttl_ms));
4975                row.insert(
4976                    "lsp_idle_ttl_ms".to_string(),
4977                    json!(executor
4978                        .actor_context(root_id)
4979                        .map(|ctx| ctx
4980                            .config()
4981                            .idle
4982                            .lsp_ttl()
4983                            .as_millis()
4984                            .min(u128::from(u64::MAX)) as u64)
4985                        .unwrap_or(0)),
4986                );
4987                row.insert(
4988                    "evictable_in_ms".to_string(),
4989                    crate::commands::memory_census::evictable_in_ms(bound_routes, ttl_ms, age_ms)
4990                        .map_or(Value::Null, |value| json!(value)),
4991                );
4992                row.insert(
4993                    "evictable_bytes".to_string(),
4994                    if bound_routes == 0 {
4995                        evictable_bytes
4996                    } else {
4997                        json!(0)
4998                    },
4999                );
5000                row.insert(
5001                    "lsp_children".to_string(),
5002                    json!({
5003                        "count": lsp.map_or(0, |child| child.count),
5004                        "rss_bytes": lsp.map_or(0, |child| child.rss_bytes),
5005                    }),
5006                );
5007            }
5008        }
5009    }
5010    census
5011}
5012
5013async fn send_management_response(
5014    tx: &WriterSender,
5015    request: &Frame,
5016    operation: &str,
5017    result: Response,
5018    metrics: &DispatchPathMetrics,
5019) -> Result<(), SubcError> {
5020    let status = if result.success { "ok" } else { "error" };
5021    let body = json!({ "op": operation, "status": status, "data": result.data });
5022    let response = Frame::build_with_version(
5023        request.header.ver,
5024        FrameType::Response,
5025        request.header.flags,
5026        request.header.channel,
5027        request.header.epoch,
5028        request.header.corr,
5029        serde_json::to_vec(&body).map_err(SubcError::Json)?,
5030    )
5031    .map_err(SubcError::FrameBuild)?;
5032    send_reliable_writer_frame(tx, metrics, response, "management response").await
5033}
5034
5035fn install_bash_compressor(ctx: &AppContext) {
5036    // Mirrors main.rs per-actor compressor installation for subc-created actors.
5037    let filter_registry_handle = ctx.shared_filter_registry();
5038    let compress_flag = ctx.bash_compress_flag();
5039    ctx.bash_background().set_compressor_with_exit_code(
5040        move |command: &str, output: String, exit_code: Option<i32>| {
5041            if !compress_flag.load(std::sync::atomic::Ordering::Relaxed) {
5042                return crate::compress::CompressionResult::new(output);
5043            }
5044            let registry_guard = match filter_registry_handle.read() {
5045                Ok(g) => g,
5046                Err(poisoned) => poisoned.into_inner(),
5047            };
5048            crate::compress::compress_with_registry_exit_code(
5049                command,
5050                &output,
5051                exit_code,
5052                &registry_guard,
5053            )
5054        },
5055    );
5056}
5057
5058async fn send_route_bind_ack(
5059    tx: &WriterSender,
5060    ver: u8,
5061    corr: u64,
5062    flags: Flags,
5063    metrics: &DispatchPathMetrics,
5064) -> Result<(), SubcError> {
5065    let body =
5066        serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).map_err(SubcError::Json)?;
5067    let response = Frame::build_with_version(ver, FrameType::Response, flags, 0, 0, corr, body)
5068        .map_err(SubcError::FrameBuild)?;
5069    send_reliable_writer_frame(tx, metrics, response, "RouteBindAck").await
5070}
5071
5072async fn send_route_bind_error(
5073    tx: &WriterSender,
5074    frame: &Frame,
5075    code: &str,
5076    message: &str,
5077    metrics: &DispatchPathMetrics,
5078) -> Result<(), SubcError> {
5079    send_route_bind_error_parts(
5080        tx,
5081        frame.header.ver,
5082        frame.header.corr,
5083        frame.header.flags,
5084        code,
5085        message,
5086        metrics,
5087    )
5088    .await
5089}
5090
5091async fn send_route_bind_error_parts(
5092    tx: &WriterSender,
5093    ver: u8,
5094    corr: u64,
5095    flags: Flags,
5096    code: &str,
5097    message: &str,
5098    metrics: &DispatchPathMetrics,
5099) -> Result<(), SubcError> {
5100    let response = build_error_frame(ver, 0, 0, corr, flags, code, message)?;
5101    send_reliable_writer_frame(tx, metrics, response, "RouteBind error").await?;
5102    log_route_bind_rejection(code, message);
5103    Ok(())
5104}
5105
5106/// Per-message rate limit for the bind-rejection warn line. A caller that
5107/// re-attaches a dead root forever turns this line into the entire readable
5108/// tail of the SHARED daemon log (measured 2026-08-09: 2.08M copies, ~40/sec,
5109/// 936MB log — other modules' incident lines pushed out of the tail).
5110/// The line itself stays byte-identical so external counters keep matching;
5111/// repeats inside the window are summarized with a suppressed count on the
5112/// next emission (volume stays diagnosable, per the log-diet convention).
5113fn log_route_bind_rejection(code: &str, message: &str) {
5114    const WINDOW: Duration = Duration::from_secs(60);
5115    static SUPPRESSED: OnceLock<StdMutex<HashMap<String, (Instant, u64)>>> = OnceLock::new();
5116    let map = SUPPRESSED.get_or_init(|| StdMutex::new(HashMap::new()));
5117    let mut map = match map.try_lock() {
5118        Ok(map) => map,
5119        // Contended: log unsuppressed rather than blocking or dropping.
5120        Err(_) => {
5121            log::warn!("subc attach: route bind rejected ({code}): {message}");
5122            return;
5123        }
5124    };
5125    let now = Instant::now();
5126    // Bound the map: dead roots churn, and an unbounded suppression map is
5127    // its own leak. Sweep expired entries once it grows past a fleet-sized
5128    // number of distinct rejection messages.
5129    if map.len() > 512 {
5130        map.retain(|_, (start, _)| now.duration_since(*start) < WINDOW);
5131    }
5132    match map.get_mut(message) {
5133        Some((window_start, suppressed)) if now.duration_since(*window_start) < WINDOW => {
5134            *suppressed += 1;
5135        }
5136        Some((window_start, suppressed)) => {
5137            if *suppressed > 0 {
5138                log::warn!(
5139                    "subc attach: route bind rejected ({code}): {message} (repeated {}x in last 60s)",
5140                    *suppressed
5141                );
5142            } else {
5143                log::warn!("subc attach: route bind rejected ({code}): {message}");
5144            }
5145            *window_start = now;
5146            *suppressed = 0;
5147        }
5148        None => {
5149            log::warn!("subc attach: route bind rejected ({code}): {message}");
5150            map.insert(message.to_string(), (now, 0));
5151        }
5152    }
5153}
5154
5155/// Route-channel tool call: `{name, arguments}` → executor lane → dispatch to
5156/// the sync command core → wrap the structured Response in a CallToolResult
5157/// `{content, isError}`. Tool-result mapping: the whole `{success, ...}` Response
5158/// serialized into ONE text block; `isError` carries `success == false`.
5159async fn handle_tool_call(
5160    tx: &WriterSender,
5161    frame: &Frame,
5162    mut phase_trace: PhaseTrace,
5163    routes: &HashMap<RouteChannel, RouteIdentity>,
5164    pending_binds: &HashMap<RouteChannel, PendingBind>,
5165    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
5166    executor: &Arc<Executor>,
5167    active_tool_calls: &ActiveToolCalls,
5168    pending_deferred_setups: &Arc<AtomicUsize>,
5169    shutdown: &Arc<Notify>,
5170    connection_cancel: &PersistentCancelSignal,
5171    bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
5172    bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
5173    metrics: &Arc<DispatchPathMetrics>,
5174    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
5175    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
5176    next_bash_ask_corr: &mut u64,
5177    bg_subs: &mut HashMap<RouteChannel, BgSub>,
5178    bg_sub_by_session: &mut BgSubsBySession,
5179    bg_wake_pending: &mut BgWakePending,
5180    bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
5181    dispatch: DispatchFn,
5182    deferred_response_tx: &mpsc::UnboundedSender<PendingSubcResponse>,
5183    allow_native_passthrough: bool,
5184    tool_response_body_limit: usize,
5185) -> Result<(), SubcError> {
5186    let route_id = route_key(frame.header.channel, frame.header.epoch);
5187    if pending_binds.contains_key(&route_id) {
5188        let error = build_error_frame(
5189            frame.header.ver,
5190            frame.header.channel,
5191            frame.header.epoch,
5192            frame.header.corr,
5193            frame.header.flags,
5194            "route_not_bound",
5195            "route is not bound before tool call",
5196        )?;
5197        return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
5198    }
5199
5200    let Some(identity) = routes.get(&route_id).cloned() else {
5201        let error = build_error_frame(
5202            frame.header.ver,
5203            frame.header.channel,
5204            frame.header.epoch,
5205            frame.header.corr,
5206            frame.header.flags,
5207            "route_not_bound",
5208            "route is not bound before tool call",
5209        )?;
5210        return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
5211    };
5212    let restore_watcher = live_roots
5213        .get(&identity.root)
5214        .is_some_and(|meta| meta.idle_artifacts_evicted);
5215    if let Some(meta) = live_roots.get_mut(&identity.root) {
5216        meta.reactivate_bound();
5217    }
5218    if restore_watcher {
5219        if let Some(ctx) = executor.actor_context(&identity.root) {
5220            crate::commands::configure::ensure_project_watcher(&ctx);
5221        }
5222    }
5223
5224    let route_request = match serde_json::from_slice::<RouteRequest>(&frame.body) {
5225        Ok(request) => request,
5226        Err(error) => {
5227            let management_envelope = serde_json::from_slice::<Value>(&frame.body).ok();
5228            let Some(operation) = management_envelope
5229                .as_ref()
5230                .and_then(|value| value.get("op"))
5231                .and_then(Value::as_str)
5232            else {
5233                return Err(SubcError::Json(error));
5234            };
5235            RouteRequest::ToolCall(ToolCallRequest {
5236                name: operation.to_string(),
5237                arguments: management_envelope
5238                    .and_then(|value| value.get("params").cloned())
5239                    .unwrap_or_else(|| json!({})),
5240                edit_slot_survives: None,
5241                preview: false,
5242            })
5243        }
5244    };
5245    if matches!(
5246        route_request,
5247        RouteRequest::BgEvents(BgEventsRequest {
5248            op: BgEventsOp::BgEvents
5249        })
5250    ) {
5251        if let Some(old_sub) = bg_subs.get(&route_id).cloned() {
5252            metrics.record_bg_subscription_ended(
5253                &old_sub.root,
5254                &old_sub.session,
5255                route_id,
5256                "resubscribe",
5257            );
5258            push::send_reliable_bg_stream_end(tx, metrics, route_id, &old_sub).await?;
5259        }
5260        if !identity.trust.allows_bash_observation() {
5261            bg_subs.remove(&route_id);
5262            bg_wake_pending.remove(&route_id);
5263            remove_bg_subscription_index(bg_sub_by_session, route_id, Some(&identity));
5264            let denied_sub = BgSub {
5265                corr: frame.header.corr,
5266                ver: frame.header.ver,
5267                flags: frame.header.flags,
5268                root: identity.root.clone(),
5269                session: identity.session.clone(),
5270            };
5271            metrics.record_bg_subscription_ended(
5272                &identity.root,
5273                &identity.session,
5274                route_id,
5275                "subscribe-denied",
5276            );
5277            push::send_reliable_bg_stream_end(tx, metrics, route_id, &denied_sub).await?;
5278            return Ok(());
5279        }
5280        bg_subs.insert(
5281            route_id,
5282            BgSub {
5283                corr: frame.header.corr,
5284                ver: frame.header.ver,
5285                flags: frame.header.flags,
5286                root: identity.root.clone(),
5287                session: identity.session.clone(),
5288            },
5289        );
5290        insert_bg_subscription_index(
5291            bg_sub_by_session,
5292            identity.root.clone(),
5293            identity.session.clone(),
5294            route_id,
5295        );
5296        metrics.record_bg_subscription_installed(&identity.root, &identity.session, route_id);
5297        push::arm_bg_wake(
5298            identity.root.clone(),
5299            identity.session.clone(),
5300            route_id,
5301            bg_wake_pending,
5302            bg_wake_epoch,
5303            metrics,
5304        );
5305        return Ok(());
5306    }
5307
5308    let RouteRequest::ToolCall(call) = route_request else {
5309        unreachable!("background event subscription returned above")
5310    };
5311    let bare_name = call.name;
5312    let arguments = strip_agent_preview_arg_owned(call.arguments);
5313    let format_context = crate::subc_format::FormatContext::from_tool_call(
5314        &bare_name,
5315        &arguments,
5316        identity.project_root.as_path(),
5317    );
5318
5319    let request_id = format!("subc-{}-{}", frame.header.channel, frame.header.corr);
5320    let bind_trust = identity.trust;
5321    let diagnostics_on_edit = live_roots
5322        .get(&identity.root)
5323        .map(|meta| meta.diagnostics_on_edit)
5324        .unwrap_or(false);
5325
5326    let requests_host = matches!(bare_name.as_str(), "bash" | "powershell")
5327        && arguments
5328            .get("sandbox")
5329            .or_else(|| {
5330                arguments
5331                    .get("params")
5332                    .and_then(|params| params.get("sandbox"))
5333            })
5334            .and_then(Value::as_str)
5335            == Some("host");
5336    if matches!(bind_trust, BindTrust::Untrusted) && requests_host {
5337        let response = Response::error(
5338            request_id.clone(),
5339            "sandbox_escalation_denied",
5340            "sandbox host escalation is unavailable to untrusted principals",
5341        );
5342        let text = crate::subc_format::format_response_with_context(
5343            &bare_name,
5344            &response,
5345            &format_context,
5346        );
5347        let result = ToolCallResult { text, response };
5348        let response_frame = build_tool_response_frame_with_limit(
5349            frame.header.ver,
5350            route_id,
5351            frame.header.corr,
5352            frame.header.flags,
5353            &result,
5354            bind_trust,
5355            tool_response_body_limit,
5356        )?;
5357        return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
5358    }
5359
5360    if matches!(bind_trust, BindTrust::Untrusted)
5361        && is_bash_family_tool(&bare_name)
5362        && (!matches!(bare_name.as_str(), "bash" | "powershell")
5363            || !identity.consumer_elicitation_capable)
5364    {
5365        let response = bash::bash_denied_untrusted_response(request_id.clone());
5366        let text = crate::subc_format::format_response_with_context(
5367            &bare_name,
5368            &response,
5369            &format_context,
5370        );
5371        let result = ToolCallResult { text, response };
5372        let response_frame = build_tool_response_frame_with_limit(
5373            frame.header.ver,
5374            route_id,
5375            frame.header.corr,
5376            frame.header.flags,
5377            &result,
5378            bind_trust,
5379            tool_response_body_limit,
5380        )?;
5381        return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
5382    }
5383
5384    // A non-core name is NOT in the tool manifest. AFT fails closed and
5385    // does not trust subc to enforce the manifest: rejecting here is the
5386    // defense-in-depth backstop that prevents a forwarded native command
5387    // (e.g. `configure`, which would reach handle_configure and bypass
5388    // the RouteBind config-trust cap) from ever reaching dispatch. Only
5389    // the integration-test harness (run_subc_mode_for_test) opens this to
5390    // drive synthetic native commands through the executor.
5391    if !is_subc_agent_core_tool(&bare_name)
5392        && !is_subc_native_plumbing_tool(&bare_name)
5393        && !allow_native_passthrough
5394    {
5395        log::warn!(
5396            "subc tool call: rejecting non-manifest tool name {:?} on route {} (fail-closed)",
5397            bare_name,
5398            frame.header.channel
5399        );
5400        let response = Response::error(
5401            request_id.clone(),
5402            "unknown_tool",
5403            format!("tool {:?} is not in the AFT tool manifest", bare_name),
5404        );
5405        let text = crate::subc_format::format_response_with_context(
5406            &bare_name,
5407            &response,
5408            &format_context,
5409        );
5410        let result = ToolCallResult { text, response };
5411        let response_frame = build_tool_response_frame_with_limit(
5412            frame.header.ver,
5413            route_id,
5414            frame.header.corr,
5415            frame.header.flags,
5416            &result,
5417            bind_trust,
5418            tool_response_body_limit,
5419        )?;
5420        return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
5421    }
5422
5423    if matches!(bare_name.as_str(), "bash" | "powershell") {
5424        if matches!(bind_trust, BindTrust::Untrusted) {
5425            let plan = match bash::prepare_bash_elicitation_plan(
5426                &arguments,
5427                identity.project_root.as_path(),
5428            ) {
5429                Ok(plan) => plan,
5430                Err(error) => {
5431                    let response = Response::error(request_id.clone(), error.code, error.message);
5432                    let text = crate::subc_format::format_response_with_context(
5433                        &bare_name,
5434                        &response,
5435                        &format_context,
5436                    );
5437                    let result = ToolCallResult { text, response };
5438                    let response_frame = build_tool_response_frame_with_limit(
5439                        frame.header.ver,
5440                        route_id,
5441                        frame.header.corr,
5442                        frame.header.flags,
5443                        &result,
5444                        bind_trust,
5445                        tool_response_body_limit,
5446                    )?;
5447                    return send_reliable_writer_frame(
5448                        tx,
5449                        metrics,
5450                        response_frame,
5451                        "tool response",
5452                    )
5453                    .await;
5454                }
5455            };
5456
5457            let reverse_corr =
5458                allocate_reverse_corr(pending_bash_asks, route_id, next_bash_ask_corr);
5459            let ask_frame = build_bash_elicitation_request_frame(
5460                frame.header.ver,
5461                route_id,
5462                reverse_corr,
5463                frame.header.flags,
5464                &plan.command,
5465                &plan.asks,
5466            )?;
5467
5468            let meta = live_roots
5469                .entry(identity.root.clone())
5470                .or_insert_with(|| RootMeta::new(Instant::now()));
5471            meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
5472            meta.reactivate_bound();
5473
5474            let route_cancel =
5475                route_bash_cancels
5476                    .entry(route_id)
5477                    .or_insert_with(|| bash::RouteBashCancel {
5478                        token: PersistentCancelSignal::new(),
5479                        active_waits: 0,
5480                    });
5481            route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
5482            let cancel = bash::BashWaitCancel {
5483                connection: connection_cancel.clone(),
5484                route: route_cancel.token.clone(),
5485            };
5486            pending_bash_asks.insert(
5487                ReverseCorrKey {
5488                    route: route_id,
5489                    corr: reverse_corr,
5490                },
5491                PendingBashAsk {
5492                    route: route_id,
5493                    tool_corr: frame.header.corr,
5494                    tool_flags: frame.header.flags,
5495                    tool_ver: frame.header.ver,
5496                    root: identity.root.clone(),
5497                    project_root: identity.project_root.clone(),
5498                    session_id: identity.session.clone(),
5499                    spawn_principal: identity.spawn_principal.clone(),
5500                    edit_slot_survives: call.edit_slot_survives,
5501                    request_id,
5502                    arguments,
5503                    format_context,
5504                    cancel,
5505                    grants: plan.grants,
5506                    expires_at: Instant::now() + bash_elicitation_timeout(),
5507                },
5508            );
5509            return send_reliable_writer_frame(tx, metrics, ask_frame, "bash elicitation request")
5510                .await;
5511        }
5512
5513        let meta = live_roots
5514            .entry(identity.root.clone())
5515            .or_insert_with(|| RootMeta::new(Instant::now()));
5516        meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
5517        meta.reactivate_bound();
5518
5519        let route_cancel =
5520            route_bash_cancels
5521                .entry(route_id)
5522                .or_insert_with(|| bash::RouteBashCancel {
5523                    token: PersistentCancelSignal::new(),
5524                    active_waits: 0,
5525                });
5526        route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
5527        let cancel = bash::BashWaitCancel {
5528            connection: connection_cancel.clone(),
5529            route: route_cancel.token.clone(),
5530        };
5531
5532        bash::submit_deferred_bash(
5533            executor,
5534            bash_deferred_tx,
5535            bash_poll_touch_tx,
5536            metrics,
5537            dispatch,
5538            identity.root.clone(),
5539            identity.project_root.clone(),
5540            identity.session.clone(),
5541            request_id,
5542            route_id,
5543            frame.header.corr,
5544            frame.header.flags,
5545            frame.header.ver,
5546            arguments,
5547            format_context,
5548            cancel,
5549            bind_trust,
5550            identity.spawn_principal.clone(),
5551            call.edit_slot_survives,
5552            None,
5553        );
5554        return Ok(());
5555    }
5556
5557    let lane = command_lane(&bare_name);
5558    let tool_call_context = ToolCallContext {
5559        project_root: identity.project_root.clone(),
5560        session_id: Some(identity.session.clone()),
5561        request_id: request_id.clone(),
5562        diagnostics_on_edit,
5563        preview: call.preview,
5564        edit_slot_survives: call.edit_slot_survives,
5565        report_registration_downgrade: true,
5566    };
5567
5568    let uses_deferred_response_seam = bare_name == "inspect"
5569        || crate::commands::lsp_navigation::is_lsp_navigation_command(&bare_name);
5570    if uses_deferred_response_seam {
5571        let Some(deferred_ctx) = executor.actor_context(&identity.root) else {
5572            let response = Response::error(
5573                &request_id,
5574                "actor_not_registered",
5575                "executor actor is not registered",
5576            );
5577            let text = crate::subc_format::format_response_with_context(
5578                &bare_name,
5579                &response,
5580                &format_context,
5581            );
5582            let result = ToolCallResult { text, response };
5583            let response_frame = build_tool_response_frame_with_limit(
5584                frame.header.ver,
5585                route_id,
5586                frame.header.corr,
5587                frame.header.flags,
5588                &result,
5589                bind_trust,
5590                tool_response_body_limit,
5591            )?;
5592            return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
5593        };
5594        let identity_for_run = identity.clone();
5595        let request_id_for_force = request_id.clone();
5596        let format_context_for_run = format_context.clone();
5597        let bare_name_for_run = bare_name.clone();
5598        let (setup_tx, setup_rx) = oneshot::channel::<DeferredSetupOutcome>();
5599        phase_trace.mark_executor_submitted();
5600        let job: crate::executor::ExecutorJob = Box::new(move |ctx| {
5601            phase_trace.mark_job_admitted();
5602            log_ctx::with_session(Some(identity_for_run.session.clone()), || {
5603                let run = || match prepare_tool_call(
5604                    &bare_name_for_run,
5605                    arguments,
5606                    &format_context_for_run,
5607                    &tool_call_context,
5608                    ctx,
5609                    Some(&mut phase_trace),
5610                ) {
5611                    Err(result) => {
5612                        let response = result.response;
5613                        let _ = setup_tx.send(DeferredSetupOutcome::Immediate {
5614                            text: result.text,
5615                            phase_trace,
5616                        });
5617                        response
5618                    }
5619                    Ok(prepared) => {
5620                        let outcome = if bare_name_for_run == "inspect" {
5621                            crate::commands::inspect::handle_inspect_deferred_with_restriction(
5622                                &prepared.request,
5623                                Arc::clone(&deferred_ctx),
5624                                matches!(bind_trust, BindTrust::Untrusted),
5625                            )
5626                        } else {
5627                            crate::commands::lsp_navigation::handle_lsp_navigation_deferred_with_restriction(
5628                                &prepared.request,
5629                                Arc::clone(&deferred_ctx),
5630                                matches!(bind_trust, BindTrust::Untrusted),
5631                            )
5632                        };
5633                        match outcome {
5634                            DispatchOutcome::Deferred(pending) => {
5635                                let _ = setup_tx.send(DeferredSetupOutcome::Deferred {
5636                                    pending,
5637                                    surface_downgraded: prepared.surface_downgraded,
5638                                    phase_trace,
5639                                });
5640                                Response::success(
5641                                    request_id_for_force.clone(),
5642                                    json!({ "response_deferred": true }),
5643                                )
5644                            }
5645                            DispatchOutcome::Immediate(response) => {
5646                                phase_trace.mark_execute_done();
5647                                let finalizer = |response: &mut Response| {
5648                                    crate::response_finalize::finalize_response_with_bg_completions(
5649                                        response,
5650                                        ctx,
5651                                        &identity_for_run.session,
5652                                        &bare_name_for_run,
5653                                        bind_trust.allows_bash_observation(),
5654                                    );
5655                                };
5656                                let result = finish_tool_call_response(
5657                                    &bare_name_for_run,
5658                                    &format_context_for_run,
5659                                    response,
5660                                    prepared.surface_downgraded,
5661                                    Some(&finalizer),
5662                                    Some(&mut phase_trace),
5663                                );
5664                                let response = result.response;
5665                                let _ = setup_tx.send(DeferredSetupOutcome::Immediate {
5666                                    text: result.text,
5667                                    phase_trace,
5668                                });
5669                                response
5670                            }
5671                        }
5672                    }
5673                };
5674                if matches!(bind_trust, BindTrust::Untrusted) {
5675                    ctx.with_force_restrict(&request_id_for_force, run)
5676                } else {
5677                    run()
5678                }
5679            })
5680        });
5681        let deferred_setup_guard =
5682            PendingDeferredSetupGuard::new(Arc::clone(pending_deferred_setups));
5683        let rx = submit_active_tool_call(
5684            executor.as_ref(),
5685            active_tool_calls,
5686            route_id,
5687            frame.header.corr,
5688            identity.root.clone(),
5689            lane,
5690            request_id.clone(),
5691            RouteDetachPolicy::CancelOnDetach,
5692            job,
5693        );
5694
5695        let completion_tx = tx.clone();
5696        let completion_shutdown = Arc::clone(shutdown);
5697        let completion_metrics = Arc::clone(metrics);
5698        let active_tool_calls = Arc::clone(active_tool_calls);
5699        let deferred_response_tx = deferred_response_tx.clone();
5700        let route = route_id;
5701        let corr = frame.header.corr;
5702        let flags = frame.header.flags;
5703        let ver = frame.header.ver;
5704        let root = identity.root.clone();
5705        let session_id = identity.session.clone();
5706        tokio::spawn(async move {
5707            let _response_task = ResponseTaskGuard::new(&completion_metrics);
5708            let _deferred_setup = deferred_setup_guard;
5709            let response = await_executor_response(rx, request_id.clone()).await;
5710            match setup_rx.await {
5711                Ok(DeferredSetupOutcome::Deferred {
5712                    pending,
5713                    surface_downgraded,
5714                    phase_trace,
5715                }) => {
5716                    let pending = PendingSubcResponse {
5717                        route,
5718                        corr,
5719                        flags,
5720                        ver,
5721                        root,
5722                        session_id,
5723                        bare_name,
5724                        format_context,
5725                        bind_trust,
5726                        pending,
5727                        surface_downgraded,
5728                        phase_trace,
5729                    };
5730                    if let Err(error) = deferred_response_tx.send(pending) {
5731                        if let Some(cancellation) = &error.0.pending.cancellation {
5732                            cancellation.request_cancel();
5733                        }
5734                        finish_active_tool_call(&active_tool_calls, route, corr);
5735                    }
5736                }
5737                Ok(DeferredSetupOutcome::Immediate { text, phase_trace }) => {
5738                    if !finish_active_tool_call(&active_tool_calls, route, corr) {
5739                        return;
5740                    }
5741                    let result = ToolCallResult { text, response };
5742                    let fatal = note_fatal_panic_response(&result.response);
5743                    match build_tool_response_frame_with_limit(
5744                        ver,
5745                        route,
5746                        corr,
5747                        flags,
5748                        &result,
5749                        bind_trust,
5750                        tool_response_body_limit,
5751                    ) {
5752                        Ok(response_frame) => {
5753                            let trace = ToolResponseWriteTrace::new(
5754                                phase_trace,
5755                                bare_name.clone(),
5756                                identity.project_root.clone(),
5757                                identity.session.clone(),
5758                                route.channel,
5759                                corr,
5760                            );
5761                            if let Err(error) = send_traced_tool_response_frame(
5762                                &completion_tx,
5763                                &completion_metrics,
5764                                response_frame,
5765                                trace,
5766                            )
5767                            .await
5768                            {
5769                                log::warn!(
5770                                    "subc attach: failed to queue deferred-seam setup response: {error}"
5771                                );
5772                            }
5773                        }
5774                        Err(error) => {
5775                            log::error!(
5776                                "subc attach: failed to build deferred-seam setup response: {error}"
5777                            );
5778                        }
5779                    }
5780                    if fatal {
5781                        signal_fatal_teardown(
5782                            &completion_tx,
5783                            Some(route),
5784                            ver,
5785                            corr,
5786                            &completion_shutdown,
5787                            &completion_metrics,
5788                        )
5789                        .await;
5790                    }
5791                }
5792                Err(_) => {
5793                    if !finish_active_tool_call(&active_tool_calls, route, corr) {
5794                        return;
5795                    }
5796                    let text = crate::subc_format::format_response_with_context(
5797                        &bare_name,
5798                        &response,
5799                        &format_context,
5800                    );
5801                    let result = ToolCallResult { text, response };
5802                    if let Ok(response_frame) = build_tool_response_frame_with_limit(
5803                        ver,
5804                        route,
5805                        corr,
5806                        flags,
5807                        &result,
5808                        bind_trust,
5809                        tool_response_body_limit,
5810                    ) {
5811                        let _ = send_reliable_writer_frame(
5812                            &completion_tx,
5813                            &completion_metrics,
5814                            response_frame,
5815                            "deferred setup failure",
5816                        )
5817                        .await;
5818                    }
5819                }
5820            }
5821        });
5822        return Ok(());
5823    }
5824
5825    let bare_name_for_frame = bare_name.clone();
5826    let identity_for_run = identity.clone();
5827    let completion_session = identity.session.clone();
5828    let completion_root = identity.project_root.clone();
5829    let request_id_for_force = request_id.clone();
5830    let format_context_for_frame = format_context.clone();
5831    let (tool_call_tx, tool_call_rx) = oneshot::channel::<ToolCallCompletion>();
5832    phase_trace.mark_executor_submitted();
5833    let job: crate::executor::ExecutorJob = Box::new(move |ctx| {
5834        phase_trace.mark_job_admitted();
5835        log_ctx::with_session(Some(identity_for_run.session.clone()), || {
5836            let run = || {
5837                let finalizer = |response: &mut Response| {
5838                    crate::response_finalize::finalize_response_with_bg_completions(
5839                        response,
5840                        ctx,
5841                        &identity_for_run.session,
5842                        &bare_name,
5843                        bind_trust.allows_bash_observation(),
5844                    );
5845                };
5846                match run_tool_call(
5847                    &bare_name,
5848                    arguments,
5849                    &format_context,
5850                    &tool_call_context,
5851                    ctx,
5852                    &dispatch,
5853                    Some(&finalizer),
5854                    Some(&mut phase_trace),
5855                ) {
5856                    ToolCallOutcome::Unary(result) => {
5857                        let response = result.response;
5858                        let _ = tool_call_tx.send(ToolCallCompletion {
5859                            text: result.text,
5860                            phase_trace,
5861                        });
5862                        response
5863                    }
5864                }
5865            };
5866            if matches!(bind_trust, BindTrust::Untrusted) {
5867                ctx.with_force_restrict(&request_id_for_force, run)
5868            } else {
5869                run()
5870            }
5871        })
5872    });
5873    let rx = submit_active_tool_call(
5874        executor.as_ref(),
5875        active_tool_calls,
5876        route_id,
5877        frame.header.corr,
5878        identity.root.clone(),
5879        lane,
5880        request_id.clone(),
5881        RouteDetachPolicy::RetainForReplay,
5882        job,
5883    );
5884    let completion_tx = tx.clone();
5885    let completion_shutdown = Arc::clone(shutdown);
5886    let route = route_id;
5887    let corr = frame.header.corr;
5888    let flags = frame.header.flags;
5889    let ver = frame.header.ver;
5890    let completion_metrics = Arc::clone(metrics);
5891    let active_tool_calls = Arc::clone(active_tool_calls);
5892    tokio::spawn(async move {
5893        let _response_task = ResponseTaskGuard::new(&completion_metrics);
5894        let response = await_executor_response(rx, request_id.clone()).await;
5895        let (text, phase_trace) = match tool_call_rx.await {
5896            Ok(completion) => (completion.text, Some(completion.phase_trace)),
5897            Err(_) => (
5898                crate::subc_format::format_response_with_context(
5899                    &bare_name_for_frame,
5900                    &response,
5901                    &format_context_for_frame,
5902                ),
5903                None,
5904            ),
5905        };
5906        if !finish_active_tool_call(&active_tool_calls, route, corr) {
5907            return;
5908        }
5909        let result = ToolCallResult { text, response };
5910        let fatal = note_fatal_panic_response(&result.response);
5911        match build_tool_response_frame_with_limit(
5912            ver,
5913            route,
5914            corr,
5915            flags,
5916            &result,
5917            bind_trust,
5918            tool_response_body_limit,
5919        ) {
5920            Ok(response_frame) => {
5921                let send_result = if let Some(phase_trace) = phase_trace {
5922                    let trace = ToolResponseWriteTrace::new(
5923                        phase_trace,
5924                        bare_name_for_frame,
5925                        completion_root,
5926                        completion_session,
5927                        route.channel,
5928                        corr,
5929                    );
5930                    send_traced_tool_response_frame(
5931                        &completion_tx,
5932                        &completion_metrics,
5933                        response_frame,
5934                        trace,
5935                    )
5936                    .await
5937                } else {
5938                    send_reliable_writer_frame(
5939                        &completion_tx,
5940                        &completion_metrics,
5941                        response_frame,
5942                        "tool response",
5943                    )
5944                    .await
5945                };
5946                if let Err(error) = send_result {
5947                    log::warn!("subc attach: failed to queue tool response frame: {error}");
5948                }
5949            }
5950            Err(error) => {
5951                log::error!("subc attach: failed to build tool response frame: {error}");
5952            }
5953        }
5954        if fatal {
5955            signal_fatal_teardown(
5956                &completion_tx,
5957                Some(route),
5958                ver,
5959                corr,
5960                &completion_shutdown,
5961                &completion_metrics,
5962            )
5963            .await;
5964        }
5965    });
5966    Ok(())
5967}
5968
5969fn submit_maintenance_job(
5970    executor: &Arc<Executor>,
5971    root_id: ProjectRootId,
5972    kind: MaintenanceDrainKind,
5973    bg_sessions_to_check: Vec<(String, u64)>,
5974    completion_tx: &mpsc::Sender<MaintenanceCompletion>,
5975    metrics: &Arc<DispatchPathMetrics>,
5976) {
5977    let request_id = format!(
5978        "subc-maintenance-drain-{}-{}",
5979        kind.label(),
5980        root_id.as_path().to_string_lossy()
5981    );
5982    let response_id = request_id.clone();
5983    let completion_root_id = root_id.clone();
5984    let maintenance_generation = executor
5985        .actor_context(&root_id)
5986        .map(|ctx| ctx.configure_generation())
5987        .unwrap_or(0);
5988    let (outcome_tx, outcome_rx) = oneshot::channel::<MaintenanceJobOutcome>();
5989    // Deferred drains mutate subsystem state behind each subsystem's own lock.
5990    // Keeping every drain on MaintenanceCommit lets interactive reads and lazy
5991    // HeavyInit queries run while maintenance converges after the bind ack.
5992    let lane = Lane::MaintenanceCommit;
5993    let job: crate::executor::ExecutorJob = Box::new(move |ctx: &AppContext| {
5994        let outcome = match kind {
5995            MaintenanceDrainKind::Watcher => {
5996                let drained = runtime_drain::drain_watcher_events_bounded(
5997                    ctx,
5998                    runtime_drain::WATCHER_PATH_DRAIN_BATCH_CAP,
5999                );
6000                MaintenanceJobOutcome {
6001                    empty_bg_sessions: Vec::new(),
6002                    unacked_bg_keys: None,
6003                    requeue_kind: drained.has_more.then_some(kind),
6004                }
6005            }
6006            MaintenanceDrainKind::Lsp => {
6007                let drained = runtime_drain::drain_lsp_events_bounded(
6008                    ctx,
6009                    runtime_drain::LSP_EVENT_DRAIN_BATCH_CAP,
6010                );
6011                MaintenanceJobOutcome {
6012                    empty_bg_sessions: Vec::new(),
6013                    unacked_bg_keys: None,
6014                    requeue_kind: drained.has_more.then_some(kind),
6015                }
6016            }
6017            MaintenanceDrainKind::ConfigureTail => {
6018                runtime_drain::drain_deferred_configure_maintenance(ctx);
6019                runtime_drain::drain_configure_warning_events(ctx);
6020                MaintenanceJobOutcome::default()
6021            }
6022            MaintenanceDrainKind::CompletionDrains => {
6023                runtime_drain::drain_search_index_events(ctx);
6024                runtime_drain::drain_callgraph_store_events(ctx);
6025                runtime_drain::drain_semantic_index_events(ctx);
6026                runtime_drain::drain_semantic_refresh_events(ctx);
6027                runtime_drain::drain_inspect_events_for_generation(ctx, maintenance_generation);
6028                let empty_bg_sessions = bg_sessions_to_check
6029                    .into_iter()
6030                    .filter(|(session, _)| {
6031                        !ctx.bash_background().has_unacked_wakes_for_session(session)
6032                    })
6033                    .collect();
6034                MaintenanceJobOutcome {
6035                    empty_bg_sessions,
6036                    unacked_bg_keys: Some(ctx.bash_background().unacked_wake_keys()),
6037                    requeue_kind: None,
6038                }
6039            }
6040        };
6041        let requeued = outcome.requeue_kind.is_some();
6042        let _ = outcome_tx.send(outcome);
6043        Response::success(
6044            response_id,
6045            json!({ "drained": true, "kind": kind.label(), "requeued": requeued }),
6046        )
6047    });
6048    let rx = match kind {
6049        MaintenanceDrainKind::Watcher => executor.submit_coalescable_maintenance_async(
6050            root_id,
6051            lane,
6052            request_id.clone(),
6053            crate::executor::MaintenanceCoalesceKey::WatcherDrain,
6054            job,
6055        ),
6056        MaintenanceDrainKind::Lsp => executor.submit_coalescable_maintenance_async(
6057            root_id,
6058            lane,
6059            request_id.clone(),
6060            crate::executor::MaintenanceCoalesceKey::LspDrain,
6061            job,
6062        ),
6063        MaintenanceDrainKind::ConfigureTail | MaintenanceDrainKind::CompletionDrains => {
6064            executor.submit_maintenance_async(root_id, lane, request_id.clone(), job)
6065        }
6066    };
6067    let completion_tx = completion_tx.clone();
6068    let completion_metrics = Arc::clone(metrics);
6069    tokio::spawn(async move {
6070        let _response_task = ResponseTaskGuard::new(&completion_metrics);
6071        let response = await_executor_response(rx, request_id).await;
6072        let outcome = outcome_rx.await.unwrap_or_default();
6073        let _ = send_counted_channel(
6074            &completion_tx,
6075            &completion_metrics.maintenance_queued,
6076            MaintenanceCompletion {
6077                root_id: completion_root_id,
6078                kind,
6079                response,
6080                empty_bg_sessions: outcome.empty_bg_sessions,
6081                unacked_bg_keys: outcome.unacked_bg_keys,
6082                requeue_kind: outcome.requeue_kind,
6083            },
6084        )
6085        .await;
6086    });
6087}
6088
6089async fn await_executor_response(rx: oneshot::Receiver<Response>, request_id: String) -> Response {
6090    rx.await
6091        .unwrap_or_else(|_| Response::error(request_id, "internal_error", "executor dropped"))
6092}
6093
6094async fn deliver_resolved_subc_response(
6095    tx: &WriterSender,
6096    mut resolved: ResolvedSubcResponse,
6097    routes: &HashMap<RouteChannel, RouteIdentity>,
6098    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
6099    executor: &Executor,
6100    active_tool_calls: &ActiveToolCalls,
6101    shutdown: &Arc<Notify>,
6102    metrics: &DispatchPathMetrics,
6103    tool_response_body_limit: usize,
6104) -> Result<(), SubcError> {
6105    let entry = &mut resolved.entry;
6106    finish_active_tool_call(active_tool_calls, entry.route, entry.corr);
6107    if let Some(meta) = live_roots.get_mut(&entry.root) {
6108        meta.note_activity();
6109    }
6110
6111    let Some(identity) = routes.get(&entry.route) else {
6112        log::debug!(
6113            "subc attach: dropping deferred {} response {} for unbound route {}",
6114            entry.bare_name,
6115            entry.pending.request_id,
6116            entry.route
6117        );
6118        return Ok(());
6119    };
6120    let Some(ctx) = executor.actor_context(&entry.root) else {
6121        return Ok(());
6122    };
6123    entry.phase_trace.mark_execute_done();
6124    let finalizer = |response: &mut Response| {
6125        crate::response_finalize::finalize_response_with_bg_completions(
6126            response,
6127            &ctx,
6128            &entry.session_id,
6129            &entry.bare_name,
6130            entry.bind_trust.allows_bash_observation(),
6131        );
6132    };
6133    let result = finish_tool_call_response(
6134        &entry.bare_name,
6135        &entry.format_context,
6136        resolved.response,
6137        entry.surface_downgraded,
6138        Some(&finalizer),
6139        Some(&mut entry.phase_trace),
6140    );
6141    let fatal = note_fatal_panic_response(&result.response);
6142    let response_frame = build_tool_response_frame_with_limit(
6143        entry.ver,
6144        entry.route,
6145        entry.corr,
6146        entry.flags,
6147        &result,
6148        identity.trust,
6149        tool_response_body_limit,
6150    )?;
6151    let trace = ToolResponseWriteTrace::new(
6152        std::mem::replace(&mut entry.phase_trace, PhaseTrace::new(Instant::now())),
6153        entry.bare_name.clone(),
6154        identity.project_root.clone(),
6155        entry.session_id.clone(),
6156        entry.route.channel,
6157        entry.corr,
6158    );
6159    send_traced_tool_response_frame(tx, metrics, response_frame, trace).await?;
6160    if fatal {
6161        signal_fatal_teardown(
6162            tx,
6163            Some(entry.route),
6164            entry.ver,
6165            entry.corr,
6166            shutdown,
6167            metrics,
6168        )
6169        .await;
6170    }
6171    Ok(())
6172}
6173
6174async fn signal_fatal_teardown(
6175    tx: &WriterSender,
6176    route: Option<RouteChannel>,
6177    ver: u8,
6178    corr: u64,
6179    shutdown: &Arc<Notify>,
6180    metrics: &DispatchPathMetrics,
6181) {
6182    if let Some(route) = route {
6183        if let Ok(frame) = build_goodbye_frame(ver, route.channel, route.epoch, corr) {
6184            if let Err(error) = send_frame(tx, metrics, frame).await {
6185                log::warn!(
6186                    "subc attach: failed to queue fatal route Goodbye for route {route}: {error}"
6187                );
6188            }
6189        }
6190    }
6191    if let Ok(frame) = build_goodbye_frame(ver, 0, 0, 0) {
6192        if let Err(error) = send_frame(tx, metrics, frame).await {
6193            log::warn!("subc attach: failed to queue fatal channel-0 Goodbye: {error}");
6194        }
6195    }
6196    shutdown.notify_one();
6197}
6198#[derive(Debug, Deserialize)]
6199#[serde(untagged)]
6200enum RouteRequest {
6201    BgEvents(BgEventsRequest),
6202    ToolCall(ToolCallRequest),
6203}
6204
6205#[derive(Debug, Deserialize)]
6206struct BgEventsRequest {
6207    op: BgEventsOp,
6208}
6209
6210#[derive(Debug, Deserialize)]
6211#[serde(rename_all = "snake_case")]
6212enum BgEventsOp {
6213    BgEvents,
6214}
6215
6216#[derive(Debug, Deserialize)]
6217struct ToolCallRequest {
6218    name: String,
6219    #[serde(default)]
6220    arguments: Value,
6221    /// Host-computed registration fact; kept outside agent-controlled arguments.
6222    #[serde(default)]
6223    edit_slot_survives: Option<bool>,
6224    /// Server-owned preview control (B1c-0): the plugin's mutation flow is
6225    /// preview -> permission ask -> apply. Dropping this field made "preview"
6226    /// calls mutate disk before the permission prompt and the subsequent
6227    /// apply fail with not-found.
6228    #[serde(default)]
6229    preview: bool,
6230}
6231
6232#[cfg(test)]
6233pub(crate) mod test_support {
6234    use super::*;
6235    use crate::bash_background::BgTaskStatus;
6236    use crate::protocol::{
6237        BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, ConfigureWarningsFrame,
6238        ProgressFrame, StatusChangedFrame,
6239    };
6240    use serde_json::json;
6241
6242    pub(super) fn test_root(name: &str) -> (tempfile::TempDir, ProjectRootId) {
6243        let dir = tempfile::Builder::new()
6244            .prefix(name)
6245            .tempdir()
6246            .expect("temp root");
6247        let root = ProjectRootId::from_path(dir.path()).expect("project root id");
6248        (dir, root)
6249    }
6250
6251    pub(super) fn test_ctx() -> Arc<AppContext> {
6252        Arc::new(AppContext::new(
6253            Box::new(crate::parser::TreeSitterProvider::new()),
6254            crate::config::Config::default(),
6255        ))
6256    }
6257
6258    fn inspect_context(root: &Path) -> Arc<AppContext> {
6259        inspect_context_with_timeout(root, None)
6260    }
6261
6262    fn inspect_context_with_timeout(root: &Path, timeout_ms: Option<u64>) -> Arc<AppContext> {
6263        let mut config = crate::config::Config::default();
6264        config.project_root = Some(root.to_path_buf());
6265        if let Some(timeout_ms) = timeout_ms {
6266            config.inspect.diagnostics_timeout_ms = timeout_ms;
6267        }
6268        let ctx = Arc::new(AppContext::new(
6269            Box::new(crate::parser::TreeSitterProvider::new()),
6270            config,
6271        ));
6272        ctx.set_harness(crate::harness::Harness::Opencode);
6273        ctx
6274    }
6275
6276    fn inspect_request(id: &str) -> RawRequest {
6277        serde_json::from_value(json!({ "id": id, "command": "inspect" })).expect("inspect request")
6278    }
6279
6280    fn submit_deferred_inspect_setup(
6281        executor: &Arc<Executor>,
6282        root: &ProjectRootId,
6283        ctx: &Arc<AppContext>,
6284        request_id: &str,
6285    ) -> (PendingResponse, JobCancellation) {
6286        let (pending_tx, pending_rx) = std::sync::mpsc::sync_channel(1);
6287        let request = inspect_request(request_id);
6288        let inspect_ctx = Arc::clone(ctx);
6289        let (_rx, cancellation) = executor.submit_cancellable_async(
6290            root.clone(),
6291            Lane::SerialLspStatus,
6292            request_id.to_string(),
6293            Box::new(move |_| {
6294                let DispatchOutcome::Deferred(pending) =
6295                    crate::commands::inspect::handle_inspect_deferred_with_restriction(
6296                        &request,
6297                        inspect_ctx,
6298                        true,
6299                    )
6300                else {
6301                    panic!("inspect setup must defer")
6302                };
6303                pending_tx.send(pending).expect("send pending inspect");
6304                Response::success("inspect-setup", json!({}))
6305            }),
6306        );
6307        let pending = pending_rx
6308            .recv_timeout(Duration::from_secs(1))
6309            .expect("inspect setup leaves the executor lane");
6310        let deadline = Instant::now() + Duration::from_secs(1);
6311        while !executor.actor_is_idle(root) {
6312            assert!(
6313                Instant::now() < deadline,
6314                "inspect setup kept lane counters live"
6315            );
6316            std::thread::sleep(Duration::from_millis(5));
6317        }
6318        (pending, cancellation)
6319    }
6320
6321    fn wait_for_inspect_terminal(pending: &mut PendingResponse, ctx: &AppContext) -> Response {
6322        let deadline = Instant::now() + Duration::from_secs(60);
6323        loop {
6324            if let Some(response) = (pending.poll)(ctx) {
6325                return response;
6326            }
6327            assert!(Instant::now() < deadline, "inspect terminal timed out");
6328            std::thread::sleep(Duration::from_millis(5));
6329        }
6330    }
6331
6332    fn cold_navigation_context(root: &Path) -> (Arc<AppContext>, PathBuf) {
6333        let source_dir = root.join("src");
6334        std::fs::create_dir_all(&source_dir).expect("create source dir");
6335        std::fs::write(root.join("Cargo.toml"), "[package]\nname = \"fixture\"\n")
6336            .expect("write Cargo manifest");
6337        let source = source_dir.join("main.rs");
6338        std::fs::write(&source, "fn main() {}\n").expect("write source");
6339        let binary = root.join("cold-navigation-server");
6340        std::fs::write(&binary, b"fixture").expect("write server placeholder");
6341
6342        let ctx = inspect_context(root);
6343        ctx.lsp()
6344            .override_binary(crate::lsp::registry::ServerKind::Rust, binary);
6345        (ctx, source)
6346    }
6347
6348    fn navigation_request(id: &str, source: &Path) -> RawRequest {
6349        serde_json::from_value(json!({
6350            "id": id,
6351            "command": "lsp_hover",
6352            "file": source.display().to_string(),
6353            "line": 1,
6354            "character": 1,
6355        }))
6356        .expect("navigation request")
6357    }
6358
6359    fn submit_deferred_navigation_setup(
6360        executor: &Arc<Executor>,
6361        root: &ProjectRootId,
6362        ctx: &Arc<AppContext>,
6363        source: &Path,
6364        request_id: &str,
6365    ) -> (PendingResponse, JobCancellation) {
6366        let (pending_tx, pending_rx) = std::sync::mpsc::sync_channel(1);
6367        let request = navigation_request(request_id, source);
6368        let navigation_ctx = Arc::clone(ctx);
6369        let (_rx, cancellation) = executor.submit_cancellable_async(
6370            root.clone(),
6371            Lane::SerialLspStatus,
6372            request_id.to_string(),
6373            Box::new(move |_| {
6374                let DispatchOutcome::Deferred(pending) = crate::commands::lsp_navigation::
6375                    handle_lsp_navigation_deferred_with_restriction(
6376                        &request,
6377                        navigation_ctx,
6378                        true,
6379                    )
6380                else {
6381                    panic!("cold navigation setup must defer")
6382                };
6383                pending_tx.send(pending).expect("send pending navigation");
6384                Response::success("navigation-setup", json!({}))
6385            }),
6386        );
6387        let pending = pending_rx
6388            .recv_timeout(Duration::from_secs(1))
6389            .expect("navigation setup leaves the executor lane");
6390        let deadline = Instant::now() + Duration::from_secs(1);
6391        while !executor.actor_is_idle(root) {
6392            assert!(
6393                Instant::now() < deadline,
6394                "navigation setup kept lane counters live"
6395            );
6396            std::thread::sleep(Duration::from_millis(5));
6397        }
6398        (pending, cancellation)
6399    }
6400
6401    fn wait_for_navigation_terminal(pending: &mut PendingResponse, ctx: &AppContext) -> Response {
6402        let deadline = Instant::now() + Duration::from_secs(1);
6403        loop {
6404            if let Some(response) = (pending.poll)(ctx) {
6405                return response;
6406            }
6407            assert!(Instant::now() < deadline, "navigation terminal timed out");
6408            std::thread::sleep(Duration::from_millis(5));
6409        }
6410    }
6411
6412    #[test]
6413    fn deferred_inspect_releases_lane_for_bind_and_mutation() {
6414        let _serial = crate::commands::inspect::deferred_inspect_test_lock();
6415        let executor = Arc::new(Executor::new());
6416        let (dir, root) = test_root("deferred-inspect-storm");
6417        std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
6418        let ctx = inspect_context(dir.path());
6419        executor.register_actor(root.clone(), Arc::clone(&ctx));
6420        let (started_rx, release_tx) =
6421            crate::commands::inspect::install_deferred_inspect_stat_gate_for_test();
6422        let (mut pending, _cancellation) =
6423            submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-storm");
6424        started_rx
6425            .recv_timeout(Duration::from_secs(1))
6426            .expect("deferred inspect body starts");
6427
6428        for request_id in ["subc-bind-other-session", "subc-edit-other-session"] {
6429            let response = executor.submit(
6430                root.clone(),
6431                Lane::Mutating,
6432                request_id.to_string(),
6433                Box::new(move |_| Response::success(request_id, json!({ "admitted": true }))),
6434            );
6435            assert!(
6436                response
6437                    .recv_timeout(Duration::from_secs(1))
6438                    .expect("writer admits while inspect remains deferred")
6439                    .success
6440            );
6441        }
6442        assert_eq!(
6443            crate::commands::inspect::deferred_inspect_root_count_for_test(),
6444            1,
6445            "writer admissions must not finish the detached inspect"
6446        );
6447
6448        release_tx.send(()).expect("release inspect body");
6449        let terminal = wait_for_inspect_terminal(&mut pending, &ctx);
6450        assert!(
6451            terminal.data.get("inspect_terminal").is_some(),
6452            "inspect must still produce its terminal: {:?}",
6453            terminal.data
6454        );
6455    }
6456
6457    #[test]
6458    fn deferred_inspect_honors_the_shared_request_deadline() {
6459        let _serial = crate::commands::inspect::deferred_inspect_test_lock();
6460        let executor = Arc::new(Executor::new());
6461        let (dir, root) = test_root("deferred-inspect-deadline");
6462        std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
6463        let ctx = inspect_context_with_timeout(dir.path(), Some(10_000));
6464        executor.register_actor(root.clone(), Arc::clone(&ctx));
6465        let (started_rx, _release_tx) =
6466            crate::commands::inspect::install_deferred_inspect_body_gate_for_test();
6467        let started = Instant::now();
6468        let (mut pending, _cancellation) =
6469            submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-deadline");
6470        started_rx
6471            .recv_timeout(Duration::from_secs(1))
6472            .expect("deferred inspect body starts");
6473
6474        let terminal = wait_for_inspect_terminal(&mut pending, &ctx);
6475        assert_eq!(terminal.data["inspect_terminal"], "phase_failed");
6476        assert_eq!(terminal.data["failure_reason"], "inspect_request_timeout");
6477        assert_eq!(terminal.data["failed_phase"], "tier2_rescan");
6478        assert!(
6479            started.elapsed() < Duration::from_secs(8),
6480            "deferred inspect missed its terminal reserve: {:?}",
6481            started.elapsed()
6482        );
6483    }
6484
6485    #[test]
6486    fn deferred_cold_navigation_releases_lsp_lane_for_reads_and_mutation() {
6487        let _serial = crate::commands::lsp_navigation::deferred_navigation_test_lock();
6488        let executor = Arc::new(Executor::new());
6489        let (dir, root) = test_root("deferred-cold-navigation");
6490        let (ctx, source) = cold_navigation_context(dir.path());
6491        executor.register_actor(root.clone(), Arc::clone(&ctx));
6492        let (started_rx, _release_tx) =
6493            crate::commands::lsp_navigation::install_deferred_navigation_gate_for_test();
6494        let (mut pending, cancellation) = submit_deferred_navigation_setup(
6495            &executor,
6496            &root,
6497            &ctx,
6498            &source,
6499            "subc-cold-navigation",
6500        );
6501        started_rx
6502            .recv_timeout(Duration::from_secs(1))
6503            .expect("detached navigation reaches its cold-start gate");
6504
6505        for (lane, request_id) in [
6506            (Lane::PureRead, "subc-navigation-read"),
6507            (Lane::Mutating, "subc-navigation-write"),
6508        ] {
6509            let response = executor.submit(
6510                root.clone(),
6511                lane,
6512                request_id.to_string(),
6513                Box::new(move |_| Response::success(request_id, json!({ "admitted": true }))),
6514            );
6515            assert!(
6516                response
6517                    .recv_timeout(Duration::from_secs(1))
6518                    .expect("unrelated work admits while navigation remains deferred")
6519                    .success
6520            );
6521        }
6522        assert_eq!(
6523            crate::commands::lsp_navigation::deferred_navigation_worker_count_for_test(),
6524            1,
6525            "lane admissions must not finish the detached navigation"
6526        );
6527
6528        cancellation.request_cancel();
6529        let terminal = wait_for_navigation_terminal(&mut pending, &ctx);
6530        assert!(!terminal.success);
6531        let deadline = Instant::now() + Duration::from_secs(1);
6532        while crate::commands::lsp_navigation::deferred_navigation_worker_count_for_test() != 0 {
6533            assert!(
6534                Instant::now() < deadline,
6535                "cancelled navigation worker did not settle"
6536            );
6537            std::thread::sleep(Duration::from_millis(5));
6538        }
6539        assert!(executor.actor_is_idle(&root));
6540    }
6541
6542    #[test]
6543    fn cancelling_pending_navigation_removes_entry_without_reply() {
6544        let _serial = crate::commands::lsp_navigation::deferred_navigation_test_lock();
6545        let executor = Arc::new(Executor::new());
6546        let (dir, root) = test_root("cancelled-pending-navigation");
6547        let (ctx, source) = cold_navigation_context(dir.path());
6548        executor.register_actor(root.clone(), Arc::clone(&ctx));
6549        let (started_rx, _release_tx) =
6550            crate::commands::lsp_navigation::install_deferred_navigation_gate_for_test();
6551        let (pending, _cancellation) = submit_deferred_navigation_setup(
6552            &executor,
6553            &root,
6554            &ctx,
6555            &source,
6556            "subc-cancel-navigation",
6557        );
6558        started_rx
6559            .recv_timeout(Duration::from_secs(1))
6560            .expect("detached navigation reaches its cancellation gate");
6561
6562        let route = RouteChannel {
6563            channel: 17,
6564            epoch: 1,
6565        };
6566        let mut registry = PendingSubcResponses::default();
6567        registry.register(PendingSubcResponse {
6568            route,
6569            corr: 71,
6570            flags: Flags::new(false, Priority::Passive, false),
6571            ver: PROTOCOL_VERSION,
6572            root: root.clone(),
6573            session_id: "navigation-cancel-session".to_string(),
6574            bare_name: "lsp_hover".to_string(),
6575            format_context: crate::subc_format::FormatContext::from_tool_call(
6576                "lsp_hover",
6577                &json!({}),
6578                dir.path(),
6579            ),
6580            bind_trust: BindTrust::FirstParty,
6581            pending,
6582            surface_downgraded: false,
6583            phase_trace: PhaseTrace::new(Instant::now()),
6584        });
6585
6586        assert!(registry.cancel_request(route, 71));
6587        assert!(registry.is_empty());
6588        let deadline = Instant::now() + Duration::from_secs(1);
6589        while crate::commands::lsp_navigation::deferred_navigation_worker_count_for_test() != 0 {
6590            assert!(
6591                Instant::now() < deadline,
6592                "pending cancellation did not settle the detached worker"
6593            );
6594            std::thread::sleep(Duration::from_millis(5));
6595        }
6596        assert!(
6597            registry.poll_ready(executor.as_ref()).is_empty(),
6598            "a cancelled navigation must not leak a reply"
6599        );
6600    }
6601
6602    #[test]
6603    fn same_root_deferred_inspects_are_single_flight() {
6604        let _serial = crate::commands::inspect::deferred_inspect_test_lock();
6605        let executor = Arc::new(Executor::new());
6606        let (dir, root) = test_root("single-flight-deferred-inspect");
6607        std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
6608        let ctx = inspect_context(dir.path());
6609        executor.register_actor(root.clone(), Arc::clone(&ctx));
6610        let (started_rx, release_tx) =
6611            crate::commands::inspect::install_deferred_inspect_stat_gate_for_test();
6612        let (mut first, _first_cancellation) =
6613            submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-first");
6614        started_rx
6615            .recv_timeout(Duration::from_secs(1))
6616            .expect("first inspect owns the root flight");
6617        let (mut second, second_cancellation) =
6618            submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-second");
6619
6620        assert_eq!(
6621            crate::commands::inspect::deferred_inspect_root_count_for_test(),
6622            1,
6623            "only one detached body may run for a root"
6624        );
6625        assert!((second.poll)(&ctx).is_none(), "second inspect must queue");
6626        second_cancellation.request_cancel();
6627        let second_terminal = wait_for_inspect_terminal(&mut second, &ctx);
6628        assert_eq!(second_terminal.data["inspect_terminal"], "interrupted");
6629        assert_eq!(
6630            crate::commands::inspect::deferred_inspect_root_count_for_test(),
6631            1,
6632            "cancelling the queued request must not release the active flight"
6633        );
6634
6635        release_tx.send(()).expect("release first inspect");
6636        let first_terminal = wait_for_inspect_terminal(&mut first, &ctx);
6637        assert_eq!(first_terminal.data["inspect_terminal"], "fresh");
6638        assert_eq!(
6639            crate::commands::inspect::deferred_inspect_root_count_for_test(),
6640            0
6641        );
6642    }
6643
6644    #[test]
6645    fn route_abandonment_cancels_detached_inspect_thread() {
6646        let _serial = crate::commands::inspect::deferred_inspect_test_lock();
6647        let executor = Arc::new(Executor::new());
6648        let (dir, root) = test_root("cancelled-deferred-inspect");
6649        std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
6650        let ctx = inspect_context(dir.path());
6651        executor.register_actor(root.clone(), Arc::clone(&ctx));
6652        let active: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
6653        let route = RouteChannel {
6654            channel: 7,
6655            epoch: 1,
6656        };
6657        let (started_rx, _release_tx) =
6658            crate::commands::inspect::install_deferred_inspect_body_gate_for_test();
6659        let (mut pending, cancellation) =
6660            submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-abandoned");
6661        active.lock().expect("active tool call map").insert(
6662            (route, 41),
6663            ActiveToolCall {
6664                root_id: root.clone(),
6665                cancellation,
6666                detach_policy: RouteDetachPolicy::CancelOnDetach,
6667            },
6668        );
6669        started_rx
6670            .recv_timeout(Duration::from_secs(1))
6671            .expect("detached inspect reaches cancellation gate");
6672        assert!(ctx.request_force_restrict("subc-inspect-abandoned"));
6673
6674        assert!(cancel_active_tool_call(
6675            &active,
6676            executor.as_ref(),
6677            route,
6678            41,
6679            "test route abandonment"
6680        ));
6681        let terminal = wait_for_inspect_terminal(&mut pending, &ctx);
6682        assert_eq!(terminal.data["inspect_terminal"], "interrupted");
6683        assert_eq!(
6684            crate::commands::inspect::deferred_inspect_root_count_for_test(),
6685            0
6686        );
6687        assert!(executor.actor_is_idle(&root));
6688        assert!(active.lock().expect("active tool call map").is_empty());
6689        let restriction_deadline = Instant::now() + Duration::from_secs(1);
6690        while ctx.request_force_restrict("subc-inspect-abandoned") {
6691            assert!(
6692                Instant::now() < restriction_deadline,
6693                "detached force-restrict guard leaked"
6694            );
6695            std::thread::sleep(Duration::from_millis(5));
6696        }
6697    }
6698
6699    #[test]
6700    fn true_abandonment_cancels_but_route_detach_retains_interactive_search() {
6701        // This scenario requires the replayable and terminal calls to run at the
6702        // same time. Pin two read slots instead of deriving the topology from
6703        // the host, where a small runner can queue the terminal call forever
6704        // behind the intentionally retained call.
6705        let executor = Arc::new(Executor::with_config(crate::executor::ExecutorConfig {
6706            pool_size: 3,
6707            read_cap: 2,
6708            actor_cap: 2,
6709            heavy_permits: 1,
6710            drr_quantum: 1,
6711        }));
6712        let (_dir, root) = test_root("cancelled-interactive-search");
6713        executor.register_actor(root.clone(), test_ctx());
6714        let active: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
6715        let route = RouteChannel {
6716            channel: 9,
6717            epoch: 1,
6718        };
6719
6720        let disabled_iterations = Arc::new(AtomicUsize::new(0));
6721        let disabled_probe = Arc::clone(&disabled_iterations);
6722        let (disabled_started_tx, disabled_started_rx) = std::sync::mpsc::sync_channel(1);
6723        let (disabled_rx, disabled_cancellation) = executor.submit_cancellable_async(
6724            root.clone(),
6725            Lane::PureRead,
6726            "untracked-search".to_string(),
6727            Box::new(move |_| {
6728                disabled_started_tx
6729                    .send(())
6730                    .expect("signal untracked search");
6731                let deadline = Instant::now() + Duration::from_secs(5);
6732                while !crate::commands::semantic_search::search_cancellation_requested() {
6733                    if Instant::now() >= deadline {
6734                        return Response::error(
6735                            "untracked-search",
6736                            "test_timeout",
6737                            "untracked search did not receive cancellation",
6738                        );
6739                    }
6740                    disabled_probe.fetch_add(1, Ordering::Relaxed);
6741                    std::thread::yield_now();
6742                }
6743                Response::error(
6744                    "untracked-search",
6745                    "request_cancelled",
6746                    "cancelled at search checkpoint",
6747                )
6748            }),
6749        );
6750        disabled_started_rx
6751            .recv_timeout(Duration::from_secs(1))
6752            .expect("untracked search starts");
6753        assert_eq!(
6754            apply_route_work_disposition(
6755                &active,
6756                executor.as_ref(),
6757                route,
6758                RouteWorkDisposition::Abandon,
6759                "disabled cancellation wiring",
6760            ),
6761            0
6762        );
6763        let iterations_before = disabled_iterations.load(Ordering::Relaxed);
6764        std::thread::sleep(Duration::from_millis(10));
6765        assert!(
6766            disabled_iterations.load(Ordering::Relaxed) > iterations_before,
6767            "without route registration the search keeps computing"
6768        );
6769        disabled_cancellation.request_cancel();
6770        let disabled_response = disabled_rx
6771            .blocking_recv()
6772            .expect("untracked search settles after explicit cleanup");
6773        assert_eq!(disabled_response.data["code"], "request_cancelled");
6774
6775        let tracked_iterations = Arc::new(AtomicUsize::new(0));
6776        let tracked_probe = Arc::clone(&tracked_iterations);
6777        let (tracked_started_tx, tracked_started_rx) = std::sync::mpsc::sync_channel(1);
6778        let tracked_rx = submit_active_tool_call(
6779            executor.as_ref(),
6780            &active,
6781            route,
6782            42,
6783            root.clone(),
6784            Lane::PureRead,
6785            "tracked-search".to_string(),
6786            RouteDetachPolicy::RetainForReplay,
6787            Box::new(move |_| {
6788                tracked_started_tx.send(()).expect("signal tracked search");
6789                let deadline = Instant::now() + Duration::from_secs(5);
6790                while !crate::commands::semantic_search::search_cancellation_requested() {
6791                    if Instant::now() >= deadline {
6792                        return Response::error(
6793                            "tracked-search",
6794                            "test_timeout",
6795                            "tracked search did not receive cancellation",
6796                        );
6797                    }
6798                    tracked_probe.fetch_add(1, Ordering::Relaxed);
6799                    std::thread::yield_now();
6800                }
6801                Response::error(
6802                    "tracked-search",
6803                    "request_cancelled",
6804                    "cancelled at search checkpoint",
6805                )
6806            }),
6807        );
6808        tracked_started_rx
6809            .recv_timeout(Duration::from_secs(1))
6810            .expect("tracked search starts");
6811
6812        let (terminal_started_tx, terminal_started_rx) = std::sync::mpsc::sync_channel(1);
6813        let terminal_rx = submit_active_tool_call(
6814            executor.as_ref(),
6815            &active,
6816            route,
6817            43,
6818            root.clone(),
6819            Lane::PureRead,
6820            "teardown-terminal".to_string(),
6821            RouteDetachPolicy::CancelOnDetach,
6822            Box::new(move |_| {
6823                terminal_started_tx
6824                    .send(())
6825                    .expect("signal teardown-terminal call");
6826                let deadline = Instant::now() + Duration::from_secs(5);
6827                while !crate::executor::current_job_cancelled() {
6828                    if Instant::now() >= deadline {
6829                        return Response::error(
6830                            "teardown-terminal",
6831                            "test_timeout",
6832                            "terminal call did not receive cancellation",
6833                        );
6834                    }
6835                    std::thread::yield_now();
6836                }
6837                Response::error(
6838                    "teardown-terminal",
6839                    "request_cancelled",
6840                    "cancelled for terminal-emitting teardown",
6841                )
6842            }),
6843        );
6844        terminal_started_rx
6845            .recv_timeout(Duration::from_secs(1))
6846            .expect("teardown-terminal call starts");
6847        assert_eq!(
6848            apply_route_work_disposition(
6849                &active,
6850                executor.as_ref(),
6851                route,
6852                RouteWorkDisposition::RetainForReplay,
6853                "test route detach",
6854            ),
6855            1,
6856            "only the replayable search remains active after route detach"
6857        );
6858        let terminal_response = terminal_rx
6859            .blocking_recv()
6860            .expect("teardown-terminal call stops at cancellation checkpoint");
6861        assert_eq!(terminal_response.data["code"], "request_cancelled");
6862        let iterations_before_detach = tracked_iterations.load(Ordering::Relaxed);
6863        std::thread::sleep(Duration::from_millis(10));
6864        assert!(
6865            tracked_iterations.load(Ordering::Relaxed) > iterations_before_detach,
6866            "route detach must retain work whose response can replay after rebind"
6867        );
6868        assert_eq!(
6869            apply_route_work_disposition(
6870                &active,
6871                executor.as_ref(),
6872                route,
6873                RouteWorkDisposition::Abandon,
6874                "test session purge",
6875            ),
6876            1
6877        );
6878        let tracked_response = tracked_rx
6879            .blocking_recv()
6880            .expect("tracked search stops at cancellation checkpoint");
6881        assert_eq!(tracked_response.data["code"], "request_cancelled");
6882        assert!(active.lock().expect("active tool calls").is_empty());
6883
6884        let deadline = Instant::now() + Duration::from_secs(1);
6885        while !executor.actor_is_idle(&root) {
6886            assert!(
6887                Instant::now() < deadline,
6888                "cancelled search must release the PureRead lane"
6889            );
6890            std::thread::sleep(Duration::from_millis(2));
6891        }
6892    }
6893
6894    #[test]
6895    fn shutdown_drain_emits_terminal_and_clears_pending_inspect() {
6896        let _serial = crate::commands::inspect::deferred_inspect_test_lock();
6897        let executor = Arc::new(Executor::new());
6898        let (dir, root) = test_root("shutdown-deferred-inspect");
6899        std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
6900        let ctx = inspect_context(dir.path());
6901        executor.register_actor(root.clone(), Arc::clone(&ctx));
6902        let (started_rx, _release_tx) =
6903            crate::commands::inspect::install_deferred_inspect_body_gate_for_test();
6904        let (pending, cancellation) =
6905            submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-shutdown");
6906        let route = RouteChannel {
6907            channel: 8,
6908            epoch: 1,
6909        };
6910        started_rx
6911            .recv_timeout(Duration::from_secs(1))
6912            .expect("detached inspect reaches shutdown gate");
6913        let mut registry = PendingSubcResponses::default();
6914        registry.register(PendingSubcResponse {
6915            route,
6916            corr: 42,
6917            flags: Flags::new(false, Priority::Passive, false),
6918            ver: PROTOCOL_VERSION,
6919            root: root.clone(),
6920            session_id: "shutdown-session".to_string(),
6921            bare_name: "inspect".to_string(),
6922            format_context: crate::subc_format::FormatContext::from_tool_call(
6923                "inspect",
6924                &json!({}),
6925                dir.path(),
6926            ),
6927            bind_trust: BindTrust::FirstParty,
6928            pending,
6929            surface_downgraded: false,
6930            phase_trace: PhaseTrace::new(Instant::now()),
6931        });
6932        let active: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::from([(
6933            (route, 42),
6934            ActiveToolCall {
6935                root_id: root.clone(),
6936                cancellation,
6937                detach_policy: RouteDetachPolicy::CancelOnDetach,
6938            },
6939        )])));
6940
6941        let resolved = registry.drain_on_shutdown(executor.as_ref());
6942        assert!(registry.is_empty());
6943        assert_eq!(resolved.len(), 1);
6944        assert_eq!(
6945            resolved[0].response.data["failure_reason"],
6946            "daemon_shutdown"
6947        );
6948        finish_active_tool_call(&active, route, 42);
6949        let deadline = Instant::now() + Duration::from_secs(1);
6950        while crate::commands::inspect::deferred_inspect_root_count_for_test() != 0 {
6951            assert!(Instant::now() < deadline, "shutdown cancellation was inert");
6952            std::thread::sleep(Duration::from_millis(5));
6953        }
6954        assert!(active.lock().expect("active calls").is_empty());
6955        assert!(executor.actor_is_idle(&root));
6956    }
6957
6958    pub(super) fn wait_for_watcher_count(ctx: &AppContext, expected: usize) {
6959        let deadline = Instant::now() + Duration::from_secs(30);
6960        loop {
6961            let observed = ctx.watcher_registry_count();
6962            if observed == expected {
6963                return;
6964            }
6965            assert!(
6966                Instant::now() < deadline,
6967                "watcher count did not settle before deadline: expected={expected}, observed={observed}"
6968            );
6969            std::thread::sleep(Duration::from_millis(50));
6970        }
6971    }
6972
6973    /// Sweep until `root` is forgotten, mirroring how production reaps.
6974    ///
6975    /// `reap_idle_roots` probes actor idleness with a try-lock and retains the
6976    /// root when the scheduler holds that lock — correct behavior, since the
6977    /// real caller sweeps on a timer and simply catches the root next tick. A
6978    /// test that asserts a single sweep succeeds is therefore asserting it wins
6979    /// a lock race that nothing in production depends on: `register_actor` wakes
6980    /// the scheduler, which grabs the same lock, and on a loaded runner that
6981    /// window is wide enough to lose. Sweep to the outcome instead.
6982    pub(super) fn reap_until_forgotten(
6983        root: &ProjectRootId,
6984        live_roots: &mut HashMap<ProjectRootId, RootMeta>,
6985        pending_binds: &HashMap<RouteChannel, PendingBind>,
6986        root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
6987        executor: &Arc<Executor>,
6988        metrics: &DispatchPathMetrics,
6989    ) -> IdleReapOutcome {
6990        let deadline = Instant::now() + Duration::from_secs(30);
6991        loop {
6992            let outcome = reap_idle_roots(
6993                Instant::now(),
6994                live_roots,
6995                pending_binds,
6996                root_channels,
6997                executor,
6998                metrics,
6999            );
7000            if outcome.forgotten_deleted_roots.contains(root) {
7001                return outcome;
7002            }
7003            assert!(
7004                Instant::now() < deadline,
7005                "deleted root was never forgotten: {root:?}"
7006            );
7007            std::thread::sleep(Duration::from_millis(10));
7008        }
7009    }
7010
7011    pub(super) fn wait_for_actor_root_count(app: &App, expected: usize) {
7012        let deadline = Instant::now() + Duration::from_secs(30);
7013        loop {
7014            let observed = app.actor_root_count();
7015            if observed == expected {
7016                return;
7017            }
7018            assert!(
7019                Instant::now() < deadline,
7020                "actor root count did not settle before deadline: expected={expected}, observed={observed}"
7021            );
7022            std::thread::sleep(Duration::from_millis(50));
7023        }
7024    }
7025
7026    pub(super) fn status_frame(seq: u64) -> PushFrame {
7027        status_frame_with_session(seq, None)
7028    }
7029
7030    pub(super) fn status_frame_with_session(seq: u64, session_id: Option<&str>) -> PushFrame {
7031        PushFrame::StatusChanged(StatusChangedFrame {
7032            frame_type: "status_changed",
7033            session_id: session_id.map(str::to_string),
7034            snapshot: json!({ "seq": seq }),
7035        })
7036    }
7037
7038    pub(super) fn completion_frame(task_id: &str) -> PushFrame {
7039        completion_frame_with_session(task_id, "session-1")
7040    }
7041
7042    pub(super) fn completion_frame_with_session(task_id: &str, session_id: &str) -> PushFrame {
7043        PushFrame::BashCompleted(BashCompletedFrame {
7044            frame_type: "bash_completed",
7045            task_id: task_id.to_string(),
7046            session_id: session_id.to_string(),
7047            status: BgTaskStatus::Completed,
7048            exit_code: Some(0),
7049            command: format!("echo {task_id}"),
7050            output_preview: String::new(),
7051            bash_output_list_envelope: None,
7052            output_truncated: false,
7053            original_tokens: None,
7054            compressed_tokens: None,
7055            tokens_skipped: false,
7056            status_reason: None,
7057            live_descendants: Some(Vec::new()),
7058            live_descendants_omitted: 0,
7059            live_descendants_summary: None,
7060        })
7061    }
7062
7063    pub(super) fn long_running_frame(task_id: &str, elapsed_ms: u64) -> PushFrame {
7064        long_running_frame_with_session(task_id, "session-1", elapsed_ms)
7065    }
7066
7067    pub(super) fn long_running_frame_with_session(
7068        task_id: &str,
7069        session_id: &str,
7070        elapsed_ms: u64,
7071    ) -> PushFrame {
7072        PushFrame::BashLongRunning(BashLongRunningFrame {
7073            frame_type: "bash_long_running",
7074            task_id: task_id.to_string(),
7075            session_id: session_id.to_string(),
7076            command: format!("sleep {elapsed_ms}"),
7077            elapsed_ms,
7078        })
7079    }
7080
7081    pub(super) fn pattern_match_frame(session_id: &str) -> PushFrame {
7082        PushFrame::BashPatternMatch(BashPatternMatchFrame {
7083            frame_type: "bash_pattern_match",
7084            task_id: "task-pattern".to_string(),
7085            session_id: session_id.to_string(),
7086            watch_id: "watch-1".to_string(),
7087            match_text: "needle".to_string(),
7088            match_offset: 7,
7089            context: "haystack needle".to_string(),
7090            once: true,
7091            reason: "pattern_match",
7092        })
7093    }
7094
7095    pub(super) fn configure_warnings_frame(session_id: Option<&str>) -> PushFrame {
7096        PushFrame::ConfigureWarnings(ConfigureWarningsFrame {
7097            frame_type: "configure_warnings",
7098            session_id: session_id.map(str::to_string),
7099            project_root: "/tmp/subc-test".to_string(),
7100            warnings: Vec::new(),
7101        })
7102    }
7103
7104    pub(super) fn route_identity(root: &ProjectRootId, session_id: &str) -> RouteIdentity {
7105        route_identity_with_trust(root, session_id, BindTrust::FirstParty)
7106    }
7107
7108    pub(super) fn route_identity_with_trust(
7109        root: &ProjectRootId,
7110        session_id: &str,
7111        trust: BindTrust,
7112    ) -> RouteIdentity {
7113        RouteIdentity(Arc::new(RouteIdentityData {
7114            root: root.clone(),
7115            project_root: root.as_path().to_path_buf(),
7116            harness: "opencode".to_string(),
7117            session: session_id.to_string(),
7118            trust,
7119            spawn_principal: AuthenticatedPrincipal::RouteBind {
7120                trust: trust.sandbox_trust(),
7121                route_channel: 0,
7122                route_epoch: 0,
7123                project_root: root.as_path().to_path_buf(),
7124                harness: "opencode".to_string(),
7125                session_id: session_id.to_string(),
7126                principal_id: Some(match trust {
7127                    BindTrust::FirstParty => "direct".to_string(),
7128                    BindTrust::Untrusted => "unverified".to_string(),
7129                }),
7130            },
7131            consumer_elicitation_capable: false,
7132        }))
7133    }
7134
7135    pub(super) fn progress_frame(request_id: &str, kind: ProgressKind, chunk: &str) -> PushFrame {
7136        PushFrame::Progress(ProgressFrame::new(request_id, kind, chunk))
7137    }
7138
7139    pub(super) fn status_seq(frame: &PushFrame) -> Option<u64> {
7140        match frame {
7141            PushFrame::StatusChanged(status) => status.snapshot.get("seq").and_then(|v| v.as_u64()),
7142            _ => None,
7143        }
7144    }
7145
7146    pub(super) fn completion_task(frame: &PushFrame) -> Option<&str> {
7147        match frame {
7148            PushFrame::BashCompleted(completion) => Some(completion.task_id.as_str()),
7149            _ => None,
7150        }
7151    }
7152
7153    pub(super) fn push_frame_task_id(frame: &Frame) -> Option<String> {
7154        let body: serde_json::Value = serde_json::from_slice(&frame.body).expect("push body");
7155        body.get("task_id")
7156            .and_then(serde_json::Value::as_str)
7157            .map(str::to_string)
7158    }
7159}
7160
7161#[cfg(test)]
7162mod tests {
7163    use super::test_support::{
7164        completion_frame, reap_until_forgotten, route_identity, test_ctx, test_root,
7165        wait_for_actor_root_count, wait_for_watcher_count,
7166    };
7167    use super::*;
7168    use crate::bash_background::BgTaskStatus;
7169
7170    /// Only a daemon-requested stop may exit 0. The supervisor never respawns
7171    /// a clean exit, so a fatal-actor teardown that mapped to Ok(()) left the
7172    /// module down host-wide until a manual start (2026-09-11).
7173    #[test]
7174    fn only_a_graceful_goodbye_maps_to_a_clean_exit() {
7175        assert!(module_loop_exit_result(ModuleLoopExit::Graceful).is_ok());
7176        assert!(matches!(
7177            module_loop_exit_result(ModuleLoopExit::ConnectionLost),
7178            Err(SubcError::ConnectionLost)
7179        ));
7180        assert!(matches!(
7181            module_loop_exit_result(ModuleLoopExit::SkipSearchFlush),
7182            Err(SubcError::ActorFatal)
7183        ));
7184    }
7185
7186    #[test]
7187    fn fatal_panic_responses_are_detected_and_noted() {
7188        let panic = Response::error(
7189            "req-fatal",
7190            "actor_fatal",
7191            "start byte index 7 is not a char boundary",
7192        );
7193        assert!(note_fatal_panic_response(&panic));
7194        let ordinary = Response::error("req-ok", "invalid_request", "missing field");
7195        assert!(!note_fatal_panic_response(&ordinary));
7196    }
7197
7198    fn attach_error(kind: io::ErrorKind) -> SubcError {
7199        SubcError::Connect {
7200            endpoint: "127.0.0.1:1".to_string(),
7201            source: io::Error::new(kind, "constructed attach failure"),
7202        }
7203    }
7204
7205    fn auth_io_error(kind: io::ErrorKind) -> SubcError {
7206        SubcError::Auth {
7207            endpoint: "127.0.0.1:1".to_string(),
7208            source: subc_transport::AuthError::Io {
7209                stage: subc_transport::AuthStage::ServerProof,
7210                source: io::Error::new(kind, "constructed auth failure"),
7211            },
7212        }
7213    }
7214
7215    fn cpu_hunt_process_cpu_us() -> u64 {
7216        #[cfg(unix)]
7217        {
7218            let mut usage = std::mem::MaybeUninit::<libc::rusage>::uninit();
7219            // getrusage initializes the output on success; no pointer escapes.
7220            if unsafe { libc::getrusage(libc::RUSAGE_SELF, usage.as_mut_ptr()) } == 0 {
7221                let usage = unsafe { usage.assume_init() };
7222                return (usage.ru_utime.tv_sec + usage.ru_stime.tv_sec) as u64 * 1_000_000
7223                    + (usage.ru_utime.tv_usec + usage.ru_stime.tv_usec) as u64;
7224            }
7225        }
7226        0
7227    }
7228
7229    #[test]
7230    fn route_bind_does_not_recompute_fleet_health() {
7231        let runtime = tokio::runtime::Builder::new_current_thread()
7232            .enable_all()
7233            .build()
7234            .expect("bind runtime");
7235        runtime.block_on(async {
7236            let (dir, root) = test_root("bind-health-work-count");
7237            let app = App::default_shared();
7238            let executor = Arc::new(Executor::new());
7239            let ctx = Arc::new(AppContext::from_app(Arc::clone(&app), Config::default()));
7240            let mut fixture_dirs = Vec::new();
7241            // The opt-in probe accepts only an already-copied artifact below this
7242            // checkout's target directory; normal tests never open a live store.
7243            if let Some(copy) = std::env::var_os("AFT_CPU_HUNT_STORE_COPY") {
7244                let copy = std::fs::canonicalize(copy).expect("copied store exists");
7245                let project = Path::new(env!("CARGO_MANIFEST_DIR"))
7246                    .parent()
7247                    .expect("crates directory")
7248                    .parent()
7249                    .expect("checkout directory")
7250                    .canonicalize()
7251                    .expect("canonical checkout");
7252                assert!(copy.starts_with(project.join("target")));
7253                for index in 0..36 {
7254                    let actor = if index == 0 {
7255                        Arc::clone(&ctx)
7256                    } else {
7257                        Arc::new(AppContext::from_app(Arc::clone(&app), Config::default()))
7258                    };
7259                    actor.update_config(|config| config.project_root = Some(project.clone()));
7260                    *actor.callgraph_store().write().expect("store slot") = Some(Arc::new(
7261                        crate::callgraph_store::CallGraphStore::open_readonly(
7262                            copy.clone(),
7263                            project.clone(),
7264                        )
7265                        .expect("open copied graph")
7266                        .expect("copied graph ready"),
7267                    ));
7268                    if index > 0 {
7269                        let (fixture, id) = test_root(&format!("cpu-hunt-{index}"));
7270                        assert!(executor.register_actor(id, actor));
7271                        fixture_dirs.push(fixture);
7272                    }
7273                }
7274            }
7275            assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7276            let cache = HealthRollupCache::new();
7277            if !fixture_dirs.is_empty() {
7278                let started = Instant::now();
7279                let cpu = cpu_hunt_process_cpu_us();
7280                cache.refresh(&executor, &app);
7281                eprintln!(
7282                    "cpu_hunt health_rollup wall_us={} cpu_us={}",
7283                    started.elapsed().as_micros(),
7284                    cpu_hunt_process_cpu_us().saturating_sub(cpu)
7285                );
7286                let started = Instant::now();
7287                let cpu = cpu_hunt_process_cpu_us();
7288                std::hint::black_box(ctx.build_status_snapshot());
7289                eprintln!(
7290                    "cpu_hunt status wall_us={} cpu_us={}",
7291                    started.elapsed().as_micros(),
7292                    cpu_hunt_process_cpu_us().saturating_sub(cpu)
7293                );
7294            }
7295            let refreshes_before = cache.refresh_count_for_test();
7296            let metrics = Arc::new(DispatchPathMetrics::new());
7297            let (writer_tx, _writer_rx) = mpsc::channel(8);
7298            let (completion_tx, mut completion_rx) = mpsc::channel(8);
7299            let (lossy_tx, _lossy_rx) = mpsc::channel(8);
7300            let (reliable_tx, _reliable_rx) = mpsc::unbounded_channel();
7301            let senders = PushSenders {
7302                lossy_tx,
7303                reliable_tx,
7304                lossy_overflow: Arc::new(push::LossyOverflow::default()),
7305                lossy_seq: Arc::new(AtomicU64::new(0)),
7306                fleet_status_client: FleetStatusClient::channel(1).0,
7307            };
7308            let request = ModuleControlRequest::RouteBind {
7309                route_channel: 1,
7310                epoch: 1,
7311                target: RouteTarget::ToolProvider {
7312                    module_id: "aft".to_string(),
7313                },
7314                identity: subc_protocol::BindIdentity {
7315                    project_root: root.as_path().to_path_buf(),
7316                    harness: "opencode".to_string(),
7317                    session: "bind-health-work-count".to_string(),
7318                },
7319                principal: Some(subc_protocol::Principal::Direct),
7320                consumer_capabilities: None,
7321                admission_facts: Default::default(),
7322            };
7323            let frame = Frame::build_with_version(
7324                PROTOCOL_VERSION,
7325                FrameType::Request,
7326                control_flags(),
7327                0,
7328                0,
7329                1,
7330                serde_json::to_vec(&request).expect("bind body"),
7331            )
7332            .expect("bind frame");
7333            let mut pending_binds = HashMap::new();
7334            let started = Instant::now();
7335            let cpu = cpu_hunt_process_cpu_us();
7336            handle_control_request(
7337                &writer_tx,
7338                &frame,
7339                &app,
7340                &executor,
7341                &mut HashMap::new(),
7342                &mut pending_binds,
7343                &mut HashMap::new(),
7344                &mut HashMap::new(),
7345                &mut HashSet::new(),
7346                &mut HashMap::new(),
7347                &mut HashMap::new(),
7348                &mut HashMap::new(),
7349                &mut HashMap::new(),
7350                &mut HashMap::new(),
7351                &mut HashMap::new(),
7352                &Arc::new(StdMutex::new(HashMap::new())),
7353                &mut PendingSubcResponses::default(),
7354                &mut RetryBuffer::new(),
7355                &mut HashMap::new(),
7356                &Arc::new(Notify::new()),
7357                &completion_tx,
7358                &metrics,
7359                None,
7360                &cache,
7361                &senders,
7362                |request, _| Response::success(request.id, json!({})),
7363                Some(&dir.path().join("absent-user-config.json")),
7364                usize::MAX,
7365            )
7366            .await
7367            .expect("admit route bind");
7368            eprintln!(
7369                "route_bind_health admission_us={} cpu_us={} refreshes={}",
7370                started.elapsed().as_micros(),
7371                cpu_hunt_process_cpu_us().saturating_sub(cpu),
7372                cache.refresh_count_for_test() - refreshes_before
7373            );
7374            let completion = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
7375                .await
7376                .expect("configure completes")
7377                .expect("completion delivered");
7378            assert!(completion.configure_response.success);
7379            assert_eq!(pending_binds.len(), 1, "the bind must reach admission");
7380            assert_eq!(
7381                cache.refresh_count_for_test() - refreshes_before,
7382                0,
7383                "route admission must not perform a fleet-wide health census"
7384            );
7385        });
7386    }
7387
7388    #[test]
7389    fn channel_zero_health_response_does_not_wait_for_bash_background_db() {
7390        let (dir, root) = test_root("health-does-not-lock-bash-db");
7391        let executor = Arc::new(Executor::new());
7392        let ctx = test_ctx();
7393        ctx.set_harness(crate::harness::Harness::Opencode);
7394        let db = Arc::new(StdMutex::new(
7395            crate::db::open(&dir.path().join("health.db")).expect("open health test DB"),
7396        ));
7397        ctx.bash_background().set_db_pool(Arc::clone(&db));
7398        executor.register_actor(root, ctx);
7399
7400        let guard = db.lock().expect("hold bash-background DB mutex");
7401        let app = App::default_shared();
7402        let metrics = Arc::new(DispatchPathMetrics::new());
7403        let health_rollup_cache = Arc::new(HealthRollupCache::new());
7404        let frame = Frame::build_with_version(
7405            PROTOCOL_VERSION,
7406            FrameType::Request,
7407            control_flags(),
7408            0,
7409            0,
7410            77,
7411            Vec::new(),
7412        )
7413        .expect("health request frame");
7414        let (writer_tx, _writer_rx) = mpsc::channel::<WriterFrame>(1);
7415        let (done_tx, done_rx) = std::sync::mpsc::channel();
7416        let join = std::thread::spawn(move || {
7417            let runtime = tokio::runtime::Builder::new_current_thread()
7418                .enable_all()
7419                .build()
7420                .expect("health test runtime");
7421            let result = runtime.block_on(send_cached_health_response(
7422                &writer_tx,
7423                &frame,
7424                &app,
7425                &executor,
7426                &HashMap::new(),
7427                &metrics,
7428                &health_rollup_cache,
7429            ));
7430            done_tx.send(result).expect("report health result");
7431        });
7432
7433        let result = done_rx
7434            .recv_timeout(Duration::from_millis(500))
7435            .expect("channel-0 health blocked on bash-background DB mutex");
7436        result.expect("send cached health response");
7437        drop(guard);
7438        join.join().expect("health thread");
7439    }
7440
7441    #[test]
7442    fn maintenance_bg_runtime_refresh_deduplicates_shared_db_items() {
7443        let (_dir_a, root_a) = test_root("health-metric-root-a");
7444        let (_dir_b, root_b) = test_root("health-metric-root-b");
7445        let duplicate_key = "match\0session-1\0bash-0000000000000001\0watch-00000001";
7446        let snapshots = HashMap::from([
7447            (root_a, HashSet::from([duplicate_key.to_string()])),
7448            (root_b, HashSet::from([duplicate_key.to_string()])),
7449        ]);
7450        let metrics = DispatchPathMetrics::new();
7451
7452        record_bg_runtime_from_snapshots(&metrics, 2, 1, &snapshots);
7453
7454        assert_eq!(metrics.bg_runtime_for_test(), (2, 1, 1));
7455    }
7456
7457    #[test]
7458    fn initial_attach_error_classifier_distinguishes_transient_and_permanent_failures() {
7459        let transient_errors = vec![
7460            attach_error(io::ErrorKind::ConnectionRefused),
7461            attach_error(io::ErrorKind::TimedOut),
7462            attach_error(io::ErrorKind::ConnectionReset),
7463            auth_io_error(io::ErrorKind::ConnectionAborted),
7464            auth_io_error(io::ErrorKind::BrokenPipe),
7465            SubcError::Auth {
7466                endpoint: "127.0.0.1:1".to_string(),
7467                source: subc_transport::AuthError::UnexpectedEof {
7468                    stage: subc_transport::AuthStage::ServerProof,
7469                    expected: 4,
7470                    actual: 0,
7471                },
7472            },
7473            SubcError::Auth {
7474                endpoint: "127.0.0.1:1".to_string(),
7475                source: subc_transport::AuthError::Timeout {
7476                    stage: subc_transport::AuthStage::ServerProof,
7477                    deadline: AUTH_DEADLINE,
7478                },
7479            },
7480        ];
7481        for error in &transient_errors {
7482            assert_eq!(
7483                classify_attach_error(error),
7484                AttachErrorClass::Transient,
7485                "expected transient: {error}"
7486            );
7487        }
7488
7489        let permanent_errors = vec![
7490            attach_error(io::ErrorKind::PermissionDenied),
7491            auth_io_error(io::ErrorKind::InvalidData),
7492            SubcError::Auth {
7493                endpoint: "127.0.0.1:1".to_string(),
7494                source: subc_transport::AuthError::InvalidServerProof,
7495            },
7496            SubcError::Auth {
7497                endpoint: "127.0.0.1:1".to_string(),
7498                source: subc_transport::AuthError::DaemonIdMismatch,
7499            },
7500            SubcError::ConnectionFile {
7501                path: PathBuf::from("subc-connection.json"),
7502                source: subc_transport::ConnectionFileError::Invalid {
7503                    reason: "constructed invalid file".to_string(),
7504                },
7505            },
7506            SubcError::NoEndpoint {
7507                path: PathBuf::from("subc-connection.json"),
7508            },
7509            SubcError::InvalidEndpoint {
7510                path: PathBuf::from("subc-connection.json"),
7511                endpoint: "not-an-ip:1234".to_string(),
7512            },
7513        ];
7514        for error in &permanent_errors {
7515            assert_eq!(
7516                classify_attach_error(error),
7517                AttachErrorClass::Permanent,
7518                "expected permanent: {error}"
7519            );
7520        }
7521    }
7522
7523    #[test]
7524    fn incompatible_wire_version_is_rejected_before_tcp_connect() {
7525        let conn_dir = tempfile::tempdir().expect("connection tempdir");
7526        let conn_path = conn_dir.path().join("subc-connection.json");
7527        let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind listener");
7528        listener
7529            .set_nonblocking(true)
7530            .expect("set listener nonblocking");
7531        let port = listener.local_addr().expect("listener addr").port();
7532        connection_file::write_atomic(
7533            &conn_path,
7534            &connection_file::ConnectionInfo {
7535                schema: connection_file::SCHEMA_VERSION,
7536                wire_version: Some(PROTOCOL_VERSION.wrapping_add(1)),
7537                endpoints: vec![connection_file::Endpoint {
7538                    host: "127.0.0.1".to_string(),
7539                    port,
7540                }],
7541                key: vec![0x42; subc_transport::KEY_LEN],
7542                daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
7543                pid: std::process::id(),
7544                daemon_ver: "subc-test".to_string(),
7545            },
7546        )
7547        .expect("write connection file");
7548
7549        let runtime = tokio::runtime::Builder::new_current_thread()
7550            .enable_all()
7551            .build()
7552            .expect("test runtime");
7553        let result = runtime.block_on(connect_and_authenticate_with_policy(
7554            &conn_path,
7555            AttachRetryPolicy {
7556                budget: Duration::from_secs(1),
7557                initial_backoff: Duration::from_millis(5),
7558                max_backoff: Duration::from_millis(10),
7559                jitter_percent: 0,
7560            },
7561            None,
7562        ));
7563        assert!(matches!(
7564            result,
7565            Err(SubcError::ConnectionFile {
7566                source: connection_file::ConnectionFileError::WireVersionMismatch { .. },
7567                ..
7568            })
7569        ));
7570        assert!(matches!(
7571            listener.accept(),
7572            Err(error) if error.kind() == std::io::ErrorKind::WouldBlock
7573        ));
7574    }
7575
7576    #[test]
7577    fn initial_attach_unreachable_endpoint_retries_until_budget_then_fails_loud() {
7578        let conn_dir = tempfile::tempdir().expect("connection tempdir");
7579        let conn_path = conn_dir.path().join("subc-connection.json");
7580        let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve port");
7581        let port = listener.local_addr().expect("reserved addr").port();
7582        drop(listener);
7583        connection_file::write_atomic(
7584            &conn_path,
7585            &connection_file::ConnectionInfo {
7586                schema: connection_file::SCHEMA_VERSION,
7587                wire_version: Some(PROTOCOL_VERSION),
7588                endpoints: vec![connection_file::Endpoint {
7589                    host: "127.0.0.1".to_string(),
7590                    port,
7591                }],
7592                key: vec![0x42; subc_transport::KEY_LEN],
7593                daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
7594                pid: std::process::id(),
7595                daemon_ver: "subc-test".to_string(),
7596            },
7597        )
7598        .expect("write connection file");
7599
7600        let policy = AttachRetryPolicy {
7601            budget: Duration::from_millis(40),
7602            initial_backoff: Duration::from_millis(5),
7603            max_backoff: Duration::from_millis(10),
7604            jitter_percent: 0,
7605        };
7606        let runtime = tokio::runtime::Builder::new_current_thread()
7607            .enable_all()
7608            .build()
7609            .expect("test runtime");
7610        let started_at = Instant::now();
7611        let result = runtime.block_on(connect_and_authenticate_with_policy(
7612            &conn_path, policy, None,
7613        ));
7614        let elapsed = started_at.elapsed();
7615        let error = match result {
7616            Ok(_) => panic!("unreachable endpoint unexpectedly attached"),
7617            Err(error) => error,
7618        };
7619
7620        assert!(matches!(error, SubcError::Connect { .. }), "{error}");
7621        assert!(
7622            elapsed >= Duration::from_millis(35),
7623            "retry budget ended too early: {elapsed:?}"
7624        );
7625        assert!(
7626            elapsed < Duration::from_secs(1),
7627            "retry budget was not bounded: {elapsed:?}"
7628        );
7629    }
7630
7631    fn due_maintenance_jobs_without_actor_context(
7632        live_roots: &mut HashMap<ProjectRootId, RootMeta>,
7633        budget: usize,
7634        pending_bind_roots: &HashSet<ProjectRootId>,
7635    ) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
7636        due_maintenance_jobs(
7637            live_roots,
7638            None,
7639            &HashMap::new(),
7640            &BgWakePending::new(),
7641            budget,
7642            pending_bind_roots,
7643        )
7644    }
7645
7646    fn actor_ctx_with_dirty_search_index(
7647        root: &Path,
7648        storage: &Path,
7649        file_name: &str,
7650        old_contents: &str,
7651        new_contents: &str,
7652    ) -> (Arc<AppContext>, PathBuf, PathBuf) {
7653        let file = root.join(file_name);
7654        std::fs::write(&file, old_contents).expect("write source");
7655        let canonical_root = std::fs::canonicalize(root).expect("canonical root");
7656        let ctx = Arc::new(AppContext::new(
7657            Box::new(crate::parser::TreeSitterProvider::new()),
7658            Config {
7659                project_root: Some(root.to_path_buf()),
7660                storage_dir: Some(storage.to_path_buf()),
7661                ..Config::default()
7662            },
7663        ));
7664        ctx.set_canonical_cache_root(canonical_root.clone());
7665
7666        let cache_dir = crate::search_index::resolve_cache_dir(&canonical_root, Some(storage));
7667        let mut index = crate::search_index::SearchIndex::build(&canonical_root);
7668        let git_head = index.stored_git_head().map(str::to_owned);
7669        index.write_to_disk(&cache_dir, git_head.as_deref());
7670
7671        std::fs::write(&file, new_contents).expect("edit source");
7672        index.update_file(&file);
7673        *ctx.search_index()
7674            .write()
7675            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
7676        (ctx, canonical_root, cache_dir)
7677    }
7678
7679    #[test]
7680    fn graceful_shutdown_flushes_every_actor_search_index() {
7681        let storage = tempfile::tempdir().expect("storage tempdir");
7682        let (root1_dir, root1) = test_root("shutdown-flush-root-1");
7683        let (root2_dir, root2) = test_root("shutdown-flush-root-2");
7684        let (ctx1, canonical_root1, cache_dir1) = actor_ctx_with_dirty_search_index(
7685            root1_dir.path(),
7686            storage.path(),
7687            "alpha.txt",
7688            "old actor one token\n",
7689            "new actor one token\n",
7690        );
7691        let (ctx2, canonical_root2, cache_dir2) = actor_ctx_with_dirty_search_index(
7692            root2_dir.path(),
7693            storage.path(),
7694            "beta.txt",
7695            "old actor two token\n",
7696            "new actor two token\n",
7697        );
7698
7699        let executor = Executor::new();
7700        assert!(executor.register_actor(root1.clone(), Arc::clone(&ctx1)));
7701        assert!(executor.register_actor(root2.clone(), Arc::clone(&ctx2)));
7702
7703        flush_actor_indexes_on_graceful_shutdown(&executor.actor_contexts());
7704
7705        let mut restored1 =
7706            crate::search_index::SearchIndex::read_from_disk(&cache_dir1, &canonical_root1)
7707                .expect("load flushed root one index");
7708        restored1.ready = true;
7709        assert_eq!(
7710            restored1
7711                .grep("new actor one token", true, &[], &[], &canonical_root1, 10)
7712                .matches
7713                .len(),
7714            1,
7715            "graceful subc shutdown should flush the first root's trigram delta"
7716        );
7717
7718        let mut restored2 =
7719            crate::search_index::SearchIndex::read_from_disk(&cache_dir2, &canonical_root2)
7720                .expect("load flushed root two index");
7721        restored2.ready = true;
7722        assert_eq!(
7723            restored2
7724                .grep("new actor two token", true, &[], &[], &canonical_root2, 10)
7725                .matches
7726                .len(),
7727            1,
7728            "graceful subc shutdown should flush every registered root"
7729        );
7730    }
7731
7732    #[test]
7733    fn idle_root_reaper_closes_artifacts_and_stops_watcher() {
7734        let _ = env_logger::builder().is_test(true).try_init();
7735        let (root_dir, root) = test_root("idle-root-reaper");
7736        let storage = tempfile::tempdir().expect("storage tempdir");
7737        std::fs::write(
7738            root_dir.path().join("main.rs"),
7739            "fn entry() { leaf(); }\nfn leaf() {}\n",
7740        )
7741        .expect("source file");
7742        let canonical_root = std::fs::canonicalize(root_dir.path()).expect("canonical root");
7743        let app = App::default_shared();
7744        let ctx = Arc::new(AppContext::from_app(
7745            Arc::clone(&app),
7746            Config {
7747                project_root: Some(canonical_root.clone()),
7748                storage_dir: Some(storage.path().to_path_buf()),
7749                callgraph_store: true,
7750                search_index: true,
7751                ..Config::default()
7752            },
7753        ));
7754        ctx.set_canonical_cache_root(canonical_root.clone());
7755        let project_key = crate::search_index::artifact_cache_key(&canonical_root);
7756        crate::root_cache::configure_artifact_access(&canonical_root, &project_key, false);
7757        assert!(ctx
7758            .ensure_callgraph_store()
7759            .expect("build callgraph store")
7760            .is_some());
7761
7762        let cache_dir =
7763            crate::search_index::resolve_cache_dir(&canonical_root, Some(storage.path()));
7764        let mut index = crate::search_index::SearchIndex::build(&canonical_root);
7765        let git_head = index.stored_git_head().map(str::to_owned);
7766        index.write_to_disk(&cache_dir, git_head.as_deref());
7767        *ctx.search_index()
7768            .write()
7769            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
7770        // Seed a completed warm verification so the test can prove eviction
7771        // downgrades it to Strict rather than passing vacuously.
7772        let seeded_generation =
7773            crate::cache_freshness::artifact_generation(&cache_dir.join("cache.bin"))
7774                .expect("seeded artifact generation");
7775        crate::cache_freshness::record_verify_completed(
7776            &canonical_root,
7777            crate::cache_freshness::VerifyArtifact::Search,
7778            Some(seeded_generation),
7779        );
7780        assert!(
7781            matches!(
7782                crate::cache_freshness::warm_verify_plan(
7783                    canonical_root.as_path(),
7784                    crate::cache_freshness::VerifyArtifact::Search,
7785                    Some(seeded_generation),
7786                ),
7787                crate::cache_freshness::WarmVerifyPlan::Skip
7788            ),
7789            "memo must be warm before eviction for the downgrade assertion to bite"
7790        );
7791
7792        let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
7793        let _dispatch_tx = dispatch_tx;
7794        let shutdown = Arc::new(AtomicBool::new(false));
7795        let thread_shutdown = Arc::clone(&shutdown);
7796        let join = std::thread::spawn(move || {
7797            while !thread_shutdown.load(Ordering::SeqCst) {
7798                std::thread::yield_now();
7799            }
7800        });
7801        ctx.install_watcher_runtime(
7802            dispatch_rx,
7803            crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
7804        );
7805        wait_for_watcher_count(&ctx, 1);
7806
7807        let executor = Arc::new(Executor::new());
7808        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7809        ctx.mark_subc_unbound();
7810        let mut live_roots = HashMap::new();
7811        let mut meta = RootMeta::new(Instant::now());
7812        meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
7813        meta.unbound_quiesced = true;
7814        live_roots.insert(root.clone(), meta);
7815
7816        let message = idle_root_eviction_message(&root, &ctx.memory_root_snapshot(), None);
7817        assert!(message.contains("evicted idle root"));
7818        assert!(message.contains("freed ~"));
7819        assert!(message.contains("semantic"));
7820        assert!(!message.contains("semantic not estimated retained"));
7821        assert!(message.contains("trigram"));
7822        assert!(message.contains("retained: bash"));
7823        assert!(message.contains("parser_pool"));
7824
7825        assert_eq!(
7826            reap_idle_roots(
7827                Instant::now(),
7828                &mut live_roots,
7829                &HashMap::new(),
7830                &HashMap::new(),
7831                &executor,
7832                &DispatchPathMetrics::new(),
7833            )
7834            .evicted,
7835            1
7836        );
7837        assert!(ctx.search_index().read().unwrap().is_none());
7838        wait_for_watcher_count(&ctx, 0);
7839        // The watcher stopped with the eviction, so the idle interval is
7840        // unobserved: the pre-seeded warm-verify memo (Skip) must fall back to
7841        // strict content verification (stat-first would miss same-size,
7842        // preserved-mtime edits made while nobody was watching).
7843        assert!(
7844            matches!(
7845                crate::cache_freshness::warm_verify_plan(
7846                    canonical_root.as_path(),
7847                    crate::cache_freshness::VerifyArtifact::Search,
7848                    Some(seeded_generation),
7849                ),
7850                crate::cache_freshness::WarmVerifyPlan::Strict
7851            ),
7852            "idle eviction must force strict re-verification"
7853        );
7854        assert!(
7855            crate::search_index::SearchIndex::read_from_disk(&cache_dir, &canonical_root).is_some()
7856        );
7857        ctx.mark_subc_bound();
7858        assert!(ctx
7859            .ensure_callgraph_store()
7860            .expect("reopen callgraph store")
7861            .is_some());
7862        assert!(live_roots[&root].idle_artifacts_evicted);
7863    }
7864
7865    #[test]
7866    fn idle_root_reaper_applies_ttl_to_unbound_roots() {
7867        let (_root_dir, root) = test_root("idle-root-ttl-gate");
7868        let ctx = test_ctx();
7869        let executor = Arc::new(Executor::new());
7870        assert!(executor.register_actor(root.clone(), ctx));
7871        let ctx = executor.actor_context(&root).expect("actor context");
7872        ctx.mark_subc_unbound();
7873        let now = Instant::now();
7874        let mut meta = RootMeta::new(now);
7875        meta.unbound_quiesced = true;
7876        let mut live_roots = HashMap::from([(root.clone(), meta)]);
7877
7878        // A recently-unbound root stays warm: a transient unbind (host
7879        // restart) must not pay the strict-verify + forced-rebuild teardown
7880        // on the next maintenance sweep.
7881        assert_eq!(
7882            reap_idle_roots(
7883                now,
7884                &mut live_roots,
7885                &HashMap::new(),
7886                &HashMap::new(),
7887                &executor,
7888                &DispatchPathMetrics::new(),
7889            )
7890            .evicted,
7891            0
7892        );
7893        assert!(!live_roots[&root].idle_artifacts_evicted);
7894
7895        // Past the TTL the same unbound root pays the full teardown. The
7896        // pending reconciliation paths retained across the transient-unbind
7897        // window would block eviction forever through
7898        // `artifact_eviction_blocked`; the reaper disposes them because the
7899        // strict gap invalidation subsumes their purpose.
7900        ctx.add_pending_search_index_paths([root.as_path().join("retained.rs")]);
7901        assert_eq!(
7902            reap_idle_roots(
7903                now + IDLE_ROOT_TTL,
7904                &mut live_roots,
7905                &HashMap::new(),
7906                &HashMap::new(),
7907                &executor,
7908                &DispatchPathMetrics::new(),
7909            )
7910            .evicted,
7911            1
7912        );
7913        assert!(live_roots[&root].idle_artifacts_evicted);
7914        assert!(
7915            ctx.take_pending_search_index_paths().is_empty(),
7916            "TTL eviction must dispose retained pending reconciliation paths"
7917        );
7918    }
7919
7920    #[test]
7921    fn blocked_ttl_eviction_restores_taken_pending_reconciliation_state() {
7922        let (_root_dir, root) = test_root("ttl-eviction-blocked-restore");
7923        let ctx = test_ctx();
7924        let executor = Arc::new(Executor::new());
7925        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7926        ctx.mark_subc_unbound();
7927
7928        // Retained pending path from the transient-unbind window plus a
7929        // SECONDARY eviction blocker (a non-ready resident search index, the
7930        // dirty-index blocker in artifact_eviction_blocked). Disposal must be
7931        // transactional: the blocked eviction may be followed by a rebind, and
7932        // the path is the only repair record for its consumed watcher event.
7933        let pending = root.as_path().join("edited-while-unbound.rs");
7934        ctx.add_pending_search_index_paths([pending.clone()]);
7935        let dirty_source = root.as_path().join("dirty.rs");
7936        std::fs::write(&dirty_source, "fn dirty() {}\n").expect("dirty source");
7937        let mut dirty = crate::search_index::SearchIndex::new();
7938        dirty.ready = true;
7939        dirty.update_file(&dirty_source);
7940        *ctx.search_index()
7941            .write()
7942            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(dirty);
7943        assert!(ctx.artifact_eviction_blocked());
7944
7945        let mut live_roots = HashMap::new();
7946        let mut meta = RootMeta::new(Instant::now());
7947        meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
7948        meta.unbound_quiesced = true;
7949        live_roots.insert(root.clone(), meta);
7950
7951        assert_eq!(
7952            reap_idle_roots(
7953                Instant::now(),
7954                &mut live_roots,
7955                &HashMap::new(),
7956                &HashMap::new(),
7957                &executor,
7958                &DispatchPathMetrics::new(),
7959            )
7960            .evicted,
7961            0,
7962            "the dirty index must still block this eviction"
7963        );
7964        assert_eq!(
7965            ctx.take_pending_search_index_paths(),
7966            vec![pending],
7967            "a blocked eviction must restore the taken pending paths"
7968        );
7969    }
7970
7971    #[test]
7972    fn idle_reap_with_bound_route_keeps_watcher_running() {
7973        let (_root_dir, root) = test_root("bound-root-reap-gate");
7974        let ctx = test_ctx();
7975        let executor = Arc::new(Executor::new());
7976        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7977
7978        let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
7979        let _dispatch_tx = dispatch_tx;
7980        let shutdown = Arc::new(AtomicBool::new(false));
7981        let thread_shutdown = Arc::clone(&shutdown);
7982        let join = std::thread::spawn(move || {
7983            while !thread_shutdown.load(Ordering::SeqCst) {
7984                std::thread::yield_now();
7985            }
7986        });
7987        ctx.install_watcher_runtime(
7988            dispatch_rx,
7989            crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
7990        );
7991
7992        let mut meta = RootMeta::new(Instant::now());
7993        meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
7994        let mut live_roots = HashMap::from([(root.clone(), meta)]);
7995        let bound = HashMap::from([(root, HashSet::from([route_key(7, 1)]))]);
7996        assert_eq!(
7997            reap_idle_roots(
7998                Instant::now(),
7999                &mut live_roots,
8000                &HashMap::new(),
8001                &bound,
8002                &executor,
8003                &DispatchPathMetrics::new(),
8004            )
8005            .evicted,
8006            0
8007        );
8008        wait_for_watcher_count(&ctx, 1);
8009        ctx.stop_watcher_runtime_in_background();
8010        wait_for_watcher_count(&ctx, 0);
8011    }
8012
8013    #[test]
8014    fn deleted_root_with_bound_route_is_reclaimed_after_confirmation_and_routes_are_purged() {
8015        let (root_dir, root) = test_root("deleted-bound-root-reap");
8016        let executor = Arc::new(Executor::new());
8017        assert!(executor.register_actor(root.clone(), test_ctx()));
8018        root_dir.close().expect("delete project root");
8019
8020        let route = route_key(19, 3);
8021        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
8022        let cancel_signal = PersistentCancelSignal::new();
8023        let mut routes = HashMap::from([(route, route_identity(&root, "deleted-route"))]);
8024        let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
8025        let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
8026        let mut route_bash_cancels = HashMap::from([(
8027            route,
8028            bash::RouteBashCancel {
8029                token: cancel_signal.clone(),
8030                active_waits: 0,
8031            },
8032        )]);
8033        let metrics = DispatchPathMetrics::new();
8034
8035        let first = reap_idle_roots(
8036            Instant::now(),
8037            &mut live_roots,
8038            &HashMap::new(),
8039            &root_channels,
8040            &executor,
8041            &metrics,
8042        );
8043        assert!(first.forgotten_deleted_roots.is_empty());
8044        assert!(executor.actor_registered(&root));
8045
8046        let mut forgotten = Vec::new();
8047        for _ in 0..100 {
8048            let outcome = reap_idle_roots(
8049                Instant::now(),
8050                &mut live_roots,
8051                &HashMap::new(),
8052                &root_channels,
8053                &executor,
8054                &metrics,
8055            );
8056            if !outcome.forgotten_deleted_roots.is_empty() {
8057                forgotten = outcome.forgotten_deleted_roots;
8058                break;
8059            }
8060            std::thread::sleep(Duration::from_millis(10));
8061        }
8062        assert_eq!(forgotten, vec![root.clone()]);
8063        assert!(!executor.actor_registered(&root));
8064
8065        let mut retry_buffer = HashMap::new();
8066        let mut reclaimed_routes = ReclaimedRoutes::default();
8067        let mut session_identity = HashMap::new();
8068        let mut push_buffer = HashMap::new();
8069        let mut bg_subs = HashMap::from([(
8070            route,
8071            BgSub {
8072                corr: 77,
8073                ver: PROTOCOL_VERSION,
8074                flags: control_flags(),
8075                root: root.clone(),
8076                session: "deleted-route".to_string(),
8077            },
8078        )]);
8079        let mut bg_sub_by_session = HashMap::from([(
8080            (root.clone(), "deleted-route".to_string()),
8081            HashSet::from([route]),
8082        )]);
8083        let mut bg_wake_pending =
8084            BgWakePending::from([(route, BgWakeState::armed(Instant::now()))]);
8085        let mut bg_wake_epoch = HashMap::new();
8086        let mut pending_bash_asks = HashMap::new();
8087        let active_tool_calls: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
8088        health::take_bg_observability_logs_for_test();
8089        purge_deleted_root_residents(
8090            &root,
8091            &mut routes,
8092            &mut root_channels,
8093            &mut installed_route_epochs,
8094            &mut route_bash_cancels,
8095            &active_tool_calls,
8096            executor.as_ref(),
8097            &mut retry_buffer,
8098            &mut reclaimed_routes,
8099            &mut session_identity,
8100            &mut push_buffer,
8101            &mut bg_subs,
8102            &mut bg_sub_by_session,
8103            &mut bg_wake_pending,
8104            &mut bg_wake_epoch,
8105            &mut pending_bash_asks,
8106            &metrics,
8107        );
8108
8109        assert!(routes.is_empty());
8110        assert!(root_channels.is_empty());
8111        assert!(installed_route_epochs.is_empty());
8112        assert!(route_bash_cancels.is_empty());
8113        assert!(reclaimed_routes.contains(route));
8114        assert!(cancel_signal.is_cancelled());
8115        assert_eq!(
8116            health::take_bg_observability_logs_for_test(),
8117            vec![format!(
8118                "subc bg subscription: ended root={} session=deleted-route channel=19@3 cause=root-reclaim suppressed=0",
8119                root.as_path().display()
8120            )]
8121        );
8122    }
8123
8124    /// The control for the deleted-root reclamation above: a root whose
8125    /// directory still EXISTS must stay retained while it holds a bound route,
8126    /// even with every other reap precondition satisfied. Without this, the
8127    /// suite cannot tell "reclaim roots that are provably gone" apart from
8128    /// "reap any root that looks idle" — the second would tear down live
8129    /// sessions, and both satisfy the deleted-root tests.
8130    #[test]
8131    fn live_root_with_bound_route_is_never_reclaimed() {
8132        let (_root_dir, root) = test_root("live-bound-root-retained");
8133        let ctx = test_ctx();
8134        ctx.mark_subc_unbound();
8135        let executor = Arc::new(Executor::new());
8136        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
8137
8138        let route = RouteChannel {
8139            channel: 7,
8140            epoch: 1,
8141        };
8142        let mut meta = RootMeta::new(Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1));
8143        meta.unbound_quiesced = true;
8144        let mut live_roots = HashMap::from([(root.clone(), meta)]);
8145        let root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
8146
8147        // Sweep three times because reclamation requires the root path to be
8148        // observed missing on two CONSECUTIVE sweeps. A single sweep would pass
8149        // here even if reclamation were wrongly unconditional, since the first
8150        // absence never reclaims on its own.
8151        for _ in 0..3 {
8152            let outcome = reap_idle_roots(
8153                Instant::now(),
8154                &mut live_roots,
8155                &HashMap::new(),
8156                &root_channels,
8157                &executor,
8158                &DispatchPathMetrics::new(),
8159            );
8160            assert!(
8161                outcome.forgotten_deleted_roots.is_empty(),
8162                "a root whose directory exists must never be forgotten"
8163            );
8164        }
8165
8166        assert!(live_roots.contains_key(&root), "live root must be retained");
8167        assert!(
8168            executor.actor_registered(&root),
8169            "live root's actor must survive"
8170        );
8171        assert!(
8172            root.as_path().exists(),
8173            "test vehicle must keep the directory alive; otherwise this control proves nothing"
8174        );
8175    }
8176
8177    #[test]
8178    fn deleted_root_is_not_reclaimed_on_first_absence_observation() {
8179        let (root_dir, root) = test_root("deleted-root-first-observation");
8180        let ctx = test_ctx();
8181        ctx.mark_subc_unbound();
8182        let executor = Arc::new(Executor::new());
8183        assert!(executor.register_actor(root.clone(), ctx));
8184        root_dir.close().expect("delete project root");
8185
8186        let mut meta = RootMeta::new(Instant::now());
8187        meta.unbound_quiesced = true;
8188        let mut live_roots = HashMap::from([(root.clone(), meta)]);
8189        let outcome = reap_idle_roots(
8190            Instant::now(),
8191            &mut live_roots,
8192            &HashMap::new(),
8193            &HashMap::new(),
8194            &executor,
8195            &DispatchPathMetrics::new(),
8196        );
8197
8198        assert!(outcome.forgotten_deleted_roots.is_empty());
8199        assert!(live_roots.contains_key(&root));
8200        assert!(executor.actor_registered(&root));
8201    }
8202
8203    fn spawn_background_for_root(
8204        ctx: &AppContext,
8205        root: &ProjectRootId,
8206        storage: &tempfile::TempDir,
8207        session_id: &str,
8208    ) -> (String, u32) {
8209        // Windows refuses to delete a directory that is a running process's
8210        // cwd (ERROR_SHARING_VIOLATION), so the task must not live inside the
8211        // project root these tests delete. The kill path matches on the task's
8212        // registered project_root, not its cwd, so pointing the cwd at task
8213        // storage keeps the association under test intact.
8214        let command = if cfg!(windows) {
8215            // timeout.exe requires a console; ping is the standard sleep shim.
8216            "ping -n 31 127.0.0.1 > nul"
8217        } else {
8218            "sleep 30"
8219        };
8220        let task_id = ctx
8221            .bash_background()
8222            .spawn(
8223                crate::sandbox_spawn::SpawnPlan::Unsandboxed,
8224                command,
8225                session_id.to_string(),
8226                storage.path().to_path_buf(),
8227                HashMap::new(),
8228                Some(Duration::from_secs(60)),
8229                storage.path().to_path_buf(),
8230                8,
8231                true,
8232                false,
8233                Some(root.as_path().to_path_buf()),
8234            )
8235            .expect("spawn background task");
8236        let snapshot = ctx
8237            .bash_background()
8238            .status(
8239                &task_id,
8240                session_id,
8241                Some(root.as_path()),
8242                Some(storage.path()),
8243                0,
8244            )
8245            .expect("background task status");
8246        (task_id, snapshot.child_pid.expect("background child pid"))
8247    }
8248
8249    fn wait_for_background_exit(pid: u32) {
8250        let deadline = Instant::now() + Duration::from_secs(5);
8251        while crate::bash_background::process::is_process_alive(pid) {
8252            assert!(
8253                Instant::now() < deadline,
8254                "background task process survived kill"
8255            );
8256            std::thread::sleep(Duration::from_millis(20));
8257        }
8258    }
8259
8260    #[test]
8261    fn deleted_root_reclaims_background_task_after_two_absence_sweeps() {
8262        let (root_dir, root) = test_root("deleted-root-background-task");
8263        let storage = tempfile::tempdir().expect("task storage");
8264        let ctx = test_ctx();
8265        let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "reclaim-session");
8266        assert!(crate::bash_background::process::is_process_alive(pid));
8267
8268        let executor = Arc::new(Executor::new());
8269        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
8270        assert!(executor.actor_is_idle(&root));
8271        root_dir.close().expect("delete project root");
8272        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
8273        let pending_binds = HashMap::new();
8274        let root_channels = HashMap::new();
8275        let metrics = DispatchPathMetrics::new();
8276
8277        let first = reap_idle_roots(
8278            Instant::now(),
8279            &mut live_roots,
8280            &pending_binds,
8281            &root_channels,
8282            &executor,
8283            &metrics,
8284        );
8285        assert!(first.forgotten_deleted_roots.is_empty());
8286        assert!(crate::bash_background::process::is_process_alive(pid));
8287
8288        let outcome = reap_until_forgotten(
8289            &root,
8290            &mut live_roots,
8291            &pending_binds,
8292            &root_channels,
8293            &executor,
8294            &metrics,
8295        );
8296        assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
8297        wait_for_background_exit(pid);
8298
8299        let snapshot = ctx
8300            .bash_background()
8301            .status(
8302                &task_id,
8303                "reclaim-session",
8304                Some(root.as_path()),
8305                Some(storage.path()),
8306                0,
8307            )
8308            .expect("reclaimed task status");
8309        assert_eq!(snapshot.info.status, BgTaskStatus::Killed);
8310        assert_eq!(
8311            snapshot.info.status_reason.as_deref(),
8312            Some(crate::bash_background::registry::ROOT_RECLAIMED_REASON)
8313        );
8314        assert_eq!(
8315            serde_json::to_value(&snapshot).expect("serialize bash status")["status_reason"],
8316            crate::bash_background::registry::ROOT_RECLAIMED_REASON
8317        );
8318        let completion = ctx
8319            .bash_background()
8320            .drain_completions_for_session(Some("reclaim-session"))
8321            .pop()
8322            .expect("reclaimed task completion");
8323        assert_eq!(
8324            completion.status_reason.as_deref(),
8325            Some(crate::bash_background::registry::ROOT_RECLAIMED_REASON)
8326        );
8327    }
8328
8329    #[test]
8330    fn existing_unbound_root_keeps_background_task_alive_across_sweeps() {
8331        let (root_dir, root) = test_root("existing-root-background-task");
8332        let storage = tempfile::tempdir().expect("task storage");
8333        let ctx = test_ctx();
8334        ctx.mark_subc_unbound();
8335        let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "existing-session");
8336
8337        let executor = Arc::new(Executor::new());
8338        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
8339        let mut meta = RootMeta::new(
8340            Instant::now()
8341                .checked_sub(IDLE_ROOT_TTL + Duration::from_secs(1))
8342                .expect("old root timestamp"),
8343        );
8344        meta.unbound_quiesced = true;
8345        let mut live_roots = HashMap::from([(root.clone(), meta)]);
8346        let pending_binds = HashMap::new();
8347        let root_channels = HashMap::new();
8348        let metrics = DispatchPathMetrics::new();
8349
8350        for _ in 0..8 {
8351            reap_idle_roots(
8352                Instant::now(),
8353                &mut live_roots,
8354                &pending_binds,
8355                &root_channels,
8356                &executor,
8357                &metrics,
8358            );
8359            std::thread::sleep(Duration::from_millis(10));
8360        }
8361        assert!(root_dir.path().exists());
8362        assert!(crate::bash_background::process::is_process_alive(pid));
8363        let snapshot = ctx
8364            .bash_background()
8365            .status(
8366                &task_id,
8367                "existing-session",
8368                Some(root.as_path()),
8369                Some(storage.path()),
8370                0,
8371            )
8372            .expect("existing task status");
8373        assert_eq!(snapshot.info.status, BgTaskStatus::Running);
8374        let _ = ctx.bash_background().kill(&task_id, "existing-session");
8375        wait_for_background_exit(pid);
8376    }
8377
8378    #[test]
8379    fn restored_root_between_absence_sweeps_keeps_background_task_alive() {
8380        let (root_dir, root) = test_root("restored-root-background-task");
8381        let storage = tempfile::tempdir().expect("task storage");
8382        let ctx = test_ctx();
8383        let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "restored-session");
8384
8385        let executor = Arc::new(Executor::new());
8386        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
8387        root_dir.close().expect("delete project root");
8388        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
8389        let pending_binds = HashMap::new();
8390        let root_channels = HashMap::new();
8391        let metrics = DispatchPathMetrics::new();
8392
8393        let first = reap_idle_roots(
8394            Instant::now(),
8395            &mut live_roots,
8396            &pending_binds,
8397            &root_channels,
8398            &executor,
8399            &metrics,
8400        );
8401        assert!(first.forgotten_deleted_roots.is_empty());
8402        std::fs::create_dir_all(root.as_path()).expect("restore project root");
8403        let second = reap_idle_roots(
8404            Instant::now(),
8405            &mut live_roots,
8406            &pending_binds,
8407            &root_channels,
8408            &executor,
8409            &metrics,
8410        );
8411        assert!(second.forgotten_deleted_roots.is_empty());
8412        assert!(crate::bash_background::process::is_process_alive(pid));
8413        let _ = ctx.bash_background().kill(&task_id, "restored-session");
8414        wait_for_background_exit(pid);
8415    }
8416
8417    #[test]
8418    fn observing_root_again_resets_deleted_sweep_confirmation() {
8419        let (root_dir, root) = test_root("deleted-root-observation-reset");
8420        let ctx = test_ctx();
8421        ctx.mark_subc_unbound();
8422        let executor = Arc::new(Executor::new());
8423        assert!(executor.register_actor(root.clone(), ctx));
8424        root_dir.close().expect("delete project root");
8425
8426        let mut meta = RootMeta::new(Instant::now());
8427        meta.unbound_quiesced = true;
8428        let mut live_roots = HashMap::from([(root.clone(), meta)]);
8429        let pending_binds = HashMap::new();
8430        let root_channels = HashMap::new();
8431        let metrics = DispatchPathMetrics::new();
8432
8433        let first = reap_idle_roots(
8434            Instant::now(),
8435            &mut live_roots,
8436            &pending_binds,
8437            &root_channels,
8438            &executor,
8439            &metrics,
8440        );
8441        assert!(first.forgotten_deleted_roots.is_empty());
8442
8443        std::fs::create_dir_all(root.as_path()).expect("restore project root");
8444        reap_idle_roots(
8445            Instant::now(),
8446            &mut live_roots,
8447            &pending_binds,
8448            &root_channels,
8449            &executor,
8450            &metrics,
8451        );
8452        std::fs::remove_dir_all(root.as_path()).expect("delete project root again");
8453
8454        let after_reset = reap_idle_roots(
8455            Instant::now(),
8456            &mut live_roots,
8457            &pending_binds,
8458            &root_channels,
8459            &executor,
8460            &metrics,
8461        );
8462        assert!(after_reset.forgotten_deleted_roots.is_empty());
8463        assert!(live_roots.contains_key(&root));
8464        assert!(executor.actor_registered(&root));
8465    }
8466
8467    #[test]
8468    fn deleted_idle_root_is_fully_forgotten_and_status_counts_drop() {
8469        let (root_dir, root) = test_root("deleted-root-reap");
8470        let app = App::default_shared();
8471        let ctx = Arc::new(AppContext::from_app(
8472            Arc::clone(&app),
8473            Config {
8474                project_root: Some(root.as_path().to_path_buf()),
8475                ..Config::default()
8476            },
8477        ));
8478        ctx.set_canonical_cache_root(root.as_path().to_path_buf());
8479        ctx.mark_subc_unbound();
8480        let executor = Arc::new(Executor::new());
8481        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
8482        assert_eq!(app.actor_root_count(), 1);
8483        drop(ctx);
8484        root_dir.close().expect("delete project root");
8485
8486        let mut meta = RootMeta::new(Instant::now());
8487        meta.unbound_quiesced = true;
8488        let mut live_roots = HashMap::from([(root.clone(), meta)]);
8489        let outcome = reap_until_forgotten(
8490            &root,
8491            &mut live_roots,
8492            &HashMap::new(),
8493            &HashMap::new(),
8494            &executor,
8495            &DispatchPathMetrics::new(),
8496        );
8497        assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
8498        assert!(!executor.actor_registered(&root));
8499        assert!(!live_roots.contains_key(&root));
8500        wait_for_actor_root_count(&app, 0);
8501
8502        let status_ctx = AppContext::from_app(app, Config::default());
8503        let status = status_ctx.build_status_snapshot();
8504        assert_eq!(status["runtime"]["live_actor_roots"], 0);
8505        assert_eq!(status["runtime"]["open_routes"], 0);
8506    }
8507
8508    #[test]
8509    fn deleted_root_reap_blocker_census_is_exposed_in_health_metrics() {
8510        let (root_dir, root) = test_root("deleted-root-reap-census");
8511        let executor = Arc::new(Executor::new());
8512        assert!(executor.register_actor(root.clone(), test_ctx()));
8513        root_dir.close().expect("delete project root");
8514
8515        let mut live_roots = HashMap::from([(root, RootMeta::new(Instant::now()))]);
8516        let metrics = DispatchPathMetrics::new();
8517        let outcome = reap_idle_roots(
8518            Instant::now(),
8519            &mut live_roots,
8520            &HashMap::new(),
8521            &HashMap::new(),
8522            &executor,
8523            &metrics,
8524        );
8525        assert_eq!(outcome.evicted, 0);
8526
8527        let app = crate::context::App::default_shared();
8528        let health_rollup_cache = HealthRollupCache::new();
8529        health_rollup_cache.refresh(&executor, &app);
8530        let report = build_health_report(
8531            &health_rollup_cache,
8532            &executor,
8533            &HashMap::new(),
8534            &metrics,
8535            &app,
8536        );
8537        let reap = report
8538            .metrics
8539            .as_ref()
8540            .and_then(|metrics| metrics.get("reap"))
8541            .expect("reap health metrics");
8542        assert_eq!(reap["deleted_retained"].as_u64(), Some(1));
8543        assert_eq!(reap["blockers"]["absence_unconfirmed"].as_u64(), Some(1));
8544        assert_eq!(reap["blockers"]["unbound_quiesced"].as_u64(), Some(0));
8545        assert_eq!(reap["blockers"]["actor_busy"].as_u64(), Some(0));
8546    }
8547
8548    #[test]
8549    fn connection_exit_quiesces_queued_maintenance_and_deleted_root_is_purged() {
8550        let (root_dir, root) = test_root("connection-exit-deleted-root");
8551        let executor = Arc::new(Executor::new());
8552        assert!(executor.register_actor(root.clone(), test_ctx()));
8553
8554        let route = route_key(11, 1);
8555        let mut meta = RootMeta::new(Instant::now());
8556        meta.maintenance_pending = true;
8557        meta.maintenance_queued_kinds
8558            .push_back(MaintenanceDrainKind::CompletionDrains);
8559        let mut live_roots = HashMap::from([(root.clone(), meta)]);
8560        let mut pending_binds = HashMap::new();
8561        let mut routes = HashMap::from([(route, route_identity(&root, "abandoned"))]);
8562        let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
8563        let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
8564        let mut route_bash_cancels = HashMap::new();
8565        let active_tool_calls: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
8566
8567        quiesce_connection_roots(
8568            &mut live_roots,
8569            &mut pending_binds,
8570            &mut routes,
8571            &mut root_channels,
8572            &mut installed_route_epochs,
8573            &mut route_bash_cancels,
8574            &active_tool_calls,
8575            &executor,
8576        );
8577        assert!(live_roots[&root].unbound_quiesced);
8578        assert!(!live_roots[&root].maintenance_pending);
8579        assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
8580        assert!(routes.is_empty());
8581        assert!(root_channels.is_empty());
8582
8583        root_dir.close().expect("delete project root");
8584        let metrics = DispatchPathMetrics::new();
8585        let outcome = reap_until_forgotten(
8586            &root,
8587            &mut live_roots,
8588            &pending_binds,
8589            &root_channels,
8590            &executor,
8591            &metrics,
8592        );
8593        let mut session_identity = HashMap::new();
8594        let mut push_buffer = HashMap::new();
8595        let mut bg_subs = HashMap::new();
8596        let mut bg_sub_by_session = HashMap::new();
8597        let mut bg_wake_pending = BgWakePending::new();
8598        let mut bg_wake_epoch = HashMap::new();
8599        let mut pending_bash_asks = HashMap::new();
8600        let mut retry_buffer = HashMap::new();
8601        let mut reclaimed_routes = ReclaimedRoutes::default();
8602        for forgotten in &outcome.forgotten_deleted_roots {
8603            purge_deleted_root_residents(
8604                forgotten,
8605                &mut routes,
8606                &mut root_channels,
8607                &mut installed_route_epochs,
8608                &mut route_bash_cancels,
8609                &active_tool_calls,
8610                executor.as_ref(),
8611                &mut retry_buffer,
8612                &mut reclaimed_routes,
8613                &mut session_identity,
8614                &mut push_buffer,
8615                &mut bg_subs,
8616                &mut bg_sub_by_session,
8617                &mut bg_wake_pending,
8618                &mut bg_wake_epoch,
8619                &mut pending_bash_asks,
8620                &metrics,
8621            );
8622        }
8623
8624        assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
8625        assert!(!executor.actor_registered(&root));
8626        assert!(!live_roots.contains_key(&root));
8627    }
8628
8629    #[test]
8630    fn unbound_root_quiesces_maintenance_without_removing_actor() {
8631        let (_root_dir, root) = test_root("unbound-root-quiesce");
8632        let ctx = test_ctx();
8633        let executor = Arc::new(Executor::new());
8634        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
8635        let mut meta = RootMeta::new(Instant::now());
8636        meta.maintenance_pending = true;
8637        meta.maintenance_jobs_in_flight = 1;
8638        meta.maintenance_queued_kinds
8639            .push_back(MaintenanceDrainKind::ConfigureTail);
8640        let mut live_roots = HashMap::from([(root.clone(), meta)]);
8641        // Warm state planted before the unbind: quiesce must keep it.
8642        *ctx.search_index()
8643            .write()
8644            .unwrap_or_else(std::sync::PoisonError::into_inner) =
8645            Some(crate::search_index::SearchIndex::new());
8646        ctx.set_cache_writer_capabilities(true, true);
8647        let pending = root.as_path().join("pending.rs");
8648        ctx.add_pending_search_index_paths([pending.clone()]);
8649        // A warm verify memo must survive the transient unbind: the watcher
8650        // keeps running, so no unobserved window exists and the next warm
8651        // reload must not pay a strict full-corpus re-hash.
8652        let canonical_root = root.as_path().to_path_buf();
8653        let artifact = canonical_root.join("cache.bin");
8654        std::fs::write(&artifact, b"warm-artifact").expect("write artifact");
8655        let seeded_generation = crate::cache_freshness::artifact_generation(&artifact);
8656        crate::cache_freshness::record_verify_completed(
8657            &canonical_root,
8658            crate::cache_freshness::VerifyArtifact::Search,
8659            seeded_generation,
8660        );
8661        assert!(matches!(
8662            crate::cache_freshness::warm_verify_plan(
8663                &canonical_root,
8664                crate::cache_freshness::VerifyArtifact::Search,
8665                seeded_generation,
8666            ),
8667            crate::cache_freshness::WarmVerifyPlan::Skip
8668        ));
8669        // A live watcher runtime must survive quiesce (its events accumulate
8670        // for the rebind replay).
8671        let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
8672        let _dispatch_tx = dispatch_tx;
8673        let shutdown = Arc::new(AtomicBool::new(false));
8674        let thread_shutdown = Arc::clone(&shutdown);
8675        let join = std::thread::spawn(move || {
8676            while !thread_shutdown.load(Ordering::SeqCst) {
8677                std::thread::yield_now();
8678            }
8679        });
8680        ctx.install_watcher_runtime(
8681            dispatch_rx,
8682            crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
8683        );
8684        assert!(ctx.watcher_runtime_active());
8685
8686        quiesce_unbound_root(&root, &mut live_roots, &executor);
8687        let meta = &live_roots[&root];
8688        assert!(meta.unbound_quiesced);
8689        assert!(ctx.subc_unbound_quiesced());
8690        assert!(meta.maintenance_pending);
8691        assert!(meta.maintenance_queued_kinds.is_empty());
8692        assert!(executor.actor_registered(&root));
8693        // Transient unbind keeps the root warm: resident artifacts stay
8694        // resident, no forced callgraph rebuild is planted, and pending
8695        // reconciliation paths survive for the rebind replay.
8696        assert!(
8697            ctx.search_index()
8698                .read()
8699                .unwrap_or_else(std::sync::PoisonError::into_inner)
8700                .is_some(),
8701            "quiesce must not evict resident artifacts"
8702        );
8703        assert_eq!(
8704            ctx.pending_callgraph_store_force_token(),
8705            None,
8706            "quiesce must not force a callgraph rebuild"
8707        );
8708        assert_eq!(
8709            ctx.take_pending_search_index_paths(),
8710            vec![pending],
8711            "quiesce must retain pending watcher-derived paths"
8712        );
8713        assert!(
8714            matches!(
8715                crate::cache_freshness::warm_verify_plan(
8716                    &canonical_root,
8717                    crate::cache_freshness::VerifyArtifact::Search,
8718                    seeded_generation,
8719                ),
8720                crate::cache_freshness::WarmVerifyPlan::Skip
8721            ),
8722            "quiesce must not invalidate the warm verify memo"
8723        );
8724        assert!(
8725            ctx.watcher_runtime_active(),
8726            "quiesce must not stop a running watcher"
8727        );
8728        ctx.stop_watcher_runtime();
8729
8730        let meta = live_roots.get_mut(&root).expect("root metadata");
8731        note_maintenance_completion(
8732            meta,
8733            Some(MaintenanceDrainKind::ConfigureTail),
8734            false,
8735            meta.unbound_quiesced,
8736        );
8737        assert!(!meta.maintenance_pending);
8738        assert!(meta.maintenance_queued_kinds.is_empty());
8739    }
8740
8741    #[test]
8742    fn same_root_higher_epoch_replacement_does_not_quiesce_between_generations() {
8743        let (_dir, root) = test_root("same-root-replacement");
8744        let route = route_key(7, 1);
8745        let installed_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
8746        let root_channels = HashMap::new();
8747
8748        assert!(!route_removal_will_quiesce_root(
8749            &root,
8750            route,
8751            &installed_channels,
8752            false,
8753            Some(&root),
8754        ));
8755        assert!(route_removal_will_quiesce_root(
8756            &root,
8757            route,
8758            &installed_channels,
8759            false,
8760            None,
8761        ));
8762        assert!(!should_quiesce_removed_root(
8763            &root,
8764            &root_channels,
8765            false,
8766            Some(&root),
8767        ));
8768        assert!(should_quiesce_removed_root(
8769            &root,
8770            &root_channels,
8771            false,
8772            None,
8773        ));
8774        assert!(!should_quiesce_removed_root(
8775            &root,
8776            &root_channels,
8777            true,
8778            None,
8779        ));
8780    }
8781
8782    #[test]
8783    fn root_quiesces_only_after_its_last_route_is_removed_and_reactivates_on_bind() {
8784        let (_root_dir, root) = test_root("unbound-root-route-count");
8785        let executor = Arc::new(Executor::new());
8786        assert!(executor.register_actor(root.clone(), test_ctx()));
8787        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
8788        let mut root_channels = HashMap::from([(
8789            root.clone(),
8790            HashSet::from([route_key(7, 1), route_key(8, 1)]),
8791        )]);
8792
8793        remove_root_channel(&mut root_channels, &root, route_key(7, 1));
8794        if !root_channels.contains_key(&root) {
8795            quiesce_unbound_root(&root, &mut live_roots, &executor);
8796        }
8797        assert!(!live_roots[&root].unbound_quiesced);
8798
8799        remove_root_channel(&mut root_channels, &root, route_key(8, 1));
8800        if !root_channels.contains_key(&root) {
8801            quiesce_unbound_root(&root, &mut live_roots, &executor);
8802        }
8803        assert!(live_roots[&root].unbound_quiesced);
8804
8805        live_roots
8806            .get_mut(&root)
8807            .expect("root metadata")
8808            .note_activity();
8809        assert!(
8810            live_roots[&root].unbound_quiesced,
8811            "late asynchronous activity must not reactivate an unbound root"
8812        );
8813
8814        live_roots
8815            .get_mut(&root)
8816            .expect("root metadata")
8817            .reactivate_bound();
8818        assert!(!live_roots[&root].unbound_quiesced);
8819    }
8820
8821    #[test]
8822    fn allocator_pressure_relief_requires_every_root_to_be_idle() {
8823        let (_idle_dir, idle_root) = test_root("allocator-relief-idle");
8824        let (_active_dir, active_root) = test_root("allocator-relief-active");
8825        let now = Instant::now();
8826        let mut live_roots = HashMap::new();
8827        let mut idle = RootMeta::new(now);
8828        idle.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
8829        live_roots.insert(idle_root, idle);
8830        let executor = Executor::new();
8831        assert!(process_has_been_idle(now, &live_roots, &executor));
8832
8833        live_roots.insert(active_root.clone(), RootMeta::new(now));
8834        assert!(!process_has_been_idle(now, &live_roots, &executor));
8835
8836        let active = live_roots
8837            .get_mut(&active_root)
8838            .expect("active root metadata");
8839        active.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
8840        active.active_bash_waits = 1;
8841        assert!(!process_has_been_idle(now, &live_roots, &executor));
8842    }
8843
8844    #[test]
8845    fn pressure_relief_log_reports_before_and_after_measurements() {
8846        let allocator = crate::memory::AllocatorMemorySnapshot {
8847            status: "measured",
8848            bytes_in_use: Some(8 * 1024 * 1024),
8849            size_allocated: Some(12 * 1024 * 1024),
8850            retained_slack_bytes: Some(4 * 1024 * 1024),
8851            not_estimated: None,
8852        };
8853        let relief = crate::memory::AllocatorPressureRelief {
8854            bytes_released: 3 * 1024 * 1024,
8855            rss_before_bytes: Some(20 * 1024 * 1024),
8856            rss_after_bytes: Some(17 * 1024 * 1024),
8857            allocator_before: allocator.clone(),
8858            allocator_after: crate::memory::AllocatorMemorySnapshot {
8859                size_allocated: Some(9 * 1024 * 1024),
8860                retained_slack_bytes: Some(1024 * 1024),
8861                ..allocator
8862            },
8863        };
8864        let message = pressure_relief_label(&relief);
8865        assert!(message.contains("RSS 20.0 MB -> 17.0 MB"));
8866        assert!(message.contains("allocated 12.0 MB -> 9.0 MB"));
8867        assert!(message.contains("slack 4.0 MB -> 1.0 MB"));
8868        assert!(message.contains("reported 3.0 MB released"));
8869    }
8870
8871    #[test]
8872    fn due_maintenance_jobs_skip_poisoned_roots() {
8873        let (_healthy_dir, healthy_root) = test_root("maintenance-healthy");
8874        let (_poisoned_dir, poisoned_root) = test_root("maintenance-poisoned");
8875        let mut live_roots = HashMap::new();
8876        live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
8877        let mut poisoned_meta = RootMeta::new(Instant::now());
8878        poisoned_meta.maintenance_poisoned = true;
8879        live_roots.insert(poisoned_root.clone(), poisoned_meta);
8880
8881        let (due, deferred) = due_maintenance_jobs_without_actor_context(
8882            &mut live_roots,
8883            MAINTENANCE_SUBMIT_BUDGET,
8884            &HashSet::new(),
8885        );
8886
8887        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
8888        assert!(due.iter().all(|(root, _)| root == &healthy_root));
8889        assert!(!deferred);
8890        assert!(live_roots[&healthy_root].maintenance_pending);
8891        assert_eq!(
8892            live_roots[&healthy_root].maintenance_jobs_in_flight,
8893            INITIAL_MAINTENANCE_JOB_COUNT
8894        );
8895        assert!(!live_roots[&poisoned_root].maintenance_pending);
8896    }
8897
8898    #[test]
8899    fn due_maintenance_jobs_do_not_restart_quiesced_root_work() {
8900        let (_dir, root) = test_root("maintenance-unbound");
8901        let mut meta = RootMeta::new(Instant::now());
8902        meta.unbound_quiesced = true;
8903        let mut live_roots = HashMap::from([(root.clone(), meta)]);
8904
8905        let (due, deferred) = due_maintenance_jobs_without_actor_context(
8906            &mut live_roots,
8907            MAINTENANCE_SUBMIT_BUDGET,
8908            &HashSet::new(),
8909        );
8910
8911        assert!(due.is_empty());
8912        assert!(!deferred);
8913        assert!(!live_roots[&root].maintenance_pending);
8914    }
8915
8916    #[test]
8917    fn idle_bg_subscription_queues_no_jobs_until_a_wake_arrives() {
8918        let (_dir, root) = test_root("maintenance-idle-bg-subscription");
8919        let ctx = test_ctx();
8920        assert!(!ctx.completion_drains_have_work());
8921
8922        let executor = Executor::new();
8923        assert!(executor.register_actor(root.clone(), ctx));
8924        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
8925        let session = "idle-session".to_string();
8926        let channel = route_key(17, 1);
8927        let metrics = DispatchPathMetrics::new();
8928        let bg_sub_by_session =
8929            HashMap::from([((root.clone(), session.clone()), HashSet::from([channel]))]);
8930        let mut bg_wake_pending = BgWakePending::new();
8931
8932        let (idle_tick_jobs, deferred) = due_maintenance_jobs(
8933            &mut live_roots,
8934            Some(&executor),
8935            &bg_sub_by_session,
8936            &bg_wake_pending,
8937            MAINTENANCE_SUBMIT_BUDGET,
8938            &HashSet::new(),
8939        );
8940        assert!(idle_tick_jobs.is_empty());
8941        assert!(!deferred);
8942        assert!(!live_roots[&root].maintenance_pending);
8943
8944        // A completion can arm its wake after the idle tick's probes. The wake
8945        // remains loop-owned state, so the following tick must observe it.
8946        let mut bg_wake_epoch = HashMap::new();
8947        push::arm_bg_wake(
8948            root.clone(),
8949            session,
8950            channel,
8951            &mut bg_wake_pending,
8952            &mut bg_wake_epoch,
8953            &metrics,
8954        );
8955        let (next_tick_jobs, deferred) = due_maintenance_jobs(
8956            &mut live_roots,
8957            Some(&executor),
8958            &bg_sub_by_session,
8959            &bg_wake_pending,
8960            MAINTENANCE_SUBMIT_BUDGET,
8961            &HashSet::new(),
8962        );
8963        assert_eq!(
8964            next_tick_jobs,
8965            vec![(root, MaintenanceDrainKind::CompletionDrains)]
8966        );
8967        assert!(!deferred);
8968    }
8969
8970    async fn assert_slow_configure_tail_admission(
8971        config: crate::executor::ExecutorConfig,
8972        shape: &'static str,
8973    ) {
8974        let root_dir = tempfile::tempdir().unwrap();
8975        let root_path = std::fs::canonicalize(root_dir.path()).unwrap();
8976        let root = ProjectRootId::from_path(&root_path).unwrap();
8977        let ctx = test_ctx();
8978        ctx.mark_subc_bound();
8979        let storage_root = ctx.storage_dir();
8980        ctx.enqueue_configure_maintenance(crate::context::ConfigureMaintenanceJob {
8981            generation: ctx.configure_generation(),
8982            root_path: root_path.clone(),
8983            canonical_cache_root: root_path.clone(),
8984            harness: crate::harness::Harness::Opencode,
8985            storage_root: storage_root.clone(),
8986            harness_dir: storage_root.join("opencode"),
8987            session_id: "first-search-admission".to_string(),
8988            home_match: false,
8989            format_tool_cache_clear_needed: false,
8990            run_bash_replay: false,
8991            refresh_project_runtime: false,
8992            sync_bash_compress_flag: false,
8993            reset_filter_registry: false,
8994            clear_failed_spawns: false,
8995            warm_callgraph_store: false,
8996            supersede_search_artifact_persistence: false,
8997            supersede_callgraph_artifact_persistence: false,
8998            supersede_semantic_artifact_persistence: false,
8999            search_artifact_load_start: None,
9000            semantic_artifact_load_start: None,
9001        })
9002        .expect("queue configure maintenance");
9003        let (_gate, maintenance_reached, release_maintenance) =
9004            crate::commands::configure::gate_configure_deferred_maintenance_for_test(
9005                root_path.clone(),
9006            );
9007
9008        let expected_pool_size = config.pool_size;
9009        let expected_actor_cap = config.actor_cap;
9010        let executor = Arc::new(Executor::with_config(config));
9011        assert_eq!(executor.pool_size(), expected_pool_size);
9012        assert_eq!(executor.actor_cap(), expected_actor_cap);
9013        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
9014        let metrics = Arc::new(DispatchPathMetrics::new());
9015        let (completion_tx, mut completion_rx) = mpsc::channel(2);
9016        submit_maintenance_job(
9017            &executor,
9018            root.clone(),
9019            MaintenanceDrainKind::ConfigureTail,
9020            Vec::new(),
9021            &completion_tx,
9022            &metrics,
9023        );
9024        maintenance_reached
9025            .recv_timeout(Duration::from_secs(2))
9026            .expect("configure tail reached gate");
9027
9028        let admission_started = std::time::Instant::now();
9029        let (search_admitted_tx, search_admitted_rx) = crossbeam_channel::bounded(1);
9030        let search = executor.submit_async(
9031            root.clone(),
9032            Lane::HeavyInit,
9033            "first-search".to_string(),
9034            Box::new(move |_ctx| {
9035                search_admitted_tx
9036                    .send(())
9037                    .expect("signal search admission");
9038                Response::success("first-search", json!({}))
9039            }),
9040        );
9041        let (mutation_started_tx, mutation_started_rx) = crossbeam_channel::bounded(1);
9042        let mutation = executor.submit_async(
9043            root,
9044            Lane::Mutating,
9045            "queued-mutation".to_string(),
9046            Box::new(move |_ctx| {
9047                mutation_started_tx.send(()).expect("signal mutation start");
9048                Response::success("queued-mutation", json!({}))
9049            }),
9050        );
9051
9052        // The decision under test is ORDERING, not latency: this recv happens
9053        // while the maintenance gate is still held, so a search that could only
9054        // admit after maintenance completes can never satisfy it. The budget is
9055        // a hang catch - a tight bound here just measures runner scheduling
9056        // (the census S-class shape) and flaked on loaded macOS CI at 100ms.
9057        let search_admission = search_admitted_rx.recv_timeout(Duration::from_secs(10));
9058        let admission_elapsed = admission_started.elapsed();
9059        let mutation_waited = mutation_started_rx.try_recv().is_err();
9060        release_maintenance
9061            .send(())
9062            .expect("release configure maintenance");
9063        search_admission
9064            .expect("first search must admit while configure maintenance remains gated");
9065        eprintln!(
9066            "first-search admission while configure maintenance is gated ({shape}): {}ms",
9067            admission_elapsed.as_millis()
9068        );
9069        assert!(
9070            mutation_waited,
9071            "mutating work must wait for configure maintenance to release its read epoch"
9072        );
9073        tokio::time::timeout(Duration::from_secs(5), search)
9074            .await
9075            .expect("first search completion timed out")
9076            .expect("first search completion channel closed");
9077        tokio::time::timeout(Duration::from_secs(5), mutation)
9078            .await
9079            .expect("mutation completion timed out")
9080            .expect("mutation completion channel closed");
9081        tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
9082            .await
9083            .expect("configure-tail completion timed out")
9084            .expect("configure-tail completion channel closed");
9085    }
9086
9087    #[tokio::test]
9088    async fn slow_configure_tail_admits_first_search_but_not_mutating_work() {
9089        assert_slow_configure_tail_admission(
9090            crate::executor::ExecutorConfig {
9091                pool_size: 2,
9092                read_cap: 1,
9093                actor_cap: 1,
9094                heavy_permits: 1,
9095                drr_quantum: 1,
9096            },
9097            "pool=2 actor_cap=1",
9098        )
9099        .await;
9100        assert_slow_configure_tail_admission(
9101            crate::executor::ExecutorConfig {
9102                pool_size: 4,
9103                read_cap: 3,
9104                actor_cap: 3,
9105                heavy_permits: 3,
9106                drr_quantum: 1,
9107            },
9108            "pool=4 actor_cap=3",
9109        )
9110        .await;
9111    }
9112
9113    #[tokio::test]
9114    async fn subc_configure_tail_precedes_completed_search_install() {
9115        let root_dir = tempfile::tempdir().unwrap();
9116        let storage = tempfile::tempdir().unwrap();
9117        let root = ProjectRootId::from_path(root_dir.path()).unwrap();
9118        let (ctx, ignored_path) =
9119            runtime_drain::configure_search_order_context_for_test(root_dir.path(), storage.path());
9120        let ctx = Arc::new(ctx);
9121        assert!(!runtime_drain::watcher_path_is_ignored_by_current_matcher(
9122            &ctx,
9123            &ignored_path
9124        ));
9125
9126        let executor = Arc::new(Executor::new());
9127        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
9128        let metrics = Arc::new(DispatchPathMetrics::new());
9129        let (completion_tx, mut completion_rx) = mpsc::channel(4);
9130        submit_maintenance_job(
9131            &executor,
9132            root.clone(),
9133            MaintenanceDrainKind::ConfigureTail,
9134            Vec::new(),
9135            &completion_tx,
9136            &metrics,
9137        );
9138        submit_maintenance_job(
9139            &executor,
9140            root,
9141            MaintenanceDrainKind::CompletionDrains,
9142            Vec::new(),
9143            &completion_tx,
9144            &metrics,
9145        );
9146
9147        let first = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
9148            .await
9149            .expect("configure-tail completion timed out")
9150            .expect("configure-tail completion channel closed");
9151        let second = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
9152            .await
9153            .expect("completion-drains completion timed out")
9154            .expect("completion-drains completion channel closed");
9155        assert!(first.response.id.contains("configure-tail"));
9156        assert!(second.response.id.contains("completion-drains"));
9157        assert!(runtime_drain::watcher_path_is_ignored_by_current_matcher(
9158            &ctx,
9159            &ignored_path
9160        ));
9161        assert_eq!(
9162            ctx.search_index()
9163                .read()
9164                .unwrap_or_else(std::sync::PoisonError::into_inner)
9165                .as_ref()
9166                .expect("completed search index installed")
9167                .file_count(),
9168            0,
9169            "configure must install the ignore matcher before pending paths replay"
9170        );
9171        ctx.stop_watcher_runtime();
9172    }
9173
9174    #[test]
9175    fn post_bind_configure_and_completion_jobs_are_queued_in_order() {
9176        let (_dir, root) = test_root("maintenance-post-bind");
9177        let mut live_roots = HashMap::new();
9178        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
9179
9180        queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
9181        queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
9182
9183        let meta = live_roots.get(&root).expect("root metadata");
9184        assert!(meta.maintenance_pending);
9185        assert_eq!(meta.maintenance_jobs_in_flight, 0);
9186        assert_eq!(
9187            meta.maintenance_queued_kinds
9188                .iter()
9189                .copied()
9190                .collect::<Vec<_>>(),
9191            vec![
9192                MaintenanceDrainKind::ConfigureTail,
9193                MaintenanceDrainKind::CompletionDrains,
9194            ]
9195        );
9196
9197        let (due, deferred) = due_maintenance_jobs_without_actor_context(
9198            &mut live_roots,
9199            MAINTENANCE_SUBMIT_BUDGET,
9200            &HashSet::new(),
9201        );
9202
9203        assert_eq!(
9204            due,
9205            vec![
9206                (root.clone(), MaintenanceDrainKind::ConfigureTail),
9207                (root.clone(), MaintenanceDrainKind::CompletionDrains),
9208            ]
9209        );
9210        assert!(!deferred);
9211        assert_eq!(live_roots[&root].maintenance_jobs_in_flight, 2);
9212        assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
9213    }
9214
9215    #[test]
9216    fn due_maintenance_jobs_defers_unsubmitted_roots_without_marking_pending() {
9217        let mut live_roots = HashMap::new();
9218        let mut root_ids = Vec::new();
9219        let mut _dirs = Vec::new();
9220        for index in 0..4 {
9221            let (dir, root_id) = test_root(&format!("maintenance-budget-{index}"));
9222            live_roots.insert(root_id.clone(), RootMeta::new(Instant::now()));
9223            root_ids.push(root_id);
9224            _dirs.push(dir);
9225        }
9226
9227        let small_budget = INITIAL_MAINTENANCE_JOB_COUNT + 1;
9228        let (first_due, first_deferred) = due_maintenance_jobs_without_actor_context(
9229            &mut live_roots,
9230            small_budget,
9231            &HashSet::new(),
9232        );
9233
9234        assert_eq!(first_due.len(), small_budget);
9235        assert!(first_deferred);
9236        let first_due_set: HashSet<_> = first_due.into_iter().map(|(root, _)| root).collect();
9237        assert!(first_due_set
9238            .iter()
9239            .all(|root| live_roots[root].maintenance_pending));
9240        assert!(first_due_set
9241            .iter()
9242            .any(|root| !live_roots[root].maintenance_queued_kinds.is_empty()));
9243
9244        let all_roots: HashSet<_> = root_ids.into_iter().collect();
9245        let deferred_roots: HashSet<_> = all_roots.difference(&first_due_set).cloned().collect();
9246        assert!(deferred_roots
9247            .iter()
9248            .all(|root| !live_roots[root].maintenance_pending));
9249    }
9250
9251    #[test]
9252    fn due_maintenance_jobs_defers_pending_bind_roots() {
9253        let (_bind_dir, bind_root) = test_root("maintenance-pending-bind");
9254        let (_healthy_dir, healthy_root) = test_root("maintenance-no-bind");
9255        let mut live_roots = HashMap::new();
9256        live_roots.insert(bind_root.clone(), RootMeta::new(Instant::now()));
9257        live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
9258        let pending_bind_roots = HashSet::from([bind_root.clone()]);
9259
9260        let (due, deferred) = due_maintenance_jobs_without_actor_context(
9261            &mut live_roots,
9262            usize::MAX,
9263            &pending_bind_roots,
9264        );
9265
9266        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
9267        assert!(due.iter().all(|(root, _)| root == &healthy_root));
9268        assert!(!deferred);
9269        assert!(!live_roots[&bind_root].maintenance_pending);
9270        assert!(live_roots[&bind_root].maintenance_queued_kinds.is_empty());
9271    }
9272
9273    #[test]
9274    fn maintenance_pending_survives_requeue_and_clears_after_final_batch() {
9275        let (_dir, root) = test_root("maintenance-requeue");
9276        let mut live_roots = HashMap::new();
9277        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
9278        let (due, deferred) = due_maintenance_jobs_without_actor_context(
9279            &mut live_roots,
9280            usize::MAX,
9281            &HashSet::new(),
9282        );
9283        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
9284        assert!(due.iter().all(|(due_root, _)| due_root == &root));
9285        assert!(!deferred);
9286
9287        let meta = live_roots.get_mut(&root).unwrap();
9288        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, false);
9289        assert!(meta.maintenance_pending);
9290        assert_eq!(
9291            meta.maintenance_jobs_in_flight,
9292            INITIAL_MAINTENANCE_JOB_COUNT - 1
9293        );
9294        assert_eq!(meta.maintenance_queued_kinds.len(), 1);
9295
9296        let (requeued, deferred) =
9297            due_maintenance_jobs_without_actor_context(&mut live_roots, 1, &HashSet::new());
9298        assert_eq!(
9299            requeued,
9300            vec![(root.clone(), MaintenanceDrainKind::Watcher)]
9301        );
9302        assert!(!deferred);
9303        let meta = live_roots.get_mut(&root).unwrap();
9304        assert_eq!(
9305            meta.maintenance_jobs_in_flight,
9306            INITIAL_MAINTENANCE_JOB_COUNT
9307        );
9308        assert!(meta.maintenance_queued_kinds.is_empty());
9309
9310        for _ in 0..INITIAL_MAINTENANCE_JOB_COUNT {
9311            note_maintenance_completion(meta, None, false, false);
9312        }
9313        assert!(!meta.maintenance_pending);
9314        assert_eq!(meta.maintenance_jobs_in_flight, 0);
9315    }
9316
9317    #[test]
9318    fn maintenance_requeue_drops_while_bind_is_pending() {
9319        let (_dir, root) = test_root("maintenance-bind-requeue");
9320        let mut live_roots = HashMap::new();
9321        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
9322        let (due, _) = due_maintenance_jobs_without_actor_context(
9323            &mut live_roots,
9324            usize::MAX,
9325            &HashSet::new(),
9326        );
9327        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
9328
9329        let meta = live_roots.get_mut(&root).unwrap();
9330        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, true);
9331
9332        assert_eq!(
9333            meta.maintenance_jobs_in_flight,
9334            INITIAL_MAINTENANCE_JOB_COUNT - 1
9335        );
9336        assert!(meta.maintenance_queued_kinds.is_empty());
9337        assert!(meta.maintenance_pending);
9338    }
9339
9340    #[test]
9341    fn parked_lsp_completion_never_requiesces_or_cancels_a_pending_bind() {
9342        let mut meta = RootMeta::new(Instant::now());
9343        meta.unbound_quiesced = true;
9344
9345        assert!(!should_requiesce_after_maintenance(
9346            &meta,
9347            MaintenanceDrainKind::Lsp,
9348            false,
9349        ));
9350        assert!(!should_requiesce_after_maintenance(
9351            &meta,
9352            MaintenanceDrainKind::ConfigureTail,
9353            true,
9354        ));
9355        assert!(should_requiesce_after_maintenance(
9356            &meta,
9357            MaintenanceDrainKind::ConfigureTail,
9358            false,
9359        ));
9360    }
9361
9362    #[test]
9363    fn maintenance_pending_clears_and_poison_stops_requeue_after_fatal() {
9364        let (_dir, root) = test_root("maintenance-fatal");
9365        let mut live_roots = HashMap::new();
9366        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
9367        let (due, _) = due_maintenance_jobs_without_actor_context(
9368            &mut live_roots,
9369            usize::MAX,
9370            &HashSet::new(),
9371        );
9372        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
9373
9374        let meta = live_roots.get_mut(&root).unwrap();
9375        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), true, false);
9376        assert!(meta.maintenance_poisoned);
9377        assert!(meta.maintenance_queued_kinds.is_empty());
9378
9379        for _ in 1..INITIAL_MAINTENANCE_JOB_COUNT {
9380            note_maintenance_completion(meta, None, false, false);
9381        }
9382        assert!(!meta.maintenance_pending);
9383        assert_eq!(meta.maintenance_jobs_in_flight, 0);
9384    }
9385
9386    #[test]
9387    fn trust_for_principal_matrix() {
9388        assert_eq!(
9389            trust_for_principal(&Some(Principal::Direct)),
9390            BindTrust::FirstParty
9391        );
9392        // Every first-party reserved id is asserted BY NAME, in one loop over
9393        // the full set, so two failure classes stay distinguishable: an empty
9394        // or broken allowlist reddens every name at once, while a dropped
9395        // single entry (the rename hazard) reddens exactly the missing name.
9396        // Both halves of each transitional rename pair stay listed until the
9397        // flip settles (see the allowlist comment).
9398        for module_id in [
9399            "llm-runner",
9400            "aft",
9401            "broca",
9402            "alfonso-core",
9403            "prefrontal",
9404            "prefrontal-core",
9405        ] {
9406            assert_eq!(
9407                trust_for_principal(&Some(Principal::Reserved {
9408                    module_id: module_id.to_string(),
9409                })),
9410                BindTrust::FirstParty,
9411                "reserved module id '{module_id}' must resolve to first-party trust"
9412            );
9413        }
9414        assert_eq!(
9415            trust_for_principal(&Some(Principal::Reserved {
9416                module_id: "subc-mcp".to_string(),
9417            })),
9418            BindTrust::Untrusted
9419        );
9420        assert_eq!(
9421            trust_for_principal(&Some(Principal::Reserved {
9422                module_id: "anything-unknown".to_string(),
9423            })),
9424            BindTrust::Untrusted
9425        );
9426        assert_eq!(
9427            trust_for_principal(&Some(Principal::Unverified)),
9428            BindTrust::Untrusted
9429        );
9430        assert_eq!(trust_for_principal(&None), BindTrust::Untrusted);
9431    }
9432
9433    #[test]
9434    fn fed_harness_class_maps_to_untrusted_regardless_of_fingerprint_value() {
9435        let principal = Some(Principal::Direct);
9436        let fingerprint_a = "fed:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
9437        let fingerprint_b = "fed:0123456789abcdef111111111111111111111111111111111111111111111111";
9438
9439        assert_eq!(
9440            trust_for_bind(fingerprint_a, &principal),
9441            BindTrust::Untrusted
9442        );
9443        assert_eq!(
9444            trust_for_bind(fingerprint_b, &principal),
9445            BindTrust::Untrusted
9446        );
9447    }
9448
9449    /// The table above proves `trust_for_principal` maps correctly, and the
9450    /// test above proves the `fed:` harness override wins — but neither
9451    /// exercises the ordinary path, so an implementation that ignored the
9452    /// principal entirely for non-fed harnesses would satisfy both. Pin the
9453    /// delegation itself: on a normal harness the verdict must still come from
9454    /// the principal, in both directions.
9455    #[test]
9456    fn trust_for_bind_delegates_to_the_principal_on_ordinary_harnesses() {
9457        for harness in ["opencode", "pi", "runner", "mcp:claude"] {
9458            assert_eq!(
9459                trust_for_bind(harness, &Some(Principal::Direct)),
9460                BindTrust::FirstParty,
9461                "a direct principal must stay first-party on {harness}"
9462            );
9463            assert_eq!(
9464                trust_for_bind(harness, &Some(Principal::Unverified)),
9465                BindTrust::Untrusted,
9466                "an unverified principal must stay untrusted on {harness}"
9467            );
9468            assert_eq!(
9469                trust_for_bind(harness, &None),
9470                BindTrust::Untrusted,
9471                "an absent principal must fail closed on {harness}"
9472            );
9473            assert_eq!(
9474                trust_for_bind(
9475                    harness,
9476                    &Some(Principal::Reserved {
9477                        module_id: "subc-mcp".to_string(),
9478                    })
9479                ),
9480                BindTrust::Untrusted,
9481                "a non-allowlisted reserved module must stay untrusted on {harness}"
9482            );
9483        }
9484    }
9485
9486    #[tokio::test]
9487    async fn persistent_cancel_resolves_when_fired_before_await() {
9488        // The lost-wakeup guard: cancel() fires exactly once via notify_waiters()
9489        // (no stored permit). A waiter that registers AFTER the cancel must still
9490        // observe it via the flag; a waiter racing the cancel must still be woken.
9491        let signal = PersistentCancelSignal::new();
9492        signal.cancel();
9493        // Fired before we ever call cancelled() — must return immediately, not park.
9494        tokio::time::timeout(Duration::from_secs(1), signal.cancelled())
9495            .await
9496            .expect("cancelled() must resolve when cancel fired beforehand");
9497
9498        // A fresh signal cancelled concurrently with an in-flight cancelled().
9499        let racing = PersistentCancelSignal::new();
9500        let racing_for_task = racing.clone();
9501        let waiter = tokio::spawn(async move { racing_for_task.cancelled().await });
9502        racing.cancel();
9503        tokio::time::timeout(Duration::from_secs(1), waiter)
9504            .await
9505            .expect("cancelled() must resolve when cancel races the await")
9506            .expect("waiter task panicked");
9507    }
9508
9509    #[test]
9510    fn ingress_epoch_validation_rejects_reclaimed_requests_and_drops_other_stale_epochs() {
9511        let installed = HashMap::from([(7, 9)]);
9512        let mut reclaimed = ReclaimedRoutes::default();
9513        reclaimed.insert(route_key(8, 1));
9514        for ty in [
9515            FrameType::Request,
9516            FrameType::Response,
9517            FrameType::Error,
9518            FrameType::Push,
9519            FrameType::Cancel,
9520            FrameType::Goodbye,
9521        ] {
9522            let body = if ty.is_pure_header() {
9523                Vec::new()
9524            } else {
9525                br#"{}"#.to_vec()
9526            };
9527            let stale = Frame::build(ty, control_flags(), 7, 8, 41, body).unwrap();
9528            assert!(
9529                !ingress_route_should_be_processed(&installed, &reclaimed, &stale),
9530                "{ty:?}"
9531            );
9532        }
9533
9534        let reclaimed_request = Frame::build(
9535            FrameType::Request,
9536            control_flags(),
9537            8,
9538            1,
9539            42,
9540            br#"{}"#.to_vec(),
9541        )
9542        .unwrap();
9543        assert!(ingress_route_should_be_processed(
9544            &installed,
9545            &reclaimed,
9546            &reclaimed_request
9547        ));
9548
9549        let never_installed = Frame::build(
9550            FrameType::Request,
9551            control_flags(),
9552            9,
9553            1,
9554            43,
9555            br#"{}"#.to_vec(),
9556        )
9557        .unwrap();
9558        assert!(!ingress_route_should_be_processed(
9559            &installed,
9560            &reclaimed,
9561            &never_installed
9562        ));
9563
9564        let current = Frame::build(
9565            FrameType::Request,
9566            control_flags(),
9567            7,
9568            9,
9569            43,
9570            br#"{}"#.to_vec(),
9571        )
9572        .unwrap();
9573        let control = Frame::build(FrameType::Ping, control_flags(), 0, 0, 44, Vec::new()).unwrap();
9574        assert!(ingress_route_should_be_processed(
9575            &installed, &reclaimed, &current
9576        ));
9577        assert!(ingress_route_should_be_processed(
9578            &installed, &reclaimed, &control
9579        ));
9580        assert_eq!(installed, HashMap::from([(7, 9)]));
9581    }
9582
9583    #[tokio::test]
9584    async fn route_bind_ack_precedes_route_egress_in_writer_queue() {
9585        let (_dir, root) = test_root("route-bind-b2-ordering");
9586        let route = route_key(7, 3);
9587        let identity = RouteIdentity(Arc::new(RouteIdentityData {
9588            root: root.clone(),
9589            project_root: root.as_path().to_path_buf(),
9590            harness: "opencode".to_string(),
9591            session: "b2-session".to_string(),
9592            trust: BindTrust::FirstParty,
9593            spawn_principal: AuthenticatedPrincipal::FirstParty,
9594            consumer_elicitation_capable: false,
9595        }));
9596        let replay_key = push::ReplayKey::from_identity(&identity);
9597        let completion = RouteBindCompletion {
9598            route,
9599            identity,
9600            bind_root_id: root.clone(),
9601            inserted_new_actor: false,
9602            configure_response: Response::success("subc-bind-7", json!({})),
9603            diagnostics_on_edit: false,
9604            ver: PROTOCOL_VERSION,
9605            corr: 91,
9606            flags: control_flags(),
9607        };
9608        let mut pending_binds = HashMap::from([(
9609            route,
9610            PendingBind {
9611                bind_root_id: root,
9612                inserted_new_actor: false,
9613                cancelled: false,
9614                configure_request_id: "subc-bind-7".to_string(),
9615                started_at: Instant::now(),
9616                warned_half_deadline: false,
9617                deadline_reported: false,
9618                corr: 91,
9619                ver: PROTOCOL_VERSION,
9620                flags: control_flags(),
9621                cancellation: crate::executor::JobCancellation::new(),
9622            },
9623        )]);
9624        let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
9625        let mut push_buffer =
9626            HashMap::from([(replay_key, VecDeque::from([completion_frame("b2-replay")]))]);
9627        let (writer_tx, mut writer_rx) = mpsc::channel(8);
9628        let metrics = Arc::new(DispatchPathMetrics::new());
9629        let executor = Arc::new(Executor::new());
9630        let standing_actor =
9631            standing::StandingActor::new(App::default_shared(), Arc::clone(&executor));
9632
9633        handle_route_bind_completion(
9634            &writer_tx,
9635            completion,
9636            &mut HashMap::new(),
9637            &mut HashMap::new(),
9638            &mut HashMap::new(),
9639            &mut push_buffer,
9640            &mut HashMap::new(),
9641            &mut pending_binds,
9642            &mut installed_route_epochs,
9643            &executor,
9644            &standing_actor,
9645            &Arc::new(Notify::new()),
9646            &metrics,
9647            None,
9648        )
9649        .await
9650        .unwrap();
9651
9652        let ack = writer_rx.try_recv().expect("RouteBindAck");
9653        assert_eq!(ack.header.ty, FrameType::Response);
9654        assert_eq!((ack.header.channel, ack.header.epoch), (0, 0));
9655        let route_frame = writer_rx.try_recv().expect("post-ack route frame");
9656        assert_eq!(route_frame.header.ty, FrameType::Push);
9657        assert_eq!(
9658            (route_frame.header.channel, route_frame.header.epoch),
9659            (route.channel, route.epoch)
9660        );
9661    }
9662}