Skip to main content

aft/subc/
mod.rs

1//! subc daemon attach — transport edge.
2//!
3//! When AFT is launched as `aft --subc <connection-file>`, it does NOT run the
4//! standalone NDJSON-over-stdin loop. Instead it connects to a running subc
5//! daemon over loopback TCP, authenticates with the pre-envelope HMAC handshake
6//! (`subc-transport`), then speaks the subc frame protocol (`subc-protocol`):
7//! ModuleHello → HelloAck (register provider capabilities), then a channel-0
8//! control loop (Ping/Pong, RouteBind) plus tool and management route calls.
9//!
10//! Concurrency: subc routes tool calls through the executor. The tokio
11//! edge never dispatches against `AppContext` inline; per-actor executor lanes
12//! own the reader/mutator epoch, while a writer task serializes outbound frames.
13
14pub mod blob_store;
15
16use std::collections::{HashMap, HashSet, VecDeque};
17use std::fmt;
18use std::io;
19use std::net::{IpAddr, SocketAddr};
20use std::ops::Deref;
21use std::path::{Path, PathBuf};
22use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
23use std::sync::{Arc, LazyLock, Mutex as StdMutex, OnceLock};
24use std::time::{Duration, Instant};
25
26use serde::Deserialize;
27use serde_json::{json, Value};
28
29use crate::config::Config;
30use crate::context::{App, AppContext, ProgressSender, RootHealthSnapshot};
31use crate::executor::{Executor, JobCancellation, Lane, PreExecutionCancelOutcome};
32use crate::fleet_status::{spawn_fleet_status_dial, FleetStatusClient};
33use crate::log_ctx;
34use crate::path_identity::ProjectRootId;
35use crate::protocol::{ProgressKind, PushFrame, RawRequest, Response};
36use crate::response_finalize::{DispatchOutcome, PendingResponse};
37use crate::run_tool_call::{
38    finish_tool_call_response, prepare_tool_call, run_tool_call, strip_agent_preview_arg_owned,
39    PhaseTrace, ToolCallContext, ToolCallOutcome, ToolCallResult,
40};
41use crate::runtime_drain;
42use crate::sandbox_spawn::{AuthenticatedPrincipal, PrincipalTrust};
43
44use subc_protocol::manifest::{
45    Bindings, Concurrency, ExecutionMode, IdentityBinding, IdentityScope, ManagementOperation,
46    ManagementOperationKind, ModuleManifest, ProviderRole, StorageBinding, StorageKind,
47    StorageScope, Tool, TrustTier,
48};
49use subc_protocol::session::{
50    HealthReport, HealthStatus, ModuleControlRequest, ModuleControlResponse,
51    MODULE_CONTROL_OP_HEALTH_CHECK,
52};
53use subc_protocol::{
54    ErrorBody, Flags, Frame, FrameType, ModuleHelloBody, Principal, Priority, RouteTarget,
55    MAX_FRAME_BODY_LEN, PROTOCOL_VERSION,
56};
57use subc_transport::{authenticate_client, connection_file, read_frame, write_frame};
58use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
59use tokio::net::TcpStream;
60use tokio::sync::{mpsc, oneshot, Notify};
61use tokio::task::JoinHandle;
62
63/// Per-attempt handshake deadline. The initial attach loop has a separate total
64/// budget so a stalled peer cannot consume an unbounded supervisor launch window.
65const AUTH_DEADLINE: Duration = Duration::from_secs(5);
66const ATTACH_RETRY_BUDGET: Duration = Duration::from_secs(60);
67const ATTACH_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(250);
68const ATTACH_RETRY_MAX_BACKOFF: Duration = Duration::from_secs(5);
69const ATTACH_RETRY_JITTER_PERCENT: u64 = 20;
70
71/// Correlation id for the initial ModuleHello (channel 0).
72const HELLO_CORR: u64 = 1;
73
74/// Per-session in-memory replay cap for must-deliver Push frames. This covers
75/// detach/re-attach while AFT stays alive; cross-restart replay is phased later.
76const PUSH_BUFFER_MAX_PER_KEY: usize = 256;
77
78/// Bounded guard for control-frame sends. If the daemon stops reading and the
79/// writer queue stays full, tear the subc edge down instead of stalling the
80/// route loop indefinitely.
81const CONTROL_SEND_TIMEOUT: Duration = Duration::from_millis(250);
82
83/// Cadence for the loop's deadline-driven drain work (retry-buffer flush,
84/// bg-wake emission, maintenance submission). Checked at the top of every
85/// loop turn so busy select arms cannot starve it.
86const DRAIN_TICK_PERIOD: Duration = Duration::from_millis(250);
87
88/// Fallback unbound-root artifact TTL when a root has no actor config yet.
89const IDLE_ROOT_TTL: Duration = Duration::from_secs(30 * 60);
90
91const WRITER_QUEUE_CAPACITY: usize = 256;
92
93/// Keep reliable Push bursts from monopolizing the current-thread subc loop;
94/// any remaining must-deliver frames stay queued for the next loop turn.
95const RELIABLE_PUSH_DRAIN_BUDGET: usize = 32;
96
97/// Limit maintenance submissions per tick so background drains cannot delay
98/// control-plane work such as completed RouteBind acknowledgements.
99///
100/// The decomposed maintenance pass charges this budget by Mutating job, not by
101/// root. Size the default burst for one maintenance pass over eight live roots,
102/// while follow-up batches still re-enter the capped queue instead of bypassing
103/// the budget.
104const MAINTENANCE_SUBMIT_BUDGET: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len() * 8;
105const INITIAL_MAINTENANCE_DRAIN_KINDS: [MaintenanceDrainKind; 4] = [
106    MaintenanceDrainKind::Watcher,
107    MaintenanceDrainKind::Lsp,
108    MaintenanceDrainKind::ConfigureTail,
109    MaintenanceDrainKind::CompletionDrains,
110];
111#[cfg(test)]
112const INITIAL_MAINTENANCE_JOB_COUNT: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len();
113
114const RELIABLE_WRITER_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(10);
115const RELIABLE_WRITER_RETRY_MAX_BACKOFF: Duration = Duration::from_millis(250);
116
117const DISPATCH_PATH_BIND_WARN_AFTER: Duration = Duration::from_secs(6);
118const ROUTE_BIND_DEADLINE: Duration = Duration::from_secs(12);
119
120/// Small bounded memory of completed task ids used to suppress stale lossy
121/// long-running reminders that arrive after their reliable completion event.
122const COMPLETED_TASK_SUPPRESSION_MAX: usize = 4096;
123
124/// Bash foreground orchestration polls detached tasks with short read-lane jobs.
125/// The sleep between polls is outside the executor so no read or write worker is
126/// pinned while a foreground command is still running.
127const PENDING_POLL_INTERVAL: Duration = Duration::from_millis(100);
128
129/// Host elicitation asks fail closed if the MCP facade does not answer promptly.
130const BASH_ELICITATION_TIMEOUT: Duration = Duration::from_secs(60);
131const BASH_ELICITATION_CREATE_METHOD: &str = "elicitation/create";
132
133#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
134struct RouteChannel {
135    channel: u16,
136    epoch: u32,
137}
138
139impl fmt::Display for RouteChannel {
140    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
141        write!(f, "{}@{}", self.channel, self.epoch)
142    }
143}
144
145type PushEnvelope = (ProjectRootId, PushFrame);
146type LossyPushEnvelope = (u64, ProjectRootId, PushFrame);
147type RetryBuffer = HashMap<RouteChannel, VecDeque<(push::ReplayKey, PushFrame)>>;
148mod bash;
149mod health;
150mod manifest;
151mod push;
152mod standing;
153mod wire;
154
155use self::health::{
156    build_health_report, warn_slow_pending_binds, warn_slow_running_interactive_jobs,
157    DeferredBashWaitGuard, DispatchPathMetrics, HealthRollupCache, HealthRollupWorker,
158    ReapBlockerCensus, ResponseTaskGuard,
159};
160use self::manifest::{
161    build_manifest, command_lane, control_flags, control_ops, is_bash_family_tool,
162    is_subc_agent_core_tool, is_subc_native_plumbing_tool,
163};
164pub use self::wire::SubcError;
165
166/// Lifecycle milestones emitted only by the dedicated subc integration-test runner.
167///
168/// Production entry points never install a probe, so these notifications cannot
169/// affect routing or delivery behavior.
170#[doc(hidden)]
171#[derive(Clone, Debug, PartialEq, Eq)]
172pub enum SubcLifecycleEvent {
173    AttachDecision {
174        attempt: u32,
175        will_retry: bool,
176    },
177    RouteDetached {
178        route_channel: u16,
179        route_epoch: u32,
180        session_id: String,
181    },
182    ReliableCompletionRetained {
183        task_id: String,
184        session_id: String,
185    },
186    ReliableCompletionReplayed {
187        route_channel: u16,
188        route_epoch: u32,
189        task_id: String,
190        session_id: String,
191    },
192}
193
194/// Test-only observer for the detach/rebind lifecycle.
195#[doc(hidden)]
196#[derive(Clone)]
197pub struct SubcTestLifecycleProbe {
198    events_tx: mpsc::UnboundedSender<SubcLifecycleEvent>,
199}
200
201impl SubcTestLifecycleProbe {
202    #[doc(hidden)]
203    pub fn new(events_tx: mpsc::UnboundedSender<SubcLifecycleEvent>) -> Self {
204        Self { events_tx }
205    }
206
207    fn attach_decision(&self, attempt: u32, will_retry: bool) {
208        let _ = self.events_tx.send(SubcLifecycleEvent::AttachDecision {
209            attempt,
210            will_retry,
211        });
212    }
213
214    fn route_detached(&self, route: RouteChannel, session_id: &str) {
215        let _ = self.events_tx.send(SubcLifecycleEvent::RouteDetached {
216            route_channel: route.channel,
217            route_epoch: route.epoch,
218            session_id: session_id.to_string(),
219        });
220    }
221
222    fn reliable_completion_retained(&self, task_id: &str, session_id: &str) {
223        let _ = self
224            .events_tx
225            .send(SubcLifecycleEvent::ReliableCompletionRetained {
226                task_id: task_id.to_string(),
227                session_id: session_id.to_string(),
228            });
229    }
230
231    fn reliable_completion_replayed(&self, route: RouteChannel, task_id: &str, session_id: &str) {
232        let _ = self
233            .events_tx
234            .send(SubcLifecycleEvent::ReliableCompletionReplayed {
235                route_channel: route.channel,
236                route_epoch: route.epoch,
237                task_id: task_id.to_string(),
238                session_id: session_id.to_string(),
239            });
240    }
241}
242
243/// Test-only view of the fail-closed tool-call gate: would `name` be admitted
244/// on a bound route (as an agent tool or native plumbing)? Used by the
245/// plugin-send drift guard in `subc_plumbing_drift_test.rs`.
246pub fn is_tool_call_admitted_for_test(name: &str) -> bool {
247    manifest::is_subc_agent_core_tool(name) || manifest::is_subc_native_plumbing_tool(name)
248}
249use self::wire::{
250    build_error_frame, build_goodbye_frame, build_tool_response_frame,
251    build_tool_response_frame_with_limit, decrement_counted_channel, response_is_fatal_panic,
252    response_message, send_counted_channel, send_frame, send_reliable_writer_frame,
253    send_traced_tool_response_frame, ToolResponseWriteTrace, WriterFrame, WriterSender,
254};
255
256struct DecodedFrame {
257    frame: Frame,
258    phase_trace: PhaseTrace,
259}
260
261struct ToolCallCompletion {
262    text: String,
263    phase_trace: PhaseTrace,
264}
265
266#[derive(Clone, Copy, Debug, PartialEq, Eq)]
267enum RouteDetachPolicy {
268    RetainForReplay,
269    CancelOnDetach,
270}
271
272#[derive(Clone)]
273struct ActiveToolCall {
274    root_id: ProjectRootId,
275    cancellation: JobCancellation,
276    detach_policy: RouteDetachPolicy,
277}
278
279type ActiveToolCalls = Arc<StdMutex<HashMap<(RouteChannel, u64), ActiveToolCall>>>;
280
281struct PendingDeferredSetupGuard(Arc<AtomicUsize>);
282
283impl PendingDeferredSetupGuard {
284    fn new(count: Arc<AtomicUsize>) -> Self {
285        count.fetch_add(1, Ordering::SeqCst);
286        Self(count)
287    }
288}
289
290impl Drop for PendingDeferredSetupGuard {
291    fn drop(&mut self) {
292        self.0.fetch_sub(1, Ordering::SeqCst);
293    }
294}
295
296enum DeferredSetupOutcome {
297    Immediate {
298        text: String,
299        phase_trace: PhaseTrace,
300    },
301    Deferred {
302        pending: PendingResponse,
303        surface_downgraded: bool,
304        phase_trace: PhaseTrace,
305    },
306}
307
308struct PendingSubcResponse {
309    route: RouteChannel,
310    corr: u64,
311    flags: Flags,
312    ver: u8,
313    root: ProjectRootId,
314    session_id: String,
315    bare_name: String,
316    format_context: crate::subc_format::FormatContext,
317    bind_trust: BindTrust,
318    pending: PendingResponse,
319    surface_downgraded: bool,
320    phase_trace: PhaseTrace,
321}
322
323struct ResolvedSubcResponse {
324    entry: PendingSubcResponse,
325    response: Response,
326}
327
328#[derive(Default)]
329struct PendingSubcResponses {
330    entries: Vec<PendingSubcResponse>,
331}
332
333impl PendingSubcResponses {
334    fn register(&mut self, pending: PendingSubcResponse) {
335        self.entries.retain(|entry| {
336            let keep = entry.route != pending.route || entry.corr != pending.corr;
337            if !keep {
338                if let Some(cancellation) = &entry.pending.cancellation {
339                    cancellation.request_cancel();
340                }
341            }
342            keep
343        });
344        self.entries.push(pending);
345    }
346
347    fn poll_ready(&mut self, executor: &Executor) -> Vec<ResolvedSubcResponse> {
348        let mut ready = Vec::new();
349        let mut waiting = Vec::with_capacity(self.entries.len());
350        for mut entry in self.entries.drain(..) {
351            let response = executor
352                .actor_context(&entry.root)
353                .and_then(|ctx| (entry.pending.poll)(&ctx));
354            if let Some(response) = response {
355                ready.push(ResolvedSubcResponse { entry, response });
356            } else {
357                waiting.push(entry);
358            }
359        }
360        self.entries = waiting;
361        ready
362    }
363
364    fn cancel_request(&mut self, route: RouteChannel, corr: u64) -> bool {
365        let mut cancelled = false;
366        self.entries.retain(|entry| {
367            let keep = entry.route != route || entry.corr != corr;
368            if !keep {
369                cancelled = true;
370                if let Some(cancellation) = &entry.pending.cancellation {
371                    cancellation.request_cancel();
372                }
373            }
374            keep
375        });
376        cancelled
377    }
378
379    fn drain_route(
380        &mut self,
381        route: RouteChannel,
382        executor: &Executor,
383    ) -> Vec<ResolvedSubcResponse> {
384        self.drain_matching(executor, |entry| entry.route == route)
385    }
386
387    fn drain_on_shutdown(&mut self, executor: &Executor) -> Vec<ResolvedSubcResponse> {
388        self.drain_matching(executor, |_| true)
389    }
390
391    fn drain_matching(
392        &mut self,
393        executor: &Executor,
394        matches: impl Fn(&PendingSubcResponse) -> bool,
395    ) -> Vec<ResolvedSubcResponse> {
396        let mut resolved = Vec::new();
397        let mut waiting = Vec::with_capacity(self.entries.len());
398        for mut entry in self.entries.drain(..) {
399            if !matches(&entry) {
400                waiting.push(entry);
401                continue;
402            }
403            if let Some(cancellation) = &entry.pending.cancellation {
404                cancellation.request_cancel();
405            }
406            if let Some(ctx) = executor.actor_context(&entry.root) {
407                if let Some(on_shutdown) = entry.pending.on_shutdown.as_mut() {
408                    let response = on_shutdown(&ctx);
409                    resolved.push(ResolvedSubcResponse { entry, response });
410                }
411            }
412        }
413        self.entries = waiting;
414        resolved
415    }
416
417    fn is_empty(&self) -> bool {
418        self.entries.is_empty()
419    }
420}
421
422#[derive(Clone)]
423struct PushSenders {
424    lossy_tx: mpsc::Sender<LossyPushEnvelope>,
425    reliable_tx: mpsc::UnboundedSender<PushEnvelope>,
426    lossy_overflow: Arc<push::LossyOverflow>,
427    lossy_seq: Arc<AtomicU64>,
428    fleet_status_client: FleetStatusClient,
429}
430
431#[derive(Clone)]
432struct PersistentCancelSignal {
433    inner: Arc<PersistentCancelInner>,
434}
435
436struct PersistentCancelInner {
437    cancelled: AtomicBool,
438    notify: Notify,
439}
440
441impl PersistentCancelSignal {
442    fn new() -> Self {
443        Self {
444            inner: Arc::new(PersistentCancelInner {
445                cancelled: AtomicBool::new(false),
446                notify: Notify::new(),
447            }),
448        }
449    }
450
451    fn cancel(&self) {
452        if !self.inner.cancelled.swap(true, Ordering::SeqCst) {
453            self.inner.notify.notify_waiters();
454        }
455    }
456
457    fn is_cancelled(&self) -> bool {
458        self.inner.cancelled.load(Ordering::SeqCst)
459    }
460
461    async fn cancelled(&self) {
462        // `enable()` REGISTERS this waiter before we read the flag, closing the
463        // lost-wakeup window: `notify_waiters()` only wakes already-registered
464        // waiters and stores no permit, so without enable() a `cancel()` firing
465        // between the flag read and `.await` would be missed and the future
466        // would park forever (cancel() fires only once). With enable(), a cancel
467        // racing the flag read still wakes the registered waiter. The loop is a
468        // belt-and-suspenders re-check on spurious wakeups.
469        loop {
470            let notified = self.inner.notify.notified();
471            tokio::pin!(notified);
472            notified.as_mut().enable();
473            if self.is_cancelled() {
474                return;
475            }
476            notified.await;
477        }
478    }
479}
480
481fn submit_active_tool_call(
482    executor: &Executor,
483    active: &ActiveToolCalls,
484    route: RouteChannel,
485    corr: u64,
486    root_id: ProjectRootId,
487    lane: Lane,
488    request_id: String,
489    detach_policy: RouteDetachPolicy,
490    job: crate::executor::ExecutorJob,
491) -> oneshot::Receiver<Response> {
492    let (rx, cancellation) =
493        executor.submit_cancellable_async(root_id.clone(), lane, request_id, job);
494    active
495        .lock()
496        .unwrap_or_else(std::sync::PoisonError::into_inner)
497        .insert(
498            (route, corr),
499            ActiveToolCall {
500                root_id,
501                cancellation,
502                detach_policy,
503            },
504        );
505    rx
506}
507
508fn finish_active_tool_call(active: &ActiveToolCalls, route: RouteChannel, corr: u64) -> bool {
509    active
510        .lock()
511        .unwrap_or_else(std::sync::PoisonError::into_inner)
512        .remove(&(route, corr))
513        .is_some()
514}
515
516fn active_tool_call_is_registered(
517    active: &ActiveToolCalls,
518    route: RouteChannel,
519    corr: u64,
520) -> bool {
521    active
522        .lock()
523        .unwrap_or_else(std::sync::PoisonError::into_inner)
524        .contains_key(&(route, corr))
525}
526
527fn cancel_active_tool_call(
528    active: &ActiveToolCalls,
529    executor: &Executor,
530    route: RouteChannel,
531    corr: u64,
532    reason: &str,
533) -> bool {
534    let call = active
535        .lock()
536        .unwrap_or_else(std::sync::PoisonError::into_inner)
537        .remove(&(route, corr));
538    let Some(call) = call else {
539        return false;
540    };
541    let outcome = executor.cancel_job(&call.root_id, &call.cancellation);
542    log::debug!(
543        "subc attach: cancelled active tool call route={route} corr={corr} reason={reason} outcome={outcome:?}"
544    );
545    true
546}
547
548#[derive(Clone, Copy, Debug, PartialEq, Eq)]
549enum RouteWorkDisposition {
550    RetainForReplay,
551    RetainStartedForReplay,
552    Abandon,
553}
554
555fn apply_route_work_disposition(
556    active: &ActiveToolCalls,
557    executor: &Executor,
558    route: RouteChannel,
559    disposition: RouteWorkDisposition,
560    reason: &str,
561) -> usize {
562    if disposition != RouteWorkDisposition::Abandon {
563        let route_calls = active
564            .lock()
565            .unwrap_or_else(std::sync::PoisonError::into_inner)
566            .iter()
567            .filter(|((call_route, _), _)| *call_route == route)
568            .map(|(key, call)| (*key, call.clone()))
569            .collect::<Vec<_>>();
570        let mut retained = 0usize;
571        let mut cancelled_before_execution = 0usize;
572        let mut cancelled_terminal = 0usize;
573
574        for (key, call) in route_calls {
575            let remove = match (call.detach_policy, disposition) {
576                (RouteDetachPolicy::RetainForReplay, RouteWorkDisposition::RetainForReplay) => {
577                    retained += 1;
578                    false
579                }
580                (
581                    RouteDetachPolicy::RetainForReplay,
582                    RouteWorkDisposition::RetainStartedForReplay,
583                ) => {
584                    match executor.cancel_job_before_execution(&call.root_id, &call.cancellation) {
585                        PreExecutionCancelOutcome::AlreadyStarted => {
586                            retained += 1;
587                            false
588                        }
589                        PreExecutionCancelOutcome::QueuedRemoved
590                        | PreExecutionCancelOutcome::DispatchedCancelled => {
591                            cancelled_before_execution += 1;
592                            true
593                        }
594                    }
595                }
596                (RouteDetachPolicy::CancelOnDetach, _) => {
597                    executor.cancel_job(&call.root_id, &call.cancellation);
598                    cancelled_terminal += 1;
599                    true
600                }
601                (_, RouteWorkDisposition::Abandon) => unreachable!("handled below"),
602            };
603            if remove {
604                active
605                    .lock()
606                    .unwrap_or_else(std::sync::PoisonError::into_inner)
607                    .remove(&key);
608            }
609        }
610        log::debug!(
611            "subc attach: retained {retained} replayable tool call(s), cancelled {cancelled_before_execution} replayable call(s) before execution, and cancelled {cancelled_terminal} teardown-terminal call(s) route={route} reason={reason}"
612        );
613        return retained;
614    }
615
616    let cancelled = {
617        let mut calls = active
618            .lock()
619            .unwrap_or_else(std::sync::PoisonError::into_inner);
620        let mut cancelled = Vec::new();
621        calls.retain(|(call_route, _), call| {
622            if *call_route == route {
623                cancelled.push(call.clone());
624                false
625            } else {
626                true
627            }
628        });
629        cancelled
630    };
631    for call in &cancelled {
632        let outcome = executor.cancel_job(&call.root_id, &call.cancellation);
633        log::debug!(
634            "subc attach: cancelled active tool call route={route} reason={reason} outcome={outcome:?}"
635        );
636    }
637    cancelled.len()
638}
639
640fn cancel_all_active_tool_calls(
641    active: &ActiveToolCalls,
642    executor: &Executor,
643    reason: &str,
644) -> usize {
645    let cancelled = {
646        let mut calls = active
647            .lock()
648            .unwrap_or_else(std::sync::PoisonError::into_inner);
649        calls.drain().map(|(_, call)| call).collect::<Vec<_>>()
650    };
651    for call in &cancelled {
652        let outcome = executor.cancel_job(&call.root_id, &call.cancellation);
653        log::debug!("subc attach: cancelled active tool call reason={reason} outcome={outcome:?}");
654    }
655    cancelled.len()
656}
657
658#[derive(Debug, Clone, Copy, PartialEq, Eq)]
659pub enum BindTrust {
660    FirstParty,
661    Untrusted,
662}
663
664impl BindTrust {
665    fn allows_bash_observation(self) -> bool {
666        matches!(self, Self::FirstParty)
667    }
668
669    fn label(self) -> &'static str {
670        match self {
671            Self::FirstParty => "first_party",
672            Self::Untrusted => "untrusted",
673        }
674    }
675
676    fn sandbox_trust(self) -> PrincipalTrust {
677        match self {
678            Self::FirstParty => PrincipalTrust::FirstParty,
679            Self::Untrusted => PrincipalTrust::Untrusted,
680        }
681    }
682}
683
684pub(super) fn trust_for_principal(principal: &Option<Principal>) -> BindTrust {
685    match principal {
686        Some(Principal::Direct) => BindTrust::FirstParty,
687        // Module renames are flag-days: the daemon registry refuses duplicate
688        // active ids, so a renaming module cannot advertise both names during
689        // its transition. This allowlist is DIALLED, not dialling — it must
690        // accept a module's NEW name in a released binary before the module
691        // starts using it, and the old name stays until the flip has settled.
692        // That is why transitional pairs appear here: llm-runner/broca was the
693        // previous rename, alfonso-core/prefrontal is the current one. When
694        // retiring an old name, confirm the fleet no longer spawns it — a
695        // stale entry here is inert, but a missing one silently downgrades a
696        // first-party module to Untrusted and revokes its bash access.
697        Some(Principal::Reserved { module_id })
698            if module_id == "llm-runner"
699                || module_id == "aft"
700                || module_id == "broca"
701                || module_id == "alfonso-core"
702                || module_id == "prefrontal"
703                || module_id == "prefrontal-core" =>
704        {
705            BindTrust::FirstParty
706        }
707        Some(Principal::Reserved { .. }) | Some(Principal::Unverified) | None => {
708            BindTrust::Untrusted
709        }
710    }
711}
712
713fn harness_forces_untrusted(harness: &str) -> bool {
714    harness.starts_with("fed:")
715}
716
717pub(super) fn trust_for_bind(harness: &str, principal: &Option<Principal>) -> BindTrust {
718    if harness_forces_untrusted(harness) {
719        BindTrust::Untrusted
720    } else {
721        trust_for_principal(principal)
722    }
723}
724
725fn principal_id(principal: &Option<Principal>) -> Option<String> {
726    match principal {
727        Some(Principal::Direct) => Some("direct".to_string()),
728        Some(Principal::Reserved { module_id }) => Some(format!("reserved:{module_id}")),
729        Some(Principal::Unverified) => Some("unverified".to_string()),
730        None => None,
731    }
732}
733
734fn principal_label(principal: &Option<Principal>) -> String {
735    principal_id(principal).unwrap_or_else(|| "absent".to_string())
736}
737
738#[derive(Debug)]
739/// Per-root route metadata owned by the subc loop. The `active_bash_waits` field
740/// counts detached bash processes that are still being observed for this root.
741/// Any future logic that evicts roots based on idle time must not evict a root
742/// while this count is greater than zero, because a foreground bash response may
743/// still arrive later.
744struct RootMeta {
745    maintenance_pending: bool,
746    maintenance_jobs_in_flight: usize,
747    maintenance_queued_kinds: VecDeque<MaintenanceDrainKind>,
748    maintenance_last_submitted: Option<Instant>,
749    maintenance_poisoned: bool,
750    last_touched: Instant,
751    diagnostics_on_edit: bool,
752    active_bash_waits: usize,
753    idle_artifacts_evicted: bool,
754    unbound_quiesced: bool,
755    consecutive_missing_sweeps: u8,
756}
757
758#[derive(Debug)]
759struct PendingBind {
760    bind_root_id: ProjectRootId,
761    inserted_new_actor: bool,
762    cancelled: bool,
763    configure_request_id: String,
764    started_at: Instant,
765    warned_half_deadline: bool,
766    deadline_reported: bool,
767    corr: u64,
768    ver: u8,
769    flags: Flags,
770    /// Exact-job cancellation for the submitted configure: Goodbye and
771    /// deadline expiry cancel the executor job operationally (queued jobs are
772    /// removed, running configures return at their next checkpoint) instead of
773    /// only marking bookkeeping.
774    cancellation: crate::executor::JobCancellation,
775}
776
777struct RouteBindCompletion {
778    route: RouteChannel,
779    identity: RouteIdentity,
780    bind_root_id: ProjectRootId,
781    inserted_new_actor: bool,
782    configure_response: Response,
783    diagnostics_on_edit: bool,
784    ver: u8,
785    corr: u64,
786    flags: Flags,
787}
788
789#[derive(Debug, Clone)]
790struct RouteIdentity(Arc<RouteIdentityData>);
791
792#[derive(Debug)]
793struct RouteIdentityData {
794    root: ProjectRootId,
795    project_root: PathBuf,
796    harness: String,
797    session: String,
798    trust: BindTrust,
799    spawn_principal: AuthenticatedPrincipal,
800    consumer_elicitation_capable: bool,
801}
802
803impl Deref for RouteIdentity {
804    type Target = RouteIdentityData;
805
806    fn deref(&self) -> &Self::Target {
807        &self.0
808    }
809}
810
811#[derive(Debug, Clone)]
812struct RetainedSessionIdentity {
813    harness: String,
814    trust: BindTrust,
815}
816
817#[derive(Clone)]
818struct BgSub {
819    corr: u64,
820    ver: u8,
821    flags: Flags,
822    root: ProjectRootId,
823    session: String,
824}
825
826#[derive(Clone, Copy, Debug)]
827struct BgWakeState {
828    next_nudge_at: Instant,
829    nudges_sent: u32,
830}
831
832impl BgWakeState {
833    fn armed(now: Instant) -> Self {
834        Self {
835            next_nudge_at: now,
836            nudges_sent: 0,
837        }
838    }
839}
840
841type BgWakePending = HashMap<RouteChannel, BgWakeState>;
842
843// A session can be observed by multiple long-lived consumer records. Retain
844// every route so each wake uses the correlation captured by that route's BgSub.
845type BgSubsBySession = HashMap<(ProjectRootId, String), HashSet<RouteChannel>>;
846
847struct MaintenanceCompletion {
848    root_id: ProjectRootId,
849    kind: MaintenanceDrainKind,
850    response: Response,
851    empty_bg_sessions: Vec<(String, u64)>,
852    unacked_bg_keys: Option<HashSet<String>>,
853    requeue_kind: Option<MaintenanceDrainKind>,
854}
855
856#[derive(Clone, Copy, Debug, PartialEq, Eq)]
857enum MaintenanceDrainKind {
858    Watcher,
859    Lsp,
860    ConfigureTail,
861    CompletionDrains,
862}
863
864impl MaintenanceDrainKind {
865    fn label(self) -> &'static str {
866        match self {
867            Self::Watcher => "watcher",
868            Self::Lsp => "lsp",
869            Self::ConfigureTail => "configure-tail",
870            Self::CompletionDrains => "completion-drains",
871        }
872    }
873}
874
875#[derive(Debug, Default)]
876struct MaintenanceJobOutcome {
877    empty_bg_sessions: Vec<(String, u64)>,
878    unacked_bg_keys: Option<HashSet<String>>,
879    requeue_kind: Option<MaintenanceDrainKind>,
880}
881
882#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
883struct ReverseCorrKey {
884    route: RouteChannel,
885    corr: u64,
886}
887
888struct PendingBashAsk {
889    route: RouteChannel,
890    tool_corr: u64,
891    tool_flags: Flags,
892    tool_ver: u8,
893    root: ProjectRootId,
894    project_root: PathBuf,
895    session_id: String,
896    spawn_principal: AuthenticatedPrincipal,
897    edit_slot_survives: Option<bool>,
898    request_id: String,
899    arguments: Value,
900    format_context: crate::subc_format::FormatContext,
901    cancel: bash::BashWaitCancel,
902    grants: Vec<String>,
903    expires_at: Instant,
904}
905
906impl RootMeta {
907    fn new(now: Instant) -> Self {
908        Self {
909            maintenance_pending: false,
910            maintenance_jobs_in_flight: 0,
911            maintenance_queued_kinds: VecDeque::new(),
912            maintenance_last_submitted: None,
913            maintenance_poisoned: false,
914            last_touched: now,
915            diagnostics_on_edit: false,
916            active_bash_waits: 0,
917            idle_artifacts_evicted: false,
918            unbound_quiesced: false,
919            consecutive_missing_sweeps: 0,
920        }
921    }
922
923    fn note_activity(&mut self) {
924        self.last_touched = Instant::now();
925    }
926
927    fn reactivate_bound(&mut self) {
928        self.note_activity();
929        self.idle_artifacts_evicted = false;
930        self.unbound_quiesced = false;
931    }
932}
933
934fn due_maintenance_jobs(
935    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
936    executor: Option<&Executor>,
937    bg_sub_by_session: &BgSubsBySession,
938    bg_wake_pending: &BgWakePending,
939    budget: usize,
940    pending_bind_roots: &HashSet<ProjectRootId>,
941) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
942    let mut jobs = Vec::new();
943    let mut deferred = false;
944    let mut roots = live_roots.keys().cloned().collect::<Vec<_>>();
945    roots.sort_by(|left, right| {
946        let left_last = live_roots
947            .get(left)
948            .and_then(|meta| meta.maintenance_last_submitted);
949        let right_last = live_roots
950            .get(right)
951            .and_then(|meta| meta.maintenance_last_submitted);
952        left_last
953            .cmp(&right_last)
954            .then_with(|| left.as_path().cmp(right.as_path()))
955    });
956
957    for root_id in roots {
958        let Some(meta) = live_roots.get_mut(&root_id) else {
959            continue;
960        };
961        if meta.maintenance_poisoned {
962            continue;
963        }
964
965        if pending_bind_roots.contains(&root_id) {
966            if meta.maintenance_pending || !meta.maintenance_queued_kinds.is_empty() {
967                deferred = true;
968            }
969            continue;
970        }
971
972        if !meta.maintenance_pending {
973            if jobs.len() >= budget {
974                deferred = true;
975                continue;
976            }
977            // Only enqueue kinds with pending work. Probes are cheap and
978            // fail-open (contended sources count as pending), so an idle root
979            // costs four probes per tick instead of four dispatched jobs.
980            let executor_actor_context =
981                executor.and_then(|executor| executor.actor_context(&root_id));
982            let root_has_pending_bg_wake =
983                bg_sub_by_session.iter().any(|((sub_root, _), channels)| {
984                    sub_root == &root_id
985                        && channels
986                            .iter()
987                            .any(|channel| bg_wake_pending.contains_key(channel))
988                });
989            let kinds_with_work: Vec<MaintenanceDrainKind> = match executor_actor_context {
990                Some(ctx) => INITIAL_MAINTENANCE_DRAIN_KINDS
991                    .into_iter()
992                    .filter(|kind| {
993                        if meta.unbound_quiesced && !matches!(kind, MaintenanceDrainKind::Lsp) {
994                            return false;
995                        }
996                        match kind {
997                            MaintenanceDrainKind::Watcher => ctx.watcher_drain_has_work(),
998                            MaintenanceDrainKind::Lsp => ctx.lsp_drain_has_work(),
999                            MaintenanceDrainKind::ConfigureTail => ctx.configure_tail_has_work(),
1000                            // Every CompletionDrains source is visible at this enqueue site:
1001                            // AppContext probes completion queues, this loop owns bg wakes,
1002                            // and queued continuations bypass probing via maintenance_pending.
1003                            // New drain sources must expose a probe here rather than making
1004                            // every subscribed root fail open again.
1005                            MaintenanceDrainKind::CompletionDrains => {
1006                                root_has_pending_bg_wake || ctx.completion_drains_have_work()
1007                            }
1008                        }
1009                    })
1010                    .collect(),
1011                None if meta.unbound_quiesced => Vec::new(),
1012                // No context handle (actor gone mid-tick): enqueue everything.
1013                None => INITIAL_MAINTENANCE_DRAIN_KINDS.to_vec(),
1014            };
1015            if kinds_with_work.is_empty() {
1016                continue;
1017            }
1018            meta.maintenance_pending = true;
1019            meta.maintenance_queued_kinds.extend(kinds_with_work);
1020        }
1021
1022        while let Some(kind) = meta.maintenance_queued_kinds.pop_front() {
1023            if jobs.len() >= budget {
1024                meta.maintenance_queued_kinds.push_front(kind);
1025                deferred = true;
1026                break;
1027            }
1028            meta.maintenance_jobs_in_flight += 1;
1029            meta.maintenance_last_submitted = Some(Instant::now());
1030            jobs.push((root_id.clone(), kind));
1031        }
1032
1033        meta.maintenance_pending =
1034            meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
1035    }
1036
1037    (jobs, deferred)
1038}
1039
1040fn eviction_estimate_label(estimate: &crate::memory::MemoryEstimate) -> String {
1041    match estimate.estimated_bytes {
1042        Some(bytes) => format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
1043        None if estimate.status == "busy" => "busy".to_string(),
1044        None => "not estimated".to_string(),
1045    }
1046}
1047
1048fn optional_memory_label(bytes: Option<u64>) -> String {
1049    bytes.map_or_else(
1050        || "not estimated".to_string(),
1051        |bytes| format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
1052    )
1053}
1054
1055fn pressure_relief_label(relief: &crate::memory::AllocatorPressureRelief) -> String {
1056    format!(
1057        "; allocator pressure relief: RSS {} -> {}, in-use {} -> {}, allocated {} -> {}, slack {} -> {}, allocator reported {:.1} MB released",
1058        optional_memory_label(relief.rss_before_bytes),
1059        optional_memory_label(relief.rss_after_bytes),
1060        optional_memory_label(relief.allocator_before.bytes_in_use),
1061        optional_memory_label(relief.allocator_after.bytes_in_use),
1062        optional_memory_label(relief.allocator_before.size_allocated),
1063        optional_memory_label(relief.allocator_after.size_allocated),
1064        optional_memory_label(relief.allocator_before.retained_slack_bytes),
1065        optional_memory_label(relief.allocator_after.retained_slack_bytes),
1066        relief.bytes_released as f64 / (1024.0 * 1024.0),
1067    )
1068}
1069
1070fn idle_root_eviction_message(
1071    root_id: &ProjectRootId,
1072    memory: &crate::memory::RootMemorySnapshot,
1073    pressure_relief: Option<&crate::memory::AllocatorPressureRelief>,
1074) -> String {
1075    // Bash, LSP, and parser state remain resident. The freed total is deliberately
1076    // only the known-byte portion of handles eviction actually drops.
1077    let freed_bytes = [
1078        &memory.semantic,
1079        &memory.trigram,
1080        &memory.symbols,
1081        &memory.callgraph,
1082        &memory.inspect,
1083    ]
1084    .iter()
1085    .filter_map(|estimate| estimate.estimated_bytes)
1086    .fold(0u64, u64::saturating_add);
1087    let mut message = format!(
1088        "evicted idle root {}: freed ~{:.1} MB (semantic {}, trigram {}, symbols {}, callgraph {}, inspect {}; retained: bash {}, lsp {}, parser_pool {})",
1089        root_id.as_path().display(),
1090        freed_bytes as f64 / (1024.0 * 1024.0),
1091        eviction_estimate_label(&memory.semantic),
1092        eviction_estimate_label(&memory.trigram),
1093        eviction_estimate_label(&memory.symbols),
1094        eviction_estimate_label(&memory.callgraph),
1095        eviction_estimate_label(&memory.inspect),
1096        eviction_estimate_label(&memory.bash),
1097        eviction_estimate_label(&memory.lsp),
1098        eviction_estimate_label(&memory.parser_pool),
1099    );
1100    if let Some(pressure_relief) = pressure_relief {
1101        message.push_str(&pressure_relief_label(pressure_relief));
1102    }
1103    message
1104}
1105
1106fn root_idle_ttl(executor: &Executor, root_id: &ProjectRootId) -> Duration {
1107    executor
1108        .actor_context(root_id)
1109        .map(|ctx| ctx.config().idle.root_ttl())
1110        .unwrap_or(IDLE_ROOT_TTL)
1111}
1112
1113fn process_has_been_idle(
1114    now: Instant,
1115    live_roots: &HashMap<ProjectRootId, RootMeta>,
1116    executor: &Executor,
1117) -> bool {
1118    !live_roots.is_empty()
1119        && live_roots.iter().all(|(root_id, meta)| {
1120            now.saturating_duration_since(meta.last_touched) >= root_idle_ttl(executor, root_id)
1121                && meta.active_bash_waits == 0
1122                && !meta.maintenance_pending
1123                && meta.maintenance_queued_kinds.is_empty()
1124        })
1125}
1126
1127fn allocator_pressure_relief_after_idle_sweep(
1128    now: Instant,
1129    live_roots: &HashMap<ProjectRootId, RootMeta>,
1130    executor: &Executor,
1131) -> Option<crate::memory::AllocatorPressureRelief> {
1132    if !process_has_been_idle(now, live_roots, executor)
1133        || live_roots.keys().any(|root_id| {
1134            executor
1135                .actor_context(root_id)
1136                .is_some_and(|ctx| ctx.artifact_eviction_blocked())
1137        })
1138    {
1139        return None;
1140    }
1141
1142    #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu")))]
1143    {
1144        Some(crate::memory::relieve_allocator_pressure())
1145    }
1146    #[cfg(not(any(target_os = "macos", all(target_os = "linux", target_env = "gnu"))))]
1147    {
1148        None
1149    }
1150}
1151
1152fn quiesce_unbound_root(
1153    root_id: &ProjectRootId,
1154    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1155    executor: &Arc<Executor>,
1156) {
1157    let Some(meta) = live_roots.get_mut(root_id) else {
1158        return;
1159    };
1160
1161    let ctx = executor.actor_context(root_id);
1162    if let Some(ctx) = ctx.as_ref() {
1163        // Close lifecycle admission before touching scheduler queues. A running
1164        // ConfigureTail cannot release gates, install a watcher, or reserve a
1165        // callgraph build after this transition becomes visible.
1166        ctx.mark_subc_unbound();
1167        ctx.bash_background()
1168            .replace_live_delivery_sessions(HashSet::new());
1169    }
1170    let cancelled = executor.cancel_queued_maintenance(root_id);
1171    // Transient unbind keeps the root WARM: the watcher stays running (its
1172    // events accumulate and replay on rebind, so no unobserved gap exists) and
1173    // resident artifacts stay resident. Host restarts unbind every root and
1174    // rebind seconds later; stopping the watcher here would force strict
1175    // re-verification plus a full callgraph rebuild on every restart. The
1176    // expensive teardown (watcher stop + gap invalidation) belongs to the
1177    // idle-TTL reaper and the root-deleted path.
1178    let discarded = ctx
1179        .map(|ctx| crate::commands::configure::cancel_deferred_configure_maintenance(&ctx))
1180        .unwrap_or(0);
1181    meta.unbound_quiesced = true;
1182    meta.maintenance_queued_kinds.clear();
1183    meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
1184    log::info!(
1185        "subc attach: quiesced unbound root {} (cancelled {} queued maintenance job(s), cancelled {} configure maintenance job(s)); cause=goodbye_unbound",
1186        root_id.as_path().display(),
1187        cancelled,
1188        discarded
1189    );
1190}
1191
1192#[allow(clippy::too_many_arguments)]
1193fn quiesce_connection_roots(
1194    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1195    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
1196    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1197    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1198    installed_route_epochs: &mut HashMap<u16, u32>,
1199    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1200    active_tool_calls: &ActiveToolCalls,
1201    executor: &Arc<Executor>,
1202) {
1203    cancel_all_active_tool_calls(active_tool_calls, executor, "connection teardown");
1204    for cancel in route_bash_cancels.values() {
1205        cancel.token.cancel();
1206    }
1207    route_bash_cancels.clear();
1208
1209    let mut roots = live_roots.keys().cloned().collect::<HashSet<_>>();
1210    for pending in pending_binds.values_mut() {
1211        pending.cancelled = true;
1212        roots.insert(pending.bind_root_id.clone());
1213        let _ = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
1214    }
1215
1216    // A connection exit abandons every installed route at once. Close lifecycle
1217    // admission before cancelling maintenance so no deferred worker can restore
1218    // root activity after the loop-owned route tables disappear.
1219    for root_id in roots {
1220        if live_roots.contains_key(&root_id) {
1221            quiesce_unbound_root(&root_id, live_roots, executor);
1222        } else if let Some(ctx) = executor.actor_context(&root_id) {
1223            ctx.mark_subc_unbound();
1224            executor.cancel_queued_maintenance(&root_id);
1225            crate::commands::configure::cancel_deferred_configure_maintenance(&ctx);
1226        }
1227    }
1228
1229    routes.clear();
1230    root_channels.clear();
1231    installed_route_epochs.clear();
1232}
1233
1234/// Per-channel epoch watermarks for roots reclaimed without a client Goodbye.
1235/// The 16-bit channel space bounds this map, and no root identity or resource
1236/// handle is retained. It exists only so late requests receive a typed error.
1237#[derive(Debug, Default)]
1238struct ReclaimedRoutes {
1239    highest_epoch_by_channel: HashMap<u16, u32>,
1240}
1241
1242impl ReclaimedRoutes {
1243    fn insert(&mut self, route: RouteChannel) {
1244        self.highest_epoch_by_channel
1245            .entry(route.channel)
1246            .and_modify(|epoch| *epoch = (*epoch).max(route.epoch))
1247            .or_insert(route.epoch);
1248    }
1249
1250    fn contains(&self, route: RouteChannel) -> bool {
1251        self.highest_epoch_by_channel
1252            .get(&route.channel)
1253            .is_some_and(|epoch| route.epoch <= *epoch)
1254    }
1255}
1256
1257#[derive(Debug, Default)]
1258struct IdleReapOutcome {
1259    evicted: usize,
1260    forgotten_deleted_roots: Vec<ProjectRootId>,
1261}
1262
1263fn reap_idle_roots(
1264    now: Instant,
1265    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1266    pending_binds: &HashMap<RouteChannel, PendingBind>,
1267    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1268    executor: &Arc<Executor>,
1269    metrics: &DispatchPathMetrics,
1270) -> IdleReapOutcome {
1271    let pending_bind_roots = pending_binds
1272        .values()
1273        .map(|pending| pending.bind_root_id.clone())
1274        .collect::<HashSet<_>>();
1275    let mut census = ReapBlockerCensus::default();
1276    let mut candidates = Vec::new();
1277
1278    for (root_id, meta) in live_roots.iter_mut() {
1279        let deleted = !root_id.as_path().exists();
1280        if deleted {
1281            // A missing directory makes a bound route obsolete, but one failed
1282            // lookup is not enough evidence to tear down a client-visible actor.
1283            // Requiring two maintenance sweeps protects atomic replacement and
1284            // transient filesystem failures; observing the path resets the proof.
1285            // Absence also covers renames: a task's cwd handle can follow the
1286            // moved directory while the registered path disappears. The old
1287            // path is deliberately treated as a retired root identity, so the
1288            // reaper accepts killing such tasks; rename a project only with no
1289            // live tasks rather than relying on cwd-resolution heuristics.
1290            meta.consecutive_missing_sweeps = meta.consecutive_missing_sweeps.saturating_add(1);
1291        } else {
1292            meta.consecutive_missing_sweeps = 0;
1293        }
1294        let deletion_confirmed = meta.consecutive_missing_sweeps >= 2;
1295        let has_bound_route = root_channels
1296            .get(root_id)
1297            .is_some_and(|channels| !channels.is_empty());
1298        let has_pending_bind = pending_bind_roots.contains(root_id);
1299
1300        if deleted {
1301            let mut retained = false;
1302            if !deletion_confirmed {
1303                census.absence_unconfirmed += 1;
1304                retained = true;
1305            }
1306            // Once absence is confirmed, the directory cannot serve this route
1307            // again. Neither a stale route nor the lack of normal unbind cleanup
1308            // justifies retaining the root; purge removes the route after retirement.
1309            if meta.active_bash_waits > 0 {
1310                census.bash_waits += 1;
1311                retained = true;
1312            }
1313            if meta.maintenance_pending {
1314                census.maintenance_pending += 1;
1315                retained = true;
1316            }
1317            if !meta.maintenance_queued_kinds.is_empty() {
1318                census.maintenance_queued += 1;
1319                retained = true;
1320            }
1321            if has_pending_bind {
1322                census.pending_binds += 1;
1323                retained = true;
1324            }
1325            match executor.try_actor_is_idle(root_id) {
1326                Some(true) => {}
1327                Some(false) => {
1328                    census.actor_busy += 1;
1329                    retained = true;
1330                }
1331                None => {
1332                    census.actor_state_busy += 1;
1333                    retained = true;
1334                }
1335            }
1336            if retained {
1337                census.deleted_retained += 1;
1338                continue;
1339            }
1340        } else {
1341            // Route teardown marks the lifecycle admission gate before the last
1342            // channel disappears. Requiring zero bound channels and a quiesced
1343            // lifecycle prevents a still-bound root from losing its watcher.
1344            if has_bound_route
1345                || !meta.unbound_quiesced
1346                || meta.idle_artifacts_evicted
1347                || now.saturating_duration_since(meta.last_touched)
1348                    < root_idle_ttl(executor, root_id)
1349                || meta.active_bash_waits > 0
1350                || meta.maintenance_pending
1351                || !meta.maintenance_queued_kinds.is_empty()
1352                || has_pending_bind
1353                || !executor.actor_is_idle(root_id)
1354            {
1355                continue;
1356            }
1357        }
1358        candidates.push((root_id.clone(), deleted));
1359    }
1360
1361    let mut reaped = Vec::new();
1362    let mut forgotten_deleted_roots = Vec::new();
1363    for (root_id, deleted) in candidates {
1364        let Some(ctx) = executor.actor_context(&root_id) else {
1365            if deleted {
1366                census.deleted_retained += 1;
1367                census.actor_busy += 1;
1368            }
1369            continue;
1370        };
1371        // A TTL-aged unbound root retained its watcher-derived pending paths
1372        // across the transient-unbind window. Strict gap invalidation subsumes
1373        // them, but every abort path must restore them because a rebind can
1374        // still happen until eviction commits.
1375        //
1376        // After two consecutive directory-absence scans confirm that the
1377        // root is gone, terminate its background task before checking the
1378        // artifact-eviction gate. The task can otherwise keep the root's
1379        // artifacts in use; cleanup first lets confirmed reclamation finish
1380        // without weakening the gate for unrelated active work.
1381        if deleted {
1382            ctx.bash_background()
1383                .kill_running_tasks_for_root(root_id.as_path());
1384        }
1385        let taken_pending = Some(ctx.take_pending_reconciliation_state());
1386        if ctx.artifact_eviction_blocked() {
1387            if let Some(pending) = taken_pending {
1388                ctx.restore_pending_reconciliation_state(pending);
1389            }
1390            if deleted {
1391                census.deleted_retained += 1;
1392                census.artifact_eviction_blocked += 1;
1393            }
1394            continue;
1395        }
1396        let memory_before = ctx.memory_root_snapshot();
1397        if !ctx.evict_idle_artifacts() {
1398            if let Some(pending) = taken_pending {
1399                ctx.restore_pending_reconciliation_state(pending);
1400            }
1401            if deleted {
1402                census.deleted_retained += 1;
1403                census.artifact_eviction_failed += 1;
1404            }
1405            continue;
1406        }
1407        drop(taken_pending);
1408        ctx.stop_watcher_runtime_in_background();
1409        // Edits during watcher downtime are unobserved. Advance publication
1410        // epochs and force strict verification before any later warm reload.
1411        ctx.invalidate_artifacts_after_watcher_gap();
1412
1413        if deleted {
1414            if executor.retire_idle_actor_in_background(&root_id) {
1415                live_roots.remove(&root_id);
1416                forgotten_deleted_roots.push(root_id.clone());
1417            } else {
1418                census.deleted_retained += 1;
1419                census.actor_busy += 1;
1420            }
1421        } else {
1422            if let Some(meta) = live_roots.get_mut(&root_id) {
1423                meta.idle_artifacts_evicted = true;
1424            }
1425            ctx.release_idle_reopenable_resources_in_background();
1426        }
1427        reaped.push((root_id, memory_before));
1428    }
1429
1430    // Emit-on-change: the census is on the health surface every sweep; the log
1431    // line is for transitions, including the one back to zero retained.
1432    let census_changed = metrics.record_reap(census);
1433    if census_changed {
1434        log::info!(
1435            "subc attach: retained {} deleted root(s) during idle reap; blockers={}",
1436            census.deleted_retained,
1437            census.blocker_histogram()
1438        );
1439    }
1440
1441    let pressure_relief = (!reaped.is_empty())
1442        .then(|| allocator_pressure_relief_after_idle_sweep(now, live_roots, executor))
1443        .flatten();
1444    for (root_id, memory_before) in &reaped {
1445        log::info!(
1446            "{}",
1447            idle_root_eviction_message(root_id, memory_before, pressure_relief.as_ref())
1448        );
1449    }
1450    IdleReapOutcome {
1451        evicted: reaped.len(),
1452        forgotten_deleted_roots,
1453    }
1454}
1455
1456/// Shut down language servers for roots that have had no request for the
1457/// configured LSP idle window. This is independent of artifact eviction and
1458/// runs even while the root is still bound.
1459///
1460/// `meta.last_touched` moves on every inbound tool call for a bound root
1461/// (`reactivate_bound` in the route-request handler) and on response delivery,
1462/// so an active harness does not look idle.
1463fn reap_idle_lsp_servers(
1464    now: Instant,
1465    live_roots: &HashMap<ProjectRootId, RootMeta>,
1466    executor: &Executor,
1467) {
1468    for (root_id, meta) in live_roots {
1469        let Some(ctx) = executor.actor_context(root_id) else {
1470            continue;
1471        };
1472        crate::runtime_drain::shutdown_idle_lsp_at(&ctx, now, meta.last_touched);
1473    }
1474}
1475
1476#[allow(clippy::too_many_arguments)]
1477fn purge_deleted_root_residents(
1478    root_id: &ProjectRootId,
1479    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1480    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1481    installed_route_epochs: &mut HashMap<u16, u32>,
1482    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1483    active_tool_calls: &ActiveToolCalls,
1484    executor: &Executor,
1485    retry_buffer: &mut RetryBuffer,
1486    reclaimed_routes: &mut ReclaimedRoutes,
1487    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1488    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1489    bg_subs: &mut HashMap<RouteChannel, BgSub>,
1490    bg_sub_by_session: &mut BgSubsBySession,
1491    bg_wake_pending: &mut BgWakePending,
1492    bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
1493    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1494    metrics: &DispatchPathMetrics,
1495) {
1496    let mut stale_routes = root_channels.get(root_id).cloned().unwrap_or_default();
1497    stale_routes.extend(
1498        routes
1499            .iter()
1500            .filter_map(|(route, identity)| (&identity.root == root_id).then_some(*route)),
1501    );
1502    stale_routes.extend(
1503        bg_sub_by_session
1504            .iter()
1505            .filter(|((root, _), _)| root == root_id)
1506            .flat_map(|(_, routes)| routes.iter().copied()),
1507    );
1508    stale_routes.extend(
1509        pending_bash_asks
1510            .values()
1511            .filter_map(|ask| (&ask.root == root_id).then_some(ask.route)),
1512    );
1513
1514    for route in stale_routes {
1515        reclaimed_routes.insert(route);
1516        remove_installed_route(installed_route_epochs, route);
1517        remove_route_channel(routes, root_channels, route);
1518        if let Some(cancel) = route_bash_cancels.remove(&route) {
1519            cancel.token.cancel();
1520        }
1521        apply_route_work_disposition(
1522            active_tool_calls,
1523            executor,
1524            route,
1525            RouteWorkDisposition::Abandon,
1526            "root reclaim",
1527        );
1528        retry_buffer.remove(&route);
1529        if let Some(sub) = bg_subs.remove(&route) {
1530            metrics.record_bg_subscription_ended(&sub.root, &sub.session, route, "root-reclaim");
1531        }
1532        bg_wake_pending.remove(&route);
1533    }
1534    root_channels.remove(root_id);
1535    session_identity.retain(|(root, _), _| root != root_id);
1536    push_buffer.retain(|key, _| &key.root != root_id);
1537    bg_wake_epoch.retain(|(root, _), _| root != root_id);
1538    pending_bash_asks.retain(|_, ask| &ask.root != root_id);
1539    bg_sub_by_session.retain(|(root, _), _| root != root_id);
1540    sync_bg_live_delivery_sessions(executor, routes, Some(root_id));
1541
1542    log::info!(
1543        "subc attach: fully forgot deleted root {}; cause=absence_reclaim",
1544        root_id.as_path().display()
1545    );
1546}
1547
1548#[allow(clippy::too_many_arguments)]
1549fn submit_due_maintenance_jobs(
1550    executor: &Arc<Executor>,
1551    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1552    pending_binds: &HashMap<RouteChannel, PendingBind>,
1553    bg_sub_by_session: &BgSubsBySession,
1554    bg_wake_pending: &BgWakePending,
1555    bg_wake_epoch: &HashMap<(ProjectRootId, String), u64>,
1556    maintenance_tx: &mpsc::Sender<MaintenanceCompletion>,
1557    metrics: &Arc<DispatchPathMetrics>,
1558) {
1559    let pending_bind_roots = pending_binds
1560        .values()
1561        .map(|pending| pending.bind_root_id.clone())
1562        .collect::<HashSet<_>>();
1563    let (due_jobs, deferred_jobs) = due_maintenance_jobs(
1564        live_roots,
1565        Some(executor),
1566        bg_sub_by_session,
1567        bg_wake_pending,
1568        MAINTENANCE_SUBMIT_BUDGET,
1569        &pending_bind_roots,
1570    );
1571    if deferred_jobs {
1572        metrics
1573            .maintenance_budget_deferrals
1574            .fetch_add(1, Ordering::Relaxed);
1575    }
1576    for (root_id, kind) in due_jobs {
1577        let bg_sessions_to_check = if kind == MaintenanceDrainKind::CompletionDrains {
1578            bg_sub_by_session
1579                .iter()
1580                .filter_map(|((root, session), _)| {
1581                    if root == &root_id {
1582                        Some((
1583                            session.clone(),
1584                            bg_wake_epoch
1585                                .get(&(root_id.clone(), session.clone()))
1586                                .copied()
1587                                .unwrap_or(0),
1588                        ))
1589                    } else {
1590                        None
1591                    }
1592                })
1593                .collect()
1594        } else {
1595            Vec::new()
1596        };
1597        submit_maintenance_job(
1598            executor,
1599            root_id,
1600            kind,
1601            bg_sessions_to_check,
1602            maintenance_tx,
1603            metrics,
1604        );
1605    }
1606}
1607
1608fn should_requiesce_after_maintenance(
1609    meta: &RootMeta,
1610    completed_kind: MaintenanceDrainKind,
1611    bind_pending: bool,
1612) -> bool {
1613    meta.unbound_quiesced && completed_kind != MaintenanceDrainKind::Lsp && !bind_pending
1614}
1615
1616fn note_maintenance_completion(
1617    meta: &mut RootMeta,
1618    requeue_kind: Option<MaintenanceDrainKind>,
1619    fatal: bool,
1620    defer_requeue: bool,
1621) {
1622    if fatal {
1623        meta.maintenance_poisoned = true;
1624    }
1625
1626    if let Some(kind) = requeue_kind.filter(|_| !meta.maintenance_poisoned && !defer_requeue) {
1627        meta.maintenance_queued_kinds.push_back(kind);
1628    }
1629
1630    meta.maintenance_jobs_in_flight = meta.maintenance_jobs_in_flight.saturating_sub(1);
1631    meta.maintenance_pending =
1632        meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
1633}
1634
1635fn route_key(channel: u16, epoch: u32) -> RouteChannel {
1636    RouteChannel { channel, epoch }
1637}
1638
1639fn remove_installed_route(installed_epochs: &mut HashMap<u16, u32>, route: RouteChannel) {
1640    if installed_epochs.get(&route.channel).copied() == Some(route.epoch) {
1641        installed_epochs.remove(&route.channel);
1642    }
1643}
1644
1645fn ingress_route_should_be_processed(
1646    installed_epochs: &HashMap<u16, u32>,
1647    reclaimed_routes: &ReclaimedRoutes,
1648    frame: &Frame,
1649) -> bool {
1650    if frame.header.channel == 0
1651        || installed_epochs.get(&frame.header.channel).copied() == Some(frame.header.epoch)
1652    {
1653        return true;
1654    }
1655
1656    // A late request for a reclaimed root reaches the normal unknown-route
1657    // handler, which returns the typed `route_not_bound` error. Other stale or
1658    // never-installed generations remain silent so they cannot affect a newer
1659    // route or change the protocol's rejected-bind behavior.
1660    frame.header.ty == FrameType::Request
1661        && reclaimed_routes.contains(route_key(frame.header.channel, frame.header.epoch))
1662}
1663
1664fn bash_elicitation_timeout() -> Duration {
1665    if cfg!(debug_assertions) {
1666        if let Ok(raw) = std::env::var("AFT_TEST_SUBC_BASH_ELICITATION_TTL_MS") {
1667            if let Ok(ms) = raw.parse::<u64>() {
1668                if ms > 0 {
1669                    return Duration::from_millis(ms);
1670                }
1671            }
1672        }
1673    }
1674    BASH_ELICITATION_TIMEOUT
1675}
1676
1677fn allocate_reverse_corr(
1678    pending_bash_asks: &HashMap<ReverseCorrKey, PendingBashAsk>,
1679    route: RouteChannel,
1680    next_corr: &mut u64,
1681) -> u64 {
1682    loop {
1683        let corr = *next_corr;
1684        *next_corr = (*next_corr).wrapping_add(1).max(1);
1685        if !pending_bash_asks.contains_key(&ReverseCorrKey { route, corr }) {
1686            return corr;
1687        }
1688    }
1689}
1690
1691fn bash_permission_kind_label(kind: &crate::bash_permissions::PermissionKind) -> &'static str {
1692    match kind {
1693        crate::bash_permissions::PermissionKind::ExternalDirectory => "external directory",
1694        crate::bash_permissions::PermissionKind::Bash => "bash",
1695    }
1696}
1697
1698fn bash_elicitation_patterns(asks: &[crate::bash_permissions::PermissionAsk]) -> Vec<String> {
1699    let mut patterns = Vec::new();
1700    let mut seen = HashSet::new();
1701    for ask in asks {
1702        for pattern in ask.patterns.iter().chain(ask.always.iter()) {
1703            if seen.insert(pattern.clone()) {
1704                patterns.push(pattern.clone());
1705            }
1706        }
1707    }
1708    patterns
1709}
1710
1711fn bash_elicitation_message(
1712    command: &str,
1713    asks: &[crate::bash_permissions::PermissionAsk],
1714) -> String {
1715    let command = command.split_whitespace().collect::<Vec<_>>().join(" ");
1716    let patterns = bash_elicitation_patterns(asks);
1717    let pattern_text = if patterns.is_empty() {
1718        "no matched permission patterns".to_string()
1719    } else {
1720        patterns.join(", ")
1721    };
1722    let ask_kinds = asks
1723        .iter()
1724        .map(|ask| bash_permission_kind_label(&ask.kind))
1725        .collect::<HashSet<_>>()
1726        .into_iter()
1727        .collect::<Vec<_>>()
1728        .join(", ");
1729    if ask_kinds.is_empty() {
1730        format!("Allow bash command `{command}`? Matched patterns: {pattern_text}")
1731    } else {
1732        format!("Allow bash command `{command}`? Matched {ask_kinds} patterns: {pattern_text}")
1733    }
1734}
1735
1736fn bash_elicitation_request_body(
1737    command: &str,
1738    asks: &[crate::bash_permissions::PermissionAsk],
1739) -> Value {
1740    json!({
1741        "method": BASH_ELICITATION_CREATE_METHOD,
1742        "params": {
1743            "mode": "form",
1744            "message": bash_elicitation_message(command, asks),
1745            "requestedSchema": {
1746                "type": "object",
1747                "properties": {
1748                    "decision": {
1749                        "type": "string",
1750                        "enum": ["allow", "deny"],
1751                        "description": "Choose allow to run this bash command once, or deny to block it."
1752                    }
1753                },
1754                "required": ["decision"],
1755                "additionalProperties": false
1756            },
1757            "_meta": {
1758                "aft": {
1759                    "tool": "bash",
1760                    "command": command,
1761                    "asks": asks
1762                }
1763            }
1764        }
1765    })
1766}
1767
1768fn build_bash_elicitation_request_frame(
1769    ver: u8,
1770    route: RouteChannel,
1771    corr: u64,
1772    flags: Flags,
1773    command: &str,
1774    asks: &[crate::bash_permissions::PermissionAsk],
1775) -> Result<Frame, SubcError> {
1776    let body = bash_elicitation_request_body(command, asks);
1777    Frame::build_with_version(
1778        ver,
1779        FrameType::Request,
1780        flags,
1781        route.channel,
1782        route.epoch,
1783        corr,
1784        serde_json::to_vec(&body).map_err(SubcError::Json)?,
1785    )
1786    .map_err(SubcError::FrameBuild)
1787}
1788
1789fn bash_elicitation_reply_is_allow(body: &[u8]) -> bool {
1790    let Ok(value) = serde_json::from_slice::<Value>(body) else {
1791        return false;
1792    };
1793    flat_bash_elicitation_reply_is_allow(&value) || mcp_bash_elicitation_reply_is_allow(&value)
1794}
1795
1796fn flat_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1797    let Some(object) = value.as_object() else {
1798        return false;
1799    };
1800    object.len() == 1 && object.get("decision").and_then(Value::as_str) == Some("allow")
1801}
1802
1803fn mcp_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1804    let Some(object) = value.as_object() else {
1805        return false;
1806    };
1807    if object.len() != 2 || object.get("action").and_then(Value::as_str) != Some("accept") {
1808        return false;
1809    }
1810    let Some(content) = object.get("content").and_then(Value::as_object) else {
1811        return false;
1812    };
1813    content.len() == 1 && content.get("decision").and_then(Value::as_str) == Some("allow")
1814}
1815
1816#[allow(clippy::too_many_arguments)]
1817async fn settle_pending_bash_ask_denied(
1818    tx: &WriterSender,
1819    pending: PendingBashAsk,
1820    routes: &HashMap<RouteChannel, RouteIdentity>,
1821    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1822    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1823    shutdown: &Arc<Notify>,
1824    metrics: &DispatchPathMetrics,
1825) -> Result<(), SubcError> {
1826    let completion = bash::bash_denied_untrusted_completion(
1827        pending.route,
1828        pending.tool_corr,
1829        pending.tool_flags,
1830        pending.tool_ver,
1831        pending.root,
1832        pending.request_id,
1833        pending.format_context,
1834    );
1835    bash::handle_bash_deferred_completion(
1836        tx,
1837        completion,
1838        routes,
1839        live_roots,
1840        route_bash_cancels,
1841        shutdown,
1842        metrics,
1843    )
1844    .await
1845}
1846
1847fn take_pending_bash_asks_for_route(
1848    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1849    route: RouteChannel,
1850) -> Vec<PendingBashAsk> {
1851    let keys = pending_bash_asks
1852        .keys()
1853        .copied()
1854        .filter(|key| key.route == route)
1855        .collect::<Vec<_>>();
1856    keys.into_iter()
1857        .filter_map(|key| pending_bash_asks.remove(&key))
1858        .collect()
1859}
1860
1861#[allow(clippy::too_many_arguments)]
1862async fn settle_pending_bash_asks_for_route(
1863    tx: &WriterSender,
1864    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1865    route: RouteChannel,
1866    routes: &HashMap<RouteChannel, RouteIdentity>,
1867    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1868    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1869    shutdown: &Arc<Notify>,
1870    metrics: &DispatchPathMetrics,
1871) -> Result<(), SubcError> {
1872    for pending in take_pending_bash_asks_for_route(pending_bash_asks, route) {
1873        settle_pending_bash_ask_denied(
1874            tx,
1875            pending,
1876            routes,
1877            live_roots,
1878            route_bash_cancels,
1879            shutdown,
1880            metrics,
1881        )
1882        .await?;
1883    }
1884    Ok(())
1885}
1886
1887#[allow(clippy::too_many_arguments)]
1888async fn settle_all_pending_bash_asks(
1889    tx: &WriterSender,
1890    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1891    routes: &HashMap<RouteChannel, RouteIdentity>,
1892    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1893    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1894    shutdown: &Arc<Notify>,
1895    metrics: &DispatchPathMetrics,
1896) -> Result<(), SubcError> {
1897    let pending = pending_bash_asks
1898        .drain()
1899        .map(|(_, pending)| pending)
1900        .collect::<Vec<_>>();
1901    for pending in pending {
1902        settle_pending_bash_ask_denied(
1903            tx,
1904            pending,
1905            routes,
1906            live_roots,
1907            route_bash_cancels,
1908            shutdown,
1909            metrics,
1910        )
1911        .await?;
1912    }
1913    Ok(())
1914}
1915
1916#[allow(clippy::too_many_arguments)]
1917async fn expire_pending_bash_asks(
1918    tx: &WriterSender,
1919    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1920    routes: &HashMap<RouteChannel, RouteIdentity>,
1921    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1922    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1923    shutdown: &Arc<Notify>,
1924    metrics: &DispatchPathMetrics,
1925) -> Result<(), SubcError> {
1926    let now = Instant::now();
1927    let expired = pending_bash_asks
1928        .iter()
1929        .filter_map(|(key, pending)| (pending.expires_at <= now).then_some(*key))
1930        .collect::<Vec<_>>();
1931    for key in expired {
1932        if let Some(pending) = pending_bash_asks.remove(&key) {
1933            log::debug!(
1934                "subc attach: bash elicitation request {} on route {} expired fail-closed",
1935                key.corr,
1936                pending.route
1937            );
1938            settle_pending_bash_ask_denied(
1939                tx,
1940                pending,
1941                routes,
1942                live_roots,
1943                route_bash_cancels,
1944                shutdown,
1945                metrics,
1946            )
1947            .await?;
1948        }
1949    }
1950    Ok(())
1951}
1952
1953#[allow(clippy::too_many_arguments)]
1954async fn handle_bash_elicitation_reply(
1955    tx: &WriterSender,
1956    frame: &Frame,
1957    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1958    routes: &HashMap<RouteChannel, RouteIdentity>,
1959    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1960    executor: &Arc<Executor>,
1961    shutdown: &Arc<Notify>,
1962    bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
1963    bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
1964    metrics: &Arc<DispatchPathMetrics>,
1965    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1966    dispatch: DispatchFn,
1967) -> Result<(), SubcError> {
1968    let key = ReverseCorrKey {
1969        route: route_key(frame.header.channel, frame.header.epoch),
1970        corr: frame.header.corr,
1971    };
1972    let Some(pending) = pending_bash_asks.remove(&key) else {
1973        return Ok(());
1974    };
1975
1976    if frame.header.ty == FrameType::Response && bash_elicitation_reply_is_allow(&frame.body) {
1977        if routes.contains_key(&key.route) {
1978            bash::submit_deferred_bash(
1979                executor,
1980                bash_deferred_tx,
1981                bash_poll_touch_tx,
1982                metrics,
1983                dispatch,
1984                pending.root,
1985                pending.project_root,
1986                pending.session_id,
1987                pending.request_id,
1988                pending.route,
1989                pending.tool_corr,
1990                pending.tool_flags,
1991                pending.tool_ver,
1992                pending.arguments,
1993                pending.format_context,
1994                pending.cancel,
1995                BindTrust::Untrusted,
1996                pending.spawn_principal,
1997                pending.edit_slot_survives,
1998                Some(pending.grants),
1999            );
2000            return Ok(());
2001        }
2002        log::debug!(
2003            "subc attach: dropping allowed bash elicitation reply {} for unbound route {}",
2004            key.corr,
2005            pending.route
2006        );
2007    }
2008
2009    settle_pending_bash_ask_denied(
2010        tx,
2011        pending,
2012        routes,
2013        live_roots,
2014        route_bash_cancels,
2015        shutdown,
2016        metrics,
2017    )
2018    .await
2019}
2020
2021#[allow(clippy::too_many_arguments)]
2022async fn cancel_pending_bash_ask_for_tool_call(
2023    tx: &WriterSender,
2024    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
2025    route: RouteChannel,
2026    tool_corr: u64,
2027    routes: &HashMap<RouteChannel, RouteIdentity>,
2028    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2029    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
2030    shutdown: &Arc<Notify>,
2031    metrics: &DispatchPathMetrics,
2032) -> Result<(), SubcError> {
2033    let keys = pending_bash_asks
2034        .iter()
2035        .filter_map(|(key, pending)| {
2036            (key.route == route && pending.tool_corr == tool_corr).then_some(*key)
2037        })
2038        .collect::<Vec<_>>();
2039    for key in keys {
2040        if let Some(pending) = pending_bash_asks.remove(&key) {
2041            settle_pending_bash_ask_denied(
2042                tx,
2043                pending,
2044                routes,
2045                live_roots,
2046                route_bash_cancels,
2047                shutdown,
2048                metrics,
2049            )
2050            .await?;
2051        }
2052    }
2053    Ok(())
2054}
2055
2056fn remove_root_channel(
2057    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2058    root: &ProjectRootId,
2059    channel: RouteChannel,
2060) {
2061    let remove_root = if let Some(channels) = root_channels.get_mut(root) {
2062        channels.remove(&channel);
2063        channels.is_empty()
2064    } else {
2065        false
2066    };
2067    if remove_root {
2068        root_channels.remove(root);
2069    }
2070}
2071
2072fn remove_route_channel(
2073    routes: &mut HashMap<RouteChannel, RouteIdentity>,
2074    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2075    channel: RouteChannel,
2076) -> Option<RouteIdentity> {
2077    let removed = routes.remove(&channel);
2078    if let Some(identity) = &removed {
2079        remove_root_channel(root_channels, &identity.root, channel);
2080    }
2081    removed
2082}
2083
2084fn insert_route_channel(
2085    routes: &mut HashMap<RouteChannel, RouteIdentity>,
2086    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2087    channel: RouteChannel,
2088    identity: RouteIdentity,
2089) {
2090    if let Some(previous) = routes.insert(channel, identity.clone()) {
2091        remove_root_channel(root_channels, &previous.root, channel);
2092    }
2093    root_channels
2094        .entry(identity.root.clone())
2095        .or_default()
2096        .insert(channel);
2097}
2098
2099fn sync_bg_live_delivery_sessions(
2100    executor: &Executor,
2101    routes: &HashMap<RouteChannel, RouteIdentity>,
2102    additional_root: Option<&ProjectRootId>,
2103) {
2104    // The loop-owned installed-route table is the lifecycle source of truth:
2105    // an originating session is live exactly while the daemon has an installed,
2106    // bash-observation-capable route whose identity names that session.
2107    let sessions = routes
2108        .values()
2109        .filter(|identity| identity.trust.allows_bash_observation())
2110        .map(|identity| identity.session.clone())
2111        .collect::<HashSet<_>>();
2112    let mut roots = routes
2113        .values()
2114        .map(|identity| identity.root.clone())
2115        .collect::<HashSet<_>>();
2116    roots.extend(additional_root.cloned());
2117    for root in roots {
2118        if let Some(ctx) = executor.actor_context(&root) {
2119            ctx.bash_background()
2120                .replace_live_delivery_sessions(sessions.clone());
2121        }
2122    }
2123}
2124
2125fn insert_bg_subscription_index(
2126    bg_sub_by_session: &mut BgSubsBySession,
2127    root: ProjectRootId,
2128    session: String,
2129    channel: RouteChannel,
2130) {
2131    bg_sub_by_session
2132        .entry((root, session))
2133        .or_default()
2134        .insert(channel);
2135}
2136
2137fn remove_bg_subscription_index(
2138    bg_sub_by_session: &mut BgSubsBySession,
2139    channel: RouteChannel,
2140    identity: Option<&RouteIdentity>,
2141) {
2142    if let Some(identity) = identity {
2143        let key = (identity.root.clone(), identity.session.clone());
2144        let remove_key = bg_sub_by_session.get_mut(&key).is_some_and(|channels| {
2145            channels.remove(&channel);
2146            channels.is_empty()
2147        });
2148        if remove_key {
2149            bg_sub_by_session.remove(&key);
2150        }
2151    } else {
2152        bg_sub_by_session.retain(|_, channels| {
2153            channels.remove(&channel);
2154            !channels.is_empty()
2155        });
2156    }
2157}
2158
2159fn route_removal_will_quiesce_root(
2160    root: &ProjectRootId,
2161    route: RouteChannel,
2162    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
2163    has_pending_bind: bool,
2164    replacement_root: Option<&ProjectRootId>,
2165) -> bool {
2166    let removes_last_route = root_channels
2167        .get(root)
2168        .is_some_and(|channels| channels.len() == 1 && channels.contains(&route));
2169    removes_last_route && !has_pending_bind && replacement_root != Some(root)
2170}
2171
2172fn should_quiesce_removed_root(
2173    root: &ProjectRootId,
2174    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
2175    has_pending_bind: bool,
2176    replacement_root: Option<&ProjectRootId>,
2177) -> bool {
2178    !root_channels.contains_key(root) && !has_pending_bind && replacement_root != Some(root)
2179}
2180
2181async fn end_bg_subscription(
2182    writer_tx: &WriterSender,
2183    metrics: &DispatchPathMetrics,
2184    bg_subs: &mut HashMap<RouteChannel, BgSub>,
2185    bg_sub_by_session: &mut BgSubsBySession,
2186    bg_wake_pending: &mut BgWakePending,
2187    channel: RouteChannel,
2188    identity: Option<&RouteIdentity>,
2189    cause: &str,
2190) -> Result<(), SubcError> {
2191    if let Some(sub) = bg_subs.remove(&channel) {
2192        bg_wake_pending.remove(&channel);
2193        remove_bg_subscription_index(bg_sub_by_session, channel, identity);
2194        metrics.record_bg_subscription_ended(&sub.root, &sub.session, channel, cause);
2195        push::send_reliable_bg_stream_end(writer_tx, metrics, channel, &sub).await?;
2196    }
2197    Ok(())
2198}
2199
2200#[allow(clippy::too_many_arguments)]
2201async fn teardown_installed_route(
2202    tx: &WriterSender,
2203    metrics: &DispatchPathMetrics,
2204    executor: &Arc<Executor>,
2205    channel: RouteChannel,
2206    cancellation_reason: &str,
2207    replacement_root: Option<&ProjectRootId>,
2208    installed_route_epochs: &mut HashMap<u16, u32>,
2209    routes: &mut HashMap<RouteChannel, RouteIdentity>,
2210    management_routes: &mut HashSet<RouteChannel>,
2211    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2212    bg_subs: &mut HashMap<RouteChannel, BgSub>,
2213    bg_sub_by_session: &mut BgSubsBySession,
2214    bg_wake_pending: &mut BgWakePending,
2215    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
2216    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2217    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
2218    active_tool_calls: &ActiveToolCalls,
2219    pending_responses: &mut PendingSubcResponses,
2220    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2221    retry_buffer: &mut RetryBuffer,
2222    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2223    shutdown: &Arc<Notify>,
2224    tool_response_body_limit: usize,
2225    lifecycle_probe: Option<&SubcTestLifecycleProbe>,
2226) -> Result<(), SubcError> {
2227    remove_installed_route(installed_route_epochs, channel);
2228    management_routes.remove(&channel);
2229    let bg_end_cause = match cancellation_reason {
2230        "Goodbye" => "goodbye",
2231        "higher-epoch RouteBind" => "higher-epoch",
2232        other => other,
2233    };
2234    end_bg_subscription(
2235        tx,
2236        metrics,
2237        bg_subs,
2238        bg_sub_by_session,
2239        bg_wake_pending,
2240        channel,
2241        routes.get(&channel),
2242        bg_end_cause,
2243    )
2244    .await?;
2245    settle_pending_bash_asks_for_route(
2246        tx,
2247        pending_bash_asks,
2248        channel,
2249        routes,
2250        live_roots,
2251        route_bash_cancels,
2252        shutdown,
2253        metrics,
2254    )
2255    .await?;
2256    if let Some(cancel) = route_bash_cancels.remove(&channel) {
2257        cancel.token.cancel();
2258    }
2259    for resolved in pending_responses.drain_route(channel, executor) {
2260        deliver_resolved_subc_response(
2261            tx,
2262            resolved,
2263            routes,
2264            live_roots,
2265            executor.as_ref(),
2266            active_tool_calls,
2267            shutdown,
2268            metrics,
2269            tool_response_body_limit,
2270        )
2271        .await?;
2272    }
2273    // A higher-epoch replacement keeps replayable work because the logical
2274    // route remains live. A route that actually closes keeps only calls that
2275    // already started; pre-execution work has no response to replay and must
2276    // not retain scheduler capacity or an epoch-reader admission.
2277    let work_disposition = if replacement_root.is_some() {
2278        RouteWorkDisposition::RetainForReplay
2279    } else {
2280        RouteWorkDisposition::RetainStartedForReplay
2281    };
2282    apply_route_work_disposition(
2283        active_tool_calls,
2284        executor,
2285        channel,
2286        work_disposition,
2287        cancellation_reason,
2288    );
2289    if let Some(pending) = pending_binds.get_mut(&channel) {
2290        pending.cancelled = true;
2291        let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
2292        log::debug!(
2293            "subc attach: cancelled pending RouteBind for route {} on {cancellation_reason} (configure job: {outcome:?})",
2294            channel.channel
2295        );
2296    }
2297    let migrated = push::migrate_retry_buffer_to_push_buffer(retry_buffer, channel, push_buffer);
2298    if let Some(identity) = routes.get(&channel) {
2299        let has_pending_bind = pending_binds
2300            .values()
2301            .any(|pending| pending.bind_root_id == identity.root);
2302        if route_removal_will_quiesce_root(
2303            &identity.root,
2304            channel,
2305            root_channels,
2306            has_pending_bind,
2307            replacement_root,
2308        ) {
2309            if let Some(ctx) = executor.actor_context(&identity.root) {
2310                // Fence deferred admissions before the final route disappears
2311                // from the loop-owned routing tables.
2312                ctx.mark_subc_unbound();
2313            }
2314        }
2315    }
2316    // This test-only delay lets the lifecycle probe verify that a queued rebind
2317    // cannot run before the route is removed and a completion is recorded for replay.
2318    delay_route_detach_for_test(lifecycle_probe).await;
2319    if let Some(identity) = remove_route_channel(routes, root_channels, channel) {
2320        sync_bg_live_delivery_sessions(executor, routes, Some(&identity.root));
2321        if let Some(probe) = lifecycle_probe {
2322            probe.route_detached(channel, &identity.session);
2323        }
2324        let session_still_routed = routes
2325            .values()
2326            .any(|route| route.root == identity.root && route.session == identity.session);
2327        if !session_still_routed {
2328            if let Some(ctx) = executor.actor_context(&identity.root) {
2329                ctx.hashline_bindings()
2330                    .teardown(identity.root.as_path(), &identity.session);
2331            }
2332        }
2333        if migrated > 0 {
2334            log::debug!(
2335                "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from route {} into detach replay",
2336                channel.channel
2337            );
2338        }
2339        if let Some(meta) = live_roots.get_mut(&identity.root) {
2340            let idle_for = meta.last_touched.elapsed();
2341            meta.note_activity();
2342            log::debug!(
2343                "subc attach: route {} torn down for root {} harness {} session {} (last touched {:?} ago)",
2344                channel.channel,
2345                identity.root.as_path().display(),
2346                identity.harness,
2347                identity.session,
2348                idle_for
2349            );
2350        } else {
2351            log::debug!(
2352                "subc attach: route {} torn down for root {} harness {} session {}",
2353                channel.channel,
2354                identity.root.as_path().display(),
2355                identity.harness,
2356                identity.session
2357            );
2358        }
2359        let has_pending_bind = pending_binds
2360            .values()
2361            .any(|pending| pending.bind_root_id == identity.root);
2362        if should_quiesce_removed_root(
2363            &identity.root,
2364            root_channels,
2365            has_pending_bind,
2366            replacement_root,
2367        ) {
2368            quiesce_unbound_root(&identity.root, live_roots, executor);
2369        }
2370    } else {
2371        if migrated > 0 {
2372            log::debug!(
2373                "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from unbound route {} into detach replay",
2374                channel.channel
2375            );
2376        }
2377        log::debug!("subc attach: unbound route {} torn down", channel.channel);
2378    }
2379    Ok(())
2380}
2381
2382async fn delay_route_detach_for_test(lifecycle_probe: Option<&SubcTestLifecycleProbe>) {
2383    if lifecycle_probe.is_none() {
2384        return;
2385    }
2386    let Some(delay) = std::env::var("AFT_TEST_SUBC_ROUTE_DETACH_DELAY_MS")
2387        .ok()
2388        .and_then(|raw| raw.parse::<u64>().ok())
2389    else {
2390        return;
2391    };
2392    tokio::time::sleep(Duration::from_millis(delay)).await;
2393}
2394
2395fn remember_session_identity(
2396    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2397    identity: &RouteIdentity,
2398) {
2399    let key = (identity.root.clone(), identity.session.clone());
2400    if matches!(identity.trust, BindTrust::Untrusted)
2401        && session_identity
2402            .get(&key)
2403            .is_some_and(|retained| matches!(retained.trust, BindTrust::FirstParty))
2404    {
2405        return;
2406    }
2407
2408    // Retained after route Goodbye so reliable session-scoped frames emitted while
2409    // the session is detached can still be keyed by the full (root,harness,session)
2410    // replay triple. Untrusted binds never overwrite a retained first-party
2411    // session identity, because bash completion replay is an observation channel.
2412    session_identity.insert(
2413        key,
2414        RetainedSessionIdentity {
2415            harness: identity.harness.clone(),
2416            trust: identity.trust,
2417        },
2418    );
2419}
2420
2421fn replay_key_for_session(
2422    session_identity: &HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2423    root: &ProjectRootId,
2424    session: &str,
2425) -> Option<(push::ReplayKey, BindTrust)> {
2426    let retained = session_identity.get(&(root.clone(), session.to_string()))?;
2427    Some((
2428        push::ReplayKey {
2429            root: root.clone(),
2430            harness: retained.harness.clone(),
2431            session: session.to_string(),
2432        },
2433        retained.trust,
2434    ))
2435}
2436/// Sync command dispatch, passed in from `main` (the binary owns the command
2437/// table). Invoked only inside executor jobs in subc mode.
2438pub type DispatchFn = fn(RawRequest, &AppContext) -> Response;
2439
2440#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2441enum ModuleLoopExit {
2442    /// The daemon asked us to stop (channel-0 Goodbye). Exit 0 is correct:
2443    /// the supervisor treats a clean exit as "stopped on request".
2444    Graceful,
2445    /// The connection ended without a Goodbye. Indexes still flush, but the
2446    /// process must exit non-zero so the supervisor restarts it.
2447    ConnectionLost,
2448    /// An actor went fatal (worker panic on a mutating job) and the loop tore
2449    /// the module down. Index flushes are skipped because the state that
2450    /// panicked cannot be trusted, and the process exits non-zero for the
2451    /// same reason as `ConnectionLost`.
2452    SkipSearchFlush,
2453}
2454
2455/// Entry point for `aft --subc <connection-file>`. Synchronous on the outside;
2456/// owns an isolated current-thread tokio runtime for the async transport.
2457/// Returns `Err` (fail-loud) on any connect/auth/protocol failure — we never
2458/// fall back to the standalone loop, to avoid split-brain index state.
2459pub fn run_subc_mode(
2460    connection_file_path: &Path,
2461    ctx: Arc<AppContext>,
2462    executor: Arc<Executor>,
2463    dispatch: DispatchFn,
2464    user_config_path: Option<PathBuf>,
2465) -> Result<(), SubcError> {
2466    // Production NEVER allows non-manifest tool names on route channels: AFT
2467    // fails closed and does not trust subc to enforce the manifest. The
2468    // test-only harness sets this through `run_subc_mode_for_test`.
2469    run_subc_mode_inner(
2470        connection_file_path,
2471        ctx,
2472        executor,
2473        dispatch,
2474        user_config_path,
2475        false,
2476        MAX_FRAME_BODY_LEN as usize,
2477        None,
2478    )
2479}
2480
2481fn run_subc_mode_inner(
2482    connection_file_path: &Path,
2483    ctx: Arc<AppContext>,
2484    executor: Arc<Executor>,
2485    dispatch: DispatchFn,
2486    user_config_path: Option<PathBuf>,
2487    allow_native_passthrough: bool,
2488    tool_response_body_limit: usize,
2489    lifecycle_probe: Option<SubcTestLifecycleProbe>,
2490) -> Result<(), SubcError> {
2491    let runtime = tokio::runtime::Builder::new_current_thread()
2492        .enable_all()
2493        .build()
2494        .map_err(SubcError::Runtime)?;
2495
2496    let executor_for_loop = Arc::clone(&executor);
2497    let loop_result = runtime.block_on(async move {
2498        let shared_app = ctx.app();
2499        drop(ctx);
2500        let stream =
2501            connect_and_authenticate(connection_file_path, lifecycle_probe.as_ref()).await?;
2502        log::info!(
2503            "subc attach: authenticated to daemon via {}",
2504            connection_file_path.display()
2505        );
2506        let (read_half, write_half) = tokio::io::split(stream);
2507        run_module_loop(
2508            read_half,
2509            write_half,
2510            connection_file_path,
2511            shared_app,
2512            executor_for_loop,
2513            dispatch,
2514            user_config_path,
2515            allow_native_passthrough,
2516            tool_response_body_limit,
2517            lifecycle_probe,
2518        )
2519        .await
2520    });
2521
2522    let actor_contexts = executor.actor_contexts();
2523    if matches!(
2524        loop_result,
2525        Ok(ModuleLoopExit::Graceful | ModuleLoopExit::ConnectionLost)
2526    ) {
2527        // EOF/Goodbye teardown flushes each root's index deltas and queued
2528        // callgraph refreshes. Fatal/panic teardown skips this best-effort work.
2529        flush_actor_indexes_on_graceful_shutdown(&actor_contexts);
2530    }
2531    for actor_ctx in &actor_contexts {
2532        actor_ctx.lsp().shutdown_all();
2533        actor_ctx.bash_background().detach();
2534    }
2535
2536    match loop_result {
2537        Ok(exit) => module_loop_exit_result(exit),
2538        Err(error) => Err(error),
2539    }
2540}
2541
2542/// Maps how the module loop ended onto the process outcome the supervisor
2543/// reads. Only a daemon-requested stop may exit 0: the supervisor never
2544/// respawns a clean exit, so every other ending must surface as an error.
2545fn module_loop_exit_result(exit: ModuleLoopExit) -> Result<(), SubcError> {
2546    match exit {
2547        ModuleLoopExit::Graceful => Ok(()),
2548        ModuleLoopExit::ConnectionLost => Err(SubcError::ConnectionLost),
2549        ModuleLoopExit::SkipSearchFlush => Err(SubcError::ActorFatal),
2550    }
2551}
2552
2553/// Records a fatal panic response in the module log before the teardown it
2554/// triggers. The panic text otherwise lives only in the error frame sent to
2555/// the caller, which leaves the crash undiagnosable from the host afterwards.
2556fn note_fatal_panic_response(response: &Response) -> bool {
2557    let fatal = response_is_fatal_panic(response);
2558    if fatal {
2559        log::error!(
2560            "subc attach: request {} returned a fatal panic response; tearing the module down: {}",
2561            response.id,
2562            response
2563                .data
2564                .get("message")
2565                .and_then(Value::as_str)
2566                .unwrap_or("(no message)")
2567        );
2568    }
2569    fatal
2570}
2571
2572fn flush_actor_indexes_on_graceful_shutdown(actor_contexts: &[Arc<AppContext>]) {
2573    for actor_ctx in actor_contexts {
2574        let _ = actor_ctx.flush_search_index_on_graceful_shutdown();
2575    }
2576    let _ = crate::callgraph_store::flush_callgraph_store_refreshes_on_graceful_shutdown();
2577}
2578
2579/// Test-only entry that enables the non-manifest native-command passthrough on
2580/// route channels. Integration tests drive synthetic native commands (`glob`,
2581/// `callers`, `subc_test_echo_session`, …) through the executor to exercise
2582/// mechanics; production callers use [`run_subc_mode`], which fails closed.
2583#[doc(hidden)]
2584pub fn run_subc_mode_for_test(
2585    connection_file_path: &Path,
2586    ctx: Arc<AppContext>,
2587    executor: Arc<Executor>,
2588    dispatch: DispatchFn,
2589    user_config_path: Option<PathBuf>,
2590) -> Result<(), SubcError> {
2591    run_subc_mode_inner(
2592        connection_file_path,
2593        ctx,
2594        executor,
2595        dispatch,
2596        user_config_path,
2597        true,
2598        MAX_FRAME_BODY_LEN as usize,
2599        None,
2600    )
2601}
2602
2603/// Test-only entry that observes detach/rebind lifecycle milestones.
2604#[doc(hidden)]
2605pub fn run_subc_mode_for_test_with_lifecycle_probe(
2606    connection_file_path: &Path,
2607    ctx: Arc<AppContext>,
2608    executor: Arc<Executor>,
2609    dispatch: DispatchFn,
2610    user_config_path: Option<PathBuf>,
2611    lifecycle_probe: SubcTestLifecycleProbe,
2612) -> Result<(), SubcError> {
2613    run_subc_mode_inner(
2614        connection_file_path,
2615        ctx,
2616        executor,
2617        dispatch,
2618        user_config_path,
2619        true,
2620        MAX_FRAME_BODY_LEN as usize,
2621        Some(lifecycle_probe),
2622    )
2623}
2624
2625/// Test-only entry that lowers the effective tool-response body limit without
2626/// allocating a 64 MiB fixture. The fixed fallback envelope needs 4 KiB of room.
2627#[doc(hidden)]
2628pub fn run_subc_mode_for_test_with_response_body_limit(
2629    connection_file_path: &Path,
2630    ctx: Arc<AppContext>,
2631    executor: Arc<Executor>,
2632    dispatch: DispatchFn,
2633    user_config_path: Option<PathBuf>,
2634    tool_response_body_limit: usize,
2635) -> Result<(), SubcError> {
2636    assert!((4 * 1_024..=MAX_FRAME_BODY_LEN as usize).contains(&tool_response_body_limit));
2637    run_subc_mode_inner(
2638        connection_file_path,
2639        ctx,
2640        executor,
2641        dispatch,
2642        user_config_path,
2643        true,
2644        tool_response_body_limit,
2645        None,
2646    )
2647}
2648
2649#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2650enum AttachErrorClass {
2651    Transient,
2652    Permanent,
2653}
2654
2655impl fmt::Display for AttachErrorClass {
2656    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2657        match self {
2658            Self::Transient => f.write_str("transient"),
2659            Self::Permanent => f.write_str("permanent"),
2660        }
2661    }
2662}
2663
2664#[derive(Clone, Copy)]
2665struct AttachRetryPolicy {
2666    budget: Duration,
2667    initial_backoff: Duration,
2668    max_backoff: Duration,
2669    jitter_percent: u64,
2670}
2671
2672const ATTACH_RETRY_POLICY: AttachRetryPolicy = AttachRetryPolicy {
2673    budget: ATTACH_RETRY_BUDGET,
2674    initial_backoff: ATTACH_RETRY_INITIAL_BACKOFF,
2675    max_backoff: ATTACH_RETRY_MAX_BACKOFF,
2676    jitter_percent: ATTACH_RETRY_JITTER_PERCENT,
2677};
2678
2679/// Retry only failures that can be caused by a daemon bounce or an interrupted
2680/// handshake. Protocol and credential failures are permanent for this process.
2681fn classify_attach_error(error: &SubcError) -> AttachErrorClass {
2682    let transient = match error {
2683        SubcError::Connect { source, .. } => is_transient_attach_io(source.kind()),
2684        SubcError::Auth { source, .. } => match source {
2685            subc_transport::AuthError::Timeout { .. }
2686            | subc_transport::AuthError::UnexpectedEof { .. } => true,
2687            subc_transport::AuthError::Io { source, .. } => is_transient_attach_io(source.kind()),
2688            _ => false,
2689        },
2690        _ => false,
2691    };
2692    if transient {
2693        AttachErrorClass::Transient
2694    } else {
2695        AttachErrorClass::Permanent
2696    }
2697}
2698
2699fn is_transient_attach_io(kind: io::ErrorKind) -> bool {
2700    matches!(
2701        kind,
2702        io::ErrorKind::ConnectionRefused
2703            | io::ErrorKind::TimedOut
2704            | io::ErrorKind::ConnectionReset
2705            | io::ErrorKind::ConnectionAborted
2706            | io::ErrorKind::BrokenPipe
2707            | io::ErrorKind::UnexpectedEof
2708    )
2709}
2710
2711/// Read the connection file → resolve the first endpoint → TCP connect → HMAC
2712/// handshake. Transient initial-attach failures retry on fresh sockets and reread
2713/// the file so a daemon bounce can publish a new endpoint or authentication key.
2714async fn connect_and_authenticate(
2715    connection_file_path: &Path,
2716    lifecycle_probe: Option<&SubcTestLifecycleProbe>,
2717) -> Result<TcpStream, SubcError> {
2718    connect_and_authenticate_with_policy(connection_file_path, ATTACH_RETRY_POLICY, lifecycle_probe)
2719        .await
2720}
2721
2722async fn connect_and_authenticate_with_policy(
2723    connection_file_path: &Path,
2724    policy: AttachRetryPolicy,
2725    lifecycle_probe: Option<&SubcTestLifecycleProbe>,
2726) -> Result<TcpStream, SubcError> {
2727    let started_at = Instant::now();
2728    let deadline = started_at + policy.budget;
2729    let mut attempt = 0_u32;
2730    let mut backoff = policy.initial_backoff;
2731    let mut history = Vec::new();
2732
2733    loop {
2734        attempt = attempt.saturating_add(1);
2735        let error = match connect_and_authenticate_once(connection_file_path, deadline).await {
2736            Ok(stream) => return Ok(stream),
2737            Err(error) => error,
2738        };
2739        let class = classify_attach_error(&error);
2740        let will_retry = class != AttachErrorClass::Permanent;
2741        if let Some(probe) = lifecycle_probe {
2742            probe.attach_decision(attempt, will_retry);
2743        }
2744        let error_text = error.to_string().lines().collect::<Vec<_>>().join(" ");
2745        history.push(format!("attempt {attempt} [{class}]: {error_text}"));
2746
2747        if !will_retry {
2748            log_attach_final_failure(started_at.elapsed(), &history);
2749            return Err(error);
2750        }
2751
2752        let remaining = deadline.saturating_duration_since(Instant::now());
2753        if remaining.is_zero() {
2754            log_attach_final_failure(started_at.elapsed(), &history);
2755            return Err(error);
2756        }
2757
2758        let delay = jittered_attach_delay(backoff, policy.jitter_percent, attempt).min(remaining);
2759        log::info!(
2760            "subc attach retry: attempt {attempt} failed; error_class={class}; error={error_text}; next_delay={delay:?}"
2761        );
2762        tokio::time::sleep(delay).await;
2763
2764        if Instant::now() >= deadline {
2765            log_attach_final_failure(started_at.elapsed(), &history);
2766            return Err(error);
2767        }
2768        backoff = backoff.saturating_mul(2).min(policy.max_backoff);
2769    }
2770}
2771
2772fn jittered_attach_delay(base: Duration, jitter_percent: u64, attempt: u32) -> Duration {
2773    let jitter_percent = jitter_percent.min(100);
2774    if jitter_percent == 0 {
2775        return base;
2776    }
2777
2778    let mut random_bytes = [0_u8; 8];
2779    let random = if getrandom::fill(&mut random_bytes).is_ok() {
2780        u64::from_le_bytes(random_bytes)
2781    } else {
2782        let timestamp = std::time::SystemTime::now()
2783            .duration_since(std::time::UNIX_EPOCH)
2784            .unwrap_or_default()
2785            .subsec_nanos();
2786        u64::from(timestamp) ^ u64::from(attempt)
2787    };
2788    let span = jitter_percent.saturating_mul(2).saturating_add(1);
2789    let multiplier_percent = 100 - jitter_percent + random % span;
2790    let base_millis = u64::try_from(base.as_millis()).unwrap_or(u64::MAX);
2791    Duration::from_millis(base_millis.saturating_mul(multiplier_percent) / 100)
2792}
2793
2794fn log_attach_final_failure(elapsed: Duration, history: &[String]) {
2795    log::error!(
2796        "subc initial attach failed after {} attempt(s) in {elapsed:?}; attempt history: {}",
2797        history.len(),
2798        history.join(" | ")
2799    );
2800}
2801
2802async fn connect_and_authenticate_once(
2803    connection_file_path: &Path,
2804    deadline: Instant,
2805) -> Result<TcpStream, SubcError> {
2806    // This read intentionally lives inside the per-attempt function. The daemon
2807    // publishes connection files atomically and may change both port and key.
2808    let conn = connection_file::read_for_client(connection_file_path).map_err(|source| {
2809        SubcError::ConnectionFile {
2810            path: connection_file_path.to_path_buf(),
2811            source,
2812        }
2813    })?;
2814
2815    let endpoint = conn
2816        .endpoints
2817        .first()
2818        .ok_or_else(|| SubcError::NoEndpoint {
2819            path: connection_file_path.to_path_buf(),
2820        })?;
2821    let endpoint_label = format!("{}:{}", endpoint.host, endpoint.port);
2822    let ip = endpoint
2823        .host
2824        .parse::<IpAddr>()
2825        .map_err(|_| SubcError::InvalidEndpoint {
2826            path: connection_file_path.to_path_buf(),
2827            endpoint: endpoint_label.clone(),
2828        })?;
2829    let addr = SocketAddr::new(ip, endpoint.port);
2830
2831    let connect_budget = deadline.saturating_duration_since(Instant::now());
2832    let mut stream = tokio::time::timeout(connect_budget, TcpStream::connect(addr))
2833        .await
2834        .map_err(|_| SubcError::Connect {
2835            endpoint: endpoint_label.clone(),
2836            source: io::Error::new(
2837                io::ErrorKind::TimedOut,
2838                "initial subc attach retry budget elapsed during TCP connect",
2839            ),
2840        })?
2841        .map_err(|source| SubcError::Connect {
2842            endpoint: endpoint_label.clone(),
2843            source,
2844        })?;
2845    stream
2846        .set_nodelay(true)
2847        .map_err(|source| SubcError::Connect {
2848            endpoint: endpoint_label.clone(),
2849            source,
2850        })?;
2851
2852    let auth_budget = AUTH_DEADLINE.min(deadline.saturating_duration_since(Instant::now()));
2853    authenticate_client(&mut stream, &conn, auth_budget)
2854        .await
2855        .map_err(|source| SubcError::Auth {
2856            endpoint: endpoint_label,
2857            source,
2858        })?;
2859
2860    Ok(stream)
2861}
2862
2863#[allow(clippy::too_many_arguments)]
2864async fn process_route_bind_completion(
2865    writer_tx: &WriterSender,
2866    completion: RouteBindCompletion,
2867    routes: &mut HashMap<RouteChannel, RouteIdentity>,
2868    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2869    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2870    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2871    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2872    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2873    installed_route_epochs: &mut HashMap<u16, u32>,
2874    executor: &Arc<Executor>,
2875    standing_actor: &standing::StandingActor,
2876    shutdown: &Arc<Notify>,
2877    metrics: &Arc<DispatchPathMetrics>,
2878    lifecycle_probe: Option<&SubcTestLifecycleProbe>,
2879) -> Result<(), SubcError> {
2880    decrement_counted_channel(&metrics.control_completion_queued);
2881    handle_route_bind_completion(
2882        writer_tx,
2883        completion,
2884        routes,
2885        root_channels,
2886        session_identity,
2887        push_buffer,
2888        live_roots,
2889        pending_binds,
2890        installed_route_epochs,
2891        executor,
2892        standing_actor,
2893        shutdown,
2894        metrics,
2895        lifecycle_probe,
2896    )
2897    .await
2898}
2899
2900#[allow(clippy::too_many_arguments)]
2901async fn drain_pending_route_bind_completions(
2902    control_completion_rx: &mut mpsc::Receiver<RouteBindCompletion>,
2903    writer_tx: &WriterSender,
2904    routes: &mut HashMap<RouteChannel, RouteIdentity>,
2905    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2906    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2907    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2908    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2909    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2910    installed_route_epochs: &mut HashMap<u16, u32>,
2911    executor: &Arc<Executor>,
2912    standing_actor: &standing::StandingActor,
2913    shutdown: &Arc<Notify>,
2914    metrics: &Arc<DispatchPathMetrics>,
2915    lifecycle_probe: Option<&SubcTestLifecycleProbe>,
2916) -> Result<usize, SubcError> {
2917    let mut drained = 0;
2918    while let Ok(completion) = control_completion_rx.try_recv() {
2919        process_route_bind_completion(
2920            writer_tx,
2921            completion,
2922            routes,
2923            root_channels,
2924            session_identity,
2925            push_buffer,
2926            live_roots,
2927            pending_binds,
2928            installed_route_epochs,
2929            executor,
2930            standing_actor,
2931            shutdown,
2932            metrics,
2933            lifecycle_probe,
2934        )
2935        .await?;
2936        drained += 1;
2937    }
2938    Ok(drained)
2939}
2940
2941/// ModuleHello → HelloAck → control/route loop. Runs until the daemon closes
2942/// the connection (EOF), sends channel-0 Goodbye, or a fatal mutating executor
2943/// response requests whole-connection teardown.
2944async fn run_module_loop<R, W>(
2945    mut read: R,
2946    mut write: W,
2947    connection_file_path: &Path,
2948    shared_app: Arc<App>,
2949    executor: Arc<Executor>,
2950    dispatch: DispatchFn,
2951    user_config_path: Option<PathBuf>,
2952    allow_native_passthrough: bool,
2953    tool_response_body_limit: usize,
2954    lifecycle_probe: Option<SubcTestLifecycleProbe>,
2955) -> Result<ModuleLoopExit, SubcError>
2956where
2957    R: AsyncRead + Unpin + Send + 'static,
2958    W: AsyncWrite + Unpin + Send + 'static,
2959{
2960    // ModuleHello registers the tool and management providers and advertises
2961    // the separate channel-0 control operations.
2962    // Echo the one-time launch nonce the daemon injected via SUBC_LAUNCH_NONCE so a
2963    // reserved module_id's HELLO is accepted; absent for non-reserved/self-connect.
2964    let hello = ModuleHelloBody {
2965        manifest: build_manifest(),
2966        protocol_ver: PROTOCOL_VERSION,
2967        control_ops: control_ops(),
2968        launch_nonce: std::env::var("SUBC_LAUNCH_NONCE").ok(),
2969    };
2970    let hello_frame = Frame::build(
2971        FrameType::Hello,
2972        control_flags(),
2973        0,
2974        0,
2975        HELLO_CORR,
2976        serde_json::to_vec(&hello).map_err(SubcError::Json)?,
2977    )
2978    .map_err(SubcError::FrameBuild)?;
2979    write_frame(&mut write, &hello_frame)
2980        .await
2981        .map_err(SubcError::FrameIo)?;
2982
2983    // Expect HelloAck (registered) or a channel-0 Error (manifest/version reject).
2984    match read_frame(&mut read).await.map_err(SubcError::FrameIo)? {
2985        None => return Err(SubcError::ClosedBeforeHelloAck),
2986        Some(frame) => match frame.header.ty {
2987            FrameType::HelloAck => {
2988                log::info!("subc attach: registered (HelloAck received)");
2989            }
2990            FrameType::Error => {
2991                let body = serde_json::from_slice::<ErrorBody>(&frame.body).ok();
2992                return Err(SubcError::HelloRejected { body });
2993            }
2994            other => return Err(SubcError::UnexpectedFrame { ty: other }),
2995        },
2996    }
2997
2998    let dispatch_path_metrics = Arc::new(DispatchPathMetrics::new());
2999    let (writer_tx, writer_rx) = mpsc::channel::<WriterFrame>(WRITER_QUEUE_CAPACITY);
3000    let writer_task = spawn_writer_task(write, writer_rx, Arc::clone(&dispatch_path_metrics));
3001    // `read_frame` is NOT cancellation-safe, so it must never sit directly inside
3002    // the `select!` below: a drain-interval tick (or shutdown) firing while a
3003    // frame is mid-transit would drop the partially-consumed bytes and desync the
3004    // stream (the next read would parse a body byte as a frame header). A
3005    // dedicated reader task owns the socket, reads whole frames sequentially, and
3006    // forwards them over a channel; the loop selects on the cancel-safe `recv()`.
3007    let (reader_tx, mut reader_rx) = mpsc::channel::<Result<DecodedFrame, SubcError>>(256);
3008    let reader_task = spawn_reader_task(read, reader_tx);
3009    let shutdown = Arc::new(Notify::new());
3010    // Drain-tick deadline is tracked manually and checked at the TOP of every
3011    // loop turn rather than as an Interval select arm: the select below is
3012    // `biased` (bind completions first), and biased polling means a saturated
3013    // higher arm (sustained lossy push traffic keeps lossy_rx always-ready)
3014    // would starve every arm below it, including a timer arm — leaving
3015    // backpressured reliable frames parked in the retry buffer past their
3016    // delivery deadline. The pre-turn check cannot be starved by arm order;
3017    // the sleep_until arm below only exists to wake an otherwise-idle loop.
3018    let mut next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
3019    let mut next_maintenance_at = next_drain_at;
3020    let standing_actor =
3021        standing::StandingActor::new(Arc::clone(&shared_app), Arc::clone(&executor));
3022    // Startup reconciliation is intentionally direct; subsequent passes use
3023    // this existing maintenance timer arm and never create a standing timer.
3024    standing_actor.reconcile_at_startup();
3025    let mut next_standing_pass_at = tokio::time::Instant::now();
3026    let (maintenance_tx, mut maintenance_rx) = mpsc::channel::<MaintenanceCompletion>(256);
3027    let (bash_deferred_tx, mut bash_deferred_rx) =
3028        mpsc::channel::<bash::BashDeferredCompletion>(256);
3029    let (deferred_response_tx, mut deferred_response_rx) =
3030        mpsc::unbounded_channel::<PendingSubcResponse>();
3031    let (bash_poll_touch_tx, mut bash_poll_touch_rx) = mpsc::channel::<ProjectRootId>(256);
3032    let (control_completion_tx, mut control_completion_rx) =
3033        mpsc::channel::<RouteBindCompletion>(256);
3034    let (lossy_tx, mut lossy_rx) = mpsc::channel::<LossyPushEnvelope>(1024);
3035    let lossy_overflow = Arc::new(push::LossyOverflow::default());
3036    let lossy_seq = Arc::new(AtomicU64::new(0));
3037    let (reliable_tx, mut reliable_rx) = mpsc::unbounded_channel::<PushEnvelope>();
3038    let (fleet_status_client, fleet_status_task) =
3039        spawn_fleet_status_dial(connection_file_path, 64);
3040    let push_senders = PushSenders {
3041        lossy_tx,
3042        reliable_tx,
3043        lossy_overflow: Arc::clone(&lossy_overflow),
3044        lossy_seq,
3045        fleet_status_client: fleet_status_client.clone(),
3046    };
3047    let connection_cancel = PersistentCancelSignal::new();
3048    let mut installed_route_epochs: HashMap<u16, u32> = HashMap::new();
3049    let mut routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
3050    let mut management_routes: HashSet<RouteChannel> = HashSet::new();
3051    let mut bg_subs: HashMap<RouteChannel, BgSub> = HashMap::new();
3052    let mut bg_sub_by_session: BgSubsBySession = HashMap::new();
3053    let mut bg_wake_pending = BgWakePending::new();
3054    let mut bg_wake_epoch: HashMap<(ProjectRootId, String), u64> = HashMap::new();
3055    let mut bg_unacked_keys_by_root: HashMap<ProjectRootId, HashSet<String>> = HashMap::new();
3056    let mut root_channels: HashMap<ProjectRootId, HashSet<RouteChannel>> = HashMap::new();
3057    let mut session_identity: HashMap<(ProjectRootId, String), RetainedSessionIdentity> =
3058        HashMap::new();
3059    let mut push_buffer: HashMap<push::ReplayKey, VecDeque<PushFrame>> = HashMap::new();
3060    let mut retry_buffer: RetryBuffer = HashMap::new();
3061    let mut reclaimed_routes = ReclaimedRoutes::default();
3062    let mut completed_tasks = push::CompletedTaskIds::default();
3063    let mut live_roots: HashMap<ProjectRootId, RootMeta> = HashMap::new();
3064    let mut pending_binds: HashMap<RouteChannel, PendingBind> = HashMap::new();
3065    let mut pending_bash_asks: HashMap<ReverseCorrKey, PendingBashAsk> = HashMap::new();
3066    let mut next_bash_ask_corr: u64 = 1;
3067    let mut route_bash_cancels: HashMap<RouteChannel, bash::RouteBashCancel> = HashMap::new();
3068    let active_tool_calls: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
3069    let pending_deferred_setups = Arc::new(AtomicUsize::new(0));
3070    let mut pending_responses = PendingSubcResponses::default();
3071    let health_rollup_cache = Arc::new(HealthRollupCache::new());
3072    let health_rollup_worker = HealthRollupWorker::start(
3073        Arc::clone(&health_rollup_cache),
3074        Arc::clone(&executor),
3075        Arc::clone(&shared_app),
3076    );
3077
3078    let loop_result: Result<ModuleLoopExit, SubcError> = 'module_loop: loop {
3079        shared_app.set_open_route_count(routes.len() + management_routes.len());
3080        crate::logging::perf_tick(Some(&executor));
3081        dispatch_path_metrics.mark_frame_loop_tick();
3082        let ready_inspects = pending_responses.poll_ready(executor.as_ref());
3083        for resolved in ready_inspects {
3084            if let Err(error) = deliver_resolved_subc_response(
3085                &writer_tx,
3086                resolved,
3087                &routes,
3088                &mut live_roots,
3089                executor.as_ref(),
3090                &active_tool_calls,
3091                &shutdown,
3092                &dispatch_path_metrics,
3093                tool_response_body_limit,
3094            )
3095            .await
3096            {
3097                break 'module_loop Err(error);
3098            }
3099        }
3100        if let Err(error) = expire_pending_bash_asks(
3101            &writer_tx,
3102            &mut pending_bash_asks,
3103            &routes,
3104            &mut live_roots,
3105            &mut route_bash_cancels,
3106            &shutdown,
3107            &dispatch_path_metrics,
3108        )
3109        .await
3110        {
3111            break Err(error);
3112        }
3113
3114        // RouteBind completions are control-plane unblockers. Drain any completed
3115        // binds before entering other branch work so Push and maintenance bursts
3116        // can only add one loop-turn of latency.
3117        match drain_pending_route_bind_completions(
3118            &mut control_completion_rx,
3119            &writer_tx,
3120            &mut routes,
3121            &mut root_channels,
3122            &mut session_identity,
3123            &mut push_buffer,
3124            &mut live_roots,
3125            &mut pending_binds,
3126            &mut installed_route_epochs,
3127            &executor,
3128            &standing_actor,
3129            &shutdown,
3130            &dispatch_path_metrics,
3131            lifecycle_probe.as_ref(),
3132        )
3133        .await
3134        {
3135            Ok(drained) => {
3136                if drained > 0 {
3137                    next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
3138                    health_rollup_worker.request_refresh();
3139                }
3140            }
3141            Err(error) => break Err(error),
3142        }
3143
3144        if tokio::time::Instant::now() >= next_drain_at {
3145            push::emit_bg_event_wakes(
3146                &writer_tx,
3147                &dispatch_path_metrics,
3148                &bg_subs,
3149                &mut bg_wake_pending,
3150            );
3151            dispatch_path_metrics.warn_stuck_pending_watches(&executor, &bg_sub_by_session);
3152            warn_slow_pending_binds(&mut pending_binds, &executor);
3153            warn_slow_running_interactive_jobs(&executor);
3154            if let Err(error) = expire_overdue_route_binds(
3155                &writer_tx,
3156                &executor,
3157                &mut pending_binds,
3158                &mut installed_route_epochs,
3159                &dispatch_path_metrics,
3160            )
3161            .await
3162            {
3163                break Err(error);
3164            }
3165
3166            let retried = push::drain_retry_buffers_for_bound_routes(
3167                &writer_tx,
3168                &dispatch_path_metrics,
3169                &routes,
3170                &mut retry_buffer,
3171            );
3172            if retried > 0 {
3173                log::debug!(
3174                    "subc attach: retried {retried} reliable Push frame(s) after writer backpressure"
3175                );
3176            }
3177
3178            next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
3179        }
3180
3181        // A lossy emitter may place its newest update in the overflow buffer
3182        // when the bounded channel is full, while this receive loop is draining
3183        // the channel. Drain overflow before selecting again so that raced
3184        // update is delivered on the next timer tick instead of waiting for
3185        // another lossy enqueue.
3186        let overflow_batch = lossy_overflow.drain();
3187        if !overflow_batch.is_empty() {
3188            let (_, deferred) = push::drain_reliable_push_turn(
3189                &writer_tx,
3190                &dispatch_path_metrics,
3191                &routes,
3192                &root_channels,
3193                &session_identity,
3194                &mut retry_buffer,
3195                &mut push_buffer,
3196                &mut completed_tasks,
3197                &bg_sub_by_session,
3198                &mut bg_wake_pending,
3199                &mut bg_wake_epoch,
3200                &mut reliable_rx,
3201                None,
3202                lifecycle_probe.as_ref(),
3203            );
3204            if deferred {
3205                tokio::task::yield_now().await;
3206            }
3207
3208            let mut batch = Vec::new();
3209            while let Ok(item) = lossy_rx.try_recv() {
3210                batch.push(item);
3211            }
3212            batch.extend(overflow_batch);
3213            push::process_lossy_push_envelope_batch(
3214                &writer_tx,
3215                &dispatch_path_metrics,
3216                &routes,
3217                &root_channels,
3218                &completed_tasks,
3219                batch,
3220            );
3221        }
3222
3223        tokio::select! {
3224            biased;
3225            Some(completion) = control_completion_rx.recv() => {
3226                if let Err(error) = process_route_bind_completion(
3227                    &writer_tx,
3228                    completion,
3229                    &mut routes,
3230                    &mut root_channels,
3231                    &mut session_identity,
3232                    &mut push_buffer,
3233                    &mut live_roots,
3234                    &mut pending_binds,
3235                    &mut installed_route_epochs,
3236                    &executor,
3237                    &standing_actor,
3238                    &shutdown,
3239                    &dispatch_path_metrics,
3240                    lifecycle_probe.as_ref(),
3241                )
3242                .await
3243                {
3244                    break Err(error);
3245                }
3246                next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
3247                health_rollup_worker.request_refresh();
3248            }
3249            _ = shutdown.notified() => {
3250                log::warn!("subc attach: fatal executor response requested teardown");
3251                break Ok(ModuleLoopExit::SkipSearchFlush);
3252            }
3253            maybe_frame = reader_rx.recv() => {
3254                let frame = match maybe_frame {
3255                    None => {
3256                        log::warn!(
3257                            "subc attach: daemon connection ended without Goodbye; exiting for restart"
3258                        );
3259                        break Ok(ModuleLoopExit::ConnectionLost);
3260                    }
3261                    Some(Err(error)) => break Err(error),
3262                    Some(Ok(frame)) => frame,
3263                };
3264                let phase_trace = frame.phase_trace;
3265                let frame = frame.frame;
3266
3267                if !ingress_route_should_be_processed(
3268                    &installed_route_epochs,
3269                    &reclaimed_routes,
3270                    &frame,
3271                ) {
3272                    log::debug!(
3273                        "subc attach: silently dropping {:?} for uninstalled route {}@{}",
3274                        frame.header.ty,
3275                        frame.header.channel,
3276                        frame.header.epoch
3277                    );
3278                    continue;
3279                }
3280
3281                match frame.header.ty {
3282                    FrameType::Ping if frame.header.channel == 0 => {
3283                        let pong = match Frame::build_with_version(
3284                            frame.header.ver,
3285                            FrameType::Pong,
3286                            frame.header.flags,
3287                            0,
3288                            0,
3289                            frame.header.corr,
3290                            Vec::new(),
3291                        ) {
3292                            Ok(pong) => pong,
3293                            Err(error) => break Err(SubcError::FrameBuild(error)),
3294                        };
3295                        if let Err(error) = send_frame(&writer_tx, &dispatch_path_metrics, pong).await {
3296                            break Err(error);
3297                        }
3298                    }
3299                    FrameType::Goodbye if frame.header.channel == 0 => {
3300                        log::info!("subc attach: received channel-0 Goodbye");
3301                        break Ok(ModuleLoopExit::Graceful);
3302                    }
3303                    FrameType::Goodbye => {
3304                        let channel = route_key(frame.header.channel, frame.header.epoch);
3305                        if let Err(error) = teardown_installed_route(
3306                            &writer_tx,
3307                            &dispatch_path_metrics,
3308                            &executor,
3309                            channel,
3310                            "Goodbye",
3311                            None,
3312                            &mut installed_route_epochs,
3313                            &mut routes,
3314                            &mut management_routes,
3315                            &mut root_channels,
3316                            &mut bg_subs,
3317                            &mut bg_sub_by_session,
3318                            &mut bg_wake_pending,
3319                            &mut pending_bash_asks,
3320                            &mut live_roots,
3321                            &mut route_bash_cancels,
3322                            &active_tool_calls,
3323                            &mut pending_responses,
3324                            &mut pending_binds,
3325                            &mut retry_buffer,
3326                            &mut push_buffer,
3327                            &shutdown,
3328                            tool_response_body_limit,
3329                            lifecycle_probe.as_ref(),
3330                        )
3331                        .await
3332                        {
3333                            break Err(error);
3334                        }
3335                    }
3336                    FrameType::Response | FrameType::Error if frame.header.channel != 0 => {
3337                        if let Err(error) = handle_bash_elicitation_reply(
3338                            &writer_tx,
3339                            &frame,
3340                            &mut pending_bash_asks,
3341                            &routes,
3342                            &mut live_roots,
3343                            &executor,
3344                            &shutdown,
3345                            &bash_deferred_tx,
3346                            &bash_poll_touch_tx,
3347                            &dispatch_path_metrics,
3348                            &mut route_bash_cancels,
3349                            dispatch,
3350                        )
3351                        .await
3352                        {
3353                            break Err(error);
3354                        }
3355                    }
3356                    FrameType::Request if frame.header.channel == 0 => {
3357                        if let Err(error) = handle_control_request(
3358                            &writer_tx,
3359                            &frame,
3360                            &shared_app,
3361                            &executor,
3362                            &mut live_roots,
3363                            &mut pending_binds,
3364                            &mut installed_route_epochs,
3365                            &mut routes,
3366                            &mut management_routes,
3367                            &mut root_channels,
3368                            &mut bg_subs,
3369                            &mut bg_sub_by_session,
3370                            &mut bg_wake_pending,
3371                            &mut pending_bash_asks,
3372                            &mut route_bash_cancels,
3373                            &active_tool_calls,
3374                            &mut pending_responses,
3375                            &mut retry_buffer,
3376                            &mut push_buffer,
3377                            &shutdown,
3378                            &control_completion_tx,
3379                            &dispatch_path_metrics,
3380                            lifecycle_probe.as_ref(),
3381                            &health_rollup_cache,
3382                            &push_senders,
3383                            dispatch,
3384                            user_config_path.as_deref(),
3385                            tool_response_body_limit,
3386                        )
3387                        .await
3388                        {
3389                            break Err(error);
3390                        }
3391                    }
3392                    FrameType::Request => {
3393                        let route = route_key(frame.header.channel, frame.header.epoch);
3394                        let result = if management_routes.contains(&route) {
3395                            handle_management_request(
3396                                &writer_tx,
3397                                &frame,
3398                                &shared_app,
3399                                &executor,
3400                                &live_roots,
3401                                &root_channels,
3402                                &health_rollup_cache,
3403                                &dispatch_path_metrics,
3404                            )
3405                            .await
3406                        } else {
3407                            handle_tool_call(
3408                                &writer_tx,
3409                                &frame,
3410                                phase_trace,
3411                                &routes,
3412                                &pending_binds,
3413                                &mut live_roots,
3414                                &executor,
3415                                &active_tool_calls,
3416                                &pending_deferred_setups,
3417                                &shutdown,
3418                                &connection_cancel,
3419                                &bash_deferred_tx,
3420                                &bash_poll_touch_tx,
3421                                &dispatch_path_metrics,
3422                                &mut route_bash_cancels,
3423                                &mut pending_bash_asks,
3424                                &mut next_bash_ask_corr,
3425                                &mut bg_subs,
3426                                &mut bg_sub_by_session,
3427                                &mut bg_wake_pending,
3428                                &mut bg_wake_epoch,
3429                                dispatch,
3430                                &deferred_response_tx,
3431                                allow_native_passthrough,
3432                                tool_response_body_limit,
3433                            )
3434                            .await
3435                        };
3436                        if let Err(error) = result {
3437                            break Err(error);
3438                        }
3439                    }
3440                    FrameType::Cancel => {
3441                        let channel = route_key(frame.header.channel, frame.header.epoch);
3442                        cancel_active_tool_call(
3443                            &active_tool_calls,
3444                            executor.as_ref(),
3445                            channel,
3446                            frame.header.corr,
3447                            "Cancel frame",
3448                        );
3449                        pending_responses.cancel_request(channel, frame.header.corr);
3450                        if bg_subs.contains_key(&channel) {
3451                            if let Err(error) = end_bg_subscription(
3452                                &writer_tx,
3453                                &dispatch_path_metrics,
3454                                &mut bg_subs,
3455                                &mut bg_sub_by_session,
3456                                &mut bg_wake_pending,
3457                                channel,
3458                                routes.get(&channel),
3459                                "cancel",
3460                            )
3461                            .await
3462                            {
3463                                break Err(error);
3464                            }
3465                        }
3466                        if let Err(error) = cancel_pending_bash_ask_for_tool_call(
3467                            &writer_tx,
3468                            &mut pending_bash_asks,
3469                            channel,
3470                            frame.header.corr,
3471                            &routes,
3472                            &mut live_roots,
3473                            &mut route_bash_cancels,
3474                            &shutdown,
3475                            &dispatch_path_metrics,
3476                        )
3477                        .await
3478                        {
3479                            break Err(error);
3480                        }
3481                    }
3482                    // Incoming push messages are ignored here. Cancel frames are
3483                    // handled above for active and deferred tool calls plus pending
3484                    // bash elicitation requests.
3485                    _ => {}
3486                }
3487            }
3488            Some(pending) = deferred_response_rx.recv() => {
3489                if routes.contains_key(&pending.route)
3490                    && active_tool_call_is_registered(
3491                        &active_tool_calls,
3492                        pending.route,
3493                        pending.corr,
3494                    )
3495                {
3496                    pending_responses.register(pending);
3497                } else {
3498                    if let Some(cancellation) = &pending.pending.cancellation {
3499                        cancellation.request_cancel();
3500                    }
3501                    finish_active_tool_call(&active_tool_calls, pending.route, pending.corr);
3502                }
3503            }
3504            Some((root_id, frame)) = reliable_rx.recv() => {
3505                // Reliable Push frames are FIFO and must-deliver, but draining an
3506                // unbounded burst in one current-thread turn can starve RouteBind
3507                // completions. The budget defers excess frames, never drops them.
3508                let (_, deferred) = push::drain_reliable_push_turn(
3509                    &writer_tx,
3510                    &dispatch_path_metrics,
3511                    &routes,
3512                    &root_channels,
3513                    &session_identity,
3514                    &mut retry_buffer,
3515                    &mut push_buffer,
3516                    &mut completed_tasks,
3517                    &bg_sub_by_session,
3518                    &mut bg_wake_pending,
3519                    &mut bg_wake_epoch,
3520                    &mut reliable_rx,
3521                    Some((root_id, frame)),
3522                    lifecycle_probe.as_ref(),
3523                );
3524                if deferred {
3525                    tokio::task::yield_now().await;
3526                }
3527            }
3528            Some((order, root_id, frame)) = lossy_rx.recv() => {
3529                // When both push lanes have work, handle a small reliable slice before lossy work.
3530                // That ordering lets completed task ids suppress stale BashLongRunning frames.
3531                // The slice stays bounded so reliable bursts cannot monopolize this loop turn.
3532                let (_, deferred) = push::drain_reliable_push_turn(
3533                    &writer_tx,
3534                    &dispatch_path_metrics,
3535                    &routes,
3536                    &root_channels,
3537                    &session_identity,
3538                    &mut retry_buffer,
3539                    &mut push_buffer,
3540                    &mut completed_tasks,
3541                    &bg_sub_by_session,
3542                    &mut bg_wake_pending,
3543                    &mut bg_wake_epoch,
3544                    &mut reliable_rx,
3545                    None,
3546                    lifecycle_probe.as_ref(),
3547                );
3548                if deferred {
3549                    tokio::task::yield_now().await;
3550                }
3551
3552                // Drain the currently queued burst in one loop turn so lossy
3553                // status/progress updates can be merged before reaching subc's
3554                // shared egress queue. Each lossy frame gets a sequence number
3555                // before it goes to the channel or overflow buffer, so the
3556                // combined batch is sorted back into producer order before
3557                // coalescing drops stale updates for the same key.
3558                let mut batch = vec![(order, root_id, frame)];
3559                while let Ok(item) = lossy_rx.try_recv() {
3560                    batch.push(item);
3561                }
3562                batch.extend(lossy_overflow.drain());
3563                push::process_lossy_push_envelope_batch(
3564                    &writer_tx,
3565                    &dispatch_path_metrics,
3566                    &routes,
3567                    &root_channels,
3568                    &completed_tasks,
3569                    batch,
3570                );
3571            }
3572            Some(done) = bash_deferred_rx.recv() => {
3573                decrement_counted_channel(&dispatch_path_metrics.bash_deferred_queued);
3574                if let Err(error) = bash::handle_bash_deferred_completion(
3575                    &writer_tx,
3576                    done,
3577                    &routes,
3578                    &mut live_roots,
3579                    &mut route_bash_cancels,
3580                    &shutdown,
3581                    &dispatch_path_metrics,
3582                )
3583                .await
3584                {
3585                    break Err(error);
3586                }
3587            }
3588            Some(root_id) = bash_poll_touch_rx.recv() => {
3589                decrement_counted_channel(&dispatch_path_metrics.bash_poll_touch_queued);
3590                if let Some(meta) = live_roots.get_mut(&root_id) {
3591                    meta.note_activity();
3592                }
3593            }
3594            Some(completion) = maintenance_rx.recv() => {
3595                decrement_counted_channel(&dispatch_path_metrics.maintenance_queued);
3596                let root_id = completion.root_id.clone();
3597                let response = completion.response;
3598                let response_is_fatal = response_is_fatal_panic(&response);
3599                let bind_pending = pending_binds
3600                    .values()
3601                    .any(|pending| pending.bind_root_id == root_id);
3602                let requiesce = if let Some(meta) = live_roots.get_mut(&root_id) {
3603                    let defer_requeue = meta.unbound_quiesced || bind_pending;
3604                    note_maintenance_completion(
3605                        meta,
3606                        completion.requeue_kind,
3607                        response_is_fatal,
3608                        defer_requeue,
3609                    );
3610                    should_requiesce_after_maintenance(meta, completion.kind, bind_pending)
3611                } else {
3612                    false
3613                };
3614                if requiesce {
3615                    quiesce_unbound_root(&root_id, &mut live_roots, &executor);
3616                }
3617                push::clear_stale_bg_wakes_for_empty_sessions(
3618                    &root_id,
3619                    &completion.empty_bg_sessions,
3620                    &bg_sub_by_session,
3621                    &mut bg_wake_pending,
3622                    &bg_wake_epoch,
3623                );
3624                if let Some(keys) = completion.unacked_bg_keys {
3625                    bg_unacked_keys_by_root.insert(root_id.clone(), keys);
3626                }
3627                record_bg_runtime_from_snapshots(
3628                    &dispatch_path_metrics,
3629                    bg_subs.len(),
3630                    bg_wake_pending.len(),
3631                    &bg_unacked_keys_by_root,
3632                );
3633                if response_is_fatal {
3634                    if let Some(meta) = live_roots.get_mut(&root_id) {
3635                        meta.maintenance_poisoned = true;
3636                    }
3637                    log::warn!(
3638                        "subc attach: maintenance drain observed a fatal actor; deferring teardown until a route request can receive actor_fatal"
3639                    );
3640                }
3641            }
3642            _ = tokio::time::sleep(PENDING_POLL_INTERVAL), if !pending_responses.is_empty() => {
3643                // The next loop turn polls detached inspect completions. Keeping
3644                // the timer here lets already-ready control frames run first.
3645            }
3646            _ = tokio::time::sleep_until(next_drain_at) => {
3647                // Wakes an otherwise-idle loop so the pre-turn drain check
3648                // above runs on schedule; the drain work itself lives there.
3649            }
3650            _ = tokio::time::sleep_until(next_maintenance_at) => {
3651                // Delay cache-draining maintenance until any already-ready
3652                // inbound route/control messages and push completions have run,
3653                // so maintenance does not block the actor from handling the
3654                // first request that arrives after a route bind is acknowledged.
3655                crate::logging::maybe_sweep_logs();
3656                crate::db::compression_events::maybe_spawn_retention(shared_app.db());
3657                let reaped_lsp_children = shared_app
3658                    .lsp_child_registry()
3659                    .reap_children_with_gone_cwd_or_reclaimed_root();
3660                if reaped_lsp_children > 0 {
3661                    log::warn!(
3662                        "subc attach: reaped {reaped_lsp_children} orphaned LSP child process group(s)"
3663                    );
3664                }
3665                let now = Instant::now();
3666                reap_idle_lsp_servers(now, &live_roots, &executor);
3667                let reap = reap_idle_roots(
3668                    now,
3669                    &mut live_roots,
3670                    &pending_binds,
3671                    &root_channels,
3672                    &executor,
3673                    &dispatch_path_metrics,
3674                );
3675                for root_id in &reap.forgotten_deleted_roots {
3676                    bg_unacked_keys_by_root.remove(root_id);
3677                    purge_deleted_root_residents(
3678                        root_id,
3679                        &mut routes,
3680                        &mut root_channels,
3681                        &mut installed_route_epochs,
3682                        &mut route_bash_cancels,
3683                        &active_tool_calls,
3684                        executor.as_ref(),
3685                        &mut retry_buffer,
3686                        &mut reclaimed_routes,
3687                        &mut session_identity,
3688                        &mut push_buffer,
3689                        &mut bg_subs,
3690                        &mut bg_sub_by_session,
3691                        &mut bg_wake_pending,
3692                        &mut bg_wake_epoch,
3693                        &mut pending_bash_asks,
3694                        &dispatch_path_metrics,
3695                    );
3696                }
3697                if reap.evicted > 0 {
3698                    log::debug!("subc attach: reaped {} idle root(s)", reap.evicted);
3699                }
3700                record_bg_runtime_from_snapshots(
3701                    &dispatch_path_metrics,
3702                    bg_subs.len(),
3703                    bg_wake_pending.len(),
3704                    &bg_unacked_keys_by_root,
3705                );
3706                submit_due_maintenance_jobs(
3707                    &executor,
3708                    &mut live_roots,
3709                    &pending_binds,
3710                    &bg_sub_by_session,
3711                    &bg_wake_pending,
3712                    &bg_wake_epoch,
3713                    &maintenance_tx,
3714                    &dispatch_path_metrics,
3715                );
3716                if tokio::time::Instant::now() >= next_standing_pass_at {
3717                    standing_actor.tick();
3718                    next_standing_pass_at = tokio::time::Instant::now()
3719                        + standing::STANDING_MAINTENANCE_INTERVAL;
3720                }
3721                // Opportunistic allocator relief, independent of the idle
3722                // sweep: the sweep's whole-process idle gate never opens while
3723                // any session stays active, which let freed warm-up arenas sit
3724                // resident for the process lifetime (5.1 GB RSS over ~600 MB
3725                // live). Slack threshold + spacing live in memory.rs; the pass
3726                // itself runs on a detached thread.
3727                #[cfg(any(target_os = "macos", target_os = "linux"))]
3728                {
3729                    let now_std = std::time::Instant::now();
3730                    let _ = crate::memory::spawn_allocator_slack_relief_if_due(now_std);
3731                }
3732                next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
3733            }
3734        }
3735    };
3736
3737    shared_app.set_open_route_count(0);
3738    health_rollup_worker.shutdown();
3739
3740    connection_cancel.cancel();
3741    cancel_all_active_tool_calls(&active_tool_calls, executor.as_ref(), "connection teardown");
3742    let setup_drain_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
3743    while pending_deferred_setups.load(Ordering::SeqCst) != 0
3744        && tokio::time::Instant::now() < setup_drain_deadline
3745    {
3746        tokio::select! {
3747            biased;
3748            Some(pending) = deferred_response_rx.recv() => pending_responses.register(pending),
3749            _ = tokio::time::sleep(Duration::from_millis(5)) => {}
3750        }
3751    }
3752    if pending_deferred_setups.load(Ordering::SeqCst) != 0 {
3753        log::warn!(
3754            "subc attach: timed out waiting for deferred response setup registration during shutdown"
3755        );
3756    }
3757    while let Ok(pending) = deferred_response_rx.try_recv() {
3758        pending_responses.register(pending);
3759    }
3760    for resolved in pending_responses.drain_on_shutdown(executor.as_ref()) {
3761        if let Err(error) = deliver_resolved_subc_response(
3762            &writer_tx,
3763            resolved,
3764            &routes,
3765            &mut live_roots,
3766            executor.as_ref(),
3767            &active_tool_calls,
3768            &shutdown,
3769            &dispatch_path_metrics,
3770            tool_response_body_limit,
3771        )
3772        .await
3773        {
3774            log::warn!("subc attach: failed to emit deferred shutdown terminal: {error}");
3775        }
3776    }
3777    // Channel-0 Goodbye, EOF, and fatal exits bypass per-route Goodbye. Settle
3778    // their root lifecycle state before loop-owned routing metadata is dropped.
3779    quiesce_connection_roots(
3780        &mut live_roots,
3781        &mut pending_binds,
3782        &mut routes,
3783        &mut root_channels,
3784        &mut installed_route_epochs,
3785        &mut route_bash_cancels,
3786        &active_tool_calls,
3787        &executor,
3788    );
3789
3790    fleet_status_client.set_route_live(false);
3791    fleet_status_task.abort();
3792    let _ = fleet_status_task.await;
3793
3794    let mut loop_result = loop_result;
3795    if !pending_bash_asks.is_empty() {
3796        let no_routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
3797        if let Err(error) = settle_all_pending_bash_asks(
3798            &writer_tx,
3799            &mut pending_bash_asks,
3800            &no_routes,
3801            &mut live_roots,
3802            &mut route_bash_cancels,
3803            &shutdown,
3804            &dispatch_path_metrics,
3805        )
3806        .await
3807        {
3808            loop_result = loop_result.and(Err(error));
3809        }
3810    }
3811
3812    // The reader task may be parked on `read_frame`; abort it (we are done with
3813    // the connection) and flush the writer.
3814    reader_task.abort();
3815    drop(writer_tx);
3816    let writer_result = finish_writer_task(writer_task).await;
3817    loop_result.and_then(|exit| writer_result.map(|_| exit))
3818}
3819
3820fn spawn_writer_task<W>(
3821    mut write: W,
3822    mut rx: mpsc::Receiver<WriterFrame>,
3823    metrics: Arc<DispatchPathMetrics>,
3824) -> JoinHandle<Result<(), subc_transport::FrameIoError>>
3825where
3826    W: AsyncWrite + Unpin + Send + 'static,
3827{
3828    tokio::spawn(async move {
3829        let mut write_buffer = Vec::new();
3830        while let Some(mut queued) = rx.recv().await {
3831            let measure = queued.tool_response_trace.is_some();
3832            let dequeued = measure.then(Instant::now);
3833            metrics.writer_active.store(true, Ordering::Relaxed);
3834            decrement_counted_channel(&metrics.writer_queued);
3835            let write_timing = write_frame_contiguous(
3836                &mut write,
3837                queued.frame(),
3838                queued.body(),
3839                &mut write_buffer,
3840                measure,
3841            )
3842            .await;
3843            metrics.writer_active.store(false, Ordering::Relaxed);
3844            let write_timing = write_timing?;
3845
3846            if let (Some(trace), Some(dequeued), Some(write_timing)) =
3847                (queued.tool_response_trace.take(), dequeued, write_timing)
3848            {
3849                if let Some(completed) = trace.finish(
3850                    dequeued,
3851                    write_timing.write_started,
3852                    write_timing.write_finished,
3853                    write_timing.frame_bytes,
3854                ) {
3855                    log_ctx::with_session(Some(completed.session), || {
3856                        crate::logging::note_tool_call_trace(
3857                            &completed.name,
3858                            &completed.root,
3859                            completed.channel,
3860                            completed.corr,
3861                            completed.phases,
3862                        );
3863                    });
3864                }
3865            }
3866        }
3867        Ok(())
3868    })
3869}
3870
3871struct FrameWriteTiming {
3872    write_started: Instant,
3873    write_finished: Instant,
3874    frame_bytes: usize,
3875}
3876
3877/// Encode one complete frame into the existing reusable buffer and write it
3878/// without interleaving bytes from another channel. Timing is collected only
3879/// for tool responses, so Push and control frames add no clock reads.
3880async fn write_frame_contiguous<W>(
3881    writer: &mut W,
3882    frame: &Frame,
3883    body: &[u8],
3884    buffer: &mut Vec<u8>,
3885    measure: bool,
3886) -> Result<Option<FrameWriteTiming>, subc_transport::FrameIoError>
3887where
3888    W: AsyncWrite + Unpin,
3889{
3890    if frame.header.len as usize != body.len() {
3891        return Err(subc_transport::FrameIoError::BodyLengthMismatch {
3892            header_len: frame.header.len,
3893            body_len: body.len(),
3894        });
3895    }
3896
3897    let header = frame.header.encode();
3898    buffer.clear();
3899    buffer.reserve(header.len() + body.len());
3900    buffer.extend_from_slice(&header);
3901    buffer.extend_from_slice(body);
3902    let write_started = measure.then(Instant::now);
3903    writer
3904        .write_all(buffer)
3905        .await
3906        .map_err(subc_transport::FrameIoError::Io)?;
3907    Ok(write_started.map(|write_started| FrameWriteTiming {
3908        write_started,
3909        write_finished: Instant::now(),
3910        frame_bytes: buffer.len(),
3911    }))
3912}
3913
3914fn spawn_reader_task<R>(
3915    mut read: R,
3916    tx: mpsc::Sender<Result<DecodedFrame, SubcError>>,
3917) -> JoinHandle<()>
3918where
3919    R: AsyncRead + Unpin + Send + 'static,
3920{
3921    tokio::spawn(async move {
3922        loop {
3923            match read_frame(&mut read).await {
3924                Ok(Some(frame)) => {
3925                    let decoded = DecodedFrame {
3926                        frame,
3927                        phase_trace: PhaseTrace::new(Instant::now()),
3928                    };
3929                    if tx.send(Ok(decoded)).await.is_err() {
3930                        return;
3931                    }
3932                }
3933                Ok(None) => {
3934                    // EOF: let the loop observe channel close as "daemon closed".
3935                    return;
3936                }
3937                Err(error) => {
3938                    // A killed daemon surfaces as ConnectionReset (RST) on
3939                    // Windows where Unix delivers a clean EOF (FIN); a
3940                    // mid-teardown daemon can also abort the socket. Both mean
3941                    // "daemon went away", not a wire fault — normalize them to
3942                    // the clean-close path so module exit behavior matches
3943                    // across platforms (same class subc-core fixed in d33d9a71).
3944                    if let subc_transport::FrameIoError::Io(io_error) = &error {
3945                        if matches!(
3946                            io_error.kind(),
3947                            std::io::ErrorKind::ConnectionReset
3948                                | std::io::ErrorKind::ConnectionAborted
3949                        ) {
3950                            log::info!(
3951                                "subc attach: connection reset by daemon; treating as close"
3952                            );
3953                            return;
3954                        }
3955                    }
3956                    let _ = tx.send(Err(SubcError::FrameIo(error))).await;
3957                    return;
3958                }
3959            }
3960        }
3961    })
3962}
3963
3964async fn finish_writer_task(
3965    mut writer_task: JoinHandle<Result<(), subc_transport::FrameIoError>>,
3966) -> Result<(), SubcError> {
3967    match tokio::time::timeout(Duration::from_millis(100), &mut writer_task).await {
3968        Ok(Ok(Ok(()))) => Ok(()),
3969        Ok(Ok(Err(error))) => Err(SubcError::FrameIo(error)),
3970        Ok(Err(error)) => Err(SubcError::WriterJoin(error)),
3971        Err(_) => {
3972            writer_task.abort();
3973            Ok(())
3974        }
3975    }
3976}
3977
3978fn register_actor_for_bind(
3979    shared_app: &Arc<App>,
3980    executor: &Arc<Executor>,
3981    push_senders: &PushSenders,
3982    bind_root_id: &ProjectRootId,
3983    route_channel: u16,
3984    root_was_live: bool,
3985) -> bool {
3986    if executor.actor_registered(bind_root_id) {
3987        log::debug!(
3988            "subc attach: reusing actor for route {} root {}",
3989            route_channel,
3990            bind_root_id.as_path().display()
3991        );
3992        return false;
3993    }
3994
3995    if root_was_live {
3996        log::warn!(
3997            "subc attach: recreating missing actor for live root {} on route {}",
3998            bind_root_id.as_path().display(),
3999            route_channel
4000        );
4001    }
4002
4003    let actor_ctx = Arc::new(AppContext::from_app(
4004        Arc::clone(shared_app),
4005        Config::default(),
4006    ));
4007    install_bash_compressor(&actor_ctx);
4008    actor_ctx.install_fleet_status_client(Some(push_senders.fleet_status_client.clone()));
4009    actor_ctx.set_progress_sender(Some(push::progress_sender_for_root(
4010        push_senders.clone(),
4011        bind_root_id.clone(),
4012    )));
4013    let inserted = executor.register_actor(bind_root_id.clone(), Arc::clone(&actor_ctx));
4014    drop(actor_ctx);
4015    if inserted {
4016        // Do not insert into live_roots until configure succeeds: live_roots
4017        // drives maintenance, and a half-configured new actor must not be
4018        // maintenance-eligible before its route/session identity exists.
4019        log::debug!(
4020            "subc attach: registered actor for route {} root {}",
4021            route_channel,
4022            bind_root_id.as_path().display()
4023        );
4024    } else {
4025        log::debug!(
4026            "subc attach: actor appeared while binding route {} root {}; reusing it",
4027            route_channel,
4028            bind_root_id.as_path().display()
4029        );
4030    }
4031    inserted
4032}
4033
4034fn rollback_pending_bind_actor(
4035    executor: &Arc<Executor>,
4036    live_roots: &HashMap<ProjectRootId, RootMeta>,
4037    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
4038    root_id: &ProjectRootId,
4039    inserted_new_actor: bool,
4040) {
4041    if !inserted_new_actor || live_roots.contains_key(root_id) {
4042        return;
4043    }
4044
4045    if let Some((route, pending)) = pending_binds
4046        .iter_mut()
4047        .find(|(_, pending)| &pending.bind_root_id == root_id)
4048    {
4049        pending.inserted_new_actor = true;
4050        log::debug!(
4051            "subc attach: transferred rollback ownership for root {} to pending route {}",
4052            root_id.as_path().display(),
4053            route
4054        );
4055        return;
4056    }
4057
4058    executor.remove_actor(root_id);
4059}
4060
4061fn route_bind_error_code_for_configure_response(response: &Response) -> &'static str {
4062    match response.data.get("code").and_then(|code| code.as_str()) {
4063        // Preserve typed configure rejections across the bind boundary: a
4064        // malformed fed fingerprint means a federation-module bug or
4065        // fingerprint-format drift, and the fed side matches on the code rather
4066        // than parsing prose.
4067        Some("bad_harness_fingerprint") => "bad_harness_fingerprint",
4068        // Cache-key probe failures are transient (fd pressure, git spawn
4069        // contention); the client retries the bind rather than treating the
4070        // root as permanently divergent.
4071        Some("cache_key_probe_failed") => "cache_key_probe_failed",
4072        // Actor lifecycle gaps are transient from the daemon/client viewpoint:
4073        // a fresh bind can create or join a healthy actor, so do not classify
4074        // them as permanent config divergence.
4075        Some("actor_not_registered" | "actor_fatal") => "actor_not_ready",
4076        _ => "config_divergence",
4077    }
4078}
4079
4080fn queue_post_bind_configure_and_completion_maintenance(
4081    root_id: &ProjectRootId,
4082    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4083) {
4084    let Some(meta) = live_roots.get_mut(root_id) else {
4085        return;
4086    };
4087    if meta.maintenance_poisoned || meta.maintenance_pending {
4088        return;
4089    }
4090
4091    meta.maintenance_pending = true;
4092    meta.maintenance_queued_kinds
4093        .push_back(MaintenanceDrainKind::ConfigureTail);
4094    meta.maintenance_queued_kinds
4095        .push_back(MaintenanceDrainKind::CompletionDrains);
4096}
4097
4098#[allow(clippy::too_many_arguments)]
4099async fn handle_route_bind_completion(
4100    tx: &WriterSender,
4101    completion: RouteBindCompletion,
4102    routes: &mut HashMap<RouteChannel, RouteIdentity>,
4103    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
4104    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
4105    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
4106    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4107    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
4108    installed_route_epochs: &mut HashMap<u16, u32>,
4109    executor: &Arc<Executor>,
4110    standing_actor: &standing::StandingActor,
4111    shutdown: &Arc<Notify>,
4112    metrics: &Arc<DispatchPathMetrics>,
4113    lifecycle_probe: Option<&SubcTestLifecycleProbe>,
4114) -> Result<(), SubcError> {
4115    let route_id = completion.route;
4116    let Some(pending) = pending_binds.remove(&route_id) else {
4117        log::warn!(
4118            "subc attach: dropping RouteBind completion for non-pending route {}",
4119            completion.route
4120        );
4121        rollback_pending_bind_actor(
4122            executor,
4123            live_roots,
4124            pending_binds,
4125            &completion.bind_root_id,
4126            completion.inserted_new_actor,
4127        );
4128        let has_pending_bind = pending_binds
4129            .values()
4130            .any(|pending| pending.bind_root_id == completion.bind_root_id);
4131        if !root_channels
4132            .get(&completion.bind_root_id)
4133            .is_some_and(|channels| !channels.is_empty())
4134            && !has_pending_bind
4135        {
4136            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
4137        }
4138        remove_installed_route(installed_route_epochs, route_id);
4139        return Ok(());
4140    };
4141
4142    if pending.bind_root_id != completion.bind_root_id {
4143        log::warn!(
4144            "subc attach: pending RouteBind root mismatch for route {} (pending {} completion {})",
4145            completion.route,
4146            pending.bind_root_id.as_path().display(),
4147            completion.bind_root_id.as_path().display()
4148        );
4149    }
4150
4151    let inserted_new_actor = pending.inserted_new_actor || completion.inserted_new_actor;
4152    if pending.cancelled {
4153        rollback_pending_bind_actor(
4154            executor,
4155            live_roots,
4156            pending_binds,
4157            &completion.bind_root_id,
4158            inserted_new_actor,
4159        );
4160        let has_pending_bind = pending_binds
4161            .values()
4162            .any(|pending| pending.bind_root_id == completion.bind_root_id);
4163        if !root_channels
4164            .get(&completion.bind_root_id)
4165            .is_some_and(|channels| !channels.is_empty())
4166            && !has_pending_bind
4167        {
4168            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
4169        }
4170        log::debug!(
4171            "subc attach: discarded completed RouteBind for cancelled route {} root {}",
4172            completion.route,
4173            completion.bind_root_id.as_path().display()
4174        );
4175        remove_installed_route(installed_route_epochs, route_id);
4176        return Ok(());
4177    }
4178
4179    let failure = if !completion.configure_response.success {
4180        Some((
4181            &completion.configure_response,
4182            "configure failed during route bind",
4183        ))
4184    } else {
4185        None
4186    };
4187
4188    if let Some((response, fallback)) = failure {
4189        rollback_pending_bind_actor(
4190            executor,
4191            live_roots,
4192            pending_binds,
4193            &completion.bind_root_id,
4194            inserted_new_actor,
4195        );
4196        let has_pending_bind = pending_binds
4197            .values()
4198            .any(|pending| pending.bind_root_id == completion.bind_root_id);
4199        if !root_channels
4200            .get(&completion.bind_root_id)
4201            .is_some_and(|channels| !channels.is_empty())
4202            && !has_pending_bind
4203        {
4204            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
4205        }
4206        let message = response_message(response, fallback);
4207        let fatal = response_is_fatal_panic(response);
4208        let error_code = route_bind_error_code_for_configure_response(response);
4209        send_route_bind_error_parts(
4210            tx,
4211            completion.ver,
4212            completion.corr,
4213            completion.flags,
4214            error_code,
4215            &message,
4216            metrics,
4217        )
4218        .await?;
4219        remove_installed_route(installed_route_epochs, route_id);
4220        if fatal {
4221            signal_fatal_teardown(
4222                tx,
4223                Some(completion.route),
4224                completion.ver,
4225                completion.corr,
4226                shutdown,
4227                metrics,
4228            )
4229            .await;
4230        }
4231        return Ok(());
4232    }
4233
4234    remember_session_identity(session_identity, &completion.identity);
4235    let replay_key = push::ReplayKey::from_identity(&completion.identity);
4236    let bind_trust = completion.identity.trust;
4237    insert_route_channel(routes, root_channels, route_id, completion.identity);
4238    sync_bg_live_delivery_sessions(executor, routes, Some(&completion.bind_root_id));
4239    let restore_watcher = live_roots
4240        .get(&completion.bind_root_id)
4241        .is_some_and(|meta| meta.idle_artifacts_evicted || meta.unbound_quiesced);
4242    live_roots
4243        .entry(completion.bind_root_id.clone())
4244        .and_modify(|meta| {
4245            meta.reactivate_bound();
4246            meta.diagnostics_on_edit = completion.diagnostics_on_edit;
4247            meta.maintenance_poisoned = false;
4248        })
4249        .or_insert_with(|| RootMeta::new(Instant::now()));
4250    if let Some(meta) = live_roots.get_mut(&completion.bind_root_id) {
4251        meta.diagnostics_on_edit = completion.diagnostics_on_edit;
4252        meta.maintenance_poisoned = false;
4253    }
4254    if let Some(ctx) = executor.actor_context(&completion.bind_root_id) {
4255        // The bind transition revokes any matching unbound standing admission
4256        // before this session can select the shared artifact family.
4257        standing_actor.begin_session_bind(&ctx);
4258        ctx.mark_subc_bound();
4259        if restore_watcher {
4260            crate::commands::configure::ensure_project_watcher(&ctx);
4261        }
4262    }
4263
4264    let ack =
4265        serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).map_err(SubcError::Json)?;
4266    let response = Frame::build_with_version(
4267        completion.ver,
4268        FrameType::Response,
4269        control_flags(),
4270        0,
4271        0,
4272        completion.corr,
4273        ack,
4274    )
4275    .map_err(SubcError::FrameBuild)?;
4276    send_reliable_writer_frame(tx, metrics, response, "RouteBindAck").await?;
4277    queue_post_bind_configure_and_completion_maintenance(&completion.bind_root_id, live_roots);
4278    let replayed = push::replay_buffered_push_frames(
4279        tx,
4280        metrics,
4281        route_id,
4282        push_buffer,
4283        &replay_key,
4284        bind_trust,
4285        lifecycle_probe,
4286    );
4287    if replayed > 0 {
4288        log::debug!(
4289            "subc attach: replayed {} buffered Push frame(s) to route {} root {} harness {} session {}",
4290            replayed,
4291            completion.route,
4292            replay_key.root.as_path().display(),
4293            replay_key.harness,
4294            replay_key.session
4295        );
4296    }
4297    log::info!(
4298        "subc attach: route {} bound to root {}",
4299        completion.route,
4300        completion.bind_root_id.as_path().display()
4301    );
4302    Ok(())
4303}
4304
4305async fn expire_overdue_route_binds(
4306    tx: &WriterSender,
4307    executor: &Arc<Executor>,
4308    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
4309    installed_route_epochs: &mut HashMap<u16, u32>,
4310    metrics: &DispatchPathMetrics,
4311) -> Result<(), SubcError> {
4312    let now = Instant::now();
4313    let expired: Vec<_> = pending_binds
4314        .iter()
4315        .filter_map(|(route, pending)| {
4316            let age = now.saturating_duration_since(pending.started_at);
4317            (!pending.deadline_reported && age >= ROUTE_BIND_DEADLINE).then(|| {
4318                (
4319                    *route,
4320                    pending.corr,
4321                    pending.ver,
4322                    pending.flags,
4323                    pending.bind_root_id.clone(),
4324                    pending.configure_request_id.clone(),
4325                    age,
4326                )
4327            })
4328        })
4329        .collect();
4330
4331    for (route, corr, ver, flags, root_id, configure_request_id, age) in expired {
4332        if let Some(pending) = pending_binds.get_mut(&route) {
4333            pending.cancelled = true;
4334            pending.deadline_reported = true;
4335            let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
4336            log::debug!(
4337                "subc attach: cancelled overdue RouteBind configure for route {route} ({outcome:?})"
4338            );
4339        }
4340        remove_installed_route(installed_route_epochs, route);
4341        let age_ms = age.as_millis().min(u128::from(u64::MAX)) as u64;
4342        let deadline_ms = ROUTE_BIND_DEADLINE.as_millis();
4343        send_route_bind_error_parts(
4344            tx,
4345            ver,
4346            corr,
4347            flags,
4348            "actor_not_ready",
4349            &format!("route bind deadline exceeded after {age_ms}ms (deadline {deadline_ms}ms)"),
4350            metrics,
4351        )
4352        .await?;
4353        log::warn!(
4354            "subc attach: route {} bind for root {} exceeded {}ms deadline (configure_request_id={})",
4355            route,
4356            root_id.as_path().display(),
4357            deadline_ms,
4358            configure_request_id
4359        );
4360    }
4361
4362    Ok(())
4363}
4364
4365fn record_bg_runtime_from_snapshots(
4366    metrics: &DispatchPathMetrics,
4367    subscriptions: usize,
4368    wake_pending: usize,
4369    unacked_keys_by_root: &HashMap<ProjectRootId, HashSet<String>>,
4370) {
4371    let unacked_total = unacked_keys_by_root
4372        .values()
4373        .flatten()
4374        .collect::<HashSet<_>>()
4375        .len();
4376    metrics.record_bg_runtime(subscriptions, wake_pending, unacked_total);
4377}
4378
4379async fn send_cached_health_response(
4380    tx: &WriterSender,
4381    frame: &Frame,
4382    shared_app: &App,
4383    executor: &Executor,
4384    pending_binds: &HashMap<RouteChannel, PendingBind>,
4385    metrics: &DispatchPathMetrics,
4386    health_rollup_cache: &HealthRollupCache,
4387) -> Result<(), SubcError> {
4388    let report = build_health_report(
4389        health_rollup_cache,
4390        executor,
4391        pending_binds,
4392        metrics,
4393        shared_app,
4394    );
4395    let body = serde_json::to_vec(&ModuleControlResponse::from(report)).map_err(SubcError::Json)?;
4396    let response = Frame::build_with_version(
4397        frame.header.ver,
4398        FrameType::Response,
4399        frame.header.flags,
4400        0,
4401        0,
4402        frame.header.corr,
4403        body,
4404    )
4405    .map_err(SubcError::FrameBuild)?;
4406    send_frame(tx, metrics, response).await
4407}
4408
4409/// Channel-0 control requests: RouteBind plus the cached health probe.
4410/// Tool-provider binds reconcile RootConfig through the executor's Mutating lane;
4411/// management binds install immediately without creating project state.
4412#[allow(clippy::too_many_arguments)]
4413async fn handle_control_request(
4414    tx: &WriterSender,
4415    frame: &Frame,
4416    shared_app: &Arc<App>,
4417    executor: &Arc<Executor>,
4418    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4419    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
4420    installed_route_epochs: &mut HashMap<u16, u32>,
4421    routes: &mut HashMap<RouteChannel, RouteIdentity>,
4422    management_routes: &mut HashSet<RouteChannel>,
4423    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
4424    bg_subs: &mut HashMap<RouteChannel, BgSub>,
4425    bg_sub_by_session: &mut BgSubsBySession,
4426    bg_wake_pending: &mut BgWakePending,
4427    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
4428    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
4429    active_tool_calls: &ActiveToolCalls,
4430    pending_responses: &mut PendingSubcResponses,
4431    retry_buffer: &mut RetryBuffer,
4432    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
4433    shutdown: &Arc<Notify>,
4434    control_completion_tx: &mpsc::Sender<RouteBindCompletion>,
4435    metrics: &Arc<DispatchPathMetrics>,
4436    lifecycle_probe: Option<&SubcTestLifecycleProbe>,
4437    health_rollup_cache: &HealthRollupCache,
4438    push_senders: &PushSenders,
4439    dispatch: DispatchFn,
4440    user_config_path: Option<&Path>,
4441    tool_response_body_limit: usize,
4442) -> Result<(), SubcError> {
4443    let request =
4444        serde_json::from_slice::<ModuleControlRequest>(&frame.body).map_err(SubcError::Json)?;
4445    match request {
4446        ModuleControlRequest::RouteBind {
4447            route_channel,
4448            epoch,
4449            target,
4450            identity,
4451            principal,
4452            consumer_capabilities,
4453            admission_facts: _,
4454        } => {
4455            let route_id = route_key(route_channel, epoch);
4456            if epoch == 0 {
4457                return send_route_bind_error(
4458                    tx,
4459                    frame,
4460                    "config_divergence",
4461                    "route bind uses an invalid channel generation",
4462                    metrics,
4463                )
4464                .await;
4465            }
4466
4467            let bind_trust = trust_for_bind(&identity.harness, &principal);
4468            if let RouteTarget::ManagementSurface { module_id } = &target {
4469                if module_id != "aft" {
4470                    return send_route_bind_error(
4471                        tx,
4472                        frame,
4473                        "route_refused",
4474                        "management route target is not AFT",
4475                        metrics,
4476                    )
4477                    .await;
4478                }
4479                if !matches!(bind_trust, BindTrust::FirstParty) {
4480                    return send_route_bind_error(
4481                        tx,
4482                        frame,
4483                        "route_refused",
4484                        "AFT management routes require a first-party principal",
4485                        metrics,
4486                    )
4487                    .await;
4488                }
4489                if let Some(installed_epoch) = installed_route_epochs.get(&route_channel).copied() {
4490                    if installed_epoch >= epoch {
4491                        return send_route_bind_error(
4492                            tx,
4493                            frame,
4494                            "config_divergence",
4495                            "route bind generation is not newer than the installed generation",
4496                            metrics,
4497                        )
4498                        .await;
4499                    }
4500                    teardown_installed_route(
4501                        tx,
4502                        metrics,
4503                        executor,
4504                        route_key(route_channel, installed_epoch),
4505                        "higher-epoch RouteBind",
4506                        None,
4507                        installed_route_epochs,
4508                        routes,
4509                        management_routes,
4510                        root_channels,
4511                        bg_subs,
4512                        bg_sub_by_session,
4513                        bg_wake_pending,
4514                        pending_bash_asks,
4515                        live_roots,
4516                        route_bash_cancels,
4517                        active_tool_calls,
4518                        pending_responses,
4519                        pending_binds,
4520                        retry_buffer,
4521                        push_buffer,
4522                        shutdown,
4523                        tool_response_body_limit,
4524                        lifecycle_probe,
4525                    )
4526                    .await?;
4527                }
4528                if pending_binds.contains_key(&route_id) {
4529                    return send_route_bind_error(
4530                        tx,
4531                        frame,
4532                        "config_divergence",
4533                        "route bind is already pending for channel",
4534                        metrics,
4535                    )
4536                    .await;
4537                }
4538
4539                installed_route_epochs.insert(route_channel, epoch);
4540                management_routes.insert(route_id);
4541                return send_route_bind_ack(
4542                    tx,
4543                    frame.header.ver,
4544                    frame.header.corr,
4545                    frame.header.flags,
4546                    metrics,
4547                )
4548                .await;
4549            }
4550            if matches!(&target, RouteTarget::InternalService { .. }) {
4551                return send_route_bind_error(
4552                    tx,
4553                    frame,
4554                    "route_refused",
4555                    "AFT does not provide an internal-service route",
4556                    metrics,
4557                )
4558                .await;
4559            }
4560
4561            let mut bind_root_id = None;
4562            if let Some(installed_epoch) = installed_route_epochs.get(&route_channel).copied() {
4563                if installed_epoch >= epoch {
4564                    return send_route_bind_error(
4565                        tx,
4566                        frame,
4567                        "config_divergence",
4568                        "route bind generation is not newer than the installed generation",
4569                        metrics,
4570                    )
4571                    .await;
4572                }
4573
4574                let replacement_root = match ProjectRootId::from_path(&identity.project_root) {
4575                    Ok(root_id) => root_id,
4576                    Err(error) => {
4577                        return send_route_bind_error(
4578                            tx,
4579                            frame,
4580                            "config_divergence",
4581                            &format!("invalid route project root: {error}"),
4582                            metrics,
4583                        )
4584                        .await;
4585                    }
4586                };
4587                teardown_installed_route(
4588                    tx,
4589                    metrics,
4590                    executor,
4591                    route_key(route_channel, installed_epoch),
4592                    "higher-epoch RouteBind",
4593                    Some(&replacement_root),
4594                    installed_route_epochs,
4595                    routes,
4596                    management_routes,
4597                    root_channels,
4598                    bg_subs,
4599                    bg_sub_by_session,
4600                    bg_wake_pending,
4601                    pending_bash_asks,
4602                    live_roots,
4603                    route_bash_cancels,
4604                    active_tool_calls,
4605                    pending_responses,
4606                    pending_binds,
4607                    retry_buffer,
4608                    push_buffer,
4609                    shutdown,
4610                    tool_response_body_limit,
4611                    lifecycle_probe,
4612                )
4613                .await?;
4614                bind_root_id = Some(replacement_root);
4615            }
4616            if pending_binds.contains_key(&route_id) {
4617                return send_route_bind_error(
4618                    tx,
4619                    frame,
4620                    "config_divergence",
4621                    "route bind is already pending for channel",
4622                    metrics,
4623                )
4624                .await;
4625            }
4626            let bind_root_id = match bind_root_id {
4627                Some(root_id) => root_id,
4628                None => match ProjectRootId::from_path(&identity.project_root) {
4629                    Ok(root_id) => root_id,
4630                    Err(error) => {
4631                        return send_route_bind_error(
4632                            tx,
4633                            frame,
4634                            "config_divergence",
4635                            &format!("invalid route project root: {error}"),
4636                            metrics,
4637                        )
4638                        .await;
4639                    }
4640                },
4641            };
4642
4643            // Reconcile RootConfig: build a configure request from the bind
4644            // identity + forwarded config tiers and run it through the executor.
4645            let request_id = format!("subc-bind-{route_channel}");
4646            let bind_project_root = identity.project_root.clone();
4647            let bind_harness = identity.harness.clone();
4648            let bind_session = identity.session.clone();
4649            let bind_principal_id = principal_id(&principal);
4650            // Typed capability declaration from the consumer: the facade stamps it
4651            // from the MCP host's initialize-advertised capabilities. Absent
4652            // means no reverse-request capability — flat deny, fail-closed. A
4653            // consumer over-declaring only earns asks that TTL-deny.
4654            let consumer_elicitation_capable = consumer_capabilities
4655                .as_ref()
4656                .is_some_and(|capabilities| capabilities.iter().any(|c| c == "elicitation"));
4657            log::info!(
4658                "subc attach: route {} harness={} principal={} trust={} elicitation={}",
4659                route_channel,
4660                bind_harness,
4661                principal_label(&principal),
4662                bind_trust.label(),
4663                consumer_elicitation_capable
4664            );
4665
4666            // Config is read directly from the CortexKit user and project files;
4667            // wire-relayed tiers are ignored so a front cannot inject settings.
4668            // The resolver selects only this bind's harness override before it
4669            // applies the unchanged user/project trust boundary.
4670            let local_tiers = crate::subc_config::read_local_cortexkit_config_tiers(
4671                user_config_path,
4672                Path::new(&bind_project_root),
4673            );
4674            let config_tiers: Vec<Value> = local_tiers
4675                .iter()
4676                .map(|t| json!({ "tier": t.tier, "source": t.source, "doc": t.doc }))
4677                .collect();
4678            // Let configure return its structured invalid-harness error rather
4679            // than panicking while computing this optional registration setting.
4680            let active_harness = bind_harness.parse::<crate::harness::Harness>().ok();
4681            let diagnostics_on_edit = crate::config_resolve::resolve_config_for_harness(
4682                &local_tiers,
4683                active_harness.as_ref(),
4684            )
4685            .config
4686            .diagnostics_on_edit;
4687            let configure_json = json!({
4688                "id": request_id,
4689                "command": "configure",
4690                "project_root": bind_project_root,
4691                "harness": bind_harness,
4692                "session_id": bind_session.clone(),
4693                "config": config_tiers,
4694            });
4695            let configure_req = match serde_json::from_value::<RawRequest>(configure_json) {
4696                Ok(req) => req,
4697                Err(error) => {
4698                    return send_route_bind_error(
4699                        tx,
4700                        frame,
4701                        "config_divergence",
4702                        &format!("failed to build configure request: {error}"),
4703                        metrics,
4704                    )
4705                    .await;
4706                }
4707            };
4708
4709            let route_identity = RouteIdentity(Arc::new(RouteIdentityData {
4710                root: bind_root_id.clone(),
4711                project_root: PathBuf::from(&bind_project_root),
4712                harness: bind_harness.clone(),
4713                session: bind_session.clone(),
4714                trust: bind_trust,
4715                spawn_principal: AuthenticatedPrincipal::RouteBind {
4716                    trust: bind_trust.sandbox_trust(),
4717                    route_channel,
4718                    route_epoch: epoch,
4719                    project_root: PathBuf::from(&bind_project_root),
4720                    harness: bind_harness.clone(),
4721                    session_id: bind_session.clone(),
4722                    principal_id: bind_principal_id,
4723                },
4724                consumer_elicitation_capable,
4725            }));
4726            let configure_session = route_identity.session.clone();
4727            let root_was_live = live_roots.contains_key(&bind_root_id);
4728            let inserted_new_actor = register_actor_for_bind(
4729                shared_app,
4730                executor,
4731                push_senders,
4732                &bind_root_id,
4733                route_channel,
4734                root_was_live,
4735            );
4736
4737            sync_bg_live_delivery_sessions(executor, routes, Some(&bind_root_id));
4738            let configure_request_id = configure_req.id.clone();
4739            installed_route_epochs.insert(route_channel, epoch);
4740            if let Some(meta) = live_roots.get_mut(&bind_root_id) {
4741                meta.maintenance_queued_kinds.clear();
4742                meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
4743            }
4744            let (configure_rx, configure_cancellation) = executor.submit_cancellable_async(
4745                bind_root_id.clone(),
4746                Lane::Mutating,
4747                configure_request_id.clone(),
4748                Box::new(move |ctx| {
4749                    log_ctx::with_session(Some(configure_session.clone()), || {
4750                        dispatch(configure_req, ctx)
4751                    })
4752                }),
4753            );
4754            pending_binds.insert(
4755                route_id,
4756                PendingBind {
4757                    bind_root_id: bind_root_id.clone(),
4758                    inserted_new_actor,
4759                    cancelled: false,
4760                    configure_request_id: configure_request_id.clone(),
4761                    started_at: Instant::now(),
4762                    warned_half_deadline: false,
4763                    deadline_reported: false,
4764                    corr: frame.header.corr,
4765                    ver: frame.header.ver,
4766                    flags: frame.header.flags,
4767                    cancellation: configure_cancellation,
4768                },
4769            );
4770
4771            let completion_tx = control_completion_tx.clone();
4772            let completion_identity = route_identity;
4773            let completion_root = bind_root_id.clone();
4774            let completion_route_channel = route_channel;
4775            let completion_ver = frame.header.ver;
4776            let completion_corr = frame.header.corr;
4777            let completion_flags = frame.header.flags;
4778            let completion_metrics = Arc::clone(metrics);
4779            tokio::spawn(async move {
4780                let _response_task = ResponseTaskGuard::new(&completion_metrics);
4781                let configure_response =
4782                    await_executor_response(configure_rx, configure_request_id.clone()).await;
4783                // Send the route-bind acknowledgment as soon as configure succeeds.
4784                // Installing completed search or callgraph builds only refreshes cached
4785                // read data, so a later maintenance pass can do it without delaying the
4786                // daemon's confirmation that the route is usable.
4787                let completion = RouteBindCompletion {
4788                    route: route_key(completion_route_channel, epoch),
4789                    identity: completion_identity,
4790                    bind_root_id: completion_root,
4791                    inserted_new_actor,
4792                    configure_response,
4793                    diagnostics_on_edit,
4794                    ver: completion_ver,
4795                    corr: completion_corr,
4796                    flags: completion_flags,
4797                };
4798                if send_counted_channel(
4799                    &completion_tx,
4800                    &completion_metrics.control_completion_queued,
4801                    completion,
4802                )
4803                .await
4804                .is_err()
4805                {
4806                    log::debug!(
4807                        "subc attach: dropped RouteBind completion for route {} after loop exit",
4808                        completion_route_channel
4809                    );
4810                }
4811            });
4812
4813            // Bind completion wakes the health worker. A synchronous census here
4814            // scans every hosted root before the transport can accept another frame.
4815            Ok(())
4816        }
4817        ModuleControlRequest::HealthCheck {} => {
4818            send_cached_health_response(
4819                tx,
4820                frame,
4821                shared_app,
4822                executor,
4823                pending_binds,
4824                metrics,
4825                health_rollup_cache,
4826            )
4827            .await
4828        }
4829    }
4830}
4831
4832async fn handle_management_request(
4833    tx: &WriterSender,
4834    frame: &Frame,
4835    shared_app: &App,
4836    executor: &Executor,
4837    live_roots: &HashMap<ProjectRootId, RootMeta>,
4838    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
4839    health_rollup_cache: &HealthRollupCache,
4840    metrics: &DispatchPathMetrics,
4841) -> Result<(), SubcError> {
4842    let decoded = serde_json::from_slice::<Value>(&frame.body).ok();
4843    let operation = decoded
4844        .as_ref()
4845        .and_then(|value| value.get("op"))
4846        .and_then(Value::as_str);
4847    let Some(operation) = operation else {
4848        let error = build_error_frame(
4849            frame.header.ver,
4850            frame.header.channel,
4851            frame.header.epoch,
4852            frame.header.corr,
4853            frame.header.flags,
4854            "unknown_management_op",
4855            "management routes accept only declared operation envelopes",
4856        )?;
4857        return send_reliable_writer_frame(tx, metrics, error, "management refusal").await;
4858    };
4859
4860    let result = match operation {
4861        crate::commands::memory_census::MEMORY_CENSUS_OPERATION => Response::success(
4862            "management-memory-census",
4863            memory_census_with_lifecycle(
4864                health_rollup_cache,
4865                shared_app,
4866                executor,
4867                live_roots,
4868                root_channels,
4869            ),
4870        ),
4871        crate::commands::health_digest::HEALTH_DIGEST_OPERATION => {
4872            let params = decoded
4873                .as_ref()
4874                .and_then(|value| value.get("params"))
4875                .cloned()
4876                .unwrap_or_else(|| json!({}));
4877            let Some(params) = params.as_object() else {
4878                return send_management_response(
4879                    tx,
4880                    frame,
4881                    operation,
4882                    Response::error(
4883                        "management-health-digest",
4884                        "invalid_request",
4885                        "health.digest params must be an object",
4886                    ),
4887                    metrics,
4888                )
4889                .await;
4890            };
4891            let root = params
4892                .get("project_root")
4893                .or_else(|| params.get("root"))
4894                .and_then(Value::as_str);
4895            let Some(root) = root else {
4896                return send_management_response(
4897                    tx,
4898                    frame,
4899                    operation,
4900                    Response::error(
4901                        "management-health-digest",
4902                        "invalid_request",
4903                        "health.digest requires params.project_root",
4904                    ),
4905                    metrics,
4906                )
4907                .await;
4908            };
4909
4910            let mut request = params.clone();
4911            request.insert("id".to_string(), json!("management-health-digest"));
4912            request.insert(
4913                "command".to_string(),
4914                json!(crate::commands::health_digest::HEALTH_DIGEST_OPERATION),
4915            );
4916            let request = serde_json::from_value::<RawRequest>(Value::Object(request))
4917                .map_err(SubcError::Json)?;
4918            match ProjectRootId::from_path(Path::new(root))
4919                .ok()
4920                .and_then(|root_id| executor.actor_context(&root_id))
4921            {
4922                Some(ctx) => crate::commands::health_digest::handle_health_digest(&request, &ctx),
4923                None => crate::commands::health_digest::root_not_bound_response(&request, root),
4924            }
4925        }
4926        _ => {
4927            let error = build_error_frame(
4928                frame.header.ver,
4929                frame.header.channel,
4930                frame.header.epoch,
4931                frame.header.corr,
4932                frame.header.flags,
4933                "unknown_management_op",
4934                &format!("management operation {operation:?} is not declared by AFT"),
4935            )?;
4936            return send_reliable_writer_frame(tx, metrics, error, "management refusal").await;
4937        }
4938    };
4939
4940    send_management_response(tx, frame, operation, result, metrics).await
4941}
4942
4943fn memory_census_with_lifecycle(
4944    health_rollup_cache: &HealthRollupCache,
4945    shared_app: &App,
4946    executor: &Executor,
4947    live_roots: &HashMap<ProjectRootId, RootMeta>,
4948    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
4949) -> Value {
4950    let mut census = health_rollup_cache.memory_census();
4951    if let Some(rows) = census.get_mut("roots").and_then(Value::as_object_mut) {
4952        let lifecycle = shared_app.lifecycle_census_snapshot();
4953        for (root_id, meta) in live_roots {
4954            let root = root_id.as_path().display().to_string();
4955            let bound_routes = root_channels.get(root_id).map_or(0, HashSet::len);
4956            let age_ms = Instant::now()
4957                .saturating_duration_since(meta.last_touched)
4958                .as_millis()
4959                .min(u128::from(u64::MAX)) as u64;
4960            let ttl_ms = root_idle_ttl(executor, root_id)
4961                .as_millis()
4962                .min(u128::from(u64::MAX)) as u64;
4963            let lsp = lifecycle
4964                .lsp
4965                .children_by_root
4966                .iter()
4967                .find(|child| child.root == root);
4968            if let Some(row) = rows.get_mut(&root).and_then(Value::as_object_mut) {
4969                let evictable_bytes = row.get("evictable_bytes").cloned().unwrap_or(json!(0));
4970                row.insert("root_id".to_string(), json!(root));
4971                row.insert("bound_routes".to_string(), json!(bound_routes));
4972                row.insert("last_request_age_ms".to_string(), json!(age_ms));
4973                row.insert("idle_ttl_ms".to_string(), json!(ttl_ms));
4974                row.insert(
4975                    "lsp_idle_ttl_ms".to_string(),
4976                    json!(executor
4977                        .actor_context(root_id)
4978                        .map(|ctx| ctx
4979                            .config()
4980                            .idle
4981                            .lsp_ttl()
4982                            .as_millis()
4983                            .min(u128::from(u64::MAX)) as u64)
4984                        .unwrap_or(0)),
4985                );
4986                row.insert(
4987                    "evictable_in_ms".to_string(),
4988                    crate::commands::memory_census::evictable_in_ms(bound_routes, ttl_ms, age_ms)
4989                        .map_or(Value::Null, |value| json!(value)),
4990                );
4991                row.insert(
4992                    "evictable_bytes".to_string(),
4993                    if bound_routes == 0 {
4994                        evictable_bytes
4995                    } else {
4996                        json!(0)
4997                    },
4998                );
4999                row.insert(
5000                    "lsp_children".to_string(),
5001                    json!({
5002                        "count": lsp.map_or(0, |child| child.count),
5003                        "rss_bytes": lsp.map_or(0, |child| child.rss_bytes),
5004                    }),
5005                );
5006            }
5007        }
5008    }
5009    census
5010}
5011
5012async fn send_management_response(
5013    tx: &WriterSender,
5014    request: &Frame,
5015    operation: &str,
5016    result: Response,
5017    metrics: &DispatchPathMetrics,
5018) -> Result<(), SubcError> {
5019    let status = if result.success { "ok" } else { "error" };
5020    let body = json!({ "op": operation, "status": status, "data": result.data });
5021    let response = Frame::build_with_version(
5022        request.header.ver,
5023        FrameType::Response,
5024        request.header.flags,
5025        request.header.channel,
5026        request.header.epoch,
5027        request.header.corr,
5028        serde_json::to_vec(&body).map_err(SubcError::Json)?,
5029    )
5030    .map_err(SubcError::FrameBuild)?;
5031    send_reliable_writer_frame(tx, metrics, response, "management response").await
5032}
5033
5034fn install_bash_compressor(ctx: &AppContext) {
5035    // Mirrors main.rs per-actor compressor installation for subc-created actors.
5036    let filter_registry_handle = ctx.shared_filter_registry();
5037    let compress_flag = ctx.bash_compress_flag();
5038    ctx.bash_background().set_compressor_with_exit_code(
5039        move |command: &str, output: String, exit_code: Option<i32>| {
5040            if !compress_flag.load(std::sync::atomic::Ordering::Relaxed) {
5041                return crate::compress::CompressionResult::new(output);
5042            }
5043            let registry_guard = match filter_registry_handle.read() {
5044                Ok(g) => g,
5045                Err(poisoned) => poisoned.into_inner(),
5046            };
5047            crate::compress::compress_with_registry_exit_code(
5048                command,
5049                &output,
5050                exit_code,
5051                &registry_guard,
5052            )
5053        },
5054    );
5055}
5056
5057async fn send_route_bind_ack(
5058    tx: &WriterSender,
5059    ver: u8,
5060    corr: u64,
5061    flags: Flags,
5062    metrics: &DispatchPathMetrics,
5063) -> Result<(), SubcError> {
5064    let body =
5065        serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).map_err(SubcError::Json)?;
5066    let response = Frame::build_with_version(ver, FrameType::Response, flags, 0, 0, corr, body)
5067        .map_err(SubcError::FrameBuild)?;
5068    send_reliable_writer_frame(tx, metrics, response, "RouteBindAck").await
5069}
5070
5071async fn send_route_bind_error(
5072    tx: &WriterSender,
5073    frame: &Frame,
5074    code: &str,
5075    message: &str,
5076    metrics: &DispatchPathMetrics,
5077) -> Result<(), SubcError> {
5078    send_route_bind_error_parts(
5079        tx,
5080        frame.header.ver,
5081        frame.header.corr,
5082        frame.header.flags,
5083        code,
5084        message,
5085        metrics,
5086    )
5087    .await
5088}
5089
5090async fn send_route_bind_error_parts(
5091    tx: &WriterSender,
5092    ver: u8,
5093    corr: u64,
5094    flags: Flags,
5095    code: &str,
5096    message: &str,
5097    metrics: &DispatchPathMetrics,
5098) -> Result<(), SubcError> {
5099    let response = build_error_frame(ver, 0, 0, corr, flags, code, message)?;
5100    send_reliable_writer_frame(tx, metrics, response, "RouteBind error").await?;
5101    log_route_bind_rejection(code, message);
5102    Ok(())
5103}
5104
5105/// Per-message rate limit for the bind-rejection warn line. A caller that
5106/// re-attaches a dead root forever turns this line into the entire readable
5107/// tail of the SHARED daemon log (measured 2026-08-09: 2.08M copies, ~40/sec,
5108/// 936MB log — other modules' incident lines pushed out of the tail).
5109/// The line itself stays byte-identical so external counters keep matching;
5110/// repeats inside the window are summarized with a suppressed count on the
5111/// next emission (volume stays diagnosable, per the log-diet convention).
5112fn log_route_bind_rejection(code: &str, message: &str) {
5113    const WINDOW: Duration = Duration::from_secs(60);
5114    static SUPPRESSED: OnceLock<StdMutex<HashMap<String, (Instant, u64)>>> = OnceLock::new();
5115    let map = SUPPRESSED.get_or_init(|| StdMutex::new(HashMap::new()));
5116    let mut map = match map.try_lock() {
5117        Ok(map) => map,
5118        // Contended: log unsuppressed rather than blocking or dropping.
5119        Err(_) => {
5120            log::warn!("subc attach: route bind rejected ({code}): {message}");
5121            return;
5122        }
5123    };
5124    let now = Instant::now();
5125    // Bound the map: dead roots churn, and an unbounded suppression map is
5126    // its own leak. Sweep expired entries once it grows past a fleet-sized
5127    // number of distinct rejection messages.
5128    if map.len() > 512 {
5129        map.retain(|_, (start, _)| now.duration_since(*start) < WINDOW);
5130    }
5131    match map.get_mut(message) {
5132        Some((window_start, suppressed)) if now.duration_since(*window_start) < WINDOW => {
5133            *suppressed += 1;
5134        }
5135        Some((window_start, suppressed)) => {
5136            if *suppressed > 0 {
5137                log::warn!(
5138                    "subc attach: route bind rejected ({code}): {message} (repeated {}x in last 60s)",
5139                    *suppressed
5140                );
5141            } else {
5142                log::warn!("subc attach: route bind rejected ({code}): {message}");
5143            }
5144            *window_start = now;
5145            *suppressed = 0;
5146        }
5147        None => {
5148            log::warn!("subc attach: route bind rejected ({code}): {message}");
5149            map.insert(message.to_string(), (now, 0));
5150        }
5151    }
5152}
5153
5154/// Route-channel tool call: `{name, arguments}` → executor lane → dispatch to
5155/// the sync command core → wrap the structured Response in a CallToolResult
5156/// `{content, isError}`. Tool-result mapping: the whole `{success, ...}` Response
5157/// serialized into ONE text block; `isError` carries `success == false`.
5158async fn handle_tool_call(
5159    tx: &WriterSender,
5160    frame: &Frame,
5161    mut phase_trace: PhaseTrace,
5162    routes: &HashMap<RouteChannel, RouteIdentity>,
5163    pending_binds: &HashMap<RouteChannel, PendingBind>,
5164    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
5165    executor: &Arc<Executor>,
5166    active_tool_calls: &ActiveToolCalls,
5167    pending_deferred_setups: &Arc<AtomicUsize>,
5168    shutdown: &Arc<Notify>,
5169    connection_cancel: &PersistentCancelSignal,
5170    bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
5171    bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
5172    metrics: &Arc<DispatchPathMetrics>,
5173    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
5174    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
5175    next_bash_ask_corr: &mut u64,
5176    bg_subs: &mut HashMap<RouteChannel, BgSub>,
5177    bg_sub_by_session: &mut BgSubsBySession,
5178    bg_wake_pending: &mut BgWakePending,
5179    bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
5180    dispatch: DispatchFn,
5181    deferred_response_tx: &mpsc::UnboundedSender<PendingSubcResponse>,
5182    allow_native_passthrough: bool,
5183    tool_response_body_limit: usize,
5184) -> Result<(), SubcError> {
5185    let route_id = route_key(frame.header.channel, frame.header.epoch);
5186    if pending_binds.contains_key(&route_id) {
5187        let error = build_error_frame(
5188            frame.header.ver,
5189            frame.header.channel,
5190            frame.header.epoch,
5191            frame.header.corr,
5192            frame.header.flags,
5193            "route_not_bound",
5194            "route is not bound before tool call",
5195        )?;
5196        return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
5197    }
5198
5199    let Some(identity) = routes.get(&route_id).cloned() else {
5200        let error = build_error_frame(
5201            frame.header.ver,
5202            frame.header.channel,
5203            frame.header.epoch,
5204            frame.header.corr,
5205            frame.header.flags,
5206            "route_not_bound",
5207            "route is not bound before tool call",
5208        )?;
5209        return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
5210    };
5211    let restore_watcher = live_roots
5212        .get(&identity.root)
5213        .is_some_and(|meta| meta.idle_artifacts_evicted);
5214    if let Some(meta) = live_roots.get_mut(&identity.root) {
5215        meta.reactivate_bound();
5216    }
5217    if restore_watcher {
5218        if let Some(ctx) = executor.actor_context(&identity.root) {
5219            crate::commands::configure::ensure_project_watcher(&ctx);
5220        }
5221    }
5222
5223    let route_request = match serde_json::from_slice::<RouteRequest>(&frame.body) {
5224        Ok(request) => request,
5225        Err(error) => {
5226            let management_envelope = serde_json::from_slice::<Value>(&frame.body).ok();
5227            let Some(operation) = management_envelope
5228                .as_ref()
5229                .and_then(|value| value.get("op"))
5230                .and_then(Value::as_str)
5231            else {
5232                return Err(SubcError::Json(error));
5233            };
5234            RouteRequest::ToolCall(ToolCallRequest {
5235                name: operation.to_string(),
5236                arguments: management_envelope
5237                    .and_then(|value| value.get("params").cloned())
5238                    .unwrap_or_else(|| json!({})),
5239                edit_slot_survives: None,
5240                preview: false,
5241            })
5242        }
5243    };
5244    if matches!(
5245        route_request,
5246        RouteRequest::BgEvents(BgEventsRequest {
5247            op: BgEventsOp::BgEvents
5248        })
5249    ) {
5250        if let Some(old_sub) = bg_subs.get(&route_id).cloned() {
5251            metrics.record_bg_subscription_ended(
5252                &old_sub.root,
5253                &old_sub.session,
5254                route_id,
5255                "resubscribe",
5256            );
5257            push::send_reliable_bg_stream_end(tx, metrics, route_id, &old_sub).await?;
5258        }
5259        if !identity.trust.allows_bash_observation() {
5260            bg_subs.remove(&route_id);
5261            bg_wake_pending.remove(&route_id);
5262            remove_bg_subscription_index(bg_sub_by_session, route_id, Some(&identity));
5263            let denied_sub = BgSub {
5264                corr: frame.header.corr,
5265                ver: frame.header.ver,
5266                flags: frame.header.flags,
5267                root: identity.root.clone(),
5268                session: identity.session.clone(),
5269            };
5270            metrics.record_bg_subscription_ended(
5271                &identity.root,
5272                &identity.session,
5273                route_id,
5274                "subscribe-denied",
5275            );
5276            push::send_reliable_bg_stream_end(tx, metrics, route_id, &denied_sub).await?;
5277            return Ok(());
5278        }
5279        bg_subs.insert(
5280            route_id,
5281            BgSub {
5282                corr: frame.header.corr,
5283                ver: frame.header.ver,
5284                flags: frame.header.flags,
5285                root: identity.root.clone(),
5286                session: identity.session.clone(),
5287            },
5288        );
5289        insert_bg_subscription_index(
5290            bg_sub_by_session,
5291            identity.root.clone(),
5292            identity.session.clone(),
5293            route_id,
5294        );
5295        metrics.record_bg_subscription_installed(&identity.root, &identity.session, route_id);
5296        push::arm_bg_wake(
5297            identity.root.clone(),
5298            identity.session.clone(),
5299            route_id,
5300            bg_wake_pending,
5301            bg_wake_epoch,
5302            metrics,
5303        );
5304        return Ok(());
5305    }
5306
5307    let RouteRequest::ToolCall(call) = route_request else {
5308        unreachable!("background event subscription returned above")
5309    };
5310    let bare_name = call.name;
5311    let arguments = strip_agent_preview_arg_owned(call.arguments);
5312    let format_context = crate::subc_format::FormatContext::from_tool_call(
5313        &bare_name,
5314        &arguments,
5315        identity.project_root.as_path(),
5316    );
5317
5318    let request_id = format!("subc-{}-{}", frame.header.channel, frame.header.corr);
5319    let bind_trust = identity.trust;
5320    let diagnostics_on_edit = live_roots
5321        .get(&identity.root)
5322        .map(|meta| meta.diagnostics_on_edit)
5323        .unwrap_or(false);
5324
5325    let requests_host = matches!(bare_name.as_str(), "bash" | "powershell")
5326        && arguments
5327            .get("sandbox")
5328            .or_else(|| {
5329                arguments
5330                    .get("params")
5331                    .and_then(|params| params.get("sandbox"))
5332            })
5333            .and_then(Value::as_str)
5334            == Some("host");
5335    if matches!(bind_trust, BindTrust::Untrusted) && requests_host {
5336        let response = Response::error(
5337            request_id.clone(),
5338            "sandbox_escalation_denied",
5339            "sandbox host escalation is unavailable to untrusted principals",
5340        );
5341        let text = crate::subc_format::format_response_with_context(
5342            &bare_name,
5343            &response,
5344            &format_context,
5345        );
5346        let result = ToolCallResult { text, response };
5347        let response_frame = build_tool_response_frame_with_limit(
5348            frame.header.ver,
5349            route_id,
5350            frame.header.corr,
5351            frame.header.flags,
5352            &result,
5353            bind_trust,
5354            tool_response_body_limit,
5355        )?;
5356        return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
5357    }
5358
5359    if matches!(bind_trust, BindTrust::Untrusted)
5360        && is_bash_family_tool(&bare_name)
5361        && (!matches!(bare_name.as_str(), "bash" | "powershell")
5362            || !identity.consumer_elicitation_capable)
5363    {
5364        let response = bash::bash_denied_untrusted_response(request_id.clone());
5365        let text = crate::subc_format::format_response_with_context(
5366            &bare_name,
5367            &response,
5368            &format_context,
5369        );
5370        let result = ToolCallResult { text, response };
5371        let response_frame = build_tool_response_frame_with_limit(
5372            frame.header.ver,
5373            route_id,
5374            frame.header.corr,
5375            frame.header.flags,
5376            &result,
5377            bind_trust,
5378            tool_response_body_limit,
5379        )?;
5380        return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
5381    }
5382
5383    // A non-core name is NOT in the tool manifest. AFT fails closed and
5384    // does not trust subc to enforce the manifest: rejecting here is the
5385    // defense-in-depth backstop that prevents a forwarded native command
5386    // (e.g. `configure`, which would reach handle_configure and bypass
5387    // the RouteBind config-trust cap) from ever reaching dispatch. Only
5388    // the integration-test harness (run_subc_mode_for_test) opens this to
5389    // drive synthetic native commands through the executor.
5390    if !is_subc_agent_core_tool(&bare_name)
5391        && !is_subc_native_plumbing_tool(&bare_name)
5392        && !allow_native_passthrough
5393    {
5394        log::warn!(
5395            "subc tool call: rejecting non-manifest tool name {:?} on route {} (fail-closed)",
5396            bare_name,
5397            frame.header.channel
5398        );
5399        let response = Response::error(
5400            request_id.clone(),
5401            "unknown_tool",
5402            format!("tool {:?} is not in the AFT tool manifest", bare_name),
5403        );
5404        let text = crate::subc_format::format_response_with_context(
5405            &bare_name,
5406            &response,
5407            &format_context,
5408        );
5409        let result = ToolCallResult { text, response };
5410        let response_frame = build_tool_response_frame_with_limit(
5411            frame.header.ver,
5412            route_id,
5413            frame.header.corr,
5414            frame.header.flags,
5415            &result,
5416            bind_trust,
5417            tool_response_body_limit,
5418        )?;
5419        return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
5420    }
5421
5422    if matches!(bare_name.as_str(), "bash" | "powershell") {
5423        if matches!(bind_trust, BindTrust::Untrusted) {
5424            let plan = match bash::prepare_bash_elicitation_plan(
5425                &arguments,
5426                identity.project_root.as_path(),
5427            ) {
5428                Ok(plan) => plan,
5429                Err(error) => {
5430                    let response = Response::error(request_id.clone(), error.code, error.message);
5431                    let text = crate::subc_format::format_response_with_context(
5432                        &bare_name,
5433                        &response,
5434                        &format_context,
5435                    );
5436                    let result = ToolCallResult { text, response };
5437                    let response_frame = build_tool_response_frame_with_limit(
5438                        frame.header.ver,
5439                        route_id,
5440                        frame.header.corr,
5441                        frame.header.flags,
5442                        &result,
5443                        bind_trust,
5444                        tool_response_body_limit,
5445                    )?;
5446                    return send_reliable_writer_frame(
5447                        tx,
5448                        metrics,
5449                        response_frame,
5450                        "tool response",
5451                    )
5452                    .await;
5453                }
5454            };
5455
5456            let reverse_corr =
5457                allocate_reverse_corr(pending_bash_asks, route_id, next_bash_ask_corr);
5458            let ask_frame = build_bash_elicitation_request_frame(
5459                frame.header.ver,
5460                route_id,
5461                reverse_corr,
5462                frame.header.flags,
5463                &plan.command,
5464                &plan.asks,
5465            )?;
5466
5467            let meta = live_roots
5468                .entry(identity.root.clone())
5469                .or_insert_with(|| RootMeta::new(Instant::now()));
5470            meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
5471            meta.reactivate_bound();
5472
5473            let route_cancel =
5474                route_bash_cancels
5475                    .entry(route_id)
5476                    .or_insert_with(|| bash::RouteBashCancel {
5477                        token: PersistentCancelSignal::new(),
5478                        active_waits: 0,
5479                    });
5480            route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
5481            let cancel = bash::BashWaitCancel {
5482                connection: connection_cancel.clone(),
5483                route: route_cancel.token.clone(),
5484            };
5485            pending_bash_asks.insert(
5486                ReverseCorrKey {
5487                    route: route_id,
5488                    corr: reverse_corr,
5489                },
5490                PendingBashAsk {
5491                    route: route_id,
5492                    tool_corr: frame.header.corr,
5493                    tool_flags: frame.header.flags,
5494                    tool_ver: frame.header.ver,
5495                    root: identity.root.clone(),
5496                    project_root: identity.project_root.clone(),
5497                    session_id: identity.session.clone(),
5498                    spawn_principal: identity.spawn_principal.clone(),
5499                    edit_slot_survives: call.edit_slot_survives,
5500                    request_id,
5501                    arguments,
5502                    format_context,
5503                    cancel,
5504                    grants: plan.grants,
5505                    expires_at: Instant::now() + bash_elicitation_timeout(),
5506                },
5507            );
5508            return send_reliable_writer_frame(tx, metrics, ask_frame, "bash elicitation request")
5509                .await;
5510        }
5511
5512        let meta = live_roots
5513            .entry(identity.root.clone())
5514            .or_insert_with(|| RootMeta::new(Instant::now()));
5515        meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
5516        meta.reactivate_bound();
5517
5518        let route_cancel =
5519            route_bash_cancels
5520                .entry(route_id)
5521                .or_insert_with(|| bash::RouteBashCancel {
5522                    token: PersistentCancelSignal::new(),
5523                    active_waits: 0,
5524                });
5525        route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
5526        let cancel = bash::BashWaitCancel {
5527            connection: connection_cancel.clone(),
5528            route: route_cancel.token.clone(),
5529        };
5530
5531        bash::submit_deferred_bash(
5532            executor,
5533            bash_deferred_tx,
5534            bash_poll_touch_tx,
5535            metrics,
5536            dispatch,
5537            identity.root.clone(),
5538            identity.project_root.clone(),
5539            identity.session.clone(),
5540            request_id,
5541            route_id,
5542            frame.header.corr,
5543            frame.header.flags,
5544            frame.header.ver,
5545            arguments,
5546            format_context,
5547            cancel,
5548            bind_trust,
5549            identity.spawn_principal.clone(),
5550            call.edit_slot_survives,
5551            None,
5552        );
5553        return Ok(());
5554    }
5555
5556    let lane = command_lane(&bare_name);
5557    let tool_call_context = ToolCallContext {
5558        project_root: identity.project_root.clone(),
5559        session_id: Some(identity.session.clone()),
5560        request_id: request_id.clone(),
5561        diagnostics_on_edit,
5562        preview: call.preview,
5563        edit_slot_survives: call.edit_slot_survives,
5564        report_registration_downgrade: true,
5565    };
5566
5567    let uses_deferred_response_seam = bare_name == "inspect"
5568        || crate::commands::lsp_navigation::is_lsp_navigation_command(&bare_name);
5569    if uses_deferred_response_seam {
5570        let Some(deferred_ctx) = executor.actor_context(&identity.root) else {
5571            let response = Response::error(
5572                &request_id,
5573                "actor_not_registered",
5574                "executor actor is not registered",
5575            );
5576            let text = crate::subc_format::format_response_with_context(
5577                &bare_name,
5578                &response,
5579                &format_context,
5580            );
5581            let result = ToolCallResult { text, response };
5582            let response_frame = build_tool_response_frame_with_limit(
5583                frame.header.ver,
5584                route_id,
5585                frame.header.corr,
5586                frame.header.flags,
5587                &result,
5588                bind_trust,
5589                tool_response_body_limit,
5590            )?;
5591            return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
5592        };
5593        let identity_for_run = identity.clone();
5594        let request_id_for_force = request_id.clone();
5595        let format_context_for_run = format_context.clone();
5596        let bare_name_for_run = bare_name.clone();
5597        let (setup_tx, setup_rx) = oneshot::channel::<DeferredSetupOutcome>();
5598        phase_trace.mark_executor_submitted();
5599        let job: crate::executor::ExecutorJob = Box::new(move |ctx| {
5600            phase_trace.mark_job_admitted();
5601            log_ctx::with_session(Some(identity_for_run.session.clone()), || {
5602                let run = || match prepare_tool_call(
5603                    &bare_name_for_run,
5604                    arguments,
5605                    &format_context_for_run,
5606                    &tool_call_context,
5607                    ctx,
5608                    Some(&mut phase_trace),
5609                ) {
5610                    Err(result) => {
5611                        let response = result.response;
5612                        let _ = setup_tx.send(DeferredSetupOutcome::Immediate {
5613                            text: result.text,
5614                            phase_trace,
5615                        });
5616                        response
5617                    }
5618                    Ok(prepared) => {
5619                        let outcome = if bare_name_for_run == "inspect" {
5620                            crate::commands::inspect::handle_inspect_deferred_with_restriction(
5621                                &prepared.request,
5622                                Arc::clone(&deferred_ctx),
5623                                matches!(bind_trust, BindTrust::Untrusted),
5624                            )
5625                        } else {
5626                            crate::commands::lsp_navigation::handle_lsp_navigation_deferred_with_restriction(
5627                                &prepared.request,
5628                                Arc::clone(&deferred_ctx),
5629                                matches!(bind_trust, BindTrust::Untrusted),
5630                            )
5631                        };
5632                        match outcome {
5633                            DispatchOutcome::Deferred(pending) => {
5634                                let _ = setup_tx.send(DeferredSetupOutcome::Deferred {
5635                                    pending,
5636                                    surface_downgraded: prepared.surface_downgraded,
5637                                    phase_trace,
5638                                });
5639                                Response::success(
5640                                    request_id_for_force.clone(),
5641                                    json!({ "response_deferred": true }),
5642                                )
5643                            }
5644                            DispatchOutcome::Immediate(response) => {
5645                                phase_trace.mark_execute_done();
5646                                let finalizer = |response: &mut Response| {
5647                                    crate::response_finalize::finalize_response_with_bg_completions(
5648                                        response,
5649                                        ctx,
5650                                        &identity_for_run.session,
5651                                        &bare_name_for_run,
5652                                        bind_trust.allows_bash_observation(),
5653                                    );
5654                                };
5655                                let result = finish_tool_call_response(
5656                                    &bare_name_for_run,
5657                                    &format_context_for_run,
5658                                    response,
5659                                    prepared.surface_downgraded,
5660                                    Some(&finalizer),
5661                                    Some(&mut phase_trace),
5662                                );
5663                                let response = result.response;
5664                                let _ = setup_tx.send(DeferredSetupOutcome::Immediate {
5665                                    text: result.text,
5666                                    phase_trace,
5667                                });
5668                                response
5669                            }
5670                        }
5671                    }
5672                };
5673                if matches!(bind_trust, BindTrust::Untrusted) {
5674                    ctx.with_force_restrict(&request_id_for_force, run)
5675                } else {
5676                    run()
5677                }
5678            })
5679        });
5680        let deferred_setup_guard =
5681            PendingDeferredSetupGuard::new(Arc::clone(pending_deferred_setups));
5682        let rx = submit_active_tool_call(
5683            executor.as_ref(),
5684            active_tool_calls,
5685            route_id,
5686            frame.header.corr,
5687            identity.root.clone(),
5688            lane,
5689            request_id.clone(),
5690            RouteDetachPolicy::CancelOnDetach,
5691            job,
5692        );
5693
5694        let completion_tx = tx.clone();
5695        let completion_shutdown = Arc::clone(shutdown);
5696        let completion_metrics = Arc::clone(metrics);
5697        let active_tool_calls = Arc::clone(active_tool_calls);
5698        let deferred_response_tx = deferred_response_tx.clone();
5699        let route = route_id;
5700        let corr = frame.header.corr;
5701        let flags = frame.header.flags;
5702        let ver = frame.header.ver;
5703        let root = identity.root.clone();
5704        let session_id = identity.session.clone();
5705        tokio::spawn(async move {
5706            let _response_task = ResponseTaskGuard::new(&completion_metrics);
5707            let _deferred_setup = deferred_setup_guard;
5708            let response = await_executor_response(rx, request_id.clone()).await;
5709            match setup_rx.await {
5710                Ok(DeferredSetupOutcome::Deferred {
5711                    pending,
5712                    surface_downgraded,
5713                    phase_trace,
5714                }) => {
5715                    let pending = PendingSubcResponse {
5716                        route,
5717                        corr,
5718                        flags,
5719                        ver,
5720                        root,
5721                        session_id,
5722                        bare_name,
5723                        format_context,
5724                        bind_trust,
5725                        pending,
5726                        surface_downgraded,
5727                        phase_trace,
5728                    };
5729                    if let Err(error) = deferred_response_tx.send(pending) {
5730                        if let Some(cancellation) = &error.0.pending.cancellation {
5731                            cancellation.request_cancel();
5732                        }
5733                        finish_active_tool_call(&active_tool_calls, route, corr);
5734                    }
5735                }
5736                Ok(DeferredSetupOutcome::Immediate { text, phase_trace }) => {
5737                    if !finish_active_tool_call(&active_tool_calls, route, corr) {
5738                        return;
5739                    }
5740                    let result = ToolCallResult { text, response };
5741                    let fatal = note_fatal_panic_response(&result.response);
5742                    match build_tool_response_frame_with_limit(
5743                        ver,
5744                        route,
5745                        corr,
5746                        flags,
5747                        &result,
5748                        bind_trust,
5749                        tool_response_body_limit,
5750                    ) {
5751                        Ok(response_frame) => {
5752                            let trace = ToolResponseWriteTrace::new(
5753                                phase_trace,
5754                                bare_name.clone(),
5755                                identity.project_root.clone(),
5756                                identity.session.clone(),
5757                                route.channel,
5758                                corr,
5759                            );
5760                            if let Err(error) = send_traced_tool_response_frame(
5761                                &completion_tx,
5762                                &completion_metrics,
5763                                response_frame,
5764                                trace,
5765                            )
5766                            .await
5767                            {
5768                                log::warn!(
5769                                    "subc attach: failed to queue deferred-seam setup response: {error}"
5770                                );
5771                            }
5772                        }
5773                        Err(error) => {
5774                            log::error!(
5775                                "subc attach: failed to build deferred-seam setup response: {error}"
5776                            );
5777                        }
5778                    }
5779                    if fatal {
5780                        signal_fatal_teardown(
5781                            &completion_tx,
5782                            Some(route),
5783                            ver,
5784                            corr,
5785                            &completion_shutdown,
5786                            &completion_metrics,
5787                        )
5788                        .await;
5789                    }
5790                }
5791                Err(_) => {
5792                    if !finish_active_tool_call(&active_tool_calls, route, corr) {
5793                        return;
5794                    }
5795                    let text = crate::subc_format::format_response_with_context(
5796                        &bare_name,
5797                        &response,
5798                        &format_context,
5799                    );
5800                    let result = ToolCallResult { text, response };
5801                    if let Ok(response_frame) = build_tool_response_frame_with_limit(
5802                        ver,
5803                        route,
5804                        corr,
5805                        flags,
5806                        &result,
5807                        bind_trust,
5808                        tool_response_body_limit,
5809                    ) {
5810                        let _ = send_reliable_writer_frame(
5811                            &completion_tx,
5812                            &completion_metrics,
5813                            response_frame,
5814                            "deferred setup failure",
5815                        )
5816                        .await;
5817                    }
5818                }
5819            }
5820        });
5821        return Ok(());
5822    }
5823
5824    let bare_name_for_frame = bare_name.clone();
5825    let identity_for_run = identity.clone();
5826    let completion_session = identity.session.clone();
5827    let completion_root = identity.project_root.clone();
5828    let request_id_for_force = request_id.clone();
5829    let format_context_for_frame = format_context.clone();
5830    let (tool_call_tx, tool_call_rx) = oneshot::channel::<ToolCallCompletion>();
5831    phase_trace.mark_executor_submitted();
5832    let job: crate::executor::ExecutorJob = Box::new(move |ctx| {
5833        phase_trace.mark_job_admitted();
5834        log_ctx::with_session(Some(identity_for_run.session.clone()), || {
5835            let run = || {
5836                let finalizer = |response: &mut Response| {
5837                    crate::response_finalize::finalize_response_with_bg_completions(
5838                        response,
5839                        ctx,
5840                        &identity_for_run.session,
5841                        &bare_name,
5842                        bind_trust.allows_bash_observation(),
5843                    );
5844                };
5845                match run_tool_call(
5846                    &bare_name,
5847                    arguments,
5848                    &format_context,
5849                    &tool_call_context,
5850                    ctx,
5851                    &dispatch,
5852                    Some(&finalizer),
5853                    Some(&mut phase_trace),
5854                ) {
5855                    ToolCallOutcome::Unary(result) => {
5856                        let response = result.response;
5857                        let _ = tool_call_tx.send(ToolCallCompletion {
5858                            text: result.text,
5859                            phase_trace,
5860                        });
5861                        response
5862                    }
5863                }
5864            };
5865            if matches!(bind_trust, BindTrust::Untrusted) {
5866                ctx.with_force_restrict(&request_id_for_force, run)
5867            } else {
5868                run()
5869            }
5870        })
5871    });
5872    let rx = submit_active_tool_call(
5873        executor.as_ref(),
5874        active_tool_calls,
5875        route_id,
5876        frame.header.corr,
5877        identity.root.clone(),
5878        lane,
5879        request_id.clone(),
5880        RouteDetachPolicy::RetainForReplay,
5881        job,
5882    );
5883    let completion_tx = tx.clone();
5884    let completion_shutdown = Arc::clone(shutdown);
5885    let route = route_id;
5886    let corr = frame.header.corr;
5887    let flags = frame.header.flags;
5888    let ver = frame.header.ver;
5889    let completion_metrics = Arc::clone(metrics);
5890    let active_tool_calls = Arc::clone(active_tool_calls);
5891    tokio::spawn(async move {
5892        let _response_task = ResponseTaskGuard::new(&completion_metrics);
5893        let response = await_executor_response(rx, request_id.clone()).await;
5894        let (text, phase_trace) = match tool_call_rx.await {
5895            Ok(completion) => (completion.text, Some(completion.phase_trace)),
5896            Err(_) => (
5897                crate::subc_format::format_response_with_context(
5898                    &bare_name_for_frame,
5899                    &response,
5900                    &format_context_for_frame,
5901                ),
5902                None,
5903            ),
5904        };
5905        if !finish_active_tool_call(&active_tool_calls, route, corr) {
5906            return;
5907        }
5908        let result = ToolCallResult { text, response };
5909        let fatal = note_fatal_panic_response(&result.response);
5910        match build_tool_response_frame_with_limit(
5911            ver,
5912            route,
5913            corr,
5914            flags,
5915            &result,
5916            bind_trust,
5917            tool_response_body_limit,
5918        ) {
5919            Ok(response_frame) => {
5920                let send_result = if let Some(phase_trace) = phase_trace {
5921                    let trace = ToolResponseWriteTrace::new(
5922                        phase_trace,
5923                        bare_name_for_frame,
5924                        completion_root,
5925                        completion_session,
5926                        route.channel,
5927                        corr,
5928                    );
5929                    send_traced_tool_response_frame(
5930                        &completion_tx,
5931                        &completion_metrics,
5932                        response_frame,
5933                        trace,
5934                    )
5935                    .await
5936                } else {
5937                    send_reliable_writer_frame(
5938                        &completion_tx,
5939                        &completion_metrics,
5940                        response_frame,
5941                        "tool response",
5942                    )
5943                    .await
5944                };
5945                if let Err(error) = send_result {
5946                    log::warn!("subc attach: failed to queue tool response frame: {error}");
5947                }
5948            }
5949            Err(error) => {
5950                log::error!("subc attach: failed to build tool response frame: {error}");
5951            }
5952        }
5953        if fatal {
5954            signal_fatal_teardown(
5955                &completion_tx,
5956                Some(route),
5957                ver,
5958                corr,
5959                &completion_shutdown,
5960                &completion_metrics,
5961            )
5962            .await;
5963        }
5964    });
5965    Ok(())
5966}
5967
5968fn submit_maintenance_job(
5969    executor: &Arc<Executor>,
5970    root_id: ProjectRootId,
5971    kind: MaintenanceDrainKind,
5972    bg_sessions_to_check: Vec<(String, u64)>,
5973    completion_tx: &mpsc::Sender<MaintenanceCompletion>,
5974    metrics: &Arc<DispatchPathMetrics>,
5975) {
5976    let request_id = format!(
5977        "subc-maintenance-drain-{}-{}",
5978        kind.label(),
5979        root_id.as_path().to_string_lossy()
5980    );
5981    let response_id = request_id.clone();
5982    let completion_root_id = root_id.clone();
5983    let maintenance_generation = executor
5984        .actor_context(&root_id)
5985        .map(|ctx| ctx.configure_generation())
5986        .unwrap_or(0);
5987    let (outcome_tx, outcome_rx) = oneshot::channel::<MaintenanceJobOutcome>();
5988    // Deferred drains mutate subsystem state behind each subsystem's own lock.
5989    // Keeping every drain on MaintenanceCommit lets interactive reads and lazy
5990    // HeavyInit queries run while maintenance converges after the bind ack.
5991    let lane = Lane::MaintenanceCommit;
5992    let job: crate::executor::ExecutorJob = Box::new(move |ctx: &AppContext| {
5993        let outcome = match kind {
5994            MaintenanceDrainKind::Watcher => {
5995                let drained = runtime_drain::drain_watcher_events_bounded(
5996                    ctx,
5997                    runtime_drain::WATCHER_PATH_DRAIN_BATCH_CAP,
5998                );
5999                MaintenanceJobOutcome {
6000                    empty_bg_sessions: Vec::new(),
6001                    unacked_bg_keys: None,
6002                    requeue_kind: drained.has_more.then_some(kind),
6003                }
6004            }
6005            MaintenanceDrainKind::Lsp => {
6006                let drained = runtime_drain::drain_lsp_events_bounded(
6007                    ctx,
6008                    runtime_drain::LSP_EVENT_DRAIN_BATCH_CAP,
6009                );
6010                MaintenanceJobOutcome {
6011                    empty_bg_sessions: Vec::new(),
6012                    unacked_bg_keys: None,
6013                    requeue_kind: drained.has_more.then_some(kind),
6014                }
6015            }
6016            MaintenanceDrainKind::ConfigureTail => {
6017                runtime_drain::drain_deferred_configure_maintenance(ctx);
6018                runtime_drain::drain_configure_warning_events(ctx);
6019                MaintenanceJobOutcome::default()
6020            }
6021            MaintenanceDrainKind::CompletionDrains => {
6022                runtime_drain::drain_search_index_events(ctx);
6023                runtime_drain::drain_callgraph_store_events(ctx);
6024                runtime_drain::drain_semantic_index_events(ctx);
6025                runtime_drain::drain_semantic_refresh_events(ctx);
6026                runtime_drain::drain_inspect_events_for_generation(ctx, maintenance_generation);
6027                let empty_bg_sessions = bg_sessions_to_check
6028                    .into_iter()
6029                    .filter(|(session, _)| {
6030                        !ctx.bash_background().has_unacked_wakes_for_session(session)
6031                    })
6032                    .collect();
6033                MaintenanceJobOutcome {
6034                    empty_bg_sessions,
6035                    unacked_bg_keys: Some(ctx.bash_background().unacked_wake_keys()),
6036                    requeue_kind: None,
6037                }
6038            }
6039        };
6040        let requeued = outcome.requeue_kind.is_some();
6041        let _ = outcome_tx.send(outcome);
6042        Response::success(
6043            response_id,
6044            json!({ "drained": true, "kind": kind.label(), "requeued": requeued }),
6045        )
6046    });
6047    let rx = match kind {
6048        MaintenanceDrainKind::Watcher => executor.submit_coalescable_maintenance_async(
6049            root_id,
6050            lane,
6051            request_id.clone(),
6052            crate::executor::MaintenanceCoalesceKey::WatcherDrain,
6053            job,
6054        ),
6055        MaintenanceDrainKind::Lsp => executor.submit_coalescable_maintenance_async(
6056            root_id,
6057            lane,
6058            request_id.clone(),
6059            crate::executor::MaintenanceCoalesceKey::LspDrain,
6060            job,
6061        ),
6062        MaintenanceDrainKind::ConfigureTail | MaintenanceDrainKind::CompletionDrains => {
6063            executor.submit_maintenance_async(root_id, lane, request_id.clone(), job)
6064        }
6065    };
6066    let completion_tx = completion_tx.clone();
6067    let completion_metrics = Arc::clone(metrics);
6068    tokio::spawn(async move {
6069        let _response_task = ResponseTaskGuard::new(&completion_metrics);
6070        let response = await_executor_response(rx, request_id).await;
6071        let outcome = outcome_rx.await.unwrap_or_default();
6072        let _ = send_counted_channel(
6073            &completion_tx,
6074            &completion_metrics.maintenance_queued,
6075            MaintenanceCompletion {
6076                root_id: completion_root_id,
6077                kind,
6078                response,
6079                empty_bg_sessions: outcome.empty_bg_sessions,
6080                unacked_bg_keys: outcome.unacked_bg_keys,
6081                requeue_kind: outcome.requeue_kind,
6082            },
6083        )
6084        .await;
6085    });
6086}
6087
6088async fn await_executor_response(rx: oneshot::Receiver<Response>, request_id: String) -> Response {
6089    rx.await
6090        .unwrap_or_else(|_| Response::error(request_id, "internal_error", "executor dropped"))
6091}
6092
6093async fn deliver_resolved_subc_response(
6094    tx: &WriterSender,
6095    mut resolved: ResolvedSubcResponse,
6096    routes: &HashMap<RouteChannel, RouteIdentity>,
6097    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
6098    executor: &Executor,
6099    active_tool_calls: &ActiveToolCalls,
6100    shutdown: &Arc<Notify>,
6101    metrics: &DispatchPathMetrics,
6102    tool_response_body_limit: usize,
6103) -> Result<(), SubcError> {
6104    let entry = &mut resolved.entry;
6105    finish_active_tool_call(active_tool_calls, entry.route, entry.corr);
6106    if let Some(meta) = live_roots.get_mut(&entry.root) {
6107        meta.note_activity();
6108    }
6109
6110    let Some(identity) = routes.get(&entry.route) else {
6111        log::debug!(
6112            "subc attach: dropping deferred {} response {} for unbound route {}",
6113            entry.bare_name,
6114            entry.pending.request_id,
6115            entry.route
6116        );
6117        return Ok(());
6118    };
6119    let Some(ctx) = executor.actor_context(&entry.root) else {
6120        return Ok(());
6121    };
6122    entry.phase_trace.mark_execute_done();
6123    let finalizer = |response: &mut Response| {
6124        crate::response_finalize::finalize_response_with_bg_completions(
6125            response,
6126            &ctx,
6127            &entry.session_id,
6128            &entry.bare_name,
6129            entry.bind_trust.allows_bash_observation(),
6130        );
6131    };
6132    let result = finish_tool_call_response(
6133        &entry.bare_name,
6134        &entry.format_context,
6135        resolved.response,
6136        entry.surface_downgraded,
6137        Some(&finalizer),
6138        Some(&mut entry.phase_trace),
6139    );
6140    let fatal = note_fatal_panic_response(&result.response);
6141    let response_frame = build_tool_response_frame_with_limit(
6142        entry.ver,
6143        entry.route,
6144        entry.corr,
6145        entry.flags,
6146        &result,
6147        identity.trust,
6148        tool_response_body_limit,
6149    )?;
6150    let trace = ToolResponseWriteTrace::new(
6151        std::mem::replace(&mut entry.phase_trace, PhaseTrace::new(Instant::now())),
6152        entry.bare_name.clone(),
6153        identity.project_root.clone(),
6154        entry.session_id.clone(),
6155        entry.route.channel,
6156        entry.corr,
6157    );
6158    send_traced_tool_response_frame(tx, metrics, response_frame, trace).await?;
6159    if fatal {
6160        signal_fatal_teardown(
6161            tx,
6162            Some(entry.route),
6163            entry.ver,
6164            entry.corr,
6165            shutdown,
6166            metrics,
6167        )
6168        .await;
6169    }
6170    Ok(())
6171}
6172
6173async fn signal_fatal_teardown(
6174    tx: &WriterSender,
6175    route: Option<RouteChannel>,
6176    ver: u8,
6177    corr: u64,
6178    shutdown: &Arc<Notify>,
6179    metrics: &DispatchPathMetrics,
6180) {
6181    if let Some(route) = route {
6182        if let Ok(frame) = build_goodbye_frame(ver, route.channel, route.epoch, corr) {
6183            if let Err(error) = send_frame(tx, metrics, frame).await {
6184                log::warn!(
6185                    "subc attach: failed to queue fatal route Goodbye for route {route}: {error}"
6186                );
6187            }
6188        }
6189    }
6190    if let Ok(frame) = build_goodbye_frame(ver, 0, 0, 0) {
6191        if let Err(error) = send_frame(tx, metrics, frame).await {
6192            log::warn!("subc attach: failed to queue fatal channel-0 Goodbye: {error}");
6193        }
6194    }
6195    shutdown.notify_one();
6196}
6197#[derive(Debug, Deserialize)]
6198#[serde(untagged)]
6199enum RouteRequest {
6200    BgEvents(BgEventsRequest),
6201    ToolCall(ToolCallRequest),
6202}
6203
6204#[derive(Debug, Deserialize)]
6205struct BgEventsRequest {
6206    op: BgEventsOp,
6207}
6208
6209#[derive(Debug, Deserialize)]
6210#[serde(rename_all = "snake_case")]
6211enum BgEventsOp {
6212    BgEvents,
6213}
6214
6215#[derive(Debug, Deserialize)]
6216struct ToolCallRequest {
6217    name: String,
6218    #[serde(default)]
6219    arguments: Value,
6220    /// Host-computed registration fact; kept outside agent-controlled arguments.
6221    #[serde(default)]
6222    edit_slot_survives: Option<bool>,
6223    /// Server-owned preview control (B1c-0): the plugin's mutation flow is
6224    /// preview -> permission ask -> apply. Dropping this field made "preview"
6225    /// calls mutate disk before the permission prompt and the subsequent
6226    /// apply fail with not-found.
6227    #[serde(default)]
6228    preview: bool,
6229}
6230
6231#[cfg(test)]
6232pub(crate) mod test_support {
6233    use super::*;
6234    use crate::bash_background::BgTaskStatus;
6235    use crate::protocol::{
6236        BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, ConfigureWarningsFrame,
6237        ProgressFrame, StatusChangedFrame,
6238    };
6239    use serde_json::json;
6240
6241    pub(super) fn test_root(name: &str) -> (tempfile::TempDir, ProjectRootId) {
6242        let dir = tempfile::Builder::new()
6243            .prefix(name)
6244            .tempdir()
6245            .expect("temp root");
6246        let root = ProjectRootId::from_path(dir.path()).expect("project root id");
6247        (dir, root)
6248    }
6249
6250    pub(super) fn test_ctx() -> Arc<AppContext> {
6251        Arc::new(AppContext::new(
6252            Box::new(crate::parser::TreeSitterProvider::new()),
6253            crate::config::Config::default(),
6254        ))
6255    }
6256
6257    fn inspect_context(root: &Path) -> Arc<AppContext> {
6258        inspect_context_with_timeout(root, None)
6259    }
6260
6261    fn inspect_context_with_timeout(root: &Path, timeout_ms: Option<u64>) -> Arc<AppContext> {
6262        let mut config = crate::config::Config::default();
6263        config.project_root = Some(root.to_path_buf());
6264        if let Some(timeout_ms) = timeout_ms {
6265            config.inspect.diagnostics_timeout_ms = timeout_ms;
6266        }
6267        let ctx = Arc::new(AppContext::new(
6268            Box::new(crate::parser::TreeSitterProvider::new()),
6269            config,
6270        ));
6271        ctx.set_harness(crate::harness::Harness::Opencode);
6272        ctx
6273    }
6274
6275    fn inspect_request(id: &str) -> RawRequest {
6276        serde_json::from_value(json!({ "id": id, "command": "inspect" })).expect("inspect request")
6277    }
6278
6279    fn submit_deferred_inspect_setup(
6280        executor: &Arc<Executor>,
6281        root: &ProjectRootId,
6282        ctx: &Arc<AppContext>,
6283        request_id: &str,
6284    ) -> (PendingResponse, JobCancellation) {
6285        let (pending_tx, pending_rx) = std::sync::mpsc::sync_channel(1);
6286        let request = inspect_request(request_id);
6287        let inspect_ctx = Arc::clone(ctx);
6288        let (_rx, cancellation) = executor.submit_cancellable_async(
6289            root.clone(),
6290            Lane::SerialLspStatus,
6291            request_id.to_string(),
6292            Box::new(move |_| {
6293                let DispatchOutcome::Deferred(pending) =
6294                    crate::commands::inspect::handle_inspect_deferred_with_restriction(
6295                        &request,
6296                        inspect_ctx,
6297                        true,
6298                    )
6299                else {
6300                    panic!("inspect setup must defer")
6301                };
6302                pending_tx.send(pending).expect("send pending inspect");
6303                Response::success("inspect-setup", json!({}))
6304            }),
6305        );
6306        let pending = pending_rx
6307            .recv_timeout(Duration::from_secs(1))
6308            .expect("inspect setup leaves the executor lane");
6309        let deadline = Instant::now() + Duration::from_secs(1);
6310        while !executor.actor_is_idle(root) {
6311            assert!(
6312                Instant::now() < deadline,
6313                "inspect setup kept lane counters live"
6314            );
6315            std::thread::sleep(Duration::from_millis(5));
6316        }
6317        (pending, cancellation)
6318    }
6319
6320    fn wait_for_inspect_terminal(pending: &mut PendingResponse, ctx: &AppContext) -> Response {
6321        let deadline = Instant::now() + Duration::from_secs(60);
6322        loop {
6323            if let Some(response) = (pending.poll)(ctx) {
6324                return response;
6325            }
6326            assert!(Instant::now() < deadline, "inspect terminal timed out");
6327            std::thread::sleep(Duration::from_millis(5));
6328        }
6329    }
6330
6331    fn cold_navigation_context(root: &Path) -> (Arc<AppContext>, PathBuf) {
6332        let source_dir = root.join("src");
6333        std::fs::create_dir_all(&source_dir).expect("create source dir");
6334        std::fs::write(root.join("Cargo.toml"), "[package]\nname = \"fixture\"\n")
6335            .expect("write Cargo manifest");
6336        let source = source_dir.join("main.rs");
6337        std::fs::write(&source, "fn main() {}\n").expect("write source");
6338        let binary = root.join("cold-navigation-server");
6339        std::fs::write(&binary, b"fixture").expect("write server placeholder");
6340
6341        let ctx = inspect_context(root);
6342        ctx.lsp()
6343            .override_binary(crate::lsp::registry::ServerKind::Rust, binary);
6344        (ctx, source)
6345    }
6346
6347    fn navigation_request(id: &str, source: &Path) -> RawRequest {
6348        serde_json::from_value(json!({
6349            "id": id,
6350            "command": "lsp_hover",
6351            "file": source.display().to_string(),
6352            "line": 1,
6353            "character": 1,
6354        }))
6355        .expect("navigation request")
6356    }
6357
6358    fn submit_deferred_navigation_setup(
6359        executor: &Arc<Executor>,
6360        root: &ProjectRootId,
6361        ctx: &Arc<AppContext>,
6362        source: &Path,
6363        request_id: &str,
6364    ) -> (PendingResponse, JobCancellation) {
6365        let (pending_tx, pending_rx) = std::sync::mpsc::sync_channel(1);
6366        let request = navigation_request(request_id, source);
6367        let navigation_ctx = Arc::clone(ctx);
6368        let (_rx, cancellation) = executor.submit_cancellable_async(
6369            root.clone(),
6370            Lane::SerialLspStatus,
6371            request_id.to_string(),
6372            Box::new(move |_| {
6373                let DispatchOutcome::Deferred(pending) = crate::commands::lsp_navigation::
6374                    handle_lsp_navigation_deferred_with_restriction(
6375                        &request,
6376                        navigation_ctx,
6377                        true,
6378                    )
6379                else {
6380                    panic!("cold navigation setup must defer")
6381                };
6382                pending_tx.send(pending).expect("send pending navigation");
6383                Response::success("navigation-setup", json!({}))
6384            }),
6385        );
6386        let pending = pending_rx
6387            .recv_timeout(Duration::from_secs(1))
6388            .expect("navigation setup leaves the executor lane");
6389        let deadline = Instant::now() + Duration::from_secs(1);
6390        while !executor.actor_is_idle(root) {
6391            assert!(
6392                Instant::now() < deadline,
6393                "navigation setup kept lane counters live"
6394            );
6395            std::thread::sleep(Duration::from_millis(5));
6396        }
6397        (pending, cancellation)
6398    }
6399
6400    fn wait_for_navigation_terminal(pending: &mut PendingResponse, ctx: &AppContext) -> Response {
6401        let deadline = Instant::now() + Duration::from_secs(1);
6402        loop {
6403            if let Some(response) = (pending.poll)(ctx) {
6404                return response;
6405            }
6406            assert!(Instant::now() < deadline, "navigation terminal timed out");
6407            std::thread::sleep(Duration::from_millis(5));
6408        }
6409    }
6410
6411    #[test]
6412    fn deferred_inspect_releases_lane_for_bind_and_mutation() {
6413        let _serial = crate::commands::inspect::deferred_inspect_test_lock();
6414        let executor = Arc::new(Executor::new());
6415        let (dir, root) = test_root("deferred-inspect-storm");
6416        std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
6417        let ctx = inspect_context(dir.path());
6418        executor.register_actor(root.clone(), Arc::clone(&ctx));
6419        let (started_rx, release_tx) =
6420            crate::commands::inspect::install_deferred_inspect_stat_gate_for_test();
6421        let (mut pending, _cancellation) =
6422            submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-storm");
6423        started_rx
6424            .recv_timeout(Duration::from_secs(1))
6425            .expect("deferred inspect body starts");
6426
6427        for request_id in ["subc-bind-other-session", "subc-edit-other-session"] {
6428            let response = executor.submit(
6429                root.clone(),
6430                Lane::Mutating,
6431                request_id.to_string(),
6432                Box::new(move |_| Response::success(request_id, json!({ "admitted": true }))),
6433            );
6434            assert!(
6435                response
6436                    .recv_timeout(Duration::from_secs(1))
6437                    .expect("writer admits while inspect remains deferred")
6438                    .success
6439            );
6440        }
6441        assert_eq!(
6442            crate::commands::inspect::deferred_inspect_root_count_for_test(),
6443            1,
6444            "writer admissions must not finish the detached inspect"
6445        );
6446
6447        release_tx.send(()).expect("release inspect body");
6448        let terminal = wait_for_inspect_terminal(&mut pending, &ctx);
6449        assert!(
6450            terminal.data.get("inspect_terminal").is_some(),
6451            "inspect must still produce its terminal: {:?}",
6452            terminal.data
6453        );
6454    }
6455
6456    #[test]
6457    fn deferred_inspect_honors_the_shared_request_deadline() {
6458        let _serial = crate::commands::inspect::deferred_inspect_test_lock();
6459        let executor = Arc::new(Executor::new());
6460        let (dir, root) = test_root("deferred-inspect-deadline");
6461        std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
6462        let ctx = inspect_context_with_timeout(dir.path(), Some(10_000));
6463        executor.register_actor(root.clone(), Arc::clone(&ctx));
6464        let (started_rx, _release_tx) =
6465            crate::commands::inspect::install_deferred_inspect_body_gate_for_test();
6466        let started = Instant::now();
6467        let (mut pending, _cancellation) =
6468            submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-deadline");
6469        started_rx
6470            .recv_timeout(Duration::from_secs(1))
6471            .expect("deferred inspect body starts");
6472
6473        let terminal = wait_for_inspect_terminal(&mut pending, &ctx);
6474        assert_eq!(terminal.data["inspect_terminal"], "phase_failed");
6475        assert_eq!(terminal.data["failure_reason"], "inspect_request_timeout");
6476        assert_eq!(terminal.data["failed_phase"], "tier2_rescan");
6477        assert!(
6478            started.elapsed() < Duration::from_secs(8),
6479            "deferred inspect missed its terminal reserve: {:?}",
6480            started.elapsed()
6481        );
6482    }
6483
6484    #[test]
6485    fn deferred_cold_navigation_releases_lsp_lane_for_reads_and_mutation() {
6486        let _serial = crate::commands::lsp_navigation::deferred_navigation_test_lock();
6487        let executor = Arc::new(Executor::new());
6488        let (dir, root) = test_root("deferred-cold-navigation");
6489        let (ctx, source) = cold_navigation_context(dir.path());
6490        executor.register_actor(root.clone(), Arc::clone(&ctx));
6491        let (started_rx, _release_tx) =
6492            crate::commands::lsp_navigation::install_deferred_navigation_gate_for_test();
6493        let (mut pending, cancellation) = submit_deferred_navigation_setup(
6494            &executor,
6495            &root,
6496            &ctx,
6497            &source,
6498            "subc-cold-navigation",
6499        );
6500        started_rx
6501            .recv_timeout(Duration::from_secs(1))
6502            .expect("detached navigation reaches its cold-start gate");
6503
6504        for (lane, request_id) in [
6505            (Lane::PureRead, "subc-navigation-read"),
6506            (Lane::Mutating, "subc-navigation-write"),
6507        ] {
6508            let response = executor.submit(
6509                root.clone(),
6510                lane,
6511                request_id.to_string(),
6512                Box::new(move |_| Response::success(request_id, json!({ "admitted": true }))),
6513            );
6514            assert!(
6515                response
6516                    .recv_timeout(Duration::from_secs(1))
6517                    .expect("unrelated work admits while navigation remains deferred")
6518                    .success
6519            );
6520        }
6521        assert_eq!(
6522            crate::commands::lsp_navigation::deferred_navigation_worker_count_for_test(),
6523            1,
6524            "lane admissions must not finish the detached navigation"
6525        );
6526
6527        cancellation.request_cancel();
6528        let terminal = wait_for_navigation_terminal(&mut pending, &ctx);
6529        assert!(!terminal.success);
6530        let deadline = Instant::now() + Duration::from_secs(1);
6531        while crate::commands::lsp_navigation::deferred_navigation_worker_count_for_test() != 0 {
6532            assert!(
6533                Instant::now() < deadline,
6534                "cancelled navigation worker did not settle"
6535            );
6536            std::thread::sleep(Duration::from_millis(5));
6537        }
6538        assert!(executor.actor_is_idle(&root));
6539    }
6540
6541    #[test]
6542    fn cancelling_pending_navigation_removes_entry_without_reply() {
6543        let _serial = crate::commands::lsp_navigation::deferred_navigation_test_lock();
6544        let executor = Arc::new(Executor::new());
6545        let (dir, root) = test_root("cancelled-pending-navigation");
6546        let (ctx, source) = cold_navigation_context(dir.path());
6547        executor.register_actor(root.clone(), Arc::clone(&ctx));
6548        let (started_rx, _release_tx) =
6549            crate::commands::lsp_navigation::install_deferred_navigation_gate_for_test();
6550        let (pending, _cancellation) = submit_deferred_navigation_setup(
6551            &executor,
6552            &root,
6553            &ctx,
6554            &source,
6555            "subc-cancel-navigation",
6556        );
6557        started_rx
6558            .recv_timeout(Duration::from_secs(1))
6559            .expect("detached navigation reaches its cancellation gate");
6560
6561        let route = RouteChannel {
6562            channel: 17,
6563            epoch: 1,
6564        };
6565        let mut registry = PendingSubcResponses::default();
6566        registry.register(PendingSubcResponse {
6567            route,
6568            corr: 71,
6569            flags: Flags::new(false, Priority::Passive, false),
6570            ver: PROTOCOL_VERSION,
6571            root: root.clone(),
6572            session_id: "navigation-cancel-session".to_string(),
6573            bare_name: "lsp_hover".to_string(),
6574            format_context: crate::subc_format::FormatContext::from_tool_call(
6575                "lsp_hover",
6576                &json!({}),
6577                dir.path(),
6578            ),
6579            bind_trust: BindTrust::FirstParty,
6580            pending,
6581            surface_downgraded: false,
6582            phase_trace: PhaseTrace::new(Instant::now()),
6583        });
6584
6585        assert!(registry.cancel_request(route, 71));
6586        assert!(registry.is_empty());
6587        let deadline = Instant::now() + Duration::from_secs(1);
6588        while crate::commands::lsp_navigation::deferred_navigation_worker_count_for_test() != 0 {
6589            assert!(
6590                Instant::now() < deadline,
6591                "pending cancellation did not settle the detached worker"
6592            );
6593            std::thread::sleep(Duration::from_millis(5));
6594        }
6595        assert!(
6596            registry.poll_ready(executor.as_ref()).is_empty(),
6597            "a cancelled navigation must not leak a reply"
6598        );
6599    }
6600
6601    #[test]
6602    fn same_root_deferred_inspects_are_single_flight() {
6603        let _serial = crate::commands::inspect::deferred_inspect_test_lock();
6604        let executor = Arc::new(Executor::new());
6605        let (dir, root) = test_root("single-flight-deferred-inspect");
6606        std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
6607        let ctx = inspect_context(dir.path());
6608        executor.register_actor(root.clone(), Arc::clone(&ctx));
6609        let (started_rx, release_tx) =
6610            crate::commands::inspect::install_deferred_inspect_stat_gate_for_test();
6611        let (mut first, _first_cancellation) =
6612            submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-first");
6613        started_rx
6614            .recv_timeout(Duration::from_secs(1))
6615            .expect("first inspect owns the root flight");
6616        let (mut second, second_cancellation) =
6617            submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-second");
6618
6619        assert_eq!(
6620            crate::commands::inspect::deferred_inspect_root_count_for_test(),
6621            1,
6622            "only one detached body may run for a root"
6623        );
6624        assert!((second.poll)(&ctx).is_none(), "second inspect must queue");
6625        second_cancellation.request_cancel();
6626        let second_terminal = wait_for_inspect_terminal(&mut second, &ctx);
6627        assert_eq!(second_terminal.data["inspect_terminal"], "interrupted");
6628        assert_eq!(
6629            crate::commands::inspect::deferred_inspect_root_count_for_test(),
6630            1,
6631            "cancelling the queued request must not release the active flight"
6632        );
6633
6634        release_tx.send(()).expect("release first inspect");
6635        let first_terminal = wait_for_inspect_terminal(&mut first, &ctx);
6636        assert_eq!(first_terminal.data["inspect_terminal"], "fresh");
6637        assert_eq!(
6638            crate::commands::inspect::deferred_inspect_root_count_for_test(),
6639            0
6640        );
6641    }
6642
6643    #[test]
6644    fn route_abandonment_cancels_detached_inspect_thread() {
6645        let _serial = crate::commands::inspect::deferred_inspect_test_lock();
6646        let executor = Arc::new(Executor::new());
6647        let (dir, root) = test_root("cancelled-deferred-inspect");
6648        std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
6649        let ctx = inspect_context(dir.path());
6650        executor.register_actor(root.clone(), Arc::clone(&ctx));
6651        let active: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
6652        let route = RouteChannel {
6653            channel: 7,
6654            epoch: 1,
6655        };
6656        let (started_rx, _release_tx) =
6657            crate::commands::inspect::install_deferred_inspect_body_gate_for_test();
6658        let (mut pending, cancellation) =
6659            submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-abandoned");
6660        active.lock().expect("active tool call map").insert(
6661            (route, 41),
6662            ActiveToolCall {
6663                root_id: root.clone(),
6664                cancellation,
6665                detach_policy: RouteDetachPolicy::CancelOnDetach,
6666            },
6667        );
6668        started_rx
6669            .recv_timeout(Duration::from_secs(1))
6670            .expect("detached inspect reaches cancellation gate");
6671        assert!(ctx.request_force_restrict("subc-inspect-abandoned"));
6672
6673        assert!(cancel_active_tool_call(
6674            &active,
6675            executor.as_ref(),
6676            route,
6677            41,
6678            "test route abandonment"
6679        ));
6680        let terminal = wait_for_inspect_terminal(&mut pending, &ctx);
6681        assert_eq!(terminal.data["inspect_terminal"], "interrupted");
6682        assert_eq!(
6683            crate::commands::inspect::deferred_inspect_root_count_for_test(),
6684            0
6685        );
6686        assert!(executor.actor_is_idle(&root));
6687        assert!(active.lock().expect("active tool call map").is_empty());
6688        let restriction_deadline = Instant::now() + Duration::from_secs(1);
6689        while ctx.request_force_restrict("subc-inspect-abandoned") {
6690            assert!(
6691                Instant::now() < restriction_deadline,
6692                "detached force-restrict guard leaked"
6693            );
6694            std::thread::sleep(Duration::from_millis(5));
6695        }
6696    }
6697
6698    #[test]
6699    fn true_abandonment_cancels_but_route_detach_retains_interactive_search() {
6700        // This scenario requires the replayable and terminal calls to run at the
6701        // same time. Pin two read slots instead of deriving the topology from
6702        // the host, where a small runner can queue the terminal call forever
6703        // behind the intentionally retained call.
6704        let executor = Arc::new(Executor::with_config(crate::executor::ExecutorConfig {
6705            pool_size: 3,
6706            read_cap: 2,
6707            actor_cap: 2,
6708            heavy_permits: 1,
6709            drr_quantum: 1,
6710        }));
6711        let (_dir, root) = test_root("cancelled-interactive-search");
6712        executor.register_actor(root.clone(), test_ctx());
6713        let active: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
6714        let route = RouteChannel {
6715            channel: 9,
6716            epoch: 1,
6717        };
6718
6719        let disabled_iterations = Arc::new(AtomicUsize::new(0));
6720        let disabled_probe = Arc::clone(&disabled_iterations);
6721        let (disabled_started_tx, disabled_started_rx) = std::sync::mpsc::sync_channel(1);
6722        let (disabled_rx, disabled_cancellation) = executor.submit_cancellable_async(
6723            root.clone(),
6724            Lane::PureRead,
6725            "untracked-search".to_string(),
6726            Box::new(move |_| {
6727                disabled_started_tx
6728                    .send(())
6729                    .expect("signal untracked search");
6730                let deadline = Instant::now() + Duration::from_secs(5);
6731                while !crate::commands::semantic_search::search_cancellation_requested() {
6732                    if Instant::now() >= deadline {
6733                        return Response::error(
6734                            "untracked-search",
6735                            "test_timeout",
6736                            "untracked search did not receive cancellation",
6737                        );
6738                    }
6739                    disabled_probe.fetch_add(1, Ordering::Relaxed);
6740                    std::thread::yield_now();
6741                }
6742                Response::error(
6743                    "untracked-search",
6744                    "request_cancelled",
6745                    "cancelled at search checkpoint",
6746                )
6747            }),
6748        );
6749        disabled_started_rx
6750            .recv_timeout(Duration::from_secs(1))
6751            .expect("untracked search starts");
6752        assert_eq!(
6753            apply_route_work_disposition(
6754                &active,
6755                executor.as_ref(),
6756                route,
6757                RouteWorkDisposition::Abandon,
6758                "disabled cancellation wiring",
6759            ),
6760            0
6761        );
6762        let iterations_before = disabled_iterations.load(Ordering::Relaxed);
6763        std::thread::sleep(Duration::from_millis(10));
6764        assert!(
6765            disabled_iterations.load(Ordering::Relaxed) > iterations_before,
6766            "without route registration the search keeps computing"
6767        );
6768        disabled_cancellation.request_cancel();
6769        let disabled_response = disabled_rx
6770            .blocking_recv()
6771            .expect("untracked search settles after explicit cleanup");
6772        assert_eq!(disabled_response.data["code"], "request_cancelled");
6773
6774        let tracked_iterations = Arc::new(AtomicUsize::new(0));
6775        let tracked_probe = Arc::clone(&tracked_iterations);
6776        let (tracked_started_tx, tracked_started_rx) = std::sync::mpsc::sync_channel(1);
6777        let tracked_rx = submit_active_tool_call(
6778            executor.as_ref(),
6779            &active,
6780            route,
6781            42,
6782            root.clone(),
6783            Lane::PureRead,
6784            "tracked-search".to_string(),
6785            RouteDetachPolicy::RetainForReplay,
6786            Box::new(move |_| {
6787                tracked_started_tx.send(()).expect("signal tracked search");
6788                let deadline = Instant::now() + Duration::from_secs(5);
6789                while !crate::commands::semantic_search::search_cancellation_requested() {
6790                    if Instant::now() >= deadline {
6791                        return Response::error(
6792                            "tracked-search",
6793                            "test_timeout",
6794                            "tracked search did not receive cancellation",
6795                        );
6796                    }
6797                    tracked_probe.fetch_add(1, Ordering::Relaxed);
6798                    std::thread::yield_now();
6799                }
6800                Response::error(
6801                    "tracked-search",
6802                    "request_cancelled",
6803                    "cancelled at search checkpoint",
6804                )
6805            }),
6806        );
6807        tracked_started_rx
6808            .recv_timeout(Duration::from_secs(1))
6809            .expect("tracked search starts");
6810
6811        let (terminal_started_tx, terminal_started_rx) = std::sync::mpsc::sync_channel(1);
6812        let terminal_rx = submit_active_tool_call(
6813            executor.as_ref(),
6814            &active,
6815            route,
6816            43,
6817            root.clone(),
6818            Lane::PureRead,
6819            "teardown-terminal".to_string(),
6820            RouteDetachPolicy::CancelOnDetach,
6821            Box::new(move |_| {
6822                terminal_started_tx
6823                    .send(())
6824                    .expect("signal teardown-terminal call");
6825                let deadline = Instant::now() + Duration::from_secs(5);
6826                while !crate::executor::current_job_cancelled() {
6827                    if Instant::now() >= deadline {
6828                        return Response::error(
6829                            "teardown-terminal",
6830                            "test_timeout",
6831                            "terminal call did not receive cancellation",
6832                        );
6833                    }
6834                    std::thread::yield_now();
6835                }
6836                Response::error(
6837                    "teardown-terminal",
6838                    "request_cancelled",
6839                    "cancelled for terminal-emitting teardown",
6840                )
6841            }),
6842        );
6843        terminal_started_rx
6844            .recv_timeout(Duration::from_secs(1))
6845            .expect("teardown-terminal call starts");
6846        assert_eq!(
6847            apply_route_work_disposition(
6848                &active,
6849                executor.as_ref(),
6850                route,
6851                RouteWorkDisposition::RetainForReplay,
6852                "test route detach",
6853            ),
6854            1,
6855            "only the replayable search remains active after route detach"
6856        );
6857        let terminal_response = terminal_rx
6858            .blocking_recv()
6859            .expect("teardown-terminal call stops at cancellation checkpoint");
6860        assert_eq!(terminal_response.data["code"], "request_cancelled");
6861        let iterations_before_detach = tracked_iterations.load(Ordering::Relaxed);
6862        std::thread::sleep(Duration::from_millis(10));
6863        assert!(
6864            tracked_iterations.load(Ordering::Relaxed) > iterations_before_detach,
6865            "route detach must retain work whose response can replay after rebind"
6866        );
6867        assert_eq!(
6868            apply_route_work_disposition(
6869                &active,
6870                executor.as_ref(),
6871                route,
6872                RouteWorkDisposition::Abandon,
6873                "test session purge",
6874            ),
6875            1
6876        );
6877        let tracked_response = tracked_rx
6878            .blocking_recv()
6879            .expect("tracked search stops at cancellation checkpoint");
6880        assert_eq!(tracked_response.data["code"], "request_cancelled");
6881        assert!(active.lock().expect("active tool calls").is_empty());
6882
6883        let deadline = Instant::now() + Duration::from_secs(1);
6884        while !executor.actor_is_idle(&root) {
6885            assert!(
6886                Instant::now() < deadline,
6887                "cancelled search must release the PureRead lane"
6888            );
6889            std::thread::sleep(Duration::from_millis(2));
6890        }
6891    }
6892
6893    #[test]
6894    fn shutdown_drain_emits_terminal_and_clears_pending_inspect() {
6895        let _serial = crate::commands::inspect::deferred_inspect_test_lock();
6896        let executor = Arc::new(Executor::new());
6897        let (dir, root) = test_root("shutdown-deferred-inspect");
6898        std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
6899        let ctx = inspect_context(dir.path());
6900        executor.register_actor(root.clone(), Arc::clone(&ctx));
6901        let (started_rx, _release_tx) =
6902            crate::commands::inspect::install_deferred_inspect_body_gate_for_test();
6903        let (pending, cancellation) =
6904            submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-shutdown");
6905        let route = RouteChannel {
6906            channel: 8,
6907            epoch: 1,
6908        };
6909        started_rx
6910            .recv_timeout(Duration::from_secs(1))
6911            .expect("detached inspect reaches shutdown gate");
6912        let mut registry = PendingSubcResponses::default();
6913        registry.register(PendingSubcResponse {
6914            route,
6915            corr: 42,
6916            flags: Flags::new(false, Priority::Passive, false),
6917            ver: PROTOCOL_VERSION,
6918            root: root.clone(),
6919            session_id: "shutdown-session".to_string(),
6920            bare_name: "inspect".to_string(),
6921            format_context: crate::subc_format::FormatContext::from_tool_call(
6922                "inspect",
6923                &json!({}),
6924                dir.path(),
6925            ),
6926            bind_trust: BindTrust::FirstParty,
6927            pending,
6928            surface_downgraded: false,
6929            phase_trace: PhaseTrace::new(Instant::now()),
6930        });
6931        let active: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::from([(
6932            (route, 42),
6933            ActiveToolCall {
6934                root_id: root.clone(),
6935                cancellation,
6936                detach_policy: RouteDetachPolicy::CancelOnDetach,
6937            },
6938        )])));
6939
6940        let resolved = registry.drain_on_shutdown(executor.as_ref());
6941        assert!(registry.is_empty());
6942        assert_eq!(resolved.len(), 1);
6943        assert_eq!(
6944            resolved[0].response.data["failure_reason"],
6945            "daemon_shutdown"
6946        );
6947        finish_active_tool_call(&active, route, 42);
6948        let deadline = Instant::now() + Duration::from_secs(1);
6949        while crate::commands::inspect::deferred_inspect_root_count_for_test() != 0 {
6950            assert!(Instant::now() < deadline, "shutdown cancellation was inert");
6951            std::thread::sleep(Duration::from_millis(5));
6952        }
6953        assert!(active.lock().expect("active calls").is_empty());
6954        assert!(executor.actor_is_idle(&root));
6955    }
6956
6957    pub(super) fn wait_for_watcher_count(ctx: &AppContext, expected: usize) {
6958        let deadline = Instant::now() + Duration::from_secs(30);
6959        loop {
6960            let observed = ctx.watcher_registry_count();
6961            if observed == expected {
6962                return;
6963            }
6964            assert!(
6965                Instant::now() < deadline,
6966                "watcher count did not settle before deadline: expected={expected}, observed={observed}"
6967            );
6968            std::thread::sleep(Duration::from_millis(50));
6969        }
6970    }
6971
6972    /// Sweep until `root` is forgotten, mirroring how production reaps.
6973    ///
6974    /// `reap_idle_roots` probes actor idleness with a try-lock and retains the
6975    /// root when the scheduler holds that lock — correct behavior, since the
6976    /// real caller sweeps on a timer and simply catches the root next tick. A
6977    /// test that asserts a single sweep succeeds is therefore asserting it wins
6978    /// a lock race that nothing in production depends on: `register_actor` wakes
6979    /// the scheduler, which grabs the same lock, and on a loaded runner that
6980    /// window is wide enough to lose. Sweep to the outcome instead.
6981    pub(super) fn reap_until_forgotten(
6982        root: &ProjectRootId,
6983        live_roots: &mut HashMap<ProjectRootId, RootMeta>,
6984        pending_binds: &HashMap<RouteChannel, PendingBind>,
6985        root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
6986        executor: &Arc<Executor>,
6987        metrics: &DispatchPathMetrics,
6988    ) -> IdleReapOutcome {
6989        let deadline = Instant::now() + Duration::from_secs(30);
6990        loop {
6991            let outcome = reap_idle_roots(
6992                Instant::now(),
6993                live_roots,
6994                pending_binds,
6995                root_channels,
6996                executor,
6997                metrics,
6998            );
6999            if outcome.forgotten_deleted_roots.contains(root) {
7000                return outcome;
7001            }
7002            assert!(
7003                Instant::now() < deadline,
7004                "deleted root was never forgotten: {root:?}"
7005            );
7006            std::thread::sleep(Duration::from_millis(10));
7007        }
7008    }
7009
7010    pub(super) fn wait_for_actor_root_count(app: &App, expected: usize) {
7011        let deadline = Instant::now() + Duration::from_secs(30);
7012        loop {
7013            let observed = app.actor_root_count();
7014            if observed == expected {
7015                return;
7016            }
7017            assert!(
7018                Instant::now() < deadline,
7019                "actor root count did not settle before deadline: expected={expected}, observed={observed}"
7020            );
7021            std::thread::sleep(Duration::from_millis(50));
7022        }
7023    }
7024
7025    pub(super) fn status_frame(seq: u64) -> PushFrame {
7026        status_frame_with_session(seq, None)
7027    }
7028
7029    pub(super) fn status_frame_with_session(seq: u64, session_id: Option<&str>) -> PushFrame {
7030        PushFrame::StatusChanged(StatusChangedFrame {
7031            frame_type: "status_changed",
7032            session_id: session_id.map(str::to_string),
7033            snapshot: json!({ "seq": seq }),
7034        })
7035    }
7036
7037    pub(super) fn completion_frame(task_id: &str) -> PushFrame {
7038        completion_frame_with_session(task_id, "session-1")
7039    }
7040
7041    pub(super) fn completion_frame_with_session(task_id: &str, session_id: &str) -> PushFrame {
7042        PushFrame::BashCompleted(BashCompletedFrame {
7043            frame_type: "bash_completed",
7044            task_id: task_id.to_string(),
7045            session_id: session_id.to_string(),
7046            status: BgTaskStatus::Completed,
7047            exit_code: Some(0),
7048            command: format!("echo {task_id}"),
7049            output_preview: String::new(),
7050            bash_output_list_envelope: None,
7051            output_truncated: false,
7052            original_tokens: None,
7053            compressed_tokens: None,
7054            tokens_skipped: false,
7055            status_reason: None,
7056            live_descendants: Some(Vec::new()),
7057            live_descendants_omitted: 0,
7058            live_descendants_summary: None,
7059        })
7060    }
7061
7062    pub(super) fn long_running_frame(task_id: &str, elapsed_ms: u64) -> PushFrame {
7063        long_running_frame_with_session(task_id, "session-1", elapsed_ms)
7064    }
7065
7066    pub(super) fn long_running_frame_with_session(
7067        task_id: &str,
7068        session_id: &str,
7069        elapsed_ms: u64,
7070    ) -> PushFrame {
7071        PushFrame::BashLongRunning(BashLongRunningFrame {
7072            frame_type: "bash_long_running",
7073            task_id: task_id.to_string(),
7074            session_id: session_id.to_string(),
7075            command: format!("sleep {elapsed_ms}"),
7076            elapsed_ms,
7077        })
7078    }
7079
7080    pub(super) fn pattern_match_frame(session_id: &str) -> PushFrame {
7081        PushFrame::BashPatternMatch(BashPatternMatchFrame {
7082            frame_type: "bash_pattern_match",
7083            task_id: "task-pattern".to_string(),
7084            session_id: session_id.to_string(),
7085            watch_id: "watch-1".to_string(),
7086            match_text: "needle".to_string(),
7087            match_offset: 7,
7088            context: "haystack needle".to_string(),
7089            once: true,
7090            reason: "pattern_match",
7091        })
7092    }
7093
7094    pub(super) fn configure_warnings_frame(session_id: Option<&str>) -> PushFrame {
7095        PushFrame::ConfigureWarnings(ConfigureWarningsFrame {
7096            frame_type: "configure_warnings",
7097            session_id: session_id.map(str::to_string),
7098            project_root: "/tmp/subc-test".to_string(),
7099            warnings: Vec::new(),
7100        })
7101    }
7102
7103    pub(super) fn route_identity(root: &ProjectRootId, session_id: &str) -> RouteIdentity {
7104        route_identity_with_trust(root, session_id, BindTrust::FirstParty)
7105    }
7106
7107    pub(super) fn route_identity_with_trust(
7108        root: &ProjectRootId,
7109        session_id: &str,
7110        trust: BindTrust,
7111    ) -> RouteIdentity {
7112        RouteIdentity(Arc::new(RouteIdentityData {
7113            root: root.clone(),
7114            project_root: root.as_path().to_path_buf(),
7115            harness: "opencode".to_string(),
7116            session: session_id.to_string(),
7117            trust,
7118            spawn_principal: AuthenticatedPrincipal::RouteBind {
7119                trust: trust.sandbox_trust(),
7120                route_channel: 0,
7121                route_epoch: 0,
7122                project_root: root.as_path().to_path_buf(),
7123                harness: "opencode".to_string(),
7124                session_id: session_id.to_string(),
7125                principal_id: Some(match trust {
7126                    BindTrust::FirstParty => "direct".to_string(),
7127                    BindTrust::Untrusted => "unverified".to_string(),
7128                }),
7129            },
7130            consumer_elicitation_capable: false,
7131        }))
7132    }
7133
7134    pub(super) fn progress_frame(request_id: &str, kind: ProgressKind, chunk: &str) -> PushFrame {
7135        PushFrame::Progress(ProgressFrame::new(request_id, kind, chunk))
7136    }
7137
7138    pub(super) fn status_seq(frame: &PushFrame) -> Option<u64> {
7139        match frame {
7140            PushFrame::StatusChanged(status) => status.snapshot.get("seq").and_then(|v| v.as_u64()),
7141            _ => None,
7142        }
7143    }
7144
7145    pub(super) fn completion_task(frame: &PushFrame) -> Option<&str> {
7146        match frame {
7147            PushFrame::BashCompleted(completion) => Some(completion.task_id.as_str()),
7148            _ => None,
7149        }
7150    }
7151
7152    pub(super) fn push_frame_task_id(frame: &Frame) -> Option<String> {
7153        let body: serde_json::Value = serde_json::from_slice(&frame.body).expect("push body");
7154        body.get("task_id")
7155            .and_then(serde_json::Value::as_str)
7156            .map(str::to_string)
7157    }
7158}
7159
7160#[cfg(test)]
7161mod tests {
7162    use super::test_support::{
7163        completion_frame, reap_until_forgotten, route_identity, test_ctx, test_root,
7164        wait_for_actor_root_count, wait_for_watcher_count,
7165    };
7166    use super::*;
7167    use crate::bash_background::BgTaskStatus;
7168
7169    /// Only a daemon-requested stop may exit 0. The supervisor never respawns
7170    /// a clean exit, so a fatal-actor teardown that mapped to Ok(()) left the
7171    /// module down host-wide until a manual start (2026-09-11).
7172    #[test]
7173    fn only_a_graceful_goodbye_maps_to_a_clean_exit() {
7174        assert!(module_loop_exit_result(ModuleLoopExit::Graceful).is_ok());
7175        assert!(matches!(
7176            module_loop_exit_result(ModuleLoopExit::ConnectionLost),
7177            Err(SubcError::ConnectionLost)
7178        ));
7179        assert!(matches!(
7180            module_loop_exit_result(ModuleLoopExit::SkipSearchFlush),
7181            Err(SubcError::ActorFatal)
7182        ));
7183    }
7184
7185    #[test]
7186    fn fatal_panic_responses_are_detected_and_noted() {
7187        let panic = Response::error(
7188            "req-fatal",
7189            "actor_fatal",
7190            "start byte index 7 is not a char boundary",
7191        );
7192        assert!(note_fatal_panic_response(&panic));
7193        let ordinary = Response::error("req-ok", "invalid_request", "missing field");
7194        assert!(!note_fatal_panic_response(&ordinary));
7195    }
7196
7197    fn attach_error(kind: io::ErrorKind) -> SubcError {
7198        SubcError::Connect {
7199            endpoint: "127.0.0.1:1".to_string(),
7200            source: io::Error::new(kind, "constructed attach failure"),
7201        }
7202    }
7203
7204    fn auth_io_error(kind: io::ErrorKind) -> SubcError {
7205        SubcError::Auth {
7206            endpoint: "127.0.0.1:1".to_string(),
7207            source: subc_transport::AuthError::Io {
7208                stage: subc_transport::AuthStage::ServerProof,
7209                source: io::Error::new(kind, "constructed auth failure"),
7210            },
7211        }
7212    }
7213
7214    fn cpu_hunt_process_cpu_us() -> u64 {
7215        #[cfg(unix)]
7216        {
7217            let mut usage = std::mem::MaybeUninit::<libc::rusage>::uninit();
7218            // getrusage initializes the output on success; no pointer escapes.
7219            if unsafe { libc::getrusage(libc::RUSAGE_SELF, usage.as_mut_ptr()) } == 0 {
7220                let usage = unsafe { usage.assume_init() };
7221                return (usage.ru_utime.tv_sec + usage.ru_stime.tv_sec) as u64 * 1_000_000
7222                    + (usage.ru_utime.tv_usec + usage.ru_stime.tv_usec) as u64;
7223            }
7224        }
7225        0
7226    }
7227
7228    #[test]
7229    fn route_bind_does_not_recompute_fleet_health() {
7230        let runtime = tokio::runtime::Builder::new_current_thread()
7231            .enable_all()
7232            .build()
7233            .expect("bind runtime");
7234        runtime.block_on(async {
7235            let (dir, root) = test_root("bind-health-work-count");
7236            let app = App::default_shared();
7237            let executor = Arc::new(Executor::new());
7238            let ctx = Arc::new(AppContext::from_app(Arc::clone(&app), Config::default()));
7239            let mut fixture_dirs = Vec::new();
7240            // The opt-in probe accepts only an already-copied artifact below this
7241            // checkout's target directory; normal tests never open a live store.
7242            if let Some(copy) = std::env::var_os("AFT_CPU_HUNT_STORE_COPY") {
7243                let copy = std::fs::canonicalize(copy).expect("copied store exists");
7244                let project = Path::new(env!("CARGO_MANIFEST_DIR"))
7245                    .parent()
7246                    .expect("crates directory")
7247                    .parent()
7248                    .expect("checkout directory")
7249                    .canonicalize()
7250                    .expect("canonical checkout");
7251                assert!(copy.starts_with(project.join("target")));
7252                for index in 0..36 {
7253                    let actor = if index == 0 {
7254                        Arc::clone(&ctx)
7255                    } else {
7256                        Arc::new(AppContext::from_app(Arc::clone(&app), Config::default()))
7257                    };
7258                    actor.update_config(|config| config.project_root = Some(project.clone()));
7259                    *actor.callgraph_store().write().expect("store slot") = Some(Arc::new(
7260                        crate::callgraph_store::CallGraphStore::open_readonly(
7261                            copy.clone(),
7262                            project.clone(),
7263                        )
7264                        .expect("open copied graph")
7265                        .expect("copied graph ready"),
7266                    ));
7267                    if index > 0 {
7268                        let (fixture, id) = test_root(&format!("cpu-hunt-{index}"));
7269                        assert!(executor.register_actor(id, actor));
7270                        fixture_dirs.push(fixture);
7271                    }
7272                }
7273            }
7274            assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7275            let cache = HealthRollupCache::new();
7276            if !fixture_dirs.is_empty() {
7277                let started = Instant::now();
7278                let cpu = cpu_hunt_process_cpu_us();
7279                cache.refresh(&executor, &app);
7280                eprintln!(
7281                    "cpu_hunt health_rollup wall_us={} cpu_us={}",
7282                    started.elapsed().as_micros(),
7283                    cpu_hunt_process_cpu_us().saturating_sub(cpu)
7284                );
7285                let started = Instant::now();
7286                let cpu = cpu_hunt_process_cpu_us();
7287                std::hint::black_box(ctx.build_status_snapshot());
7288                eprintln!(
7289                    "cpu_hunt status wall_us={} cpu_us={}",
7290                    started.elapsed().as_micros(),
7291                    cpu_hunt_process_cpu_us().saturating_sub(cpu)
7292                );
7293            }
7294            let refreshes_before = cache.refresh_count_for_test();
7295            let metrics = Arc::new(DispatchPathMetrics::new());
7296            let (writer_tx, _writer_rx) = mpsc::channel(8);
7297            let (completion_tx, mut completion_rx) = mpsc::channel(8);
7298            let (lossy_tx, _lossy_rx) = mpsc::channel(8);
7299            let (reliable_tx, _reliable_rx) = mpsc::unbounded_channel();
7300            let senders = PushSenders {
7301                lossy_tx,
7302                reliable_tx,
7303                lossy_overflow: Arc::new(push::LossyOverflow::default()),
7304                lossy_seq: Arc::new(AtomicU64::new(0)),
7305                fleet_status_client: FleetStatusClient::channel(1).0,
7306            };
7307            let request = ModuleControlRequest::RouteBind {
7308                route_channel: 1,
7309                epoch: 1,
7310                target: RouteTarget::ToolProvider {
7311                    module_id: "aft".to_string(),
7312                },
7313                identity: subc_protocol::BindIdentity {
7314                    project_root: root.as_path().to_path_buf(),
7315                    harness: "opencode".to_string(),
7316                    session: "bind-health-work-count".to_string(),
7317                },
7318                principal: Some(subc_protocol::Principal::Direct),
7319                consumer_capabilities: None,
7320                admission_facts: Default::default(),
7321            };
7322            let frame = Frame::build_with_version(
7323                PROTOCOL_VERSION,
7324                FrameType::Request,
7325                control_flags(),
7326                0,
7327                0,
7328                1,
7329                serde_json::to_vec(&request).expect("bind body"),
7330            )
7331            .expect("bind frame");
7332            let mut pending_binds = HashMap::new();
7333            let started = Instant::now();
7334            let cpu = cpu_hunt_process_cpu_us();
7335            handle_control_request(
7336                &writer_tx,
7337                &frame,
7338                &app,
7339                &executor,
7340                &mut HashMap::new(),
7341                &mut pending_binds,
7342                &mut HashMap::new(),
7343                &mut HashMap::new(),
7344                &mut HashSet::new(),
7345                &mut HashMap::new(),
7346                &mut HashMap::new(),
7347                &mut HashMap::new(),
7348                &mut HashMap::new(),
7349                &mut HashMap::new(),
7350                &mut HashMap::new(),
7351                &Arc::new(StdMutex::new(HashMap::new())),
7352                &mut PendingSubcResponses::default(),
7353                &mut RetryBuffer::new(),
7354                &mut HashMap::new(),
7355                &Arc::new(Notify::new()),
7356                &completion_tx,
7357                &metrics,
7358                None,
7359                &cache,
7360                &senders,
7361                |request, _| Response::success(request.id, json!({})),
7362                Some(&dir.path().join("absent-user-config.json")),
7363                usize::MAX,
7364            )
7365            .await
7366            .expect("admit route bind");
7367            eprintln!(
7368                "route_bind_health admission_us={} cpu_us={} refreshes={}",
7369                started.elapsed().as_micros(),
7370                cpu_hunt_process_cpu_us().saturating_sub(cpu),
7371                cache.refresh_count_for_test() - refreshes_before
7372            );
7373            let completion = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
7374                .await
7375                .expect("configure completes")
7376                .expect("completion delivered");
7377            assert!(completion.configure_response.success);
7378            assert_eq!(pending_binds.len(), 1, "the bind must reach admission");
7379            assert_eq!(
7380                cache.refresh_count_for_test() - refreshes_before,
7381                0,
7382                "route admission must not perform a fleet-wide health census"
7383            );
7384        });
7385    }
7386
7387    #[test]
7388    fn channel_zero_health_response_does_not_wait_for_bash_background_db() {
7389        let (dir, root) = test_root("health-does-not-lock-bash-db");
7390        let executor = Arc::new(Executor::new());
7391        let ctx = test_ctx();
7392        ctx.set_harness(crate::harness::Harness::Opencode);
7393        let db = Arc::new(StdMutex::new(
7394            crate::db::open(&dir.path().join("health.db")).expect("open health test DB"),
7395        ));
7396        ctx.bash_background().set_db_pool(Arc::clone(&db));
7397        executor.register_actor(root, ctx);
7398
7399        let guard = db.lock().expect("hold bash-background DB mutex");
7400        let app = App::default_shared();
7401        let metrics = Arc::new(DispatchPathMetrics::new());
7402        let health_rollup_cache = Arc::new(HealthRollupCache::new());
7403        let frame = Frame::build_with_version(
7404            PROTOCOL_VERSION,
7405            FrameType::Request,
7406            control_flags(),
7407            0,
7408            0,
7409            77,
7410            Vec::new(),
7411        )
7412        .expect("health request frame");
7413        let (writer_tx, _writer_rx) = mpsc::channel::<WriterFrame>(1);
7414        let (done_tx, done_rx) = std::sync::mpsc::channel();
7415        let join = std::thread::spawn(move || {
7416            let runtime = tokio::runtime::Builder::new_current_thread()
7417                .enable_all()
7418                .build()
7419                .expect("health test runtime");
7420            let result = runtime.block_on(send_cached_health_response(
7421                &writer_tx,
7422                &frame,
7423                &app,
7424                &executor,
7425                &HashMap::new(),
7426                &metrics,
7427                &health_rollup_cache,
7428            ));
7429            done_tx.send(result).expect("report health result");
7430        });
7431
7432        let result = done_rx
7433            .recv_timeout(Duration::from_millis(500))
7434            .expect("channel-0 health blocked on bash-background DB mutex");
7435        result.expect("send cached health response");
7436        drop(guard);
7437        join.join().expect("health thread");
7438    }
7439
7440    #[test]
7441    fn maintenance_bg_runtime_refresh_deduplicates_shared_db_items() {
7442        let (_dir_a, root_a) = test_root("health-metric-root-a");
7443        let (_dir_b, root_b) = test_root("health-metric-root-b");
7444        let duplicate_key = "match\0session-1\0bash-0000000000000001\0watch-00000001";
7445        let snapshots = HashMap::from([
7446            (root_a, HashSet::from([duplicate_key.to_string()])),
7447            (root_b, HashSet::from([duplicate_key.to_string()])),
7448        ]);
7449        let metrics = DispatchPathMetrics::new();
7450
7451        record_bg_runtime_from_snapshots(&metrics, 2, 1, &snapshots);
7452
7453        assert_eq!(metrics.bg_runtime_for_test(), (2, 1, 1));
7454    }
7455
7456    #[test]
7457    fn initial_attach_error_classifier_distinguishes_transient_and_permanent_failures() {
7458        let transient_errors = vec![
7459            attach_error(io::ErrorKind::ConnectionRefused),
7460            attach_error(io::ErrorKind::TimedOut),
7461            attach_error(io::ErrorKind::ConnectionReset),
7462            auth_io_error(io::ErrorKind::ConnectionAborted),
7463            auth_io_error(io::ErrorKind::BrokenPipe),
7464            SubcError::Auth {
7465                endpoint: "127.0.0.1:1".to_string(),
7466                source: subc_transport::AuthError::UnexpectedEof {
7467                    stage: subc_transport::AuthStage::ServerProof,
7468                    expected: 4,
7469                    actual: 0,
7470                },
7471            },
7472            SubcError::Auth {
7473                endpoint: "127.0.0.1:1".to_string(),
7474                source: subc_transport::AuthError::Timeout {
7475                    stage: subc_transport::AuthStage::ServerProof,
7476                    deadline: AUTH_DEADLINE,
7477                },
7478            },
7479        ];
7480        for error in &transient_errors {
7481            assert_eq!(
7482                classify_attach_error(error),
7483                AttachErrorClass::Transient,
7484                "expected transient: {error}"
7485            );
7486        }
7487
7488        let permanent_errors = vec![
7489            attach_error(io::ErrorKind::PermissionDenied),
7490            auth_io_error(io::ErrorKind::InvalidData),
7491            SubcError::Auth {
7492                endpoint: "127.0.0.1:1".to_string(),
7493                source: subc_transport::AuthError::InvalidServerProof,
7494            },
7495            SubcError::Auth {
7496                endpoint: "127.0.0.1:1".to_string(),
7497                source: subc_transport::AuthError::DaemonIdMismatch,
7498            },
7499            SubcError::ConnectionFile {
7500                path: PathBuf::from("subc-connection.json"),
7501                source: subc_transport::ConnectionFileError::Invalid {
7502                    reason: "constructed invalid file".to_string(),
7503                },
7504            },
7505            SubcError::NoEndpoint {
7506                path: PathBuf::from("subc-connection.json"),
7507            },
7508            SubcError::InvalidEndpoint {
7509                path: PathBuf::from("subc-connection.json"),
7510                endpoint: "not-an-ip:1234".to_string(),
7511            },
7512        ];
7513        for error in &permanent_errors {
7514            assert_eq!(
7515                classify_attach_error(error),
7516                AttachErrorClass::Permanent,
7517                "expected permanent: {error}"
7518            );
7519        }
7520    }
7521
7522    #[test]
7523    fn incompatible_wire_version_is_rejected_before_tcp_connect() {
7524        let conn_dir = tempfile::tempdir().expect("connection tempdir");
7525        let conn_path = conn_dir.path().join("subc-connection.json");
7526        let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind listener");
7527        listener
7528            .set_nonblocking(true)
7529            .expect("set listener nonblocking");
7530        let port = listener.local_addr().expect("listener addr").port();
7531        connection_file::write_atomic(
7532            &conn_path,
7533            &connection_file::ConnectionInfo {
7534                schema: connection_file::SCHEMA_VERSION,
7535                wire_version: Some(PROTOCOL_VERSION.wrapping_add(1)),
7536                endpoints: vec![connection_file::Endpoint {
7537                    host: "127.0.0.1".to_string(),
7538                    port,
7539                }],
7540                key: vec![0x42; subc_transport::KEY_LEN],
7541                daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
7542                pid: std::process::id(),
7543                daemon_ver: "subc-test".to_string(),
7544            },
7545        )
7546        .expect("write connection file");
7547
7548        let runtime = tokio::runtime::Builder::new_current_thread()
7549            .enable_all()
7550            .build()
7551            .expect("test runtime");
7552        let result = runtime.block_on(connect_and_authenticate_with_policy(
7553            &conn_path,
7554            AttachRetryPolicy {
7555                budget: Duration::from_secs(1),
7556                initial_backoff: Duration::from_millis(5),
7557                max_backoff: Duration::from_millis(10),
7558                jitter_percent: 0,
7559            },
7560            None,
7561        ));
7562        assert!(matches!(
7563            result,
7564            Err(SubcError::ConnectionFile {
7565                source: connection_file::ConnectionFileError::WireVersionMismatch { .. },
7566                ..
7567            })
7568        ));
7569        assert!(matches!(
7570            listener.accept(),
7571            Err(error) if error.kind() == std::io::ErrorKind::WouldBlock
7572        ));
7573    }
7574
7575    #[test]
7576    fn initial_attach_unreachable_endpoint_retries_until_budget_then_fails_loud() {
7577        let conn_dir = tempfile::tempdir().expect("connection tempdir");
7578        let conn_path = conn_dir.path().join("subc-connection.json");
7579        let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve port");
7580        let port = listener.local_addr().expect("reserved addr").port();
7581        drop(listener);
7582        connection_file::write_atomic(
7583            &conn_path,
7584            &connection_file::ConnectionInfo {
7585                schema: connection_file::SCHEMA_VERSION,
7586                wire_version: Some(PROTOCOL_VERSION),
7587                endpoints: vec![connection_file::Endpoint {
7588                    host: "127.0.0.1".to_string(),
7589                    port,
7590                }],
7591                key: vec![0x42; subc_transport::KEY_LEN],
7592                daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
7593                pid: std::process::id(),
7594                daemon_ver: "subc-test".to_string(),
7595            },
7596        )
7597        .expect("write connection file");
7598
7599        let policy = AttachRetryPolicy {
7600            budget: Duration::from_millis(40),
7601            initial_backoff: Duration::from_millis(5),
7602            max_backoff: Duration::from_millis(10),
7603            jitter_percent: 0,
7604        };
7605        let runtime = tokio::runtime::Builder::new_current_thread()
7606            .enable_all()
7607            .build()
7608            .expect("test runtime");
7609        let started_at = Instant::now();
7610        let result = runtime.block_on(connect_and_authenticate_with_policy(
7611            &conn_path, policy, None,
7612        ));
7613        let elapsed = started_at.elapsed();
7614        let error = match result {
7615            Ok(_) => panic!("unreachable endpoint unexpectedly attached"),
7616            Err(error) => error,
7617        };
7618
7619        assert!(matches!(error, SubcError::Connect { .. }), "{error}");
7620        assert!(
7621            elapsed >= Duration::from_millis(35),
7622            "retry budget ended too early: {elapsed:?}"
7623        );
7624        assert!(
7625            elapsed < Duration::from_secs(1),
7626            "retry budget was not bounded: {elapsed:?}"
7627        );
7628    }
7629
7630    fn due_maintenance_jobs_without_actor_context(
7631        live_roots: &mut HashMap<ProjectRootId, RootMeta>,
7632        budget: usize,
7633        pending_bind_roots: &HashSet<ProjectRootId>,
7634    ) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
7635        due_maintenance_jobs(
7636            live_roots,
7637            None,
7638            &HashMap::new(),
7639            &BgWakePending::new(),
7640            budget,
7641            pending_bind_roots,
7642        )
7643    }
7644
7645    fn actor_ctx_with_dirty_search_index(
7646        root: &Path,
7647        storage: &Path,
7648        file_name: &str,
7649        old_contents: &str,
7650        new_contents: &str,
7651    ) -> (Arc<AppContext>, PathBuf, PathBuf) {
7652        let file = root.join(file_name);
7653        std::fs::write(&file, old_contents).expect("write source");
7654        let canonical_root = std::fs::canonicalize(root).expect("canonical root");
7655        let ctx = Arc::new(AppContext::new(
7656            Box::new(crate::parser::TreeSitterProvider::new()),
7657            Config {
7658                project_root: Some(root.to_path_buf()),
7659                storage_dir: Some(storage.to_path_buf()),
7660                ..Config::default()
7661            },
7662        ));
7663        ctx.set_canonical_cache_root(canonical_root.clone());
7664
7665        let cache_dir = crate::search_index::resolve_cache_dir(&canonical_root, Some(storage));
7666        let mut index = crate::search_index::SearchIndex::build(&canonical_root);
7667        let git_head = index.stored_git_head().map(str::to_owned);
7668        index.write_to_disk(&cache_dir, git_head.as_deref());
7669
7670        std::fs::write(&file, new_contents).expect("edit source");
7671        index.update_file(&file);
7672        *ctx.search_index()
7673            .write()
7674            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
7675        (ctx, canonical_root, cache_dir)
7676    }
7677
7678    #[test]
7679    fn graceful_shutdown_flushes_every_actor_search_index() {
7680        let storage = tempfile::tempdir().expect("storage tempdir");
7681        let (root1_dir, root1) = test_root("shutdown-flush-root-1");
7682        let (root2_dir, root2) = test_root("shutdown-flush-root-2");
7683        let (ctx1, canonical_root1, cache_dir1) = actor_ctx_with_dirty_search_index(
7684            root1_dir.path(),
7685            storage.path(),
7686            "alpha.txt",
7687            "old actor one token\n",
7688            "new actor one token\n",
7689        );
7690        let (ctx2, canonical_root2, cache_dir2) = actor_ctx_with_dirty_search_index(
7691            root2_dir.path(),
7692            storage.path(),
7693            "beta.txt",
7694            "old actor two token\n",
7695            "new actor two token\n",
7696        );
7697
7698        let executor = Executor::new();
7699        assert!(executor.register_actor(root1.clone(), Arc::clone(&ctx1)));
7700        assert!(executor.register_actor(root2.clone(), Arc::clone(&ctx2)));
7701
7702        flush_actor_indexes_on_graceful_shutdown(&executor.actor_contexts());
7703
7704        let mut restored1 =
7705            crate::search_index::SearchIndex::read_from_disk(&cache_dir1, &canonical_root1)
7706                .expect("load flushed root one index");
7707        restored1.ready = true;
7708        assert_eq!(
7709            restored1
7710                .grep("new actor one token", true, &[], &[], &canonical_root1, 10)
7711                .matches
7712                .len(),
7713            1,
7714            "graceful subc shutdown should flush the first root's trigram delta"
7715        );
7716
7717        let mut restored2 =
7718            crate::search_index::SearchIndex::read_from_disk(&cache_dir2, &canonical_root2)
7719                .expect("load flushed root two index");
7720        restored2.ready = true;
7721        assert_eq!(
7722            restored2
7723                .grep("new actor two token", true, &[], &[], &canonical_root2, 10)
7724                .matches
7725                .len(),
7726            1,
7727            "graceful subc shutdown should flush every registered root"
7728        );
7729    }
7730
7731    #[test]
7732    fn idle_root_reaper_closes_artifacts_and_stops_watcher() {
7733        let _ = env_logger::builder().is_test(true).try_init();
7734        let (root_dir, root) = test_root("idle-root-reaper");
7735        let storage = tempfile::tempdir().expect("storage tempdir");
7736        std::fs::write(
7737            root_dir.path().join("main.rs"),
7738            "fn entry() { leaf(); }\nfn leaf() {}\n",
7739        )
7740        .expect("source file");
7741        let canonical_root = std::fs::canonicalize(root_dir.path()).expect("canonical root");
7742        let app = App::default_shared();
7743        let ctx = Arc::new(AppContext::from_app(
7744            Arc::clone(&app),
7745            Config {
7746                project_root: Some(canonical_root.clone()),
7747                storage_dir: Some(storage.path().to_path_buf()),
7748                callgraph_store: true,
7749                search_index: true,
7750                ..Config::default()
7751            },
7752        ));
7753        ctx.set_canonical_cache_root(canonical_root.clone());
7754        let project_key = crate::search_index::artifact_cache_key(&canonical_root);
7755        crate::root_cache::configure_artifact_access(&canonical_root, &project_key, false);
7756        assert!(ctx
7757            .ensure_callgraph_store()
7758            .expect("build callgraph store")
7759            .is_some());
7760
7761        let cache_dir =
7762            crate::search_index::resolve_cache_dir(&canonical_root, Some(storage.path()));
7763        let mut index = crate::search_index::SearchIndex::build(&canonical_root);
7764        let git_head = index.stored_git_head().map(str::to_owned);
7765        index.write_to_disk(&cache_dir, git_head.as_deref());
7766        *ctx.search_index()
7767            .write()
7768            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
7769        // Seed a completed warm verification so the test can prove eviction
7770        // downgrades it to Strict rather than passing vacuously.
7771        let seeded_generation =
7772            crate::cache_freshness::artifact_generation(&cache_dir.join("cache.bin"))
7773                .expect("seeded artifact generation");
7774        crate::cache_freshness::record_verify_completed(
7775            &canonical_root,
7776            crate::cache_freshness::VerifyArtifact::Search,
7777            Some(seeded_generation),
7778        );
7779        assert!(
7780            matches!(
7781                crate::cache_freshness::warm_verify_plan(
7782                    canonical_root.as_path(),
7783                    crate::cache_freshness::VerifyArtifact::Search,
7784                    Some(seeded_generation),
7785                ),
7786                crate::cache_freshness::WarmVerifyPlan::Skip
7787            ),
7788            "memo must be warm before eviction for the downgrade assertion to bite"
7789        );
7790
7791        let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
7792        let _dispatch_tx = dispatch_tx;
7793        let shutdown = Arc::new(AtomicBool::new(false));
7794        let thread_shutdown = Arc::clone(&shutdown);
7795        let join = std::thread::spawn(move || {
7796            while !thread_shutdown.load(Ordering::SeqCst) {
7797                std::thread::yield_now();
7798            }
7799        });
7800        ctx.install_watcher_runtime(
7801            dispatch_rx,
7802            crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
7803        );
7804        wait_for_watcher_count(&ctx, 1);
7805
7806        let executor = Arc::new(Executor::new());
7807        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7808        ctx.mark_subc_unbound();
7809        let mut live_roots = HashMap::new();
7810        let mut meta = RootMeta::new(Instant::now());
7811        meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
7812        meta.unbound_quiesced = true;
7813        live_roots.insert(root.clone(), meta);
7814
7815        let message = idle_root_eviction_message(&root, &ctx.memory_root_snapshot(), None);
7816        assert!(message.contains("evicted idle root"));
7817        assert!(message.contains("freed ~"));
7818        assert!(message.contains("semantic"));
7819        assert!(!message.contains("semantic not estimated retained"));
7820        assert!(message.contains("trigram"));
7821        assert!(message.contains("retained: bash"));
7822        assert!(message.contains("parser_pool"));
7823
7824        assert_eq!(
7825            reap_idle_roots(
7826                Instant::now(),
7827                &mut live_roots,
7828                &HashMap::new(),
7829                &HashMap::new(),
7830                &executor,
7831                &DispatchPathMetrics::new(),
7832            )
7833            .evicted,
7834            1
7835        );
7836        assert!(ctx.search_index().read().unwrap().is_none());
7837        wait_for_watcher_count(&ctx, 0);
7838        // The watcher stopped with the eviction, so the idle interval is
7839        // unobserved: the pre-seeded warm-verify memo (Skip) must fall back to
7840        // strict content verification (stat-first would miss same-size,
7841        // preserved-mtime edits made while nobody was watching).
7842        assert!(
7843            matches!(
7844                crate::cache_freshness::warm_verify_plan(
7845                    canonical_root.as_path(),
7846                    crate::cache_freshness::VerifyArtifact::Search,
7847                    Some(seeded_generation),
7848                ),
7849                crate::cache_freshness::WarmVerifyPlan::Strict
7850            ),
7851            "idle eviction must force strict re-verification"
7852        );
7853        assert!(
7854            crate::search_index::SearchIndex::read_from_disk(&cache_dir, &canonical_root).is_some()
7855        );
7856        ctx.mark_subc_bound();
7857        assert!(ctx
7858            .ensure_callgraph_store()
7859            .expect("reopen callgraph store")
7860            .is_some());
7861        assert!(live_roots[&root].idle_artifacts_evicted);
7862    }
7863
7864    #[test]
7865    fn idle_root_reaper_applies_ttl_to_unbound_roots() {
7866        let (_root_dir, root) = test_root("idle-root-ttl-gate");
7867        let ctx = test_ctx();
7868        let executor = Arc::new(Executor::new());
7869        assert!(executor.register_actor(root.clone(), ctx));
7870        let ctx = executor.actor_context(&root).expect("actor context");
7871        ctx.mark_subc_unbound();
7872        let now = Instant::now();
7873        let mut meta = RootMeta::new(now);
7874        meta.unbound_quiesced = true;
7875        let mut live_roots = HashMap::from([(root.clone(), meta)]);
7876
7877        // A recently-unbound root stays warm: a transient unbind (host
7878        // restart) must not pay the strict-verify + forced-rebuild teardown
7879        // on the next maintenance sweep.
7880        assert_eq!(
7881            reap_idle_roots(
7882                now,
7883                &mut live_roots,
7884                &HashMap::new(),
7885                &HashMap::new(),
7886                &executor,
7887                &DispatchPathMetrics::new(),
7888            )
7889            .evicted,
7890            0
7891        );
7892        assert!(!live_roots[&root].idle_artifacts_evicted);
7893
7894        // Past the TTL the same unbound root pays the full teardown. The
7895        // pending reconciliation paths retained across the transient-unbind
7896        // window would block eviction forever through
7897        // `artifact_eviction_blocked`; the reaper disposes them because the
7898        // strict gap invalidation subsumes their purpose.
7899        ctx.add_pending_search_index_paths([root.as_path().join("retained.rs")]);
7900        assert_eq!(
7901            reap_idle_roots(
7902                now + IDLE_ROOT_TTL,
7903                &mut live_roots,
7904                &HashMap::new(),
7905                &HashMap::new(),
7906                &executor,
7907                &DispatchPathMetrics::new(),
7908            )
7909            .evicted,
7910            1
7911        );
7912        assert!(live_roots[&root].idle_artifacts_evicted);
7913        assert!(
7914            ctx.take_pending_search_index_paths().is_empty(),
7915            "TTL eviction must dispose retained pending reconciliation paths"
7916        );
7917    }
7918
7919    #[test]
7920    fn blocked_ttl_eviction_restores_taken_pending_reconciliation_state() {
7921        let (_root_dir, root) = test_root("ttl-eviction-blocked-restore");
7922        let ctx = test_ctx();
7923        let executor = Arc::new(Executor::new());
7924        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7925        ctx.mark_subc_unbound();
7926
7927        // Retained pending path from the transient-unbind window plus a
7928        // SECONDARY eviction blocker (a non-ready resident search index, the
7929        // dirty-index blocker in artifact_eviction_blocked). Disposal must be
7930        // transactional: the blocked eviction may be followed by a rebind, and
7931        // the path is the only repair record for its consumed watcher event.
7932        let pending = root.as_path().join("edited-while-unbound.rs");
7933        ctx.add_pending_search_index_paths([pending.clone()]);
7934        let dirty_source = root.as_path().join("dirty.rs");
7935        std::fs::write(&dirty_source, "fn dirty() {}\n").expect("dirty source");
7936        let mut dirty = crate::search_index::SearchIndex::new();
7937        dirty.ready = true;
7938        dirty.update_file(&dirty_source);
7939        *ctx.search_index()
7940            .write()
7941            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(dirty);
7942        assert!(ctx.artifact_eviction_blocked());
7943
7944        let mut live_roots = HashMap::new();
7945        let mut meta = RootMeta::new(Instant::now());
7946        meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
7947        meta.unbound_quiesced = true;
7948        live_roots.insert(root.clone(), meta);
7949
7950        assert_eq!(
7951            reap_idle_roots(
7952                Instant::now(),
7953                &mut live_roots,
7954                &HashMap::new(),
7955                &HashMap::new(),
7956                &executor,
7957                &DispatchPathMetrics::new(),
7958            )
7959            .evicted,
7960            0,
7961            "the dirty index must still block this eviction"
7962        );
7963        assert_eq!(
7964            ctx.take_pending_search_index_paths(),
7965            vec![pending],
7966            "a blocked eviction must restore the taken pending paths"
7967        );
7968    }
7969
7970    #[test]
7971    fn idle_reap_with_bound_route_keeps_watcher_running() {
7972        let (_root_dir, root) = test_root("bound-root-reap-gate");
7973        let ctx = test_ctx();
7974        let executor = Arc::new(Executor::new());
7975        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7976
7977        let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
7978        let _dispatch_tx = dispatch_tx;
7979        let shutdown = Arc::new(AtomicBool::new(false));
7980        let thread_shutdown = Arc::clone(&shutdown);
7981        let join = std::thread::spawn(move || {
7982            while !thread_shutdown.load(Ordering::SeqCst) {
7983                std::thread::yield_now();
7984            }
7985        });
7986        ctx.install_watcher_runtime(
7987            dispatch_rx,
7988            crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
7989        );
7990
7991        let mut meta = RootMeta::new(Instant::now());
7992        meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
7993        let mut live_roots = HashMap::from([(root.clone(), meta)]);
7994        let bound = HashMap::from([(root, HashSet::from([route_key(7, 1)]))]);
7995        assert_eq!(
7996            reap_idle_roots(
7997                Instant::now(),
7998                &mut live_roots,
7999                &HashMap::new(),
8000                &bound,
8001                &executor,
8002                &DispatchPathMetrics::new(),
8003            )
8004            .evicted,
8005            0
8006        );
8007        wait_for_watcher_count(&ctx, 1);
8008        ctx.stop_watcher_runtime_in_background();
8009        wait_for_watcher_count(&ctx, 0);
8010    }
8011
8012    #[test]
8013    fn deleted_root_with_bound_route_is_reclaimed_after_confirmation_and_routes_are_purged() {
8014        let (root_dir, root) = test_root("deleted-bound-root-reap");
8015        let executor = Arc::new(Executor::new());
8016        assert!(executor.register_actor(root.clone(), test_ctx()));
8017        root_dir.close().expect("delete project root");
8018
8019        let route = route_key(19, 3);
8020        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
8021        let cancel_signal = PersistentCancelSignal::new();
8022        let mut routes = HashMap::from([(route, route_identity(&root, "deleted-route"))]);
8023        let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
8024        let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
8025        let mut route_bash_cancels = HashMap::from([(
8026            route,
8027            bash::RouteBashCancel {
8028                token: cancel_signal.clone(),
8029                active_waits: 0,
8030            },
8031        )]);
8032        let metrics = DispatchPathMetrics::new();
8033
8034        let first = reap_idle_roots(
8035            Instant::now(),
8036            &mut live_roots,
8037            &HashMap::new(),
8038            &root_channels,
8039            &executor,
8040            &metrics,
8041        );
8042        assert!(first.forgotten_deleted_roots.is_empty());
8043        assert!(executor.actor_registered(&root));
8044
8045        let mut forgotten = Vec::new();
8046        for _ in 0..100 {
8047            let outcome = reap_idle_roots(
8048                Instant::now(),
8049                &mut live_roots,
8050                &HashMap::new(),
8051                &root_channels,
8052                &executor,
8053                &metrics,
8054            );
8055            if !outcome.forgotten_deleted_roots.is_empty() {
8056                forgotten = outcome.forgotten_deleted_roots;
8057                break;
8058            }
8059            std::thread::sleep(Duration::from_millis(10));
8060        }
8061        assert_eq!(forgotten, vec![root.clone()]);
8062        assert!(!executor.actor_registered(&root));
8063
8064        let mut retry_buffer = HashMap::new();
8065        let mut reclaimed_routes = ReclaimedRoutes::default();
8066        let mut session_identity = HashMap::new();
8067        let mut push_buffer = HashMap::new();
8068        let mut bg_subs = HashMap::from([(
8069            route,
8070            BgSub {
8071                corr: 77,
8072                ver: PROTOCOL_VERSION,
8073                flags: control_flags(),
8074                root: root.clone(),
8075                session: "deleted-route".to_string(),
8076            },
8077        )]);
8078        let mut bg_sub_by_session = HashMap::from([(
8079            (root.clone(), "deleted-route".to_string()),
8080            HashSet::from([route]),
8081        )]);
8082        let mut bg_wake_pending =
8083            BgWakePending::from([(route, BgWakeState::armed(Instant::now()))]);
8084        let mut bg_wake_epoch = HashMap::new();
8085        let mut pending_bash_asks = HashMap::new();
8086        let active_tool_calls: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
8087        health::take_bg_observability_logs_for_test();
8088        purge_deleted_root_residents(
8089            &root,
8090            &mut routes,
8091            &mut root_channels,
8092            &mut installed_route_epochs,
8093            &mut route_bash_cancels,
8094            &active_tool_calls,
8095            executor.as_ref(),
8096            &mut retry_buffer,
8097            &mut reclaimed_routes,
8098            &mut session_identity,
8099            &mut push_buffer,
8100            &mut bg_subs,
8101            &mut bg_sub_by_session,
8102            &mut bg_wake_pending,
8103            &mut bg_wake_epoch,
8104            &mut pending_bash_asks,
8105            &metrics,
8106        );
8107
8108        assert!(routes.is_empty());
8109        assert!(root_channels.is_empty());
8110        assert!(installed_route_epochs.is_empty());
8111        assert!(route_bash_cancels.is_empty());
8112        assert!(reclaimed_routes.contains(route));
8113        assert!(cancel_signal.is_cancelled());
8114        assert_eq!(
8115            health::take_bg_observability_logs_for_test(),
8116            vec![format!(
8117                "subc bg subscription: ended root={} session=deleted-route channel=19@3 cause=root-reclaim suppressed=0",
8118                root.as_path().display()
8119            )]
8120        );
8121    }
8122
8123    /// The control for the deleted-root reclamation above: a root whose
8124    /// directory still EXISTS must stay retained while it holds a bound route,
8125    /// even with every other reap precondition satisfied. Without this, the
8126    /// suite cannot tell "reclaim roots that are provably gone" apart from
8127    /// "reap any root that looks idle" — the second would tear down live
8128    /// sessions, and both satisfy the deleted-root tests.
8129    #[test]
8130    fn live_root_with_bound_route_is_never_reclaimed() {
8131        let (_root_dir, root) = test_root("live-bound-root-retained");
8132        let ctx = test_ctx();
8133        ctx.mark_subc_unbound();
8134        let executor = Arc::new(Executor::new());
8135        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
8136
8137        let route = RouteChannel {
8138            channel: 7,
8139            epoch: 1,
8140        };
8141        let mut meta = RootMeta::new(Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1));
8142        meta.unbound_quiesced = true;
8143        let mut live_roots = HashMap::from([(root.clone(), meta)]);
8144        let root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
8145
8146        // Sweep three times because reclamation requires the root path to be
8147        // observed missing on two CONSECUTIVE sweeps. A single sweep would pass
8148        // here even if reclamation were wrongly unconditional, since the first
8149        // absence never reclaims on its own.
8150        for _ in 0..3 {
8151            let outcome = reap_idle_roots(
8152                Instant::now(),
8153                &mut live_roots,
8154                &HashMap::new(),
8155                &root_channels,
8156                &executor,
8157                &DispatchPathMetrics::new(),
8158            );
8159            assert!(
8160                outcome.forgotten_deleted_roots.is_empty(),
8161                "a root whose directory exists must never be forgotten"
8162            );
8163        }
8164
8165        assert!(live_roots.contains_key(&root), "live root must be retained");
8166        assert!(
8167            executor.actor_registered(&root),
8168            "live root's actor must survive"
8169        );
8170        assert!(
8171            root.as_path().exists(),
8172            "test vehicle must keep the directory alive; otherwise this control proves nothing"
8173        );
8174    }
8175
8176    #[test]
8177    fn deleted_root_is_not_reclaimed_on_first_absence_observation() {
8178        let (root_dir, root) = test_root("deleted-root-first-observation");
8179        let ctx = test_ctx();
8180        ctx.mark_subc_unbound();
8181        let executor = Arc::new(Executor::new());
8182        assert!(executor.register_actor(root.clone(), ctx));
8183        root_dir.close().expect("delete project root");
8184
8185        let mut meta = RootMeta::new(Instant::now());
8186        meta.unbound_quiesced = true;
8187        let mut live_roots = HashMap::from([(root.clone(), meta)]);
8188        let outcome = reap_idle_roots(
8189            Instant::now(),
8190            &mut live_roots,
8191            &HashMap::new(),
8192            &HashMap::new(),
8193            &executor,
8194            &DispatchPathMetrics::new(),
8195        );
8196
8197        assert!(outcome.forgotten_deleted_roots.is_empty());
8198        assert!(live_roots.contains_key(&root));
8199        assert!(executor.actor_registered(&root));
8200    }
8201
8202    fn spawn_background_for_root(
8203        ctx: &AppContext,
8204        root: &ProjectRootId,
8205        storage: &tempfile::TempDir,
8206        session_id: &str,
8207    ) -> (String, u32) {
8208        // Windows refuses to delete a directory that is a running process's
8209        // cwd (ERROR_SHARING_VIOLATION), so the task must not live inside the
8210        // project root these tests delete. The kill path matches on the task's
8211        // registered project_root, not its cwd, so pointing the cwd at task
8212        // storage keeps the association under test intact.
8213        let command = if cfg!(windows) {
8214            // timeout.exe requires a console; ping is the standard sleep shim.
8215            "ping -n 31 127.0.0.1 > nul"
8216        } else {
8217            "sleep 30"
8218        };
8219        let task_id = ctx
8220            .bash_background()
8221            .spawn(
8222                crate::sandbox_spawn::SpawnPlan::Unsandboxed,
8223                command,
8224                session_id.to_string(),
8225                storage.path().to_path_buf(),
8226                HashMap::new(),
8227                Some(Duration::from_secs(60)),
8228                storage.path().to_path_buf(),
8229                8,
8230                true,
8231                false,
8232                Some(root.as_path().to_path_buf()),
8233            )
8234            .expect("spawn background task");
8235        let snapshot = ctx
8236            .bash_background()
8237            .status(
8238                &task_id,
8239                session_id,
8240                Some(root.as_path()),
8241                Some(storage.path()),
8242                0,
8243            )
8244            .expect("background task status");
8245        (task_id, snapshot.child_pid.expect("background child pid"))
8246    }
8247
8248    fn wait_for_background_exit(pid: u32) {
8249        let deadline = Instant::now() + Duration::from_secs(5);
8250        while crate::bash_background::process::is_process_alive(pid) {
8251            assert!(
8252                Instant::now() < deadline,
8253                "background task process survived kill"
8254            );
8255            std::thread::sleep(Duration::from_millis(20));
8256        }
8257    }
8258
8259    #[test]
8260    fn deleted_root_reclaims_background_task_after_two_absence_sweeps() {
8261        let (root_dir, root) = test_root("deleted-root-background-task");
8262        let storage = tempfile::tempdir().expect("task storage");
8263        let ctx = test_ctx();
8264        let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "reclaim-session");
8265        assert!(crate::bash_background::process::is_process_alive(pid));
8266
8267        let executor = Arc::new(Executor::new());
8268        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
8269        assert!(executor.actor_is_idle(&root));
8270        root_dir.close().expect("delete project root");
8271        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
8272        let pending_binds = HashMap::new();
8273        let root_channels = HashMap::new();
8274        let metrics = DispatchPathMetrics::new();
8275
8276        let first = reap_idle_roots(
8277            Instant::now(),
8278            &mut live_roots,
8279            &pending_binds,
8280            &root_channels,
8281            &executor,
8282            &metrics,
8283        );
8284        assert!(first.forgotten_deleted_roots.is_empty());
8285        assert!(crate::bash_background::process::is_process_alive(pid));
8286
8287        let outcome = reap_until_forgotten(
8288            &root,
8289            &mut live_roots,
8290            &pending_binds,
8291            &root_channels,
8292            &executor,
8293            &metrics,
8294        );
8295        assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
8296        wait_for_background_exit(pid);
8297
8298        let snapshot = ctx
8299            .bash_background()
8300            .status(
8301                &task_id,
8302                "reclaim-session",
8303                Some(root.as_path()),
8304                Some(storage.path()),
8305                0,
8306            )
8307            .expect("reclaimed task status");
8308        assert_eq!(snapshot.info.status, BgTaskStatus::Killed);
8309        assert_eq!(
8310            snapshot.info.status_reason.as_deref(),
8311            Some(crate::bash_background::registry::ROOT_RECLAIMED_REASON)
8312        );
8313        assert_eq!(
8314            serde_json::to_value(&snapshot).expect("serialize bash status")["status_reason"],
8315            crate::bash_background::registry::ROOT_RECLAIMED_REASON
8316        );
8317        let completion = ctx
8318            .bash_background()
8319            .drain_completions_for_session(Some("reclaim-session"))
8320            .pop()
8321            .expect("reclaimed task completion");
8322        assert_eq!(
8323            completion.status_reason.as_deref(),
8324            Some(crate::bash_background::registry::ROOT_RECLAIMED_REASON)
8325        );
8326    }
8327
8328    #[test]
8329    fn existing_unbound_root_keeps_background_task_alive_across_sweeps() {
8330        let (root_dir, root) = test_root("existing-root-background-task");
8331        let storage = tempfile::tempdir().expect("task storage");
8332        let ctx = test_ctx();
8333        ctx.mark_subc_unbound();
8334        let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "existing-session");
8335
8336        let executor = Arc::new(Executor::new());
8337        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
8338        let mut meta = RootMeta::new(
8339            Instant::now()
8340                .checked_sub(IDLE_ROOT_TTL + Duration::from_secs(1))
8341                .expect("old root timestamp"),
8342        );
8343        meta.unbound_quiesced = true;
8344        let mut live_roots = HashMap::from([(root.clone(), meta)]);
8345        let pending_binds = HashMap::new();
8346        let root_channels = HashMap::new();
8347        let metrics = DispatchPathMetrics::new();
8348
8349        for _ in 0..8 {
8350            reap_idle_roots(
8351                Instant::now(),
8352                &mut live_roots,
8353                &pending_binds,
8354                &root_channels,
8355                &executor,
8356                &metrics,
8357            );
8358            std::thread::sleep(Duration::from_millis(10));
8359        }
8360        assert!(root_dir.path().exists());
8361        assert!(crate::bash_background::process::is_process_alive(pid));
8362        let snapshot = ctx
8363            .bash_background()
8364            .status(
8365                &task_id,
8366                "existing-session",
8367                Some(root.as_path()),
8368                Some(storage.path()),
8369                0,
8370            )
8371            .expect("existing task status");
8372        assert_eq!(snapshot.info.status, BgTaskStatus::Running);
8373        let _ = ctx.bash_background().kill(&task_id, "existing-session");
8374        wait_for_background_exit(pid);
8375    }
8376
8377    #[test]
8378    fn restored_root_between_absence_sweeps_keeps_background_task_alive() {
8379        let (root_dir, root) = test_root("restored-root-background-task");
8380        let storage = tempfile::tempdir().expect("task storage");
8381        let ctx = test_ctx();
8382        let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "restored-session");
8383
8384        let executor = Arc::new(Executor::new());
8385        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
8386        root_dir.close().expect("delete project root");
8387        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
8388        let pending_binds = HashMap::new();
8389        let root_channels = HashMap::new();
8390        let metrics = DispatchPathMetrics::new();
8391
8392        let first = reap_idle_roots(
8393            Instant::now(),
8394            &mut live_roots,
8395            &pending_binds,
8396            &root_channels,
8397            &executor,
8398            &metrics,
8399        );
8400        assert!(first.forgotten_deleted_roots.is_empty());
8401        std::fs::create_dir_all(root.as_path()).expect("restore project root");
8402        let second = reap_idle_roots(
8403            Instant::now(),
8404            &mut live_roots,
8405            &pending_binds,
8406            &root_channels,
8407            &executor,
8408            &metrics,
8409        );
8410        assert!(second.forgotten_deleted_roots.is_empty());
8411        assert!(crate::bash_background::process::is_process_alive(pid));
8412        let _ = ctx.bash_background().kill(&task_id, "restored-session");
8413        wait_for_background_exit(pid);
8414    }
8415
8416    #[test]
8417    fn observing_root_again_resets_deleted_sweep_confirmation() {
8418        let (root_dir, root) = test_root("deleted-root-observation-reset");
8419        let ctx = test_ctx();
8420        ctx.mark_subc_unbound();
8421        let executor = Arc::new(Executor::new());
8422        assert!(executor.register_actor(root.clone(), ctx));
8423        root_dir.close().expect("delete project root");
8424
8425        let mut meta = RootMeta::new(Instant::now());
8426        meta.unbound_quiesced = true;
8427        let mut live_roots = HashMap::from([(root.clone(), meta)]);
8428        let pending_binds = HashMap::new();
8429        let root_channels = HashMap::new();
8430        let metrics = DispatchPathMetrics::new();
8431
8432        let first = reap_idle_roots(
8433            Instant::now(),
8434            &mut live_roots,
8435            &pending_binds,
8436            &root_channels,
8437            &executor,
8438            &metrics,
8439        );
8440        assert!(first.forgotten_deleted_roots.is_empty());
8441
8442        std::fs::create_dir_all(root.as_path()).expect("restore project root");
8443        reap_idle_roots(
8444            Instant::now(),
8445            &mut live_roots,
8446            &pending_binds,
8447            &root_channels,
8448            &executor,
8449            &metrics,
8450        );
8451        std::fs::remove_dir_all(root.as_path()).expect("delete project root again");
8452
8453        let after_reset = reap_idle_roots(
8454            Instant::now(),
8455            &mut live_roots,
8456            &pending_binds,
8457            &root_channels,
8458            &executor,
8459            &metrics,
8460        );
8461        assert!(after_reset.forgotten_deleted_roots.is_empty());
8462        assert!(live_roots.contains_key(&root));
8463        assert!(executor.actor_registered(&root));
8464    }
8465
8466    #[test]
8467    fn deleted_idle_root_is_fully_forgotten_and_status_counts_drop() {
8468        let (root_dir, root) = test_root("deleted-root-reap");
8469        let app = App::default_shared();
8470        let ctx = Arc::new(AppContext::from_app(
8471            Arc::clone(&app),
8472            Config {
8473                project_root: Some(root.as_path().to_path_buf()),
8474                ..Config::default()
8475            },
8476        ));
8477        ctx.set_canonical_cache_root(root.as_path().to_path_buf());
8478        ctx.mark_subc_unbound();
8479        let executor = Arc::new(Executor::new());
8480        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
8481        assert_eq!(app.actor_root_count(), 1);
8482        drop(ctx);
8483        root_dir.close().expect("delete project root");
8484
8485        let mut meta = RootMeta::new(Instant::now());
8486        meta.unbound_quiesced = true;
8487        let mut live_roots = HashMap::from([(root.clone(), meta)]);
8488        let outcome = reap_until_forgotten(
8489            &root,
8490            &mut live_roots,
8491            &HashMap::new(),
8492            &HashMap::new(),
8493            &executor,
8494            &DispatchPathMetrics::new(),
8495        );
8496        assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
8497        assert!(!executor.actor_registered(&root));
8498        assert!(!live_roots.contains_key(&root));
8499        wait_for_actor_root_count(&app, 0);
8500
8501        let status_ctx = AppContext::from_app(app, Config::default());
8502        let status = status_ctx.build_status_snapshot();
8503        assert_eq!(status["runtime"]["live_actor_roots"], 0);
8504        assert_eq!(status["runtime"]["open_routes"], 0);
8505    }
8506
8507    #[test]
8508    fn deleted_root_reap_blocker_census_is_exposed_in_health_metrics() {
8509        let (root_dir, root) = test_root("deleted-root-reap-census");
8510        let executor = Arc::new(Executor::new());
8511        assert!(executor.register_actor(root.clone(), test_ctx()));
8512        root_dir.close().expect("delete project root");
8513
8514        let mut live_roots = HashMap::from([(root, RootMeta::new(Instant::now()))]);
8515        let metrics = DispatchPathMetrics::new();
8516        let outcome = reap_idle_roots(
8517            Instant::now(),
8518            &mut live_roots,
8519            &HashMap::new(),
8520            &HashMap::new(),
8521            &executor,
8522            &metrics,
8523        );
8524        assert_eq!(outcome.evicted, 0);
8525
8526        let app = crate::context::App::default_shared();
8527        let health_rollup_cache = HealthRollupCache::new();
8528        health_rollup_cache.refresh(&executor, &app);
8529        let report = build_health_report(
8530            &health_rollup_cache,
8531            &executor,
8532            &HashMap::new(),
8533            &metrics,
8534            &app,
8535        );
8536        let reap = report
8537            .metrics
8538            .as_ref()
8539            .and_then(|metrics| metrics.get("reap"))
8540            .expect("reap health metrics");
8541        assert_eq!(reap["deleted_retained"].as_u64(), Some(1));
8542        assert_eq!(reap["blockers"]["absence_unconfirmed"].as_u64(), Some(1));
8543        assert_eq!(reap["blockers"]["unbound_quiesced"].as_u64(), Some(0));
8544        assert_eq!(reap["blockers"]["actor_busy"].as_u64(), Some(0));
8545    }
8546
8547    #[test]
8548    fn connection_exit_quiesces_queued_maintenance_and_deleted_root_is_purged() {
8549        let (root_dir, root) = test_root("connection-exit-deleted-root");
8550        let executor = Arc::new(Executor::new());
8551        assert!(executor.register_actor(root.clone(), test_ctx()));
8552
8553        let route = route_key(11, 1);
8554        let mut meta = RootMeta::new(Instant::now());
8555        meta.maintenance_pending = true;
8556        meta.maintenance_queued_kinds
8557            .push_back(MaintenanceDrainKind::CompletionDrains);
8558        let mut live_roots = HashMap::from([(root.clone(), meta)]);
8559        let mut pending_binds = HashMap::new();
8560        let mut routes = HashMap::from([(route, route_identity(&root, "abandoned"))]);
8561        let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
8562        let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
8563        let mut route_bash_cancels = HashMap::new();
8564        let active_tool_calls: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
8565
8566        quiesce_connection_roots(
8567            &mut live_roots,
8568            &mut pending_binds,
8569            &mut routes,
8570            &mut root_channels,
8571            &mut installed_route_epochs,
8572            &mut route_bash_cancels,
8573            &active_tool_calls,
8574            &executor,
8575        );
8576        assert!(live_roots[&root].unbound_quiesced);
8577        assert!(!live_roots[&root].maintenance_pending);
8578        assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
8579        assert!(routes.is_empty());
8580        assert!(root_channels.is_empty());
8581
8582        root_dir.close().expect("delete project root");
8583        let metrics = DispatchPathMetrics::new();
8584        let outcome = reap_until_forgotten(
8585            &root,
8586            &mut live_roots,
8587            &pending_binds,
8588            &root_channels,
8589            &executor,
8590            &metrics,
8591        );
8592        let mut session_identity = HashMap::new();
8593        let mut push_buffer = HashMap::new();
8594        let mut bg_subs = HashMap::new();
8595        let mut bg_sub_by_session = HashMap::new();
8596        let mut bg_wake_pending = BgWakePending::new();
8597        let mut bg_wake_epoch = HashMap::new();
8598        let mut pending_bash_asks = HashMap::new();
8599        let mut retry_buffer = HashMap::new();
8600        let mut reclaimed_routes = ReclaimedRoutes::default();
8601        for forgotten in &outcome.forgotten_deleted_roots {
8602            purge_deleted_root_residents(
8603                forgotten,
8604                &mut routes,
8605                &mut root_channels,
8606                &mut installed_route_epochs,
8607                &mut route_bash_cancels,
8608                &active_tool_calls,
8609                executor.as_ref(),
8610                &mut retry_buffer,
8611                &mut reclaimed_routes,
8612                &mut session_identity,
8613                &mut push_buffer,
8614                &mut bg_subs,
8615                &mut bg_sub_by_session,
8616                &mut bg_wake_pending,
8617                &mut bg_wake_epoch,
8618                &mut pending_bash_asks,
8619                &metrics,
8620            );
8621        }
8622
8623        assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
8624        assert!(!executor.actor_registered(&root));
8625        assert!(!live_roots.contains_key(&root));
8626    }
8627
8628    #[test]
8629    fn unbound_root_quiesces_maintenance_without_removing_actor() {
8630        let (_root_dir, root) = test_root("unbound-root-quiesce");
8631        let ctx = test_ctx();
8632        let executor = Arc::new(Executor::new());
8633        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
8634        let mut meta = RootMeta::new(Instant::now());
8635        meta.maintenance_pending = true;
8636        meta.maintenance_jobs_in_flight = 1;
8637        meta.maintenance_queued_kinds
8638            .push_back(MaintenanceDrainKind::ConfigureTail);
8639        let mut live_roots = HashMap::from([(root.clone(), meta)]);
8640        // Warm state planted before the unbind: quiesce must keep it.
8641        *ctx.search_index()
8642            .write()
8643            .unwrap_or_else(std::sync::PoisonError::into_inner) =
8644            Some(crate::search_index::SearchIndex::new());
8645        ctx.set_cache_writer_capabilities(true, true);
8646        let pending = root.as_path().join("pending.rs");
8647        ctx.add_pending_search_index_paths([pending.clone()]);
8648        // A warm verify memo must survive the transient unbind: the watcher
8649        // keeps running, so no unobserved window exists and the next warm
8650        // reload must not pay a strict full-corpus re-hash.
8651        let canonical_root = root.as_path().to_path_buf();
8652        let artifact = canonical_root.join("cache.bin");
8653        std::fs::write(&artifact, b"warm-artifact").expect("write artifact");
8654        let seeded_generation = crate::cache_freshness::artifact_generation(&artifact);
8655        crate::cache_freshness::record_verify_completed(
8656            &canonical_root,
8657            crate::cache_freshness::VerifyArtifact::Search,
8658            seeded_generation,
8659        );
8660        assert!(matches!(
8661            crate::cache_freshness::warm_verify_plan(
8662                &canonical_root,
8663                crate::cache_freshness::VerifyArtifact::Search,
8664                seeded_generation,
8665            ),
8666            crate::cache_freshness::WarmVerifyPlan::Skip
8667        ));
8668        // A live watcher runtime must survive quiesce (its events accumulate
8669        // for the rebind replay).
8670        let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
8671        let _dispatch_tx = dispatch_tx;
8672        let shutdown = Arc::new(AtomicBool::new(false));
8673        let thread_shutdown = Arc::clone(&shutdown);
8674        let join = std::thread::spawn(move || {
8675            while !thread_shutdown.load(Ordering::SeqCst) {
8676                std::thread::yield_now();
8677            }
8678        });
8679        ctx.install_watcher_runtime(
8680            dispatch_rx,
8681            crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
8682        );
8683        assert!(ctx.watcher_runtime_active());
8684
8685        quiesce_unbound_root(&root, &mut live_roots, &executor);
8686        let meta = &live_roots[&root];
8687        assert!(meta.unbound_quiesced);
8688        assert!(ctx.subc_unbound_quiesced());
8689        assert!(meta.maintenance_pending);
8690        assert!(meta.maintenance_queued_kinds.is_empty());
8691        assert!(executor.actor_registered(&root));
8692        // Transient unbind keeps the root warm: resident artifacts stay
8693        // resident, no forced callgraph rebuild is planted, and pending
8694        // reconciliation paths survive for the rebind replay.
8695        assert!(
8696            ctx.search_index()
8697                .read()
8698                .unwrap_or_else(std::sync::PoisonError::into_inner)
8699                .is_some(),
8700            "quiesce must not evict resident artifacts"
8701        );
8702        assert_eq!(
8703            ctx.pending_callgraph_store_force_token(),
8704            None,
8705            "quiesce must not force a callgraph rebuild"
8706        );
8707        assert_eq!(
8708            ctx.take_pending_search_index_paths(),
8709            vec![pending],
8710            "quiesce must retain pending watcher-derived paths"
8711        );
8712        assert!(
8713            matches!(
8714                crate::cache_freshness::warm_verify_plan(
8715                    &canonical_root,
8716                    crate::cache_freshness::VerifyArtifact::Search,
8717                    seeded_generation,
8718                ),
8719                crate::cache_freshness::WarmVerifyPlan::Skip
8720            ),
8721            "quiesce must not invalidate the warm verify memo"
8722        );
8723        assert!(
8724            ctx.watcher_runtime_active(),
8725            "quiesce must not stop a running watcher"
8726        );
8727        ctx.stop_watcher_runtime();
8728
8729        let meta = live_roots.get_mut(&root).expect("root metadata");
8730        note_maintenance_completion(
8731            meta,
8732            Some(MaintenanceDrainKind::ConfigureTail),
8733            false,
8734            meta.unbound_quiesced,
8735        );
8736        assert!(!meta.maintenance_pending);
8737        assert!(meta.maintenance_queued_kinds.is_empty());
8738    }
8739
8740    #[test]
8741    fn same_root_higher_epoch_replacement_does_not_quiesce_between_generations() {
8742        let (_dir, root) = test_root("same-root-replacement");
8743        let route = route_key(7, 1);
8744        let installed_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
8745        let root_channels = HashMap::new();
8746
8747        assert!(!route_removal_will_quiesce_root(
8748            &root,
8749            route,
8750            &installed_channels,
8751            false,
8752            Some(&root),
8753        ));
8754        assert!(route_removal_will_quiesce_root(
8755            &root,
8756            route,
8757            &installed_channels,
8758            false,
8759            None,
8760        ));
8761        assert!(!should_quiesce_removed_root(
8762            &root,
8763            &root_channels,
8764            false,
8765            Some(&root),
8766        ));
8767        assert!(should_quiesce_removed_root(
8768            &root,
8769            &root_channels,
8770            false,
8771            None,
8772        ));
8773        assert!(!should_quiesce_removed_root(
8774            &root,
8775            &root_channels,
8776            true,
8777            None,
8778        ));
8779    }
8780
8781    #[test]
8782    fn root_quiesces_only_after_its_last_route_is_removed_and_reactivates_on_bind() {
8783        let (_root_dir, root) = test_root("unbound-root-route-count");
8784        let executor = Arc::new(Executor::new());
8785        assert!(executor.register_actor(root.clone(), test_ctx()));
8786        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
8787        let mut root_channels = HashMap::from([(
8788            root.clone(),
8789            HashSet::from([route_key(7, 1), route_key(8, 1)]),
8790        )]);
8791
8792        remove_root_channel(&mut root_channels, &root, route_key(7, 1));
8793        if !root_channels.contains_key(&root) {
8794            quiesce_unbound_root(&root, &mut live_roots, &executor);
8795        }
8796        assert!(!live_roots[&root].unbound_quiesced);
8797
8798        remove_root_channel(&mut root_channels, &root, route_key(8, 1));
8799        if !root_channels.contains_key(&root) {
8800            quiesce_unbound_root(&root, &mut live_roots, &executor);
8801        }
8802        assert!(live_roots[&root].unbound_quiesced);
8803
8804        live_roots
8805            .get_mut(&root)
8806            .expect("root metadata")
8807            .note_activity();
8808        assert!(
8809            live_roots[&root].unbound_quiesced,
8810            "late asynchronous activity must not reactivate an unbound root"
8811        );
8812
8813        live_roots
8814            .get_mut(&root)
8815            .expect("root metadata")
8816            .reactivate_bound();
8817        assert!(!live_roots[&root].unbound_quiesced);
8818    }
8819
8820    #[test]
8821    fn allocator_pressure_relief_requires_every_root_to_be_idle() {
8822        let (_idle_dir, idle_root) = test_root("allocator-relief-idle");
8823        let (_active_dir, active_root) = test_root("allocator-relief-active");
8824        let now = Instant::now();
8825        let mut live_roots = HashMap::new();
8826        let mut idle = RootMeta::new(now);
8827        idle.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
8828        live_roots.insert(idle_root, idle);
8829        let executor = Executor::new();
8830        assert!(process_has_been_idle(now, &live_roots, &executor));
8831
8832        live_roots.insert(active_root.clone(), RootMeta::new(now));
8833        assert!(!process_has_been_idle(now, &live_roots, &executor));
8834
8835        let active = live_roots
8836            .get_mut(&active_root)
8837            .expect("active root metadata");
8838        active.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
8839        active.active_bash_waits = 1;
8840        assert!(!process_has_been_idle(now, &live_roots, &executor));
8841    }
8842
8843    #[test]
8844    fn pressure_relief_log_reports_before_and_after_measurements() {
8845        let allocator = crate::memory::AllocatorMemorySnapshot {
8846            status: "measured",
8847            bytes_in_use: Some(8 * 1024 * 1024),
8848            size_allocated: Some(12 * 1024 * 1024),
8849            retained_slack_bytes: Some(4 * 1024 * 1024),
8850            not_estimated: None,
8851        };
8852        let relief = crate::memory::AllocatorPressureRelief {
8853            bytes_released: 3 * 1024 * 1024,
8854            rss_before_bytes: Some(20 * 1024 * 1024),
8855            rss_after_bytes: Some(17 * 1024 * 1024),
8856            allocator_before: allocator.clone(),
8857            allocator_after: crate::memory::AllocatorMemorySnapshot {
8858                size_allocated: Some(9 * 1024 * 1024),
8859                retained_slack_bytes: Some(1024 * 1024),
8860                ..allocator
8861            },
8862        };
8863        let message = pressure_relief_label(&relief);
8864        assert!(message.contains("RSS 20.0 MB -> 17.0 MB"));
8865        assert!(message.contains("allocated 12.0 MB -> 9.0 MB"));
8866        assert!(message.contains("slack 4.0 MB -> 1.0 MB"));
8867        assert!(message.contains("reported 3.0 MB released"));
8868    }
8869
8870    #[test]
8871    fn due_maintenance_jobs_skip_poisoned_roots() {
8872        let (_healthy_dir, healthy_root) = test_root("maintenance-healthy");
8873        let (_poisoned_dir, poisoned_root) = test_root("maintenance-poisoned");
8874        let mut live_roots = HashMap::new();
8875        live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
8876        let mut poisoned_meta = RootMeta::new(Instant::now());
8877        poisoned_meta.maintenance_poisoned = true;
8878        live_roots.insert(poisoned_root.clone(), poisoned_meta);
8879
8880        let (due, deferred) = due_maintenance_jobs_without_actor_context(
8881            &mut live_roots,
8882            MAINTENANCE_SUBMIT_BUDGET,
8883            &HashSet::new(),
8884        );
8885
8886        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
8887        assert!(due.iter().all(|(root, _)| root == &healthy_root));
8888        assert!(!deferred);
8889        assert!(live_roots[&healthy_root].maintenance_pending);
8890        assert_eq!(
8891            live_roots[&healthy_root].maintenance_jobs_in_flight,
8892            INITIAL_MAINTENANCE_JOB_COUNT
8893        );
8894        assert!(!live_roots[&poisoned_root].maintenance_pending);
8895    }
8896
8897    #[test]
8898    fn due_maintenance_jobs_do_not_restart_quiesced_root_work() {
8899        let (_dir, root) = test_root("maintenance-unbound");
8900        let mut meta = RootMeta::new(Instant::now());
8901        meta.unbound_quiesced = true;
8902        let mut live_roots = HashMap::from([(root.clone(), meta)]);
8903
8904        let (due, deferred) = due_maintenance_jobs_without_actor_context(
8905            &mut live_roots,
8906            MAINTENANCE_SUBMIT_BUDGET,
8907            &HashSet::new(),
8908        );
8909
8910        assert!(due.is_empty());
8911        assert!(!deferred);
8912        assert!(!live_roots[&root].maintenance_pending);
8913    }
8914
8915    #[test]
8916    fn idle_bg_subscription_queues_no_jobs_until_a_wake_arrives() {
8917        let (_dir, root) = test_root("maintenance-idle-bg-subscription");
8918        let ctx = test_ctx();
8919        assert!(!ctx.completion_drains_have_work());
8920
8921        let executor = Executor::new();
8922        assert!(executor.register_actor(root.clone(), ctx));
8923        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
8924        let session = "idle-session".to_string();
8925        let channel = route_key(17, 1);
8926        let metrics = DispatchPathMetrics::new();
8927        let bg_sub_by_session =
8928            HashMap::from([((root.clone(), session.clone()), HashSet::from([channel]))]);
8929        let mut bg_wake_pending = BgWakePending::new();
8930
8931        let (idle_tick_jobs, deferred) = due_maintenance_jobs(
8932            &mut live_roots,
8933            Some(&executor),
8934            &bg_sub_by_session,
8935            &bg_wake_pending,
8936            MAINTENANCE_SUBMIT_BUDGET,
8937            &HashSet::new(),
8938        );
8939        assert!(idle_tick_jobs.is_empty());
8940        assert!(!deferred);
8941        assert!(!live_roots[&root].maintenance_pending);
8942
8943        // A completion can arm its wake after the idle tick's probes. The wake
8944        // remains loop-owned state, so the following tick must observe it.
8945        let mut bg_wake_epoch = HashMap::new();
8946        push::arm_bg_wake(
8947            root.clone(),
8948            session,
8949            channel,
8950            &mut bg_wake_pending,
8951            &mut bg_wake_epoch,
8952            &metrics,
8953        );
8954        let (next_tick_jobs, deferred) = due_maintenance_jobs(
8955            &mut live_roots,
8956            Some(&executor),
8957            &bg_sub_by_session,
8958            &bg_wake_pending,
8959            MAINTENANCE_SUBMIT_BUDGET,
8960            &HashSet::new(),
8961        );
8962        assert_eq!(
8963            next_tick_jobs,
8964            vec![(root, MaintenanceDrainKind::CompletionDrains)]
8965        );
8966        assert!(!deferred);
8967    }
8968
8969    async fn assert_slow_configure_tail_admission(
8970        config: crate::executor::ExecutorConfig,
8971        shape: &'static str,
8972    ) {
8973        let root_dir = tempfile::tempdir().unwrap();
8974        let root_path = std::fs::canonicalize(root_dir.path()).unwrap();
8975        let root = ProjectRootId::from_path(&root_path).unwrap();
8976        let ctx = test_ctx();
8977        ctx.mark_subc_bound();
8978        let storage_root = ctx.storage_dir();
8979        ctx.enqueue_configure_maintenance(crate::context::ConfigureMaintenanceJob {
8980            generation: ctx.configure_generation(),
8981            root_path: root_path.clone(),
8982            canonical_cache_root: root_path.clone(),
8983            harness: crate::harness::Harness::Opencode,
8984            storage_root: storage_root.clone(),
8985            harness_dir: storage_root.join("opencode"),
8986            session_id: "first-search-admission".to_string(),
8987            home_match: false,
8988            format_tool_cache_clear_needed: false,
8989            run_bash_replay: false,
8990            refresh_project_runtime: false,
8991            sync_bash_compress_flag: false,
8992            reset_filter_registry: false,
8993            clear_failed_spawns: false,
8994            warm_callgraph_store: false,
8995            supersede_search_artifact_persistence: false,
8996            supersede_callgraph_artifact_persistence: false,
8997            supersede_semantic_artifact_persistence: false,
8998            search_artifact_load_start: None,
8999            semantic_artifact_load_start: None,
9000        })
9001        .expect("queue configure maintenance");
9002        let (_gate, maintenance_reached, release_maintenance) =
9003            crate::commands::configure::gate_configure_deferred_maintenance_for_test(
9004                root_path.clone(),
9005            );
9006
9007        let expected_pool_size = config.pool_size;
9008        let expected_actor_cap = config.actor_cap;
9009        let executor = Arc::new(Executor::with_config(config));
9010        assert_eq!(executor.pool_size(), expected_pool_size);
9011        assert_eq!(executor.actor_cap(), expected_actor_cap);
9012        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
9013        let metrics = Arc::new(DispatchPathMetrics::new());
9014        let (completion_tx, mut completion_rx) = mpsc::channel(2);
9015        submit_maintenance_job(
9016            &executor,
9017            root.clone(),
9018            MaintenanceDrainKind::ConfigureTail,
9019            Vec::new(),
9020            &completion_tx,
9021            &metrics,
9022        );
9023        maintenance_reached
9024            .recv_timeout(Duration::from_secs(2))
9025            .expect("configure tail reached gate");
9026
9027        let admission_started = std::time::Instant::now();
9028        let (search_admitted_tx, search_admitted_rx) = crossbeam_channel::bounded(1);
9029        let search = executor.submit_async(
9030            root.clone(),
9031            Lane::HeavyInit,
9032            "first-search".to_string(),
9033            Box::new(move |_ctx| {
9034                search_admitted_tx
9035                    .send(())
9036                    .expect("signal search admission");
9037                Response::success("first-search", json!({}))
9038            }),
9039        );
9040        let (mutation_started_tx, mutation_started_rx) = crossbeam_channel::bounded(1);
9041        let mutation = executor.submit_async(
9042            root,
9043            Lane::Mutating,
9044            "queued-mutation".to_string(),
9045            Box::new(move |_ctx| {
9046                mutation_started_tx.send(()).expect("signal mutation start");
9047                Response::success("queued-mutation", json!({}))
9048            }),
9049        );
9050
9051        // The decision under test is ORDERING, not latency: this recv happens
9052        // while the maintenance gate is still held, so a search that could only
9053        // admit after maintenance completes can never satisfy it. The budget is
9054        // a hang catch - a tight bound here just measures runner scheduling
9055        // (the census S-class shape) and flaked on loaded macOS CI at 100ms.
9056        let search_admission = search_admitted_rx.recv_timeout(Duration::from_secs(10));
9057        let admission_elapsed = admission_started.elapsed();
9058        let mutation_waited = mutation_started_rx.try_recv().is_err();
9059        release_maintenance
9060            .send(())
9061            .expect("release configure maintenance");
9062        search_admission
9063            .expect("first search must admit while configure maintenance remains gated");
9064        eprintln!(
9065            "first-search admission while configure maintenance is gated ({shape}): {}ms",
9066            admission_elapsed.as_millis()
9067        );
9068        assert!(
9069            mutation_waited,
9070            "mutating work must wait for configure maintenance to release its read epoch"
9071        );
9072        tokio::time::timeout(Duration::from_secs(5), search)
9073            .await
9074            .expect("first search completion timed out")
9075            .expect("first search completion channel closed");
9076        tokio::time::timeout(Duration::from_secs(5), mutation)
9077            .await
9078            .expect("mutation completion timed out")
9079            .expect("mutation completion channel closed");
9080        tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
9081            .await
9082            .expect("configure-tail completion timed out")
9083            .expect("configure-tail completion channel closed");
9084    }
9085
9086    #[tokio::test]
9087    async fn slow_configure_tail_admits_first_search_but_not_mutating_work() {
9088        assert_slow_configure_tail_admission(
9089            crate::executor::ExecutorConfig {
9090                pool_size: 2,
9091                read_cap: 1,
9092                actor_cap: 1,
9093                heavy_permits: 1,
9094                drr_quantum: 1,
9095            },
9096            "pool=2 actor_cap=1",
9097        )
9098        .await;
9099        assert_slow_configure_tail_admission(
9100            crate::executor::ExecutorConfig {
9101                pool_size: 4,
9102                read_cap: 3,
9103                actor_cap: 3,
9104                heavy_permits: 3,
9105                drr_quantum: 1,
9106            },
9107            "pool=4 actor_cap=3",
9108        )
9109        .await;
9110    }
9111
9112    #[tokio::test]
9113    async fn subc_configure_tail_precedes_completed_search_install() {
9114        let root_dir = tempfile::tempdir().unwrap();
9115        let storage = tempfile::tempdir().unwrap();
9116        let root = ProjectRootId::from_path(root_dir.path()).unwrap();
9117        let (ctx, ignored_path) =
9118            runtime_drain::configure_search_order_context_for_test(root_dir.path(), storage.path());
9119        let ctx = Arc::new(ctx);
9120        assert!(!runtime_drain::watcher_path_is_ignored_by_current_matcher(
9121            &ctx,
9122            &ignored_path
9123        ));
9124
9125        let executor = Arc::new(Executor::new());
9126        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
9127        let metrics = Arc::new(DispatchPathMetrics::new());
9128        let (completion_tx, mut completion_rx) = mpsc::channel(4);
9129        submit_maintenance_job(
9130            &executor,
9131            root.clone(),
9132            MaintenanceDrainKind::ConfigureTail,
9133            Vec::new(),
9134            &completion_tx,
9135            &metrics,
9136        );
9137        submit_maintenance_job(
9138            &executor,
9139            root,
9140            MaintenanceDrainKind::CompletionDrains,
9141            Vec::new(),
9142            &completion_tx,
9143            &metrics,
9144        );
9145
9146        let first = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
9147            .await
9148            .expect("configure-tail completion timed out")
9149            .expect("configure-tail completion channel closed");
9150        let second = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
9151            .await
9152            .expect("completion-drains completion timed out")
9153            .expect("completion-drains completion channel closed");
9154        assert!(first.response.id.contains("configure-tail"));
9155        assert!(second.response.id.contains("completion-drains"));
9156        assert!(runtime_drain::watcher_path_is_ignored_by_current_matcher(
9157            &ctx,
9158            &ignored_path
9159        ));
9160        assert_eq!(
9161            ctx.search_index()
9162                .read()
9163                .unwrap_or_else(std::sync::PoisonError::into_inner)
9164                .as_ref()
9165                .expect("completed search index installed")
9166                .file_count(),
9167            0,
9168            "configure must install the ignore matcher before pending paths replay"
9169        );
9170        ctx.stop_watcher_runtime();
9171    }
9172
9173    #[test]
9174    fn post_bind_configure_and_completion_jobs_are_queued_in_order() {
9175        let (_dir, root) = test_root("maintenance-post-bind");
9176        let mut live_roots = HashMap::new();
9177        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
9178
9179        queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
9180        queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
9181
9182        let meta = live_roots.get(&root).expect("root metadata");
9183        assert!(meta.maintenance_pending);
9184        assert_eq!(meta.maintenance_jobs_in_flight, 0);
9185        assert_eq!(
9186            meta.maintenance_queued_kinds
9187                .iter()
9188                .copied()
9189                .collect::<Vec<_>>(),
9190            vec![
9191                MaintenanceDrainKind::ConfigureTail,
9192                MaintenanceDrainKind::CompletionDrains,
9193            ]
9194        );
9195
9196        let (due, deferred) = due_maintenance_jobs_without_actor_context(
9197            &mut live_roots,
9198            MAINTENANCE_SUBMIT_BUDGET,
9199            &HashSet::new(),
9200        );
9201
9202        assert_eq!(
9203            due,
9204            vec![
9205                (root.clone(), MaintenanceDrainKind::ConfigureTail),
9206                (root.clone(), MaintenanceDrainKind::CompletionDrains),
9207            ]
9208        );
9209        assert!(!deferred);
9210        assert_eq!(live_roots[&root].maintenance_jobs_in_flight, 2);
9211        assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
9212    }
9213
9214    #[test]
9215    fn due_maintenance_jobs_defers_unsubmitted_roots_without_marking_pending() {
9216        let mut live_roots = HashMap::new();
9217        let mut root_ids = Vec::new();
9218        let mut _dirs = Vec::new();
9219        for index in 0..4 {
9220            let (dir, root_id) = test_root(&format!("maintenance-budget-{index}"));
9221            live_roots.insert(root_id.clone(), RootMeta::new(Instant::now()));
9222            root_ids.push(root_id);
9223            _dirs.push(dir);
9224        }
9225
9226        let small_budget = INITIAL_MAINTENANCE_JOB_COUNT + 1;
9227        let (first_due, first_deferred) = due_maintenance_jobs_without_actor_context(
9228            &mut live_roots,
9229            small_budget,
9230            &HashSet::new(),
9231        );
9232
9233        assert_eq!(first_due.len(), small_budget);
9234        assert!(first_deferred);
9235        let first_due_set: HashSet<_> = first_due.into_iter().map(|(root, _)| root).collect();
9236        assert!(first_due_set
9237            .iter()
9238            .all(|root| live_roots[root].maintenance_pending));
9239        assert!(first_due_set
9240            .iter()
9241            .any(|root| !live_roots[root].maintenance_queued_kinds.is_empty()));
9242
9243        let all_roots: HashSet<_> = root_ids.into_iter().collect();
9244        let deferred_roots: HashSet<_> = all_roots.difference(&first_due_set).cloned().collect();
9245        assert!(deferred_roots
9246            .iter()
9247            .all(|root| !live_roots[root].maintenance_pending));
9248    }
9249
9250    #[test]
9251    fn due_maintenance_jobs_defers_pending_bind_roots() {
9252        let (_bind_dir, bind_root) = test_root("maintenance-pending-bind");
9253        let (_healthy_dir, healthy_root) = test_root("maintenance-no-bind");
9254        let mut live_roots = HashMap::new();
9255        live_roots.insert(bind_root.clone(), RootMeta::new(Instant::now()));
9256        live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
9257        let pending_bind_roots = HashSet::from([bind_root.clone()]);
9258
9259        let (due, deferred) = due_maintenance_jobs_without_actor_context(
9260            &mut live_roots,
9261            usize::MAX,
9262            &pending_bind_roots,
9263        );
9264
9265        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
9266        assert!(due.iter().all(|(root, _)| root == &healthy_root));
9267        assert!(!deferred);
9268        assert!(!live_roots[&bind_root].maintenance_pending);
9269        assert!(live_roots[&bind_root].maintenance_queued_kinds.is_empty());
9270    }
9271
9272    #[test]
9273    fn maintenance_pending_survives_requeue_and_clears_after_final_batch() {
9274        let (_dir, root) = test_root("maintenance-requeue");
9275        let mut live_roots = HashMap::new();
9276        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
9277        let (due, deferred) = due_maintenance_jobs_without_actor_context(
9278            &mut live_roots,
9279            usize::MAX,
9280            &HashSet::new(),
9281        );
9282        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
9283        assert!(due.iter().all(|(due_root, _)| due_root == &root));
9284        assert!(!deferred);
9285
9286        let meta = live_roots.get_mut(&root).unwrap();
9287        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, false);
9288        assert!(meta.maintenance_pending);
9289        assert_eq!(
9290            meta.maintenance_jobs_in_flight,
9291            INITIAL_MAINTENANCE_JOB_COUNT - 1
9292        );
9293        assert_eq!(meta.maintenance_queued_kinds.len(), 1);
9294
9295        let (requeued, deferred) =
9296            due_maintenance_jobs_without_actor_context(&mut live_roots, 1, &HashSet::new());
9297        assert_eq!(
9298            requeued,
9299            vec![(root.clone(), MaintenanceDrainKind::Watcher)]
9300        );
9301        assert!(!deferred);
9302        let meta = live_roots.get_mut(&root).unwrap();
9303        assert_eq!(
9304            meta.maintenance_jobs_in_flight,
9305            INITIAL_MAINTENANCE_JOB_COUNT
9306        );
9307        assert!(meta.maintenance_queued_kinds.is_empty());
9308
9309        for _ in 0..INITIAL_MAINTENANCE_JOB_COUNT {
9310            note_maintenance_completion(meta, None, false, false);
9311        }
9312        assert!(!meta.maintenance_pending);
9313        assert_eq!(meta.maintenance_jobs_in_flight, 0);
9314    }
9315
9316    #[test]
9317    fn maintenance_requeue_drops_while_bind_is_pending() {
9318        let (_dir, root) = test_root("maintenance-bind-requeue");
9319        let mut live_roots = HashMap::new();
9320        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
9321        let (due, _) = due_maintenance_jobs_without_actor_context(
9322            &mut live_roots,
9323            usize::MAX,
9324            &HashSet::new(),
9325        );
9326        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
9327
9328        let meta = live_roots.get_mut(&root).unwrap();
9329        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, true);
9330
9331        assert_eq!(
9332            meta.maintenance_jobs_in_flight,
9333            INITIAL_MAINTENANCE_JOB_COUNT - 1
9334        );
9335        assert!(meta.maintenance_queued_kinds.is_empty());
9336        assert!(meta.maintenance_pending);
9337    }
9338
9339    #[test]
9340    fn parked_lsp_completion_never_requiesces_or_cancels_a_pending_bind() {
9341        let mut meta = RootMeta::new(Instant::now());
9342        meta.unbound_quiesced = true;
9343
9344        assert!(!should_requiesce_after_maintenance(
9345            &meta,
9346            MaintenanceDrainKind::Lsp,
9347            false,
9348        ));
9349        assert!(!should_requiesce_after_maintenance(
9350            &meta,
9351            MaintenanceDrainKind::ConfigureTail,
9352            true,
9353        ));
9354        assert!(should_requiesce_after_maintenance(
9355            &meta,
9356            MaintenanceDrainKind::ConfigureTail,
9357            false,
9358        ));
9359    }
9360
9361    #[test]
9362    fn maintenance_pending_clears_and_poison_stops_requeue_after_fatal() {
9363        let (_dir, root) = test_root("maintenance-fatal");
9364        let mut live_roots = HashMap::new();
9365        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
9366        let (due, _) = due_maintenance_jobs_without_actor_context(
9367            &mut live_roots,
9368            usize::MAX,
9369            &HashSet::new(),
9370        );
9371        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
9372
9373        let meta = live_roots.get_mut(&root).unwrap();
9374        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), true, false);
9375        assert!(meta.maintenance_poisoned);
9376        assert!(meta.maintenance_queued_kinds.is_empty());
9377
9378        for _ in 1..INITIAL_MAINTENANCE_JOB_COUNT {
9379            note_maintenance_completion(meta, None, false, false);
9380        }
9381        assert!(!meta.maintenance_pending);
9382        assert_eq!(meta.maintenance_jobs_in_flight, 0);
9383    }
9384
9385    #[test]
9386    fn trust_for_principal_matrix() {
9387        assert_eq!(
9388            trust_for_principal(&Some(Principal::Direct)),
9389            BindTrust::FirstParty
9390        );
9391        // Every first-party reserved id is asserted BY NAME, in one loop over
9392        // the full set, so two failure classes stay distinguishable: an empty
9393        // or broken allowlist reddens every name at once, while a dropped
9394        // single entry (the rename hazard) reddens exactly the missing name.
9395        // Both halves of each transitional rename pair stay listed until the
9396        // flip settles (see the allowlist comment).
9397        for module_id in [
9398            "llm-runner",
9399            "aft",
9400            "broca",
9401            "alfonso-core",
9402            "prefrontal",
9403            "prefrontal-core",
9404        ] {
9405            assert_eq!(
9406                trust_for_principal(&Some(Principal::Reserved {
9407                    module_id: module_id.to_string(),
9408                })),
9409                BindTrust::FirstParty,
9410                "reserved module id '{module_id}' must resolve to first-party trust"
9411            );
9412        }
9413        assert_eq!(
9414            trust_for_principal(&Some(Principal::Reserved {
9415                module_id: "subc-mcp".to_string(),
9416            })),
9417            BindTrust::Untrusted
9418        );
9419        assert_eq!(
9420            trust_for_principal(&Some(Principal::Reserved {
9421                module_id: "anything-unknown".to_string(),
9422            })),
9423            BindTrust::Untrusted
9424        );
9425        assert_eq!(
9426            trust_for_principal(&Some(Principal::Unverified)),
9427            BindTrust::Untrusted
9428        );
9429        assert_eq!(trust_for_principal(&None), BindTrust::Untrusted);
9430    }
9431
9432    #[test]
9433    fn fed_harness_class_maps_to_untrusted_regardless_of_fingerprint_value() {
9434        let principal = Some(Principal::Direct);
9435        let fingerprint_a = "fed:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
9436        let fingerprint_b = "fed:0123456789abcdef111111111111111111111111111111111111111111111111";
9437
9438        assert_eq!(
9439            trust_for_bind(fingerprint_a, &principal),
9440            BindTrust::Untrusted
9441        );
9442        assert_eq!(
9443            trust_for_bind(fingerprint_b, &principal),
9444            BindTrust::Untrusted
9445        );
9446    }
9447
9448    /// The table above proves `trust_for_principal` maps correctly, and the
9449    /// test above proves the `fed:` harness override wins — but neither
9450    /// exercises the ordinary path, so an implementation that ignored the
9451    /// principal entirely for non-fed harnesses would satisfy both. Pin the
9452    /// delegation itself: on a normal harness the verdict must still come from
9453    /// the principal, in both directions.
9454    #[test]
9455    fn trust_for_bind_delegates_to_the_principal_on_ordinary_harnesses() {
9456        for harness in ["opencode", "pi", "runner", "mcp:claude"] {
9457            assert_eq!(
9458                trust_for_bind(harness, &Some(Principal::Direct)),
9459                BindTrust::FirstParty,
9460                "a direct principal must stay first-party on {harness}"
9461            );
9462            assert_eq!(
9463                trust_for_bind(harness, &Some(Principal::Unverified)),
9464                BindTrust::Untrusted,
9465                "an unverified principal must stay untrusted on {harness}"
9466            );
9467            assert_eq!(
9468                trust_for_bind(harness, &None),
9469                BindTrust::Untrusted,
9470                "an absent principal must fail closed on {harness}"
9471            );
9472            assert_eq!(
9473                trust_for_bind(
9474                    harness,
9475                    &Some(Principal::Reserved {
9476                        module_id: "subc-mcp".to_string(),
9477                    })
9478                ),
9479                BindTrust::Untrusted,
9480                "a non-allowlisted reserved module must stay untrusted on {harness}"
9481            );
9482        }
9483    }
9484
9485    #[tokio::test]
9486    async fn persistent_cancel_resolves_when_fired_before_await() {
9487        // The lost-wakeup guard: cancel() fires exactly once via notify_waiters()
9488        // (no stored permit). A waiter that registers AFTER the cancel must still
9489        // observe it via the flag; a waiter racing the cancel must still be woken.
9490        let signal = PersistentCancelSignal::new();
9491        signal.cancel();
9492        // Fired before we ever call cancelled() — must return immediately, not park.
9493        tokio::time::timeout(Duration::from_secs(1), signal.cancelled())
9494            .await
9495            .expect("cancelled() must resolve when cancel fired beforehand");
9496
9497        // A fresh signal cancelled concurrently with an in-flight cancelled().
9498        let racing = PersistentCancelSignal::new();
9499        let racing_for_task = racing.clone();
9500        let waiter = tokio::spawn(async move { racing_for_task.cancelled().await });
9501        racing.cancel();
9502        tokio::time::timeout(Duration::from_secs(1), waiter)
9503            .await
9504            .expect("cancelled() must resolve when cancel races the await")
9505            .expect("waiter task panicked");
9506    }
9507
9508    #[test]
9509    fn ingress_epoch_validation_rejects_reclaimed_requests_and_drops_other_stale_epochs() {
9510        let installed = HashMap::from([(7, 9)]);
9511        let mut reclaimed = ReclaimedRoutes::default();
9512        reclaimed.insert(route_key(8, 1));
9513        for ty in [
9514            FrameType::Request,
9515            FrameType::Response,
9516            FrameType::Error,
9517            FrameType::Push,
9518            FrameType::Cancel,
9519            FrameType::Goodbye,
9520        ] {
9521            let body = if ty.is_pure_header() {
9522                Vec::new()
9523            } else {
9524                br#"{}"#.to_vec()
9525            };
9526            let stale = Frame::build(ty, control_flags(), 7, 8, 41, body).unwrap();
9527            assert!(
9528                !ingress_route_should_be_processed(&installed, &reclaimed, &stale),
9529                "{ty:?}"
9530            );
9531        }
9532
9533        let reclaimed_request = Frame::build(
9534            FrameType::Request,
9535            control_flags(),
9536            8,
9537            1,
9538            42,
9539            br#"{}"#.to_vec(),
9540        )
9541        .unwrap();
9542        assert!(ingress_route_should_be_processed(
9543            &installed,
9544            &reclaimed,
9545            &reclaimed_request
9546        ));
9547
9548        let never_installed = Frame::build(
9549            FrameType::Request,
9550            control_flags(),
9551            9,
9552            1,
9553            43,
9554            br#"{}"#.to_vec(),
9555        )
9556        .unwrap();
9557        assert!(!ingress_route_should_be_processed(
9558            &installed,
9559            &reclaimed,
9560            &never_installed
9561        ));
9562
9563        let current = Frame::build(
9564            FrameType::Request,
9565            control_flags(),
9566            7,
9567            9,
9568            43,
9569            br#"{}"#.to_vec(),
9570        )
9571        .unwrap();
9572        let control = Frame::build(FrameType::Ping, control_flags(), 0, 0, 44, Vec::new()).unwrap();
9573        assert!(ingress_route_should_be_processed(
9574            &installed, &reclaimed, &current
9575        ));
9576        assert!(ingress_route_should_be_processed(
9577            &installed, &reclaimed, &control
9578        ));
9579        assert_eq!(installed, HashMap::from([(7, 9)]));
9580    }
9581
9582    #[tokio::test]
9583    async fn route_bind_ack_precedes_route_egress_in_writer_queue() {
9584        let (_dir, root) = test_root("route-bind-b2-ordering");
9585        let route = route_key(7, 3);
9586        let identity = RouteIdentity(Arc::new(RouteIdentityData {
9587            root: root.clone(),
9588            project_root: root.as_path().to_path_buf(),
9589            harness: "opencode".to_string(),
9590            session: "b2-session".to_string(),
9591            trust: BindTrust::FirstParty,
9592            spawn_principal: AuthenticatedPrincipal::FirstParty,
9593            consumer_elicitation_capable: false,
9594        }));
9595        let replay_key = push::ReplayKey::from_identity(&identity);
9596        let completion = RouteBindCompletion {
9597            route,
9598            identity,
9599            bind_root_id: root.clone(),
9600            inserted_new_actor: false,
9601            configure_response: Response::success("subc-bind-7", json!({})),
9602            diagnostics_on_edit: false,
9603            ver: PROTOCOL_VERSION,
9604            corr: 91,
9605            flags: control_flags(),
9606        };
9607        let mut pending_binds = HashMap::from([(
9608            route,
9609            PendingBind {
9610                bind_root_id: root,
9611                inserted_new_actor: false,
9612                cancelled: false,
9613                configure_request_id: "subc-bind-7".to_string(),
9614                started_at: Instant::now(),
9615                warned_half_deadline: false,
9616                deadline_reported: false,
9617                corr: 91,
9618                ver: PROTOCOL_VERSION,
9619                flags: control_flags(),
9620                cancellation: crate::executor::JobCancellation::new(),
9621            },
9622        )]);
9623        let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
9624        let mut push_buffer =
9625            HashMap::from([(replay_key, VecDeque::from([completion_frame("b2-replay")]))]);
9626        let (writer_tx, mut writer_rx) = mpsc::channel(8);
9627        let metrics = Arc::new(DispatchPathMetrics::new());
9628        let executor = Arc::new(Executor::new());
9629        let standing_actor =
9630            standing::StandingActor::new(App::default_shared(), Arc::clone(&executor));
9631
9632        handle_route_bind_completion(
9633            &writer_tx,
9634            completion,
9635            &mut HashMap::new(),
9636            &mut HashMap::new(),
9637            &mut HashMap::new(),
9638            &mut push_buffer,
9639            &mut HashMap::new(),
9640            &mut pending_binds,
9641            &mut installed_route_epochs,
9642            &executor,
9643            &standing_actor,
9644            &Arc::new(Notify::new()),
9645            &metrics,
9646            None,
9647        )
9648        .await
9649        .unwrap();
9650
9651        let ack = writer_rx.try_recv().expect("RouteBindAck");
9652        assert_eq!(ack.header.ty, FrameType::Response);
9653        assert_eq!((ack.header.channel, ack.header.epoch), (0, 0));
9654        let route_frame = writer_rx.try_recv().expect("post-ack route frame");
9655        assert_eq!(route_frame.header.ty, FrameType::Push);
9656        assert_eq!(
9657            (route_frame.header.channel, route_frame.header.epoch),
9658            (route.channel, route.epoch)
9659        );
9660    }
9661}