Skip to main content

aft/subc/
mod.rs

1//! subc daemon attach — transport edge.
2//!
3//! When AFT is launched as `aft --subc <connection-file>`, it does NOT run the
4//! standalone NDJSON-over-stdin loop. Instead it connects to a running subc
5//! daemon over loopback TCP, authenticates with the pre-envelope HMAC handshake
6//! (`subc-transport`), then speaks the subc frame protocol (`subc-protocol`):
7//! ModuleHello → HelloAck (register as a tool provider), then a channel-0
8//! control loop (Ping/Pong, RouteBind) plus route-channel tool calls.
9//!
10//! Concurrency: subc routes tool calls through the executor. The tokio
11//! edge never dispatches against `AppContext` inline; per-actor executor lanes
12//! own the reader/mutator epoch, while a writer task serializes outbound frames.
13
14use std::collections::{HashMap, HashSet, VecDeque};
15use std::fmt;
16use std::io;
17use std::net::{IpAddr, SocketAddr};
18use std::ops::Deref;
19use std::path::{Path, PathBuf};
20use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
21use std::sync::{Arc, LazyLock, Mutex as StdMutex, OnceLock};
22use std::time::{Duration, Instant};
23
24use serde::Deserialize;
25use serde_json::{json, Value};
26
27use crate::config::Config;
28use crate::config_resolve::ConfigTier;
29use crate::context::{App, AppContext, ProgressSender, RootHealthSnapshot};
30use crate::executor::{Executor, JobCancellation, Lane};
31use crate::fleet_status::{spawn_fleet_status_dial, FleetStatusClient};
32use crate::jsonc::strip_jsonc;
33use crate::log_ctx;
34use crate::path_identity::ProjectRootId;
35use crate::protocol::{ProgressKind, PushFrame, RawRequest, Response};
36use crate::response_finalize::{DispatchOutcome, PendingResponse};
37use crate::run_tool_call::{
38    finish_tool_call_response, prepare_tool_call, run_tool_call, strip_agent_preview_arg_owned,
39    PhaseTrace, ToolCallContext, ToolCallOutcome, ToolCallResult,
40};
41use crate::runtime_drain;
42use crate::sandbox_spawn::{AuthenticatedPrincipal, PrincipalTrust};
43
44use subc_protocol::manifest::{
45    Bindings, Concurrency, ExecutionMode, IdentityBinding, IdentityScope, ModuleManifest,
46    ProviderRole, StorageBinding, StorageKind, StorageScope, Tool, TrustTier,
47};
48use subc_protocol::session::{
49    HealthReport, HealthStatus, ModuleControlRequest, ModuleControlResponse,
50    MODULE_CONTROL_OP_HEALTH_CHECK,
51};
52use subc_protocol::{
53    ErrorBody, Flags, Frame, FrameType, ModuleHelloBody, Principal, Priority, MAX_FRAME_BODY_LEN,
54    PROTOCOL_VERSION,
55};
56use subc_transport::{authenticate_client, connection_file, read_frame, write_frame};
57use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
58use tokio::net::TcpStream;
59use tokio::sync::{mpsc, oneshot, Notify};
60use tokio::task::JoinHandle;
61
62/// Per-attempt handshake deadline. The initial attach loop has a separate total
63/// budget so a stalled peer cannot consume an unbounded supervisor launch window.
64const AUTH_DEADLINE: Duration = Duration::from_secs(5);
65const ATTACH_RETRY_BUDGET: Duration = Duration::from_secs(60);
66const ATTACH_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(250);
67const ATTACH_RETRY_MAX_BACKOFF: Duration = Duration::from_secs(5);
68const ATTACH_RETRY_JITTER_PERCENT: u64 = 20;
69
70/// Correlation id for the initial ModuleHello (channel 0).
71const HELLO_CORR: u64 = 1;
72
73/// Per-session in-memory replay cap for must-deliver Push frames. This covers
74/// detach/re-attach while AFT stays alive; cross-restart replay is phased later.
75const PUSH_BUFFER_MAX_PER_KEY: usize = 256;
76
77/// Bounded guard for control-frame sends. If the daemon stops reading and the
78/// writer queue stays full, tear the subc edge down instead of stalling the
79/// route loop indefinitely.
80const CONTROL_SEND_TIMEOUT: Duration = Duration::from_millis(250);
81
82/// Cadence for the loop's deadline-driven drain work (retry-buffer flush,
83/// bg-wake emission, maintenance submission). Checked at the top of every
84/// loop turn so busy select arms cannot starve it.
85const DRAIN_TICK_PERIOD: Duration = Duration::from_millis(250);
86
87/// Root-scoped stores and watcher runtimes are reopened lazily after this
88/// period without tool traffic. Keeping the value fixed avoids per-client
89/// eviction policies competing inside the module loop.
90const IDLE_ROOT_TTL: Duration = Duration::from_secs(30 * 60);
91
92const WRITER_QUEUE_CAPACITY: usize = 256;
93
94/// Keep reliable Push bursts from monopolizing the current-thread subc loop;
95/// any remaining must-deliver frames stay queued for the next loop turn.
96const RELIABLE_PUSH_DRAIN_BUDGET: usize = 32;
97
98/// Limit maintenance submissions per tick so background drains cannot delay
99/// control-plane work such as completed RouteBind acknowledgements.
100///
101/// The decomposed maintenance pass charges this budget by Mutating job, not by
102/// root. Size the default burst for one maintenance pass over eight live roots,
103/// while follow-up batches still re-enter the capped queue instead of bypassing
104/// the budget.
105const MAINTENANCE_SUBMIT_BUDGET: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len() * 8;
106const INITIAL_MAINTENANCE_DRAIN_KINDS: [MaintenanceDrainKind; 4] = [
107    MaintenanceDrainKind::Watcher,
108    MaintenanceDrainKind::Lsp,
109    MaintenanceDrainKind::ConfigureTail,
110    MaintenanceDrainKind::CompletionDrains,
111];
112#[cfg(test)]
113const INITIAL_MAINTENANCE_JOB_COUNT: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len();
114
115const RELIABLE_WRITER_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(10);
116const RELIABLE_WRITER_RETRY_MAX_BACKOFF: Duration = Duration::from_millis(250);
117
118const DISPATCH_PATH_BIND_WARN_AFTER: Duration = Duration::from_secs(6);
119const ROUTE_BIND_DEADLINE: Duration = Duration::from_secs(12);
120
121/// Small bounded memory of completed task ids used to suppress stale lossy
122/// long-running reminders that arrive after their reliable completion event.
123const COMPLETED_TASK_SUPPRESSION_MAX: usize = 4096;
124
125/// Bash foreground orchestration polls detached tasks with short read-lane jobs.
126/// The sleep between polls is outside the executor so no read or write worker is
127/// pinned while a foreground command is still running.
128const PENDING_POLL_INTERVAL: Duration = Duration::from_millis(100);
129
130/// Host elicitation asks fail closed if the MCP facade does not answer promptly.
131const BASH_ELICITATION_TIMEOUT: Duration = Duration::from_secs(60);
132const BASH_ELICITATION_CREATE_METHOD: &str = "elicitation/create";
133
134#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
135struct RouteChannel {
136    channel: u16,
137    epoch: u32,
138}
139
140impl fmt::Display for RouteChannel {
141    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
142        write!(f, "{}@{}", self.channel, self.epoch)
143    }
144}
145
146type PushEnvelope = (ProjectRootId, PushFrame);
147type LossyPushEnvelope = (u64, ProjectRootId, PushFrame);
148type RetryBuffer = HashMap<RouteChannel, VecDeque<(push::ReplayKey, PushFrame)>>;
149mod bash;
150mod health;
151mod manifest;
152mod push;
153mod wire;
154
155use self::health::{
156    build_health_report, warn_slow_pending_binds, DispatchPathMetrics, HealthRollupCache,
157    ReapBlockerCensus, ResponseTaskGuard, HEALTH_ROLLUP_TTL,
158};
159use self::manifest::{
160    build_manifest, command_lane, control_flags, control_ops, is_bash_family_tool,
161    is_subc_agent_core_tool, is_subc_native_plumbing_tool,
162};
163pub use self::wire::SubcError;
164
165/// Test-only view of the fail-closed tool-call gate: would `name` be admitted
166/// on a bound route (as an agent tool or native plumbing)? Used by the
167/// plugin-send drift guard in `subc_plumbing_drift_test.rs`.
168pub fn is_tool_call_admitted_for_test(name: &str) -> bool {
169    manifest::is_subc_agent_core_tool(name) || manifest::is_subc_native_plumbing_tool(name)
170}
171use self::wire::{
172    build_error_frame, build_goodbye_frame, build_tool_response_frame,
173    build_tool_response_frame_with_limit, decrement_counted_channel, response_is_fatal_panic,
174    response_message, send_counted_channel, send_frame, send_reliable_writer_frame,
175    send_traced_tool_response_frame, ToolResponseWriteTrace, WriterFrame, WriterSender,
176};
177
178struct DecodedFrame {
179    frame: Frame,
180    phase_trace: PhaseTrace,
181}
182
183struct ToolCallCompletion {
184    text: String,
185    phase_trace: PhaseTrace,
186}
187
188#[derive(Clone)]
189struct ActiveToolCall {
190    root_id: ProjectRootId,
191    cancellation: JobCancellation,
192}
193
194type ActiveToolCalls = Arc<StdMutex<HashMap<(RouteChannel, u64), ActiveToolCall>>>;
195
196struct PendingInspectSetupGuard(Arc<AtomicUsize>);
197
198impl PendingInspectSetupGuard {
199    fn new(count: Arc<AtomicUsize>) -> Self {
200        count.fetch_add(1, Ordering::SeqCst);
201        Self(count)
202    }
203}
204
205impl Drop for PendingInspectSetupGuard {
206    fn drop(&mut self) {
207        self.0.fetch_sub(1, Ordering::SeqCst);
208    }
209}
210
211enum InspectSetupOutcome {
212    Immediate {
213        text: String,
214        phase_trace: PhaseTrace,
215    },
216    Deferred {
217        pending: PendingResponse,
218        surface_downgraded: bool,
219        phase_trace: PhaseTrace,
220    },
221}
222
223struct PendingSubcInspect {
224    route: RouteChannel,
225    corr: u64,
226    flags: Flags,
227    ver: u8,
228    root: ProjectRootId,
229    session_id: String,
230    format_context: crate::subc_format::FormatContext,
231    bind_trust: BindTrust,
232    pending: PendingResponse,
233    surface_downgraded: bool,
234    phase_trace: PhaseTrace,
235}
236
237struct ResolvedSubcInspect {
238    entry: PendingSubcInspect,
239    response: Response,
240}
241
242#[derive(Default)]
243struct PendingSubcInspects {
244    entries: Vec<PendingSubcInspect>,
245}
246
247impl PendingSubcInspects {
248    fn register(&mut self, pending: PendingSubcInspect) {
249        self.entries.retain(|entry| {
250            let keep = entry.route != pending.route || entry.corr != pending.corr;
251            if !keep {
252                if let Some(cancellation) = &entry.pending.cancellation {
253                    cancellation.request_cancel();
254                }
255            }
256            keep
257        });
258        self.entries.push(pending);
259    }
260
261    fn poll_ready(&mut self, executor: &Executor) -> Vec<ResolvedSubcInspect> {
262        let mut ready = Vec::new();
263        let mut waiting = Vec::with_capacity(self.entries.len());
264        for mut entry in self.entries.drain(..) {
265            let response = executor
266                .actor_context(&entry.root)
267                .and_then(|ctx| (entry.pending.poll)(&ctx));
268            if let Some(response) = response {
269                ready.push(ResolvedSubcInspect { entry, response });
270            } else {
271                waiting.push(entry);
272            }
273        }
274        self.entries = waiting;
275        ready
276    }
277
278    fn drain_route(
279        &mut self,
280        route: RouteChannel,
281        executor: &Executor,
282    ) -> Vec<ResolvedSubcInspect> {
283        self.drain_matching(executor, |entry| entry.route == route)
284    }
285
286    fn drain_on_shutdown(&mut self, executor: &Executor) -> Vec<ResolvedSubcInspect> {
287        self.drain_matching(executor, |_| true)
288    }
289
290    fn drain_matching(
291        &mut self,
292        executor: &Executor,
293        matches: impl Fn(&PendingSubcInspect) -> bool,
294    ) -> Vec<ResolvedSubcInspect> {
295        let mut resolved = Vec::new();
296        let mut waiting = Vec::with_capacity(self.entries.len());
297        for mut entry in self.entries.drain(..) {
298            if !matches(&entry) {
299                waiting.push(entry);
300                continue;
301            }
302            if let Some(cancellation) = &entry.pending.cancellation {
303                cancellation.request_cancel();
304            }
305            if let Some(ctx) = executor.actor_context(&entry.root) {
306                if let Some(on_shutdown) = entry.pending.on_shutdown.as_mut() {
307                    let response = on_shutdown(&ctx);
308                    resolved.push(ResolvedSubcInspect { entry, response });
309                }
310            }
311        }
312        self.entries = waiting;
313        resolved
314    }
315
316    fn is_empty(&self) -> bool {
317        self.entries.is_empty()
318    }
319}
320
321#[derive(Clone)]
322struct PushSenders {
323    lossy_tx: mpsc::Sender<LossyPushEnvelope>,
324    reliable_tx: mpsc::UnboundedSender<PushEnvelope>,
325    lossy_overflow: Arc<push::LossyOverflow>,
326    lossy_seq: Arc<AtomicU64>,
327    fleet_status_client: FleetStatusClient,
328}
329
330#[derive(Clone)]
331struct PersistentCancelSignal {
332    inner: Arc<PersistentCancelInner>,
333}
334
335struct PersistentCancelInner {
336    cancelled: AtomicBool,
337    notify: Notify,
338}
339
340impl PersistentCancelSignal {
341    fn new() -> Self {
342        Self {
343            inner: Arc::new(PersistentCancelInner {
344                cancelled: AtomicBool::new(false),
345                notify: Notify::new(),
346            }),
347        }
348    }
349
350    fn cancel(&self) {
351        if !self.inner.cancelled.swap(true, Ordering::SeqCst) {
352            self.inner.notify.notify_waiters();
353        }
354    }
355
356    fn is_cancelled(&self) -> bool {
357        self.inner.cancelled.load(Ordering::SeqCst)
358    }
359
360    async fn cancelled(&self) {
361        // `enable()` REGISTERS this waiter before we read the flag, closing the
362        // lost-wakeup window: `notify_waiters()` only wakes already-registered
363        // waiters and stores no permit, so without enable() a `cancel()` firing
364        // between the flag read and `.await` would be missed and the future
365        // would park forever (cancel() fires only once). With enable(), a cancel
366        // racing the flag read still wakes the registered waiter. The loop is a
367        // belt-and-suspenders re-check on spurious wakeups.
368        loop {
369            let notified = self.inner.notify.notified();
370            tokio::pin!(notified);
371            notified.as_mut().enable();
372            if self.is_cancelled() {
373                return;
374            }
375            notified.await;
376        }
377    }
378}
379
380fn finish_active_tool_call(active: &ActiveToolCalls, route: RouteChannel, corr: u64) {
381    active
382        .lock()
383        .unwrap_or_else(std::sync::PoisonError::into_inner)
384        .remove(&(route, corr));
385}
386
387fn cancel_active_tool_call(
388    active: &ActiveToolCalls,
389    executor: &Executor,
390    route: RouteChannel,
391    corr: u64,
392    reason: &str,
393) -> bool {
394    let call = active
395        .lock()
396        .unwrap_or_else(std::sync::PoisonError::into_inner)
397        .remove(&(route, corr));
398    let Some(call) = call else {
399        return false;
400    };
401    let outcome = executor.cancel_job(&call.root_id, &call.cancellation);
402    log::debug!(
403        "subc attach: cancelled active tool call route={route} corr={corr} reason={reason} outcome={outcome:?}"
404    );
405    true
406}
407
408fn cancel_active_tool_calls_for_route(
409    active: &ActiveToolCalls,
410    executor: &Executor,
411    route: RouteChannel,
412    reason: &str,
413) -> usize {
414    let cancelled = {
415        let mut calls = active
416            .lock()
417            .unwrap_or_else(std::sync::PoisonError::into_inner);
418        let mut cancelled = Vec::new();
419        calls.retain(|(call_route, _), call| {
420            if *call_route == route {
421                cancelled.push(call.clone());
422                false
423            } else {
424                true
425            }
426        });
427        cancelled
428    };
429    for call in &cancelled {
430        let outcome = executor.cancel_job(&call.root_id, &call.cancellation);
431        log::debug!(
432            "subc attach: cancelled active tool call route={route} reason={reason} outcome={outcome:?}"
433        );
434    }
435    cancelled.len()
436}
437
438fn cancel_all_active_tool_calls(
439    active: &ActiveToolCalls,
440    executor: &Executor,
441    reason: &str,
442) -> usize {
443    let cancelled = {
444        let mut calls = active
445            .lock()
446            .unwrap_or_else(std::sync::PoisonError::into_inner);
447        calls.drain().map(|(_, call)| call).collect::<Vec<_>>()
448    };
449    for call in &cancelled {
450        let outcome = executor.cancel_job(&call.root_id, &call.cancellation);
451        log::debug!("subc attach: cancelled active tool call reason={reason} outcome={outcome:?}");
452    }
453    cancelled.len()
454}
455
456#[derive(Debug, Clone, Copy, PartialEq, Eq)]
457pub(crate) enum BindTrust {
458    FirstParty,
459    Untrusted,
460}
461
462impl BindTrust {
463    fn allows_bash_observation(self) -> bool {
464        matches!(self, Self::FirstParty)
465    }
466
467    fn label(self) -> &'static str {
468        match self {
469            Self::FirstParty => "first_party",
470            Self::Untrusted => "untrusted",
471        }
472    }
473
474    fn sandbox_trust(self) -> PrincipalTrust {
475        match self {
476            Self::FirstParty => PrincipalTrust::FirstParty,
477            Self::Untrusted => PrincipalTrust::Untrusted,
478        }
479    }
480}
481
482pub(super) fn trust_for_principal(principal: &Option<Principal>) -> BindTrust {
483    match principal {
484        Some(Principal::Direct) => BindTrust::FirstParty,
485        // Module renames are flag-days: the daemon registry refuses duplicate
486        // active ids, so a renaming module cannot advertise both names during
487        // its transition. This allowlist is DIALLED, not dialling — it must
488        // accept a module's NEW name in a released binary before the module
489        // starts using it, and the old name stays until the flip has settled.
490        // That is why transitional pairs appear here: llm-runner/broca was the
491        // previous rename, alfonso-core/prefrontal is the current one. When
492        // retiring an old name, confirm the fleet no longer spawns it — a
493        // stale entry here is inert, but a missing one silently downgrades a
494        // first-party module to Untrusted and revokes its bash access.
495        Some(Principal::Reserved { module_id })
496            if module_id == "llm-runner"
497                || module_id == "aft"
498                || module_id == "broca"
499                || module_id == "alfonso-core"
500                || module_id == "prefrontal"
501                || module_id == "prefrontal-core" =>
502        {
503            BindTrust::FirstParty
504        }
505        Some(Principal::Reserved { .. }) | Some(Principal::Unverified) | None => {
506            BindTrust::Untrusted
507        }
508    }
509}
510
511fn harness_forces_untrusted(harness: &str) -> bool {
512    harness.starts_with("fed:")
513}
514
515pub(super) fn trust_for_bind(harness: &str, principal: &Option<Principal>) -> BindTrust {
516    if harness_forces_untrusted(harness) {
517        BindTrust::Untrusted
518    } else {
519        trust_for_principal(principal)
520    }
521}
522
523fn principal_id(principal: &Option<Principal>) -> Option<String> {
524    match principal {
525        Some(Principal::Direct) => Some("direct".to_string()),
526        Some(Principal::Reserved { module_id }) => Some(format!("reserved:{module_id}")),
527        Some(Principal::Unverified) => Some("unverified".to_string()),
528        None => None,
529    }
530}
531
532fn principal_label(principal: &Option<Principal>) -> String {
533    principal_id(principal).unwrap_or_else(|| "absent".to_string())
534}
535
536#[derive(Debug)]
537/// Per-root route metadata owned by the subc loop. The `active_bash_waits` field
538/// counts detached bash processes that are still being observed for this root.
539/// Any future logic that evicts roots based on idle time must not evict a root
540/// while this count is greater than zero, because a foreground bash response may
541/// still arrive later.
542struct RootMeta {
543    maintenance_pending: bool,
544    maintenance_jobs_in_flight: usize,
545    maintenance_queued_kinds: VecDeque<MaintenanceDrainKind>,
546    maintenance_last_submitted: Option<Instant>,
547    maintenance_poisoned: bool,
548    last_touched: Instant,
549    diagnostics_on_edit: bool,
550    active_bash_waits: usize,
551    idle_artifacts_evicted: bool,
552    unbound_quiesced: bool,
553    consecutive_missing_sweeps: u8,
554}
555
556#[derive(Debug)]
557struct PendingBind {
558    bind_root_id: ProjectRootId,
559    inserted_new_actor: bool,
560    cancelled: bool,
561    configure_request_id: String,
562    started_at: Instant,
563    warned_half_deadline: bool,
564    deadline_reported: bool,
565    corr: u64,
566    ver: u8,
567    flags: Flags,
568    /// Exact-job cancellation for the submitted configure: Goodbye and
569    /// deadline expiry cancel the executor job operationally (queued jobs are
570    /// removed, running configures return at their next checkpoint) instead of
571    /// only marking bookkeeping.
572    cancellation: crate::executor::JobCancellation,
573}
574
575struct RouteBindCompletion {
576    route: RouteChannel,
577    identity: RouteIdentity,
578    bind_root_id: ProjectRootId,
579    inserted_new_actor: bool,
580    configure_response: Response,
581    diagnostics_on_edit: bool,
582    ver: u8,
583    corr: u64,
584    flags: Flags,
585}
586
587#[derive(Debug, Clone)]
588struct RouteIdentity(Arc<RouteIdentityData>);
589
590#[derive(Debug)]
591struct RouteIdentityData {
592    root: ProjectRootId,
593    project_root: PathBuf,
594    harness: String,
595    session: String,
596    trust: BindTrust,
597    spawn_principal: AuthenticatedPrincipal,
598    consumer_elicitation_capable: bool,
599}
600
601impl Deref for RouteIdentity {
602    type Target = RouteIdentityData;
603
604    fn deref(&self) -> &Self::Target {
605        &self.0
606    }
607}
608
609#[derive(Debug, Clone)]
610struct RetainedSessionIdentity {
611    harness: String,
612    trust: BindTrust,
613}
614
615#[derive(Clone)]
616struct BgSub {
617    corr: u64,
618    ver: u8,
619    flags: Flags,
620    root: ProjectRootId,
621    session: String,
622}
623
624// A session can be observed by multiple long-lived consumer records. Retain
625// every route so each wake uses the correlation captured by that route's BgSub.
626type BgSubsBySession = HashMap<(ProjectRootId, String), HashSet<RouteChannel>>;
627
628struct MaintenanceCompletion {
629    root_id: ProjectRootId,
630    kind: MaintenanceDrainKind,
631    response: Response,
632    empty_bg_sessions: Vec<(String, u64)>,
633    requeue_kind: Option<MaintenanceDrainKind>,
634}
635
636#[derive(Clone, Copy, Debug, PartialEq, Eq)]
637enum MaintenanceDrainKind {
638    Watcher,
639    Lsp,
640    ConfigureTail,
641    CompletionDrains,
642}
643
644impl MaintenanceDrainKind {
645    fn label(self) -> &'static str {
646        match self {
647            Self::Watcher => "watcher",
648            Self::Lsp => "lsp",
649            Self::ConfigureTail => "configure-tail",
650            Self::CompletionDrains => "completion-drains",
651        }
652    }
653}
654
655#[derive(Debug, Default)]
656struct MaintenanceJobOutcome {
657    empty_bg_sessions: Vec<(String, u64)>,
658    requeue_kind: Option<MaintenanceDrainKind>,
659}
660
661#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
662struct ReverseCorrKey {
663    route: RouteChannel,
664    corr: u64,
665}
666
667struct PendingBashAsk {
668    route: RouteChannel,
669    tool_corr: u64,
670    tool_flags: Flags,
671    tool_ver: u8,
672    root: ProjectRootId,
673    project_root: PathBuf,
674    session_id: String,
675    spawn_principal: AuthenticatedPrincipal,
676    edit_slot_survives: Option<bool>,
677    request_id: String,
678    arguments: Value,
679    format_context: crate::subc_format::FormatContext,
680    cancel: bash::BashWaitCancel,
681    grants: Vec<String>,
682    expires_at: Instant,
683}
684
685impl RootMeta {
686    fn new(now: Instant) -> Self {
687        Self {
688            maintenance_pending: false,
689            maintenance_jobs_in_flight: 0,
690            maintenance_queued_kinds: VecDeque::new(),
691            maintenance_last_submitted: None,
692            maintenance_poisoned: false,
693            last_touched: now,
694            diagnostics_on_edit: false,
695            active_bash_waits: 0,
696            idle_artifacts_evicted: false,
697            unbound_quiesced: false,
698            consecutive_missing_sweeps: 0,
699        }
700    }
701
702    fn note_activity(&mut self) {
703        self.last_touched = Instant::now();
704    }
705
706    fn reactivate_bound(&mut self) {
707        self.note_activity();
708        self.idle_artifacts_evicted = false;
709        self.unbound_quiesced = false;
710    }
711}
712
713fn due_maintenance_jobs(
714    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
715    executor: Option<&Executor>,
716    bg_sub_by_session: &BgSubsBySession,
717    bg_wake_pending: &HashSet<RouteChannel>,
718    budget: usize,
719    pending_bind_roots: &HashSet<ProjectRootId>,
720) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
721    let mut jobs = Vec::new();
722    let mut deferred = false;
723    let mut roots = live_roots.keys().cloned().collect::<Vec<_>>();
724    roots.sort_by(|left, right| {
725        let left_last = live_roots
726            .get(left)
727            .and_then(|meta| meta.maintenance_last_submitted);
728        let right_last = live_roots
729            .get(right)
730            .and_then(|meta| meta.maintenance_last_submitted);
731        left_last
732            .cmp(&right_last)
733            .then_with(|| left.as_path().cmp(right.as_path()))
734    });
735
736    for root_id in roots {
737        let Some(meta) = live_roots.get_mut(&root_id) else {
738            continue;
739        };
740        if meta.maintenance_poisoned {
741            continue;
742        }
743
744        if pending_bind_roots.contains(&root_id) {
745            if meta.maintenance_pending || !meta.maintenance_queued_kinds.is_empty() {
746                deferred = true;
747            }
748            continue;
749        }
750
751        if !meta.maintenance_pending {
752            if jobs.len() >= budget {
753                deferred = true;
754                continue;
755            }
756            // Only enqueue kinds with pending work. Probes are cheap and
757            // fail-open (contended sources count as pending), so an idle root
758            // costs four probes per tick instead of four dispatched jobs.
759            let executor_actor_context =
760                executor.and_then(|executor| executor.actor_context(&root_id));
761            let root_has_pending_bg_wake =
762                bg_sub_by_session.iter().any(|((sub_root, _), channels)| {
763                    sub_root == &root_id
764                        && channels
765                            .iter()
766                            .any(|channel| bg_wake_pending.contains(channel))
767                });
768            let kinds_with_work: Vec<MaintenanceDrainKind> = match executor_actor_context {
769                Some(ctx) => INITIAL_MAINTENANCE_DRAIN_KINDS
770                    .into_iter()
771                    .filter(|kind| {
772                        if meta.unbound_quiesced && !matches!(kind, MaintenanceDrainKind::Lsp) {
773                            return false;
774                        }
775                        match kind {
776                            MaintenanceDrainKind::Watcher => ctx.watcher_drain_has_work(),
777                            MaintenanceDrainKind::Lsp => ctx.lsp_drain_has_work(),
778                            MaintenanceDrainKind::ConfigureTail => ctx.configure_tail_has_work(),
779                            // Every CompletionDrains source is visible at this enqueue site:
780                            // AppContext probes completion queues, this loop owns bg wakes,
781                            // and queued continuations bypass probing via maintenance_pending.
782                            // New drain sources must expose a probe here rather than making
783                            // every subscribed root fail open again.
784                            MaintenanceDrainKind::CompletionDrains => {
785                                root_has_pending_bg_wake || ctx.completion_drains_have_work()
786                            }
787                        }
788                    })
789                    .collect(),
790                None if meta.unbound_quiesced => Vec::new(),
791                // No context handle (actor gone mid-tick): enqueue everything.
792                None => INITIAL_MAINTENANCE_DRAIN_KINDS.to_vec(),
793            };
794            if kinds_with_work.is_empty() {
795                continue;
796            }
797            meta.maintenance_pending = true;
798            meta.maintenance_queued_kinds.extend(kinds_with_work);
799        }
800
801        while let Some(kind) = meta.maintenance_queued_kinds.pop_front() {
802            if jobs.len() >= budget {
803                meta.maintenance_queued_kinds.push_front(kind);
804                deferred = true;
805                break;
806            }
807            meta.maintenance_jobs_in_flight += 1;
808            meta.maintenance_last_submitted = Some(Instant::now());
809            jobs.push((root_id.clone(), kind));
810        }
811
812        meta.maintenance_pending =
813            meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
814    }
815
816    (jobs, deferred)
817}
818
819fn eviction_estimate_label(estimate: &crate::memory::MemoryEstimate) -> String {
820    match estimate.estimated_bytes {
821        Some(bytes) => format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
822        None if estimate.status == "busy" => "busy".to_string(),
823        None => "not estimated".to_string(),
824    }
825}
826
827fn optional_memory_label(bytes: Option<u64>) -> String {
828    bytes.map_or_else(
829        || "not estimated".to_string(),
830        |bytes| format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
831    )
832}
833
834fn pressure_relief_label(relief: &crate::memory::AllocatorPressureRelief) -> String {
835    format!(
836        "; allocator pressure relief: RSS {} -> {}, in-use {} -> {}, allocated {} -> {}, slack {} -> {}, allocator reported {:.1} MB released",
837        optional_memory_label(relief.rss_before_bytes),
838        optional_memory_label(relief.rss_after_bytes),
839        optional_memory_label(relief.allocator_before.bytes_in_use),
840        optional_memory_label(relief.allocator_after.bytes_in_use),
841        optional_memory_label(relief.allocator_before.size_allocated),
842        optional_memory_label(relief.allocator_after.size_allocated),
843        optional_memory_label(relief.allocator_before.retained_slack_bytes),
844        optional_memory_label(relief.allocator_after.retained_slack_bytes),
845        relief.bytes_released as f64 / (1024.0 * 1024.0),
846    )
847}
848
849fn idle_root_eviction_message(
850    root_id: &ProjectRootId,
851    memory: &crate::memory::RootMemorySnapshot,
852    pressure_relief: Option<&crate::memory::AllocatorPressureRelief>,
853) -> String {
854    // Bash, LSP, and parser state remain resident. The freed total is deliberately
855    // only the known-byte portion of handles eviction actually drops.
856    let freed_bytes = [
857        &memory.semantic,
858        &memory.trigram,
859        &memory.symbols,
860        &memory.callgraph,
861        &memory.inspect,
862    ]
863    .iter()
864    .filter_map(|estimate| estimate.estimated_bytes)
865    .fold(0u64, u64::saturating_add);
866    let mut message = format!(
867        "evicted idle root {}: freed ~{:.1} MB (semantic {}, trigram {}, symbols {}, callgraph {}, inspect {}; retained: bash {}, lsp {}, parser_pool {})",
868        root_id.as_path().display(),
869        freed_bytes as f64 / (1024.0 * 1024.0),
870        eviction_estimate_label(&memory.semantic),
871        eviction_estimate_label(&memory.trigram),
872        eviction_estimate_label(&memory.symbols),
873        eviction_estimate_label(&memory.callgraph),
874        eviction_estimate_label(&memory.inspect),
875        eviction_estimate_label(&memory.bash),
876        eviction_estimate_label(&memory.lsp),
877        eviction_estimate_label(&memory.parser_pool),
878    );
879    if let Some(pressure_relief) = pressure_relief {
880        message.push_str(&pressure_relief_label(pressure_relief));
881    }
882    message
883}
884
885fn process_has_been_idle(now: Instant, live_roots: &HashMap<ProjectRootId, RootMeta>) -> bool {
886    !live_roots.is_empty()
887        && live_roots.values().all(|meta| {
888            now.saturating_duration_since(meta.last_touched) >= IDLE_ROOT_TTL
889                && meta.active_bash_waits == 0
890                && !meta.maintenance_pending
891                && meta.maintenance_queued_kinds.is_empty()
892        })
893}
894
895fn allocator_pressure_relief_after_idle_sweep(
896    now: Instant,
897    live_roots: &HashMap<ProjectRootId, RootMeta>,
898    executor: &Executor,
899) -> Option<crate::memory::AllocatorPressureRelief> {
900    if !process_has_been_idle(now, live_roots)
901        || live_roots.keys().any(|root_id| {
902            executor
903                .actor_context(root_id)
904                .is_some_and(|ctx| ctx.artifact_eviction_blocked())
905        })
906    {
907        return None;
908    }
909
910    #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu")))]
911    {
912        Some(crate::memory::relieve_allocator_pressure())
913    }
914    #[cfg(not(any(target_os = "macos", all(target_os = "linux", target_env = "gnu"))))]
915    {
916        None
917    }
918}
919
920fn quiesce_unbound_root(
921    root_id: &ProjectRootId,
922    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
923    executor: &Arc<Executor>,
924) {
925    let Some(meta) = live_roots.get_mut(root_id) else {
926        return;
927    };
928
929    let ctx = executor.actor_context(root_id);
930    if let Some(ctx) = ctx.as_ref() {
931        // Close lifecycle admission before touching scheduler queues. A running
932        // ConfigureTail cannot release gates, install a watcher, or reserve a
933        // callgraph build after this transition becomes visible.
934        ctx.mark_subc_unbound();
935    }
936    let cancelled = executor.cancel_queued_maintenance(root_id);
937    // Transient unbind keeps the root WARM: the watcher stays running (its
938    // events accumulate and replay on rebind, so no unobserved gap exists) and
939    // resident artifacts stay resident. Host restarts unbind every root and
940    // rebind seconds later; stopping the watcher here would force strict
941    // re-verification plus a full callgraph rebuild on every restart. The
942    // expensive teardown (watcher stop + gap invalidation) belongs to the
943    // idle-TTL reaper and the root-deleted path.
944    let discarded = ctx
945        .map(|ctx| crate::commands::configure::cancel_deferred_configure_maintenance(&ctx))
946        .unwrap_or(0);
947    meta.unbound_quiesced = true;
948    meta.maintenance_queued_kinds.clear();
949    meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
950    log::info!(
951        "subc attach: quiesced unbound root {} (cancelled {} queued maintenance job(s), cancelled {} configure maintenance job(s)); cause=goodbye_unbound",
952        root_id.as_path().display(),
953        cancelled,
954        discarded
955    );
956}
957
958#[allow(clippy::too_many_arguments)]
959fn quiesce_connection_roots(
960    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
961    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
962    routes: &mut HashMap<RouteChannel, RouteIdentity>,
963    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
964    installed_route_epochs: &mut HashMap<u16, u32>,
965    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
966    active_tool_calls: &ActiveToolCalls,
967    executor: &Arc<Executor>,
968) {
969    cancel_all_active_tool_calls(active_tool_calls, executor, "connection teardown");
970    for cancel in route_bash_cancels.values() {
971        cancel.token.cancel();
972    }
973    route_bash_cancels.clear();
974
975    let mut roots = live_roots.keys().cloned().collect::<HashSet<_>>();
976    for pending in pending_binds.values_mut() {
977        pending.cancelled = true;
978        roots.insert(pending.bind_root_id.clone());
979        let _ = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
980    }
981
982    // A connection exit abandons every installed route at once. Close lifecycle
983    // admission before cancelling maintenance so no deferred worker can restore
984    // root activity after the loop-owned route tables disappear.
985    for root_id in roots {
986        if live_roots.contains_key(&root_id) {
987            quiesce_unbound_root(&root_id, live_roots, executor);
988        } else if let Some(ctx) = executor.actor_context(&root_id) {
989            ctx.mark_subc_unbound();
990            executor.cancel_queued_maintenance(&root_id);
991            crate::commands::configure::cancel_deferred_configure_maintenance(&ctx);
992        }
993    }
994
995    routes.clear();
996    root_channels.clear();
997    installed_route_epochs.clear();
998}
999
1000/// Per-channel epoch watermarks for roots reclaimed without a client Goodbye.
1001/// The 16-bit channel space bounds this map, and no root identity or resource
1002/// handle is retained. It exists only so late requests receive a typed error.
1003#[derive(Debug, Default)]
1004struct ReclaimedRoutes {
1005    highest_epoch_by_channel: HashMap<u16, u32>,
1006}
1007
1008impl ReclaimedRoutes {
1009    fn insert(&mut self, route: RouteChannel) {
1010        self.highest_epoch_by_channel
1011            .entry(route.channel)
1012            .and_modify(|epoch| *epoch = (*epoch).max(route.epoch))
1013            .or_insert(route.epoch);
1014    }
1015
1016    fn contains(&self, route: RouteChannel) -> bool {
1017        self.highest_epoch_by_channel
1018            .get(&route.channel)
1019            .is_some_and(|epoch| route.epoch <= *epoch)
1020    }
1021}
1022
1023#[derive(Debug, Default)]
1024struct IdleReapOutcome {
1025    evicted: usize,
1026    forgotten_deleted_roots: Vec<ProjectRootId>,
1027}
1028
1029fn reap_idle_roots(
1030    now: Instant,
1031    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1032    pending_binds: &HashMap<RouteChannel, PendingBind>,
1033    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1034    executor: &Arc<Executor>,
1035    metrics: &DispatchPathMetrics,
1036) -> IdleReapOutcome {
1037    let pending_bind_roots = pending_binds
1038        .values()
1039        .map(|pending| pending.bind_root_id.clone())
1040        .collect::<HashSet<_>>();
1041    let mut census = ReapBlockerCensus::default();
1042    let mut candidates = Vec::new();
1043
1044    for (root_id, meta) in live_roots.iter_mut() {
1045        let deleted = !root_id.as_path().exists();
1046        if deleted {
1047            // A missing directory makes a bound route obsolete, but one failed
1048            // lookup is not enough evidence to tear down a client-visible actor.
1049            // Requiring two maintenance sweeps protects atomic replacement and
1050            // transient filesystem failures; observing the path resets the proof.
1051            // Absence also covers renames: a task's cwd handle can follow the
1052            // moved directory while the registered path disappears. The old
1053            // path is deliberately treated as a retired root identity, so the
1054            // reaper accepts killing such tasks; rename a project only with no
1055            // live tasks rather than relying on cwd-resolution heuristics.
1056            meta.consecutive_missing_sweeps = meta.consecutive_missing_sweeps.saturating_add(1);
1057        } else {
1058            meta.consecutive_missing_sweeps = 0;
1059        }
1060        let deletion_confirmed = meta.consecutive_missing_sweeps >= 2;
1061        let has_bound_route = root_channels
1062            .get(root_id)
1063            .is_some_and(|channels| !channels.is_empty());
1064        let has_pending_bind = pending_bind_roots.contains(root_id);
1065
1066        if deleted {
1067            let mut retained = false;
1068            if !deletion_confirmed {
1069                census.absence_unconfirmed += 1;
1070                retained = true;
1071            }
1072            // Once absence is confirmed, the directory cannot serve this route
1073            // again. Neither a stale route nor the lack of normal unbind cleanup
1074            // justifies retaining the root; purge removes the route after retirement.
1075            if meta.active_bash_waits > 0 {
1076                census.bash_waits += 1;
1077                retained = true;
1078            }
1079            if meta.maintenance_pending {
1080                census.maintenance_pending += 1;
1081                retained = true;
1082            }
1083            if !meta.maintenance_queued_kinds.is_empty() {
1084                census.maintenance_queued += 1;
1085                retained = true;
1086            }
1087            if has_pending_bind {
1088                census.pending_binds += 1;
1089                retained = true;
1090            }
1091            match executor.try_actor_is_idle(root_id) {
1092                Some(true) => {}
1093                Some(false) => {
1094                    census.actor_busy += 1;
1095                    retained = true;
1096                }
1097                None => {
1098                    census.actor_state_busy += 1;
1099                    retained = true;
1100                }
1101            }
1102            if retained {
1103                census.deleted_retained += 1;
1104                continue;
1105            }
1106        } else {
1107            // Route teardown marks the lifecycle admission gate before the last
1108            // channel disappears. Requiring zero bound channels and a quiesced
1109            // lifecycle prevents a still-bound root from losing its watcher.
1110            if has_bound_route
1111                || !meta.unbound_quiesced
1112                || meta.idle_artifacts_evicted
1113                || now.saturating_duration_since(meta.last_touched) < IDLE_ROOT_TTL
1114                || meta.active_bash_waits > 0
1115                || meta.maintenance_pending
1116                || !meta.maintenance_queued_kinds.is_empty()
1117                || has_pending_bind
1118                || !executor.actor_is_idle(root_id)
1119            {
1120                continue;
1121            }
1122        }
1123        candidates.push((root_id.clone(), deleted));
1124    }
1125
1126    let mut reaped = Vec::new();
1127    let mut forgotten_deleted_roots = Vec::new();
1128    for (root_id, deleted) in candidates {
1129        let Some(ctx) = executor.actor_context(&root_id) else {
1130            if deleted {
1131                census.deleted_retained += 1;
1132                census.actor_busy += 1;
1133            }
1134            continue;
1135        };
1136        // A TTL-aged unbound root retained its watcher-derived pending paths
1137        // across the transient-unbind window. Strict gap invalidation subsumes
1138        // them, but every abort path must restore them because a rebind can
1139        // still happen until eviction commits.
1140        //
1141        // After two consecutive directory-absence scans confirm that the
1142        // root is gone, terminate its background task before checking the
1143        // artifact-eviction gate. The task can otherwise keep the root's
1144        // artifacts in use; cleanup first lets confirmed reclamation finish
1145        // without weakening the gate for unrelated active work.
1146        if deleted {
1147            ctx.bash_background()
1148                .kill_running_tasks_for_root(root_id.as_path());
1149        }
1150        let taken_pending = Some(ctx.take_pending_reconciliation_state());
1151        if ctx.artifact_eviction_blocked() {
1152            if let Some(pending) = taken_pending {
1153                ctx.restore_pending_reconciliation_state(pending);
1154            }
1155            if deleted {
1156                census.deleted_retained += 1;
1157                census.artifact_eviction_blocked += 1;
1158            }
1159            continue;
1160        }
1161        let memory_before = ctx.memory_root_snapshot();
1162        if !ctx.evict_idle_artifacts() {
1163            if let Some(pending) = taken_pending {
1164                ctx.restore_pending_reconciliation_state(pending);
1165            }
1166            if deleted {
1167                census.deleted_retained += 1;
1168                census.artifact_eviction_failed += 1;
1169            }
1170            continue;
1171        }
1172        drop(taken_pending);
1173        ctx.stop_watcher_runtime_in_background();
1174        // Edits during watcher downtime are unobserved. Advance publication
1175        // epochs and force strict verification before any later warm reload.
1176        ctx.invalidate_artifacts_after_watcher_gap();
1177
1178        if deleted {
1179            if executor.retire_idle_actor_in_background(&root_id) {
1180                live_roots.remove(&root_id);
1181                forgotten_deleted_roots.push(root_id.clone());
1182            } else {
1183                census.deleted_retained += 1;
1184                census.actor_busy += 1;
1185            }
1186        } else {
1187            if let Some(meta) = live_roots.get_mut(&root_id) {
1188                meta.idle_artifacts_evicted = true;
1189            }
1190            ctx.release_idle_reopenable_resources_in_background();
1191        }
1192        reaped.push((root_id, memory_before));
1193    }
1194
1195    metrics.record_reap(census);
1196    if census.deleted_retained > 0 {
1197        log::info!(
1198            "subc attach: retained {} deleted root(s) during idle reap; blockers={}",
1199            census.deleted_retained,
1200            census.blocker_histogram()
1201        );
1202    }
1203
1204    let pressure_relief = (!reaped.is_empty())
1205        .then(|| allocator_pressure_relief_after_idle_sweep(now, live_roots, executor))
1206        .flatten();
1207    for (root_id, memory_before) in &reaped {
1208        log::info!(
1209            "{}",
1210            idle_root_eviction_message(root_id, memory_before, pressure_relief.as_ref())
1211        );
1212    }
1213    IdleReapOutcome {
1214        evicted: reaped.len(),
1215        forgotten_deleted_roots,
1216    }
1217}
1218
1219#[allow(clippy::too_many_arguments)]
1220fn purge_deleted_root_residents(
1221    root_id: &ProjectRootId,
1222    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1223    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1224    installed_route_epochs: &mut HashMap<u16, u32>,
1225    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1226    active_tool_calls: &ActiveToolCalls,
1227    executor: &Executor,
1228    retry_buffer: &mut RetryBuffer,
1229    reclaimed_routes: &mut ReclaimedRoutes,
1230    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1231    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1232    bg_subs: &mut HashMap<RouteChannel, BgSub>,
1233    bg_sub_by_session: &mut BgSubsBySession,
1234    bg_wake_pending: &mut HashSet<RouteChannel>,
1235    bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
1236    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1237    metrics: &DispatchPathMetrics,
1238) {
1239    let mut stale_routes = root_channels.get(root_id).cloned().unwrap_or_default();
1240    stale_routes.extend(
1241        routes
1242            .iter()
1243            .filter_map(|(route, identity)| (&identity.root == root_id).then_some(*route)),
1244    );
1245    stale_routes.extend(
1246        bg_sub_by_session
1247            .iter()
1248            .filter(|((root, _), _)| root == root_id)
1249            .flat_map(|(_, routes)| routes.iter().copied()),
1250    );
1251    stale_routes.extend(
1252        pending_bash_asks
1253            .values()
1254            .filter_map(|ask| (&ask.root == root_id).then_some(ask.route)),
1255    );
1256
1257    for route in stale_routes {
1258        reclaimed_routes.insert(route);
1259        remove_installed_route(installed_route_epochs, route);
1260        remove_route_channel(routes, root_channels, route);
1261        if let Some(cancel) = route_bash_cancels.remove(&route) {
1262            cancel.token.cancel();
1263        }
1264        cancel_active_tool_calls_for_route(active_tool_calls, executor, route, "root reclaim");
1265        retry_buffer.remove(&route);
1266        if let Some(sub) = bg_subs.remove(&route) {
1267            metrics.record_bg_subscription_ended(&sub.root, &sub.session, route, "root-reclaim");
1268        }
1269        bg_wake_pending.remove(&route);
1270    }
1271    root_channels.remove(root_id);
1272    session_identity.retain(|(root, _), _| root != root_id);
1273    push_buffer.retain(|key, _| &key.root != root_id);
1274    bg_wake_epoch.retain(|(root, _), _| root != root_id);
1275    pending_bash_asks.retain(|_, ask| &ask.root != root_id);
1276    bg_sub_by_session.retain(|(root, _), _| root != root_id);
1277
1278    log::info!(
1279        "subc attach: fully forgot deleted root {}; cause=absence_reclaim",
1280        root_id.as_path().display()
1281    );
1282}
1283
1284#[allow(clippy::too_many_arguments)]
1285fn submit_due_maintenance_jobs(
1286    executor: &Arc<Executor>,
1287    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1288    pending_binds: &HashMap<RouteChannel, PendingBind>,
1289    bg_sub_by_session: &BgSubsBySession,
1290    bg_wake_pending: &HashSet<RouteChannel>,
1291    bg_wake_epoch: &HashMap<(ProjectRootId, String), u64>,
1292    maintenance_tx: &mpsc::Sender<MaintenanceCompletion>,
1293    metrics: &Arc<DispatchPathMetrics>,
1294) {
1295    let pending_bind_roots = pending_binds
1296        .values()
1297        .map(|pending| pending.bind_root_id.clone())
1298        .collect::<HashSet<_>>();
1299    let (due_jobs, deferred_jobs) = due_maintenance_jobs(
1300        live_roots,
1301        Some(executor),
1302        bg_sub_by_session,
1303        bg_wake_pending,
1304        MAINTENANCE_SUBMIT_BUDGET,
1305        &pending_bind_roots,
1306    );
1307    if deferred_jobs {
1308        metrics
1309            .maintenance_budget_deferrals
1310            .fetch_add(1, Ordering::Relaxed);
1311    }
1312    for (root_id, kind) in due_jobs {
1313        let bg_sessions_to_check = if kind == MaintenanceDrainKind::CompletionDrains {
1314            bg_sub_by_session
1315                .iter()
1316                .filter_map(|((root, session), _)| {
1317                    if root == &root_id {
1318                        Some((
1319                            session.clone(),
1320                            bg_wake_epoch
1321                                .get(&(root_id.clone(), session.clone()))
1322                                .copied()
1323                                .unwrap_or(0),
1324                        ))
1325                    } else {
1326                        None
1327                    }
1328                })
1329                .collect()
1330        } else {
1331            Vec::new()
1332        };
1333        submit_maintenance_job(
1334            executor,
1335            root_id,
1336            kind,
1337            bg_sessions_to_check,
1338            maintenance_tx,
1339            metrics,
1340        );
1341    }
1342}
1343
1344fn should_requiesce_after_maintenance(
1345    meta: &RootMeta,
1346    completed_kind: MaintenanceDrainKind,
1347    bind_pending: bool,
1348) -> bool {
1349    meta.unbound_quiesced && completed_kind != MaintenanceDrainKind::Lsp && !bind_pending
1350}
1351
1352fn note_maintenance_completion(
1353    meta: &mut RootMeta,
1354    requeue_kind: Option<MaintenanceDrainKind>,
1355    fatal: bool,
1356    defer_requeue: bool,
1357) {
1358    if fatal {
1359        meta.maintenance_poisoned = true;
1360    }
1361
1362    if let Some(kind) = requeue_kind.filter(|_| !meta.maintenance_poisoned && !defer_requeue) {
1363        meta.maintenance_queued_kinds.push_back(kind);
1364    }
1365
1366    meta.maintenance_jobs_in_flight = meta.maintenance_jobs_in_flight.saturating_sub(1);
1367    meta.maintenance_pending =
1368        meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
1369}
1370
1371fn route_key(channel: u16, epoch: u32) -> RouteChannel {
1372    RouteChannel { channel, epoch }
1373}
1374
1375fn remove_installed_route(installed_epochs: &mut HashMap<u16, u32>, route: RouteChannel) {
1376    if installed_epochs.get(&route.channel).copied() == Some(route.epoch) {
1377        installed_epochs.remove(&route.channel);
1378    }
1379}
1380
1381fn ingress_route_should_be_processed(
1382    installed_epochs: &HashMap<u16, u32>,
1383    reclaimed_routes: &ReclaimedRoutes,
1384    frame: &Frame,
1385) -> bool {
1386    if frame.header.channel == 0
1387        || installed_epochs.get(&frame.header.channel).copied() == Some(frame.header.epoch)
1388    {
1389        return true;
1390    }
1391
1392    // A late request for a reclaimed root reaches the normal unknown-route
1393    // handler, which returns the typed `route_not_bound` error. Other stale or
1394    // never-installed generations remain silent so they cannot affect a newer
1395    // route or change the protocol's rejected-bind behavior.
1396    frame.header.ty == FrameType::Request
1397        && reclaimed_routes.contains(route_key(frame.header.channel, frame.header.epoch))
1398}
1399
1400fn bash_elicitation_timeout() -> Duration {
1401    if cfg!(debug_assertions) {
1402        if let Ok(raw) = std::env::var("AFT_TEST_SUBC_BASH_ELICITATION_TTL_MS") {
1403            if let Ok(ms) = raw.parse::<u64>() {
1404                if ms > 0 {
1405                    return Duration::from_millis(ms);
1406                }
1407            }
1408        }
1409    }
1410    BASH_ELICITATION_TIMEOUT
1411}
1412
1413fn allocate_reverse_corr(
1414    pending_bash_asks: &HashMap<ReverseCorrKey, PendingBashAsk>,
1415    route: RouteChannel,
1416    next_corr: &mut u64,
1417) -> u64 {
1418    loop {
1419        let corr = *next_corr;
1420        *next_corr = (*next_corr).wrapping_add(1).max(1);
1421        if !pending_bash_asks.contains_key(&ReverseCorrKey { route, corr }) {
1422            return corr;
1423        }
1424    }
1425}
1426
1427fn bash_permission_kind_label(kind: &crate::bash_permissions::PermissionKind) -> &'static str {
1428    match kind {
1429        crate::bash_permissions::PermissionKind::ExternalDirectory => "external directory",
1430        crate::bash_permissions::PermissionKind::Bash => "bash",
1431    }
1432}
1433
1434fn bash_elicitation_patterns(asks: &[crate::bash_permissions::PermissionAsk]) -> Vec<String> {
1435    let mut patterns = Vec::new();
1436    let mut seen = HashSet::new();
1437    for ask in asks {
1438        for pattern in ask.patterns.iter().chain(ask.always.iter()) {
1439            if seen.insert(pattern.clone()) {
1440                patterns.push(pattern.clone());
1441            }
1442        }
1443    }
1444    patterns
1445}
1446
1447fn bash_elicitation_message(
1448    command: &str,
1449    asks: &[crate::bash_permissions::PermissionAsk],
1450) -> String {
1451    let command = command.split_whitespace().collect::<Vec<_>>().join(" ");
1452    let patterns = bash_elicitation_patterns(asks);
1453    let pattern_text = if patterns.is_empty() {
1454        "no matched permission patterns".to_string()
1455    } else {
1456        patterns.join(", ")
1457    };
1458    let ask_kinds = asks
1459        .iter()
1460        .map(|ask| bash_permission_kind_label(&ask.kind))
1461        .collect::<HashSet<_>>()
1462        .into_iter()
1463        .collect::<Vec<_>>()
1464        .join(", ");
1465    if ask_kinds.is_empty() {
1466        format!("Allow bash command `{command}`? Matched patterns: {pattern_text}")
1467    } else {
1468        format!("Allow bash command `{command}`? Matched {ask_kinds} patterns: {pattern_text}")
1469    }
1470}
1471
1472fn bash_elicitation_request_body(
1473    command: &str,
1474    asks: &[crate::bash_permissions::PermissionAsk],
1475) -> Value {
1476    json!({
1477        "method": BASH_ELICITATION_CREATE_METHOD,
1478        "params": {
1479            "mode": "form",
1480            "message": bash_elicitation_message(command, asks),
1481            "requestedSchema": {
1482                "type": "object",
1483                "properties": {
1484                    "decision": {
1485                        "type": "string",
1486                        "enum": ["allow", "deny"],
1487                        "description": "Choose allow to run this bash command once, or deny to block it."
1488                    }
1489                },
1490                "required": ["decision"],
1491                "additionalProperties": false
1492            },
1493            "_meta": {
1494                "aft": {
1495                    "tool": "bash",
1496                    "command": command,
1497                    "asks": asks
1498                }
1499            }
1500        }
1501    })
1502}
1503
1504fn build_bash_elicitation_request_frame(
1505    ver: u8,
1506    route: RouteChannel,
1507    corr: u64,
1508    flags: Flags,
1509    command: &str,
1510    asks: &[crate::bash_permissions::PermissionAsk],
1511) -> Result<Frame, SubcError> {
1512    let body = bash_elicitation_request_body(command, asks);
1513    Frame::build_with_version(
1514        ver,
1515        FrameType::Request,
1516        flags,
1517        route.channel,
1518        route.epoch,
1519        corr,
1520        serde_json::to_vec(&body).map_err(SubcError::Json)?,
1521    )
1522    .map_err(SubcError::FrameBuild)
1523}
1524
1525fn bash_elicitation_reply_is_allow(body: &[u8]) -> bool {
1526    let Ok(value) = serde_json::from_slice::<Value>(body) else {
1527        return false;
1528    };
1529    flat_bash_elicitation_reply_is_allow(&value) || mcp_bash_elicitation_reply_is_allow(&value)
1530}
1531
1532fn flat_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1533    let Some(object) = value.as_object() else {
1534        return false;
1535    };
1536    object.len() == 1 && object.get("decision").and_then(Value::as_str) == Some("allow")
1537}
1538
1539fn mcp_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1540    let Some(object) = value.as_object() else {
1541        return false;
1542    };
1543    if object.len() != 2 || object.get("action").and_then(Value::as_str) != Some("accept") {
1544        return false;
1545    }
1546    let Some(content) = object.get("content").and_then(Value::as_object) else {
1547        return false;
1548    };
1549    content.len() == 1 && content.get("decision").and_then(Value::as_str) == Some("allow")
1550}
1551
1552#[allow(clippy::too_many_arguments)]
1553async fn settle_pending_bash_ask_denied(
1554    tx: &WriterSender,
1555    pending: PendingBashAsk,
1556    routes: &HashMap<RouteChannel, RouteIdentity>,
1557    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1558    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1559    shutdown: &Arc<Notify>,
1560    metrics: &DispatchPathMetrics,
1561) -> Result<(), SubcError> {
1562    let completion = bash::bash_denied_untrusted_completion(
1563        pending.route,
1564        pending.tool_corr,
1565        pending.tool_flags,
1566        pending.tool_ver,
1567        pending.root,
1568        pending.request_id,
1569        pending.format_context,
1570    );
1571    bash::handle_bash_deferred_completion(
1572        tx,
1573        completion,
1574        routes,
1575        live_roots,
1576        route_bash_cancels,
1577        shutdown,
1578        metrics,
1579    )
1580    .await
1581}
1582
1583fn take_pending_bash_asks_for_route(
1584    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1585    route: RouteChannel,
1586) -> Vec<PendingBashAsk> {
1587    let keys = pending_bash_asks
1588        .keys()
1589        .copied()
1590        .filter(|key| key.route == route)
1591        .collect::<Vec<_>>();
1592    keys.into_iter()
1593        .filter_map(|key| pending_bash_asks.remove(&key))
1594        .collect()
1595}
1596
1597#[allow(clippy::too_many_arguments)]
1598async fn settle_pending_bash_asks_for_route(
1599    tx: &WriterSender,
1600    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1601    route: RouteChannel,
1602    routes: &HashMap<RouteChannel, RouteIdentity>,
1603    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1604    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1605    shutdown: &Arc<Notify>,
1606    metrics: &DispatchPathMetrics,
1607) -> Result<(), SubcError> {
1608    for pending in take_pending_bash_asks_for_route(pending_bash_asks, route) {
1609        settle_pending_bash_ask_denied(
1610            tx,
1611            pending,
1612            routes,
1613            live_roots,
1614            route_bash_cancels,
1615            shutdown,
1616            metrics,
1617        )
1618        .await?;
1619    }
1620    Ok(())
1621}
1622
1623#[allow(clippy::too_many_arguments)]
1624async fn settle_all_pending_bash_asks(
1625    tx: &WriterSender,
1626    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1627    routes: &HashMap<RouteChannel, RouteIdentity>,
1628    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1629    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1630    shutdown: &Arc<Notify>,
1631    metrics: &DispatchPathMetrics,
1632) -> Result<(), SubcError> {
1633    let pending = pending_bash_asks
1634        .drain()
1635        .map(|(_, pending)| pending)
1636        .collect::<Vec<_>>();
1637    for pending in pending {
1638        settle_pending_bash_ask_denied(
1639            tx,
1640            pending,
1641            routes,
1642            live_roots,
1643            route_bash_cancels,
1644            shutdown,
1645            metrics,
1646        )
1647        .await?;
1648    }
1649    Ok(())
1650}
1651
1652#[allow(clippy::too_many_arguments)]
1653async fn expire_pending_bash_asks(
1654    tx: &WriterSender,
1655    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1656    routes: &HashMap<RouteChannel, RouteIdentity>,
1657    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1658    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1659    shutdown: &Arc<Notify>,
1660    metrics: &DispatchPathMetrics,
1661) -> Result<(), SubcError> {
1662    let now = Instant::now();
1663    let expired = pending_bash_asks
1664        .iter()
1665        .filter_map(|(key, pending)| (pending.expires_at <= now).then_some(*key))
1666        .collect::<Vec<_>>();
1667    for key in expired {
1668        if let Some(pending) = pending_bash_asks.remove(&key) {
1669            log::debug!(
1670                "subc attach: bash elicitation request {} on route {} expired fail-closed",
1671                key.corr,
1672                pending.route
1673            );
1674            settle_pending_bash_ask_denied(
1675                tx,
1676                pending,
1677                routes,
1678                live_roots,
1679                route_bash_cancels,
1680                shutdown,
1681                metrics,
1682            )
1683            .await?;
1684        }
1685    }
1686    Ok(())
1687}
1688
1689#[allow(clippy::too_many_arguments)]
1690async fn handle_bash_elicitation_reply(
1691    tx: &WriterSender,
1692    frame: &Frame,
1693    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1694    routes: &HashMap<RouteChannel, RouteIdentity>,
1695    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1696    executor: &Arc<Executor>,
1697    shutdown: &Arc<Notify>,
1698    bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
1699    bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
1700    metrics: &Arc<DispatchPathMetrics>,
1701    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1702    dispatch: DispatchFn,
1703) -> Result<(), SubcError> {
1704    let key = ReverseCorrKey {
1705        route: route_key(frame.header.channel, frame.header.epoch),
1706        corr: frame.header.corr,
1707    };
1708    let Some(pending) = pending_bash_asks.remove(&key) else {
1709        return Ok(());
1710    };
1711
1712    if frame.header.ty == FrameType::Response && bash_elicitation_reply_is_allow(&frame.body) {
1713        if routes.contains_key(&key.route) {
1714            bash::submit_deferred_bash(
1715                executor,
1716                bash_deferred_tx,
1717                bash_poll_touch_tx,
1718                metrics,
1719                dispatch,
1720                pending.root,
1721                pending.project_root,
1722                pending.session_id,
1723                pending.request_id,
1724                pending.route,
1725                pending.tool_corr,
1726                pending.tool_flags,
1727                pending.tool_ver,
1728                pending.arguments,
1729                pending.format_context,
1730                pending.cancel,
1731                BindTrust::Untrusted,
1732                pending.spawn_principal,
1733                pending.edit_slot_survives,
1734                Some(pending.grants),
1735            );
1736            return Ok(());
1737        }
1738        log::debug!(
1739            "subc attach: dropping allowed bash elicitation reply {} for unbound route {}",
1740            key.corr,
1741            pending.route
1742        );
1743    }
1744
1745    settle_pending_bash_ask_denied(
1746        tx,
1747        pending,
1748        routes,
1749        live_roots,
1750        route_bash_cancels,
1751        shutdown,
1752        metrics,
1753    )
1754    .await
1755}
1756
1757#[allow(clippy::too_many_arguments)]
1758async fn cancel_pending_bash_ask_for_tool_call(
1759    tx: &WriterSender,
1760    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1761    route: RouteChannel,
1762    tool_corr: u64,
1763    routes: &HashMap<RouteChannel, RouteIdentity>,
1764    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1765    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1766    shutdown: &Arc<Notify>,
1767    metrics: &DispatchPathMetrics,
1768) -> Result<(), SubcError> {
1769    let keys = pending_bash_asks
1770        .iter()
1771        .filter_map(|(key, pending)| {
1772            (key.route == route && pending.tool_corr == tool_corr).then_some(*key)
1773        })
1774        .collect::<Vec<_>>();
1775    for key in keys {
1776        if let Some(pending) = pending_bash_asks.remove(&key) {
1777            settle_pending_bash_ask_denied(
1778                tx,
1779                pending,
1780                routes,
1781                live_roots,
1782                route_bash_cancels,
1783                shutdown,
1784                metrics,
1785            )
1786            .await?;
1787        }
1788    }
1789    Ok(())
1790}
1791
1792fn remove_root_channel(
1793    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1794    root: &ProjectRootId,
1795    channel: RouteChannel,
1796) {
1797    let remove_root = if let Some(channels) = root_channels.get_mut(root) {
1798        channels.remove(&channel);
1799        channels.is_empty()
1800    } else {
1801        false
1802    };
1803    if remove_root {
1804        root_channels.remove(root);
1805    }
1806}
1807
1808fn remove_route_channel(
1809    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1810    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1811    channel: RouteChannel,
1812) -> Option<RouteIdentity> {
1813    let removed = routes.remove(&channel);
1814    if let Some(identity) = &removed {
1815        remove_root_channel(root_channels, &identity.root, channel);
1816    }
1817    removed
1818}
1819
1820fn insert_route_channel(
1821    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1822    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1823    channel: RouteChannel,
1824    identity: RouteIdentity,
1825) {
1826    if let Some(previous) = routes.insert(channel, identity.clone()) {
1827        remove_root_channel(root_channels, &previous.root, channel);
1828    }
1829    root_channels
1830        .entry(identity.root.clone())
1831        .or_default()
1832        .insert(channel);
1833}
1834
1835fn insert_bg_subscription_index(
1836    bg_sub_by_session: &mut BgSubsBySession,
1837    root: ProjectRootId,
1838    session: String,
1839    channel: RouteChannel,
1840) {
1841    bg_sub_by_session
1842        .entry((root, session))
1843        .or_default()
1844        .insert(channel);
1845}
1846
1847fn remove_bg_subscription_index(
1848    bg_sub_by_session: &mut BgSubsBySession,
1849    channel: RouteChannel,
1850    identity: Option<&RouteIdentity>,
1851) {
1852    if let Some(identity) = identity {
1853        let key = (identity.root.clone(), identity.session.clone());
1854        let remove_key = bg_sub_by_session.get_mut(&key).is_some_and(|channels| {
1855            channels.remove(&channel);
1856            channels.is_empty()
1857        });
1858        if remove_key {
1859            bg_sub_by_session.remove(&key);
1860        }
1861    } else {
1862        bg_sub_by_session.retain(|_, channels| {
1863            channels.remove(&channel);
1864            !channels.is_empty()
1865        });
1866    }
1867}
1868
1869fn route_removal_will_quiesce_root(
1870    root: &ProjectRootId,
1871    route: RouteChannel,
1872    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1873    has_pending_bind: bool,
1874    replacement_root: Option<&ProjectRootId>,
1875) -> bool {
1876    let removes_last_route = root_channels
1877        .get(root)
1878        .is_some_and(|channels| channels.len() == 1 && channels.contains(&route));
1879    removes_last_route && !has_pending_bind && replacement_root != Some(root)
1880}
1881
1882fn should_quiesce_removed_root(
1883    root: &ProjectRootId,
1884    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1885    has_pending_bind: bool,
1886    replacement_root: Option<&ProjectRootId>,
1887) -> bool {
1888    !root_channels.contains_key(root) && !has_pending_bind && replacement_root != Some(root)
1889}
1890
1891async fn end_bg_subscription(
1892    writer_tx: &WriterSender,
1893    metrics: &DispatchPathMetrics,
1894    bg_subs: &mut HashMap<RouteChannel, BgSub>,
1895    bg_sub_by_session: &mut BgSubsBySession,
1896    bg_wake_pending: &mut HashSet<RouteChannel>,
1897    channel: RouteChannel,
1898    identity: Option<&RouteIdentity>,
1899    cause: &str,
1900) -> Result<(), SubcError> {
1901    if let Some(sub) = bg_subs.remove(&channel) {
1902        bg_wake_pending.remove(&channel);
1903        remove_bg_subscription_index(bg_sub_by_session, channel, identity);
1904        metrics.record_bg_subscription_ended(&sub.root, &sub.session, channel, cause);
1905        push::send_reliable_bg_stream_end(writer_tx, metrics, channel, &sub).await?;
1906    }
1907    Ok(())
1908}
1909
1910#[allow(clippy::too_many_arguments)]
1911async fn teardown_installed_route(
1912    tx: &WriterSender,
1913    metrics: &DispatchPathMetrics,
1914    executor: &Arc<Executor>,
1915    channel: RouteChannel,
1916    cancellation_reason: &str,
1917    replacement_root: Option<&ProjectRootId>,
1918    installed_route_epochs: &mut HashMap<u16, u32>,
1919    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1920    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1921    bg_subs: &mut HashMap<RouteChannel, BgSub>,
1922    bg_sub_by_session: &mut BgSubsBySession,
1923    bg_wake_pending: &mut HashSet<RouteChannel>,
1924    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1925    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1926    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1927    active_tool_calls: &ActiveToolCalls,
1928    pending_inspects: &mut PendingSubcInspects,
1929    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
1930    retry_buffer: &mut RetryBuffer,
1931    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1932    shutdown: &Arc<Notify>,
1933    tool_response_body_limit: usize,
1934) -> Result<(), SubcError> {
1935    remove_installed_route(installed_route_epochs, channel);
1936    let bg_end_cause = match cancellation_reason {
1937        "Goodbye" => "goodbye",
1938        "higher-epoch RouteBind" => "higher-epoch",
1939        other => other,
1940    };
1941    end_bg_subscription(
1942        tx,
1943        metrics,
1944        bg_subs,
1945        bg_sub_by_session,
1946        bg_wake_pending,
1947        channel,
1948        routes.get(&channel),
1949        bg_end_cause,
1950    )
1951    .await?;
1952    settle_pending_bash_asks_for_route(
1953        tx,
1954        pending_bash_asks,
1955        channel,
1956        routes,
1957        live_roots,
1958        route_bash_cancels,
1959        shutdown,
1960        metrics,
1961    )
1962    .await?;
1963    if let Some(cancel) = route_bash_cancels.remove(&channel) {
1964        cancel.token.cancel();
1965    }
1966    for resolved in pending_inspects.drain_route(channel, executor) {
1967        deliver_resolved_subc_inspect(
1968            tx,
1969            resolved,
1970            routes,
1971            live_roots,
1972            executor.as_ref(),
1973            active_tool_calls,
1974            shutdown,
1975            metrics,
1976            tool_response_body_limit,
1977        )
1978        .await?;
1979    }
1980    cancel_active_tool_calls_for_route(active_tool_calls, executor, channel, cancellation_reason);
1981    if let Some(pending) = pending_binds.get_mut(&channel) {
1982        pending.cancelled = true;
1983        let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
1984        log::debug!(
1985            "subc attach: cancelled pending RouteBind for route {} on {cancellation_reason} (configure job: {outcome:?})",
1986            channel.channel
1987        );
1988    }
1989    let migrated = push::migrate_retry_buffer_to_push_buffer(retry_buffer, channel, push_buffer);
1990    if let Some(identity) = routes.get(&channel) {
1991        let has_pending_bind = pending_binds
1992            .values()
1993            .any(|pending| pending.bind_root_id == identity.root);
1994        if route_removal_will_quiesce_root(
1995            &identity.root,
1996            channel,
1997            root_channels,
1998            has_pending_bind,
1999            replacement_root,
2000        ) {
2001            if let Some(ctx) = executor.actor_context(&identity.root) {
2002                // Fence deferred admissions before the final route disappears
2003                // from the loop-owned routing tables.
2004                ctx.mark_subc_unbound();
2005            }
2006        }
2007    }
2008    if let Some(identity) = remove_route_channel(routes, root_channels, channel) {
2009        let session_still_routed = routes
2010            .values()
2011            .any(|route| route.root == identity.root && route.session == identity.session);
2012        if !session_still_routed {
2013            if let Some(ctx) = executor.actor_context(&identity.root) {
2014                ctx.hashline_bindings()
2015                    .teardown(identity.root.as_path(), &identity.session);
2016            }
2017        }
2018        if migrated > 0 {
2019            log::debug!(
2020                "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from route {} into detach replay",
2021                channel.channel
2022            );
2023        }
2024        if let Some(meta) = live_roots.get_mut(&identity.root) {
2025            let idle_for = meta.last_touched.elapsed();
2026            meta.note_activity();
2027            log::debug!(
2028                "subc attach: route {} torn down for root {} harness {} session {} (last touched {:?} ago)",
2029                channel.channel,
2030                identity.root.as_path().display(),
2031                identity.harness,
2032                identity.session,
2033                idle_for
2034            );
2035        } else {
2036            log::debug!(
2037                "subc attach: route {} torn down for root {} harness {} session {}",
2038                channel.channel,
2039                identity.root.as_path().display(),
2040                identity.harness,
2041                identity.session
2042            );
2043        }
2044        let has_pending_bind = pending_binds
2045            .values()
2046            .any(|pending| pending.bind_root_id == identity.root);
2047        if should_quiesce_removed_root(
2048            &identity.root,
2049            root_channels,
2050            has_pending_bind,
2051            replacement_root,
2052        ) {
2053            quiesce_unbound_root(&identity.root, live_roots, executor);
2054        }
2055    } else {
2056        if migrated > 0 {
2057            log::debug!(
2058                "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from unbound route {} into detach replay",
2059                channel.channel
2060            );
2061        }
2062        log::debug!("subc attach: unbound route {} torn down", channel.channel);
2063    }
2064    Ok(())
2065}
2066
2067fn remember_session_identity(
2068    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2069    identity: &RouteIdentity,
2070) {
2071    let key = (identity.root.clone(), identity.session.clone());
2072    if matches!(identity.trust, BindTrust::Untrusted)
2073        && session_identity
2074            .get(&key)
2075            .is_some_and(|retained| matches!(retained.trust, BindTrust::FirstParty))
2076    {
2077        return;
2078    }
2079
2080    // Retained after route Goodbye so reliable session-scoped frames emitted while
2081    // the session is detached can still be keyed by the full (root,harness,session)
2082    // replay triple. Untrusted binds never overwrite a retained first-party
2083    // session identity, because bash completion replay is an observation channel.
2084    session_identity.insert(
2085        key,
2086        RetainedSessionIdentity {
2087            harness: identity.harness.clone(),
2088            trust: identity.trust,
2089        },
2090    );
2091}
2092
2093fn replay_key_for_session(
2094    session_identity: &HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2095    root: &ProjectRootId,
2096    session: &str,
2097) -> Option<(push::ReplayKey, BindTrust)> {
2098    let retained = session_identity.get(&(root.clone(), session.to_string()))?;
2099    Some((
2100        push::ReplayKey {
2101            root: root.clone(),
2102            harness: retained.harness.clone(),
2103            session: session.to_string(),
2104        },
2105        retained.trust,
2106    ))
2107}
2108/// Sync command dispatch, passed in from `main` (the binary owns the command
2109/// table). Invoked only inside executor jobs in subc mode.
2110pub type DispatchFn = fn(RawRequest, &AppContext) -> Response;
2111
2112#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2113enum ModuleLoopExit {
2114    Graceful,
2115    SkipSearchFlush,
2116}
2117
2118/// Entry point for `aft --subc <connection-file>`. Synchronous on the outside;
2119/// owns an isolated current-thread tokio runtime for the async transport.
2120/// Returns `Err` (fail-loud) on any connect/auth/protocol failure — we never
2121/// fall back to the standalone loop, to avoid split-brain index state.
2122pub fn run_subc_mode(
2123    connection_file_path: &Path,
2124    ctx: Arc<AppContext>,
2125    executor: Arc<Executor>,
2126    dispatch: DispatchFn,
2127    user_config_path: Option<PathBuf>,
2128) -> Result<(), SubcError> {
2129    // Production NEVER allows non-manifest tool names on route channels: AFT
2130    // fails closed and does not trust subc to enforce the manifest. The
2131    // test-only harness sets this through `run_subc_mode_for_test`.
2132    run_subc_mode_inner(
2133        connection_file_path,
2134        ctx,
2135        executor,
2136        dispatch,
2137        user_config_path,
2138        false,
2139        MAX_FRAME_BODY_LEN as usize,
2140    )
2141}
2142
2143fn run_subc_mode_inner(
2144    connection_file_path: &Path,
2145    ctx: Arc<AppContext>,
2146    executor: Arc<Executor>,
2147    dispatch: DispatchFn,
2148    user_config_path: Option<PathBuf>,
2149    allow_native_passthrough: bool,
2150    tool_response_body_limit: usize,
2151) -> Result<(), SubcError> {
2152    let runtime = tokio::runtime::Builder::new_current_thread()
2153        .enable_all()
2154        .build()
2155        .map_err(SubcError::Runtime)?;
2156
2157    let executor_for_loop = Arc::clone(&executor);
2158    let loop_result = runtime.block_on(async move {
2159        let shared_app = ctx.app();
2160        drop(ctx);
2161        let stream = connect_and_authenticate(connection_file_path).await?;
2162        log::info!(
2163            "subc attach: authenticated to daemon via {}",
2164            connection_file_path.display()
2165        );
2166        let (read_half, write_half) = tokio::io::split(stream);
2167        run_module_loop(
2168            read_half,
2169            write_half,
2170            connection_file_path,
2171            shared_app,
2172            executor_for_loop,
2173            dispatch,
2174            user_config_path,
2175            allow_native_passthrough,
2176            tool_response_body_limit,
2177        )
2178        .await
2179    });
2180
2181    let actor_contexts = executor.actor_contexts();
2182    if matches!(loop_result, Ok(ModuleLoopExit::Graceful)) {
2183        // EOF/Goodbye teardown flushes each root's index deltas and queued
2184        // callgraph refreshes. Fatal/panic teardown skips this best-effort work.
2185        flush_actor_indexes_on_graceful_shutdown(&actor_contexts);
2186    }
2187    for actor_ctx in &actor_contexts {
2188        actor_ctx.lsp().shutdown_all();
2189        actor_ctx.bash_background().detach();
2190    }
2191
2192    loop_result.map(|_| ())
2193}
2194
2195fn flush_actor_indexes_on_graceful_shutdown(actor_contexts: &[Arc<AppContext>]) {
2196    for actor_ctx in actor_contexts {
2197        let _ = actor_ctx.flush_search_index_on_graceful_shutdown();
2198    }
2199    let _ = crate::callgraph_store::flush_callgraph_store_refreshes_on_graceful_shutdown();
2200}
2201
2202/// Test-only entry that enables the non-manifest native-command passthrough on
2203/// route channels. Integration tests drive synthetic native commands (`glob`,
2204/// `callers`, `subc_test_echo_session`, …) through the executor to exercise
2205/// mechanics; production callers use [`run_subc_mode`], which fails closed.
2206#[doc(hidden)]
2207pub fn run_subc_mode_for_test(
2208    connection_file_path: &Path,
2209    ctx: Arc<AppContext>,
2210    executor: Arc<Executor>,
2211    dispatch: DispatchFn,
2212    user_config_path: Option<PathBuf>,
2213) -> Result<(), SubcError> {
2214    run_subc_mode_inner(
2215        connection_file_path,
2216        ctx,
2217        executor,
2218        dispatch,
2219        user_config_path,
2220        true,
2221        MAX_FRAME_BODY_LEN as usize,
2222    )
2223}
2224
2225/// Test-only entry that lowers the effective tool-response body limit without
2226/// allocating a 64 MiB fixture. The fixed fallback envelope needs 4 KiB of room.
2227#[doc(hidden)]
2228pub fn run_subc_mode_for_test_with_response_body_limit(
2229    connection_file_path: &Path,
2230    ctx: Arc<AppContext>,
2231    executor: Arc<Executor>,
2232    dispatch: DispatchFn,
2233    user_config_path: Option<PathBuf>,
2234    tool_response_body_limit: usize,
2235) -> Result<(), SubcError> {
2236    assert!((4 * 1_024..=MAX_FRAME_BODY_LEN as usize).contains(&tool_response_body_limit));
2237    run_subc_mode_inner(
2238        connection_file_path,
2239        ctx,
2240        executor,
2241        dispatch,
2242        user_config_path,
2243        true,
2244        tool_response_body_limit,
2245    )
2246}
2247
2248#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2249enum AttachErrorClass {
2250    Transient,
2251    Permanent,
2252}
2253
2254impl fmt::Display for AttachErrorClass {
2255    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2256        match self {
2257            Self::Transient => f.write_str("transient"),
2258            Self::Permanent => f.write_str("permanent"),
2259        }
2260    }
2261}
2262
2263#[derive(Clone, Copy)]
2264struct AttachRetryPolicy {
2265    budget: Duration,
2266    initial_backoff: Duration,
2267    max_backoff: Duration,
2268    jitter_percent: u64,
2269}
2270
2271const ATTACH_RETRY_POLICY: AttachRetryPolicy = AttachRetryPolicy {
2272    budget: ATTACH_RETRY_BUDGET,
2273    initial_backoff: ATTACH_RETRY_INITIAL_BACKOFF,
2274    max_backoff: ATTACH_RETRY_MAX_BACKOFF,
2275    jitter_percent: ATTACH_RETRY_JITTER_PERCENT,
2276};
2277
2278/// Retry only failures that can be caused by a daemon bounce or an interrupted
2279/// handshake. Protocol and credential failures are permanent for this process.
2280fn classify_attach_error(error: &SubcError) -> AttachErrorClass {
2281    let transient = match error {
2282        SubcError::Connect { source, .. } => is_transient_attach_io(source.kind()),
2283        SubcError::Auth { source, .. } => match source {
2284            subc_transport::AuthError::Timeout { .. }
2285            | subc_transport::AuthError::UnexpectedEof { .. } => true,
2286            subc_transport::AuthError::Io { source, .. } => is_transient_attach_io(source.kind()),
2287            _ => false,
2288        },
2289        _ => false,
2290    };
2291    if transient {
2292        AttachErrorClass::Transient
2293    } else {
2294        AttachErrorClass::Permanent
2295    }
2296}
2297
2298fn is_transient_attach_io(kind: io::ErrorKind) -> bool {
2299    matches!(
2300        kind,
2301        io::ErrorKind::ConnectionRefused
2302            | io::ErrorKind::TimedOut
2303            | io::ErrorKind::ConnectionReset
2304            | io::ErrorKind::ConnectionAborted
2305            | io::ErrorKind::BrokenPipe
2306            | io::ErrorKind::UnexpectedEof
2307    )
2308}
2309
2310/// Read the connection file → resolve the first endpoint → TCP connect → HMAC
2311/// handshake. Transient initial-attach failures retry on fresh sockets and reread
2312/// the file so a daemon bounce can publish a new endpoint or authentication key.
2313async fn connect_and_authenticate(connection_file_path: &Path) -> Result<TcpStream, SubcError> {
2314    connect_and_authenticate_with_policy(connection_file_path, ATTACH_RETRY_POLICY).await
2315}
2316
2317async fn connect_and_authenticate_with_policy(
2318    connection_file_path: &Path,
2319    policy: AttachRetryPolicy,
2320) -> Result<TcpStream, SubcError> {
2321    let started_at = Instant::now();
2322    let deadline = started_at + policy.budget;
2323    let mut attempt = 0_u32;
2324    let mut backoff = policy.initial_backoff;
2325    let mut history = Vec::new();
2326
2327    loop {
2328        attempt = attempt.saturating_add(1);
2329        let error = match connect_and_authenticate_once(connection_file_path, deadline).await {
2330            Ok(stream) => return Ok(stream),
2331            Err(error) => error,
2332        };
2333        let class = classify_attach_error(&error);
2334        let error_text = error.to_string().lines().collect::<Vec<_>>().join(" ");
2335        history.push(format!("attempt {attempt} [{class}]: {error_text}"));
2336
2337        if class == AttachErrorClass::Permanent {
2338            log_attach_final_failure(started_at.elapsed(), &history);
2339            return Err(error);
2340        }
2341
2342        let remaining = deadline.saturating_duration_since(Instant::now());
2343        if remaining.is_zero() {
2344            log_attach_final_failure(started_at.elapsed(), &history);
2345            return Err(error);
2346        }
2347
2348        let delay = jittered_attach_delay(backoff, policy.jitter_percent, attempt).min(remaining);
2349        log::info!(
2350            "subc attach retry: attempt {attempt} failed; error_class={class}; error={error_text}; next_delay={delay:?}"
2351        );
2352        tokio::time::sleep(delay).await;
2353
2354        if Instant::now() >= deadline {
2355            log_attach_final_failure(started_at.elapsed(), &history);
2356            return Err(error);
2357        }
2358        backoff = backoff.saturating_mul(2).min(policy.max_backoff);
2359    }
2360}
2361
2362fn jittered_attach_delay(base: Duration, jitter_percent: u64, attempt: u32) -> Duration {
2363    let jitter_percent = jitter_percent.min(100);
2364    if jitter_percent == 0 {
2365        return base;
2366    }
2367
2368    let mut random_bytes = [0_u8; 8];
2369    let random = if getrandom::fill(&mut random_bytes).is_ok() {
2370        u64::from_le_bytes(random_bytes)
2371    } else {
2372        let timestamp = std::time::SystemTime::now()
2373            .duration_since(std::time::UNIX_EPOCH)
2374            .unwrap_or_default()
2375            .subsec_nanos();
2376        u64::from(timestamp) ^ u64::from(attempt)
2377    };
2378    let span = jitter_percent.saturating_mul(2).saturating_add(1);
2379    let multiplier_percent = 100 - jitter_percent + random % span;
2380    let base_millis = u64::try_from(base.as_millis()).unwrap_or(u64::MAX);
2381    Duration::from_millis(base_millis.saturating_mul(multiplier_percent) / 100)
2382}
2383
2384fn log_attach_final_failure(elapsed: Duration, history: &[String]) {
2385    log::error!(
2386        "subc initial attach failed after {} attempt(s) in {elapsed:?}; attempt history: {}",
2387        history.len(),
2388        history.join(" | ")
2389    );
2390}
2391
2392async fn connect_and_authenticate_once(
2393    connection_file_path: &Path,
2394    deadline: Instant,
2395) -> Result<TcpStream, SubcError> {
2396    // This read intentionally lives inside the per-attempt function. The daemon
2397    // publishes connection files atomically and may change both port and key.
2398    let conn = connection_file::read_for_client(connection_file_path).map_err(|source| {
2399        SubcError::ConnectionFile {
2400            path: connection_file_path.to_path_buf(),
2401            source,
2402        }
2403    })?;
2404
2405    let endpoint = conn
2406        .endpoints
2407        .first()
2408        .ok_or_else(|| SubcError::NoEndpoint {
2409            path: connection_file_path.to_path_buf(),
2410        })?;
2411    let endpoint_label = format!("{}:{}", endpoint.host, endpoint.port);
2412    let ip = endpoint
2413        .host
2414        .parse::<IpAddr>()
2415        .map_err(|_| SubcError::InvalidEndpoint {
2416            path: connection_file_path.to_path_buf(),
2417            endpoint: endpoint_label.clone(),
2418        })?;
2419    let addr = SocketAddr::new(ip, endpoint.port);
2420
2421    let connect_budget = deadline.saturating_duration_since(Instant::now());
2422    let mut stream = tokio::time::timeout(connect_budget, TcpStream::connect(addr))
2423        .await
2424        .map_err(|_| SubcError::Connect {
2425            endpoint: endpoint_label.clone(),
2426            source: io::Error::new(
2427                io::ErrorKind::TimedOut,
2428                "initial subc attach retry budget elapsed during TCP connect",
2429            ),
2430        })?
2431        .map_err(|source| SubcError::Connect {
2432            endpoint: endpoint_label.clone(),
2433            source,
2434        })?;
2435    stream
2436        .set_nodelay(true)
2437        .map_err(|source| SubcError::Connect {
2438            endpoint: endpoint_label.clone(),
2439            source,
2440        })?;
2441
2442    let auth_budget = AUTH_DEADLINE.min(deadline.saturating_duration_since(Instant::now()));
2443    authenticate_client(&mut stream, &conn, auth_budget)
2444        .await
2445        .map_err(|source| SubcError::Auth {
2446            endpoint: endpoint_label,
2447            source,
2448        })?;
2449
2450    Ok(stream)
2451}
2452
2453#[allow(clippy::too_many_arguments)]
2454async fn process_route_bind_completion(
2455    writer_tx: &WriterSender,
2456    completion: RouteBindCompletion,
2457    routes: &mut HashMap<RouteChannel, RouteIdentity>,
2458    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2459    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2460    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2461    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2462    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2463    installed_route_epochs: &mut HashMap<u16, u32>,
2464    executor: &Arc<Executor>,
2465    shutdown: &Arc<Notify>,
2466    metrics: &Arc<DispatchPathMetrics>,
2467) -> Result<(), SubcError> {
2468    decrement_counted_channel(&metrics.control_completion_queued);
2469    handle_route_bind_completion(
2470        writer_tx,
2471        completion,
2472        routes,
2473        root_channels,
2474        session_identity,
2475        push_buffer,
2476        live_roots,
2477        pending_binds,
2478        installed_route_epochs,
2479        executor,
2480        shutdown,
2481        metrics,
2482    )
2483    .await
2484}
2485
2486#[allow(clippy::too_many_arguments)]
2487async fn drain_pending_route_bind_completions(
2488    control_completion_rx: &mut mpsc::Receiver<RouteBindCompletion>,
2489    writer_tx: &WriterSender,
2490    routes: &mut HashMap<RouteChannel, RouteIdentity>,
2491    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2492    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2493    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2494    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2495    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2496    installed_route_epochs: &mut HashMap<u16, u32>,
2497    executor: &Arc<Executor>,
2498    shutdown: &Arc<Notify>,
2499    metrics: &Arc<DispatchPathMetrics>,
2500) -> Result<usize, SubcError> {
2501    let mut drained = 0;
2502    while let Ok(completion) = control_completion_rx.try_recv() {
2503        process_route_bind_completion(
2504            writer_tx,
2505            completion,
2506            routes,
2507            root_channels,
2508            session_identity,
2509            push_buffer,
2510            live_roots,
2511            pending_binds,
2512            installed_route_epochs,
2513            executor,
2514            shutdown,
2515            metrics,
2516        )
2517        .await?;
2518        drained += 1;
2519    }
2520    Ok(drained)
2521}
2522
2523/// ModuleHello → HelloAck → control/route loop. Runs until the daemon closes
2524/// the connection (EOF), sends channel-0 Goodbye, or a fatal mutating executor
2525/// response requests whole-connection teardown.
2526async fn run_module_loop<R, W>(
2527    mut read: R,
2528    mut write: W,
2529    connection_file_path: &Path,
2530    shared_app: Arc<App>,
2531    executor: Arc<Executor>,
2532    dispatch: DispatchFn,
2533    user_config_path: Option<PathBuf>,
2534    allow_native_passthrough: bool,
2535    tool_response_body_limit: usize,
2536) -> Result<ModuleLoopExit, SubcError>
2537where
2538    R: AsyncRead + Unpin + Send + 'static,
2539    W: AsyncWrite + Unpin + Send + 'static,
2540{
2541    // ModuleHello: register as a tool provider and advertise the supported control-plane operations.
2542    // Echo the one-time launch nonce the daemon injected via SUBC_LAUNCH_NONCE so a
2543    // reserved module_id's HELLO is accepted; absent for non-reserved/self-connect.
2544    let hello = ModuleHelloBody {
2545        manifest: build_manifest(),
2546        protocol_ver: PROTOCOL_VERSION,
2547        control_ops: control_ops(),
2548        launch_nonce: std::env::var("SUBC_LAUNCH_NONCE").ok(),
2549    };
2550    let hello_frame = Frame::build(
2551        FrameType::Hello,
2552        control_flags(),
2553        0,
2554        0,
2555        HELLO_CORR,
2556        serde_json::to_vec(&hello).map_err(SubcError::Json)?,
2557    )
2558    .map_err(SubcError::FrameBuild)?;
2559    write_frame(&mut write, &hello_frame)
2560        .await
2561        .map_err(SubcError::FrameIo)?;
2562
2563    // Expect HelloAck (registered) or a channel-0 Error (manifest/version reject).
2564    match read_frame(&mut read).await.map_err(SubcError::FrameIo)? {
2565        None => return Err(SubcError::ClosedBeforeHelloAck),
2566        Some(frame) => match frame.header.ty {
2567            FrameType::HelloAck => {
2568                log::info!("subc attach: registered (HelloAck received)");
2569            }
2570            FrameType::Error => {
2571                let body = serde_json::from_slice::<ErrorBody>(&frame.body).ok();
2572                return Err(SubcError::HelloRejected { body });
2573            }
2574            other => return Err(SubcError::UnexpectedFrame { ty: other }),
2575        },
2576    }
2577
2578    let dispatch_path_metrics = Arc::new(DispatchPathMetrics::new());
2579    let (writer_tx, writer_rx) = mpsc::channel::<WriterFrame>(WRITER_QUEUE_CAPACITY);
2580    let writer_task = spawn_writer_task(write, writer_rx, Arc::clone(&dispatch_path_metrics));
2581    // `read_frame` is NOT cancellation-safe, so it must never sit directly inside
2582    // the `select!` below: a drain-interval tick (or shutdown) firing while a
2583    // frame is mid-transit would drop the partially-consumed bytes and desync the
2584    // stream (the next read would parse a body byte as a frame header). A
2585    // dedicated reader task owns the socket, reads whole frames sequentially, and
2586    // forwards them over a channel; the loop selects on the cancel-safe `recv()`.
2587    let (reader_tx, mut reader_rx) = mpsc::channel::<Result<DecodedFrame, SubcError>>(256);
2588    let reader_task = spawn_reader_task(read, reader_tx);
2589    let shutdown = Arc::new(Notify::new());
2590    // Drain-tick deadline is tracked manually and checked at the TOP of every
2591    // loop turn rather than as an Interval select arm: the select below is
2592    // `biased` (bind completions first), and biased polling means a saturated
2593    // higher arm (sustained lossy push traffic keeps lossy_rx always-ready)
2594    // would starve every arm below it, including a timer arm — leaving
2595    // backpressured reliable frames parked in the retry buffer past their
2596    // delivery deadline. The pre-turn check cannot be starved by arm order;
2597    // the sleep_until arm below only exists to wake an otherwise-idle loop.
2598    let mut next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2599    let mut next_maintenance_at = next_drain_at;
2600    // Rate-limit stamp for opportunistic allocator slack relief (checked on the
2601    // maintenance tick; policy shared with standalone via memory.rs).
2602    #[cfg(any(target_os = "macos", target_os = "linux"))]
2603    let mut last_slack_relief: Option<std::time::Instant> = None;
2604    let (maintenance_tx, mut maintenance_rx) = mpsc::channel::<MaintenanceCompletion>(256);
2605    let (bash_deferred_tx, mut bash_deferred_rx) =
2606        mpsc::channel::<bash::BashDeferredCompletion>(256);
2607    let (inspect_deferred_tx, mut inspect_deferred_rx) =
2608        mpsc::unbounded_channel::<PendingSubcInspect>();
2609    let (bash_poll_touch_tx, mut bash_poll_touch_rx) = mpsc::channel::<ProjectRootId>(256);
2610    let (control_completion_tx, mut control_completion_rx) =
2611        mpsc::channel::<RouteBindCompletion>(256);
2612    let (lossy_tx, mut lossy_rx) = mpsc::channel::<LossyPushEnvelope>(1024);
2613    let lossy_overflow = Arc::new(push::LossyOverflow::default());
2614    let lossy_seq = Arc::new(AtomicU64::new(0));
2615    let (reliable_tx, mut reliable_rx) = mpsc::unbounded_channel::<PushEnvelope>();
2616    let (fleet_status_client, fleet_status_task) =
2617        spawn_fleet_status_dial(connection_file_path, 64);
2618    let push_senders = PushSenders {
2619        lossy_tx,
2620        reliable_tx,
2621        lossy_overflow: Arc::clone(&lossy_overflow),
2622        lossy_seq,
2623        fleet_status_client: fleet_status_client.clone(),
2624    };
2625    let connection_cancel = PersistentCancelSignal::new();
2626    let mut installed_route_epochs: HashMap<u16, u32> = HashMap::new();
2627    let mut routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
2628    let mut bg_subs: HashMap<RouteChannel, BgSub> = HashMap::new();
2629    let mut bg_sub_by_session: BgSubsBySession = HashMap::new();
2630    let mut bg_wake_pending: HashSet<RouteChannel> = HashSet::new();
2631    let mut bg_wake_epoch: HashMap<(ProjectRootId, String), u64> = HashMap::new();
2632    let mut root_channels: HashMap<ProjectRootId, HashSet<RouteChannel>> = HashMap::new();
2633    let mut session_identity: HashMap<(ProjectRootId, String), RetainedSessionIdentity> =
2634        HashMap::new();
2635    let mut push_buffer: HashMap<push::ReplayKey, VecDeque<PushFrame>> = HashMap::new();
2636    let mut retry_buffer: RetryBuffer = HashMap::new();
2637    let mut reclaimed_routes = ReclaimedRoutes::default();
2638    let mut completed_tasks = push::CompletedTaskIds::default();
2639    let mut live_roots: HashMap<ProjectRootId, RootMeta> = HashMap::new();
2640    let mut pending_binds: HashMap<RouteChannel, PendingBind> = HashMap::new();
2641    let mut pending_bash_asks: HashMap<ReverseCorrKey, PendingBashAsk> = HashMap::new();
2642    let mut next_bash_ask_corr: u64 = 1;
2643    let mut route_bash_cancels: HashMap<RouteChannel, bash::RouteBashCancel> = HashMap::new();
2644    let active_tool_calls: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
2645    let pending_inspect_setups = Arc::new(AtomicUsize::new(0));
2646    let mut pending_inspects = PendingSubcInspects::default();
2647    let health_rollup_cache = HealthRollupCache::new();
2648    health_rollup_cache.refresh(&executor, &shared_app);
2649    let mut next_health_rollup_at = tokio::time::Instant::now() + HEALTH_ROLLUP_TTL;
2650
2651    let loop_result: Result<ModuleLoopExit, SubcError> = 'module_loop: loop {
2652        shared_app.set_open_route_count(routes.len());
2653        if tokio::time::Instant::now() >= next_health_rollup_at {
2654            health_rollup_cache.refresh(&executor, &shared_app);
2655            next_health_rollup_at = tokio::time::Instant::now() + HEALTH_ROLLUP_TTL;
2656        }
2657        crate::logging::perf_tick(Some(&executor));
2658        dispatch_path_metrics.mark_frame_loop_tick();
2659        let ready_inspects = pending_inspects.poll_ready(executor.as_ref());
2660        for resolved in ready_inspects {
2661            if let Err(error) = deliver_resolved_subc_inspect(
2662                &writer_tx,
2663                resolved,
2664                &routes,
2665                &mut live_roots,
2666                executor.as_ref(),
2667                &active_tool_calls,
2668                &shutdown,
2669                &dispatch_path_metrics,
2670                tool_response_body_limit,
2671            )
2672            .await
2673            {
2674                break 'module_loop Err(error);
2675            }
2676        }
2677        if let Err(error) = expire_pending_bash_asks(
2678            &writer_tx,
2679            &mut pending_bash_asks,
2680            &routes,
2681            &mut live_roots,
2682            &mut route_bash_cancels,
2683            &shutdown,
2684            &dispatch_path_metrics,
2685        )
2686        .await
2687        {
2688            break Err(error);
2689        }
2690
2691        // RouteBind completions are control-plane unblockers. Drain any completed
2692        // binds before entering other branch work so Push and maintenance bursts
2693        // can only add one loop-turn of latency.
2694        match drain_pending_route_bind_completions(
2695            &mut control_completion_rx,
2696            &writer_tx,
2697            &mut routes,
2698            &mut root_channels,
2699            &mut session_identity,
2700            &mut push_buffer,
2701            &mut live_roots,
2702            &mut pending_binds,
2703            &mut installed_route_epochs,
2704            &executor,
2705            &shutdown,
2706            &dispatch_path_metrics,
2707        )
2708        .await
2709        {
2710            Ok(drained) => {
2711                if drained > 0 {
2712                    next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2713                    health_rollup_cache.refresh(&executor, &shared_app);
2714                    next_health_rollup_at = tokio::time::Instant::now() + HEALTH_ROLLUP_TTL;
2715                }
2716            }
2717            Err(error) => break Err(error),
2718        }
2719
2720        if tokio::time::Instant::now() >= next_drain_at {
2721            push::emit_bg_event_wakes(
2722                &writer_tx,
2723                &dispatch_path_metrics,
2724                &bg_subs,
2725                &mut bg_wake_pending,
2726            );
2727            warn_slow_pending_binds(&mut pending_binds, &executor);
2728            if let Err(error) = expire_overdue_route_binds(
2729                &writer_tx,
2730                &executor,
2731                &mut pending_binds,
2732                &mut installed_route_epochs,
2733                &dispatch_path_metrics,
2734            )
2735            .await
2736            {
2737                break Err(error);
2738            }
2739
2740            let retried = push::drain_retry_buffers_for_bound_routes(
2741                &writer_tx,
2742                &dispatch_path_metrics,
2743                &routes,
2744                &mut retry_buffer,
2745            );
2746            if retried > 0 {
2747                log::debug!(
2748                    "subc attach: retried {retried} reliable Push frame(s) after writer backpressure"
2749                );
2750            }
2751
2752            next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2753        }
2754
2755        // A lossy emitter may place its newest update in the overflow buffer
2756        // when the bounded channel is full, while this receive loop is draining
2757        // the channel. Drain overflow before selecting again so that raced
2758        // update is delivered on the next timer tick instead of waiting for
2759        // another lossy enqueue.
2760        let overflow_batch = lossy_overflow.drain();
2761        if !overflow_batch.is_empty() {
2762            let (_, deferred) = push::drain_reliable_push_turn(
2763                &writer_tx,
2764                &dispatch_path_metrics,
2765                &routes,
2766                &root_channels,
2767                &session_identity,
2768                &mut retry_buffer,
2769                &mut push_buffer,
2770                &mut completed_tasks,
2771                &bg_sub_by_session,
2772                &mut bg_wake_pending,
2773                &mut bg_wake_epoch,
2774                &mut reliable_rx,
2775                None,
2776            );
2777            if deferred {
2778                tokio::task::yield_now().await;
2779            }
2780
2781            let mut batch = Vec::new();
2782            while let Ok(item) = lossy_rx.try_recv() {
2783                batch.push(item);
2784            }
2785            batch.extend(overflow_batch);
2786            push::process_lossy_push_envelope_batch(
2787                &writer_tx,
2788                &dispatch_path_metrics,
2789                &routes,
2790                &root_channels,
2791                &completed_tasks,
2792                batch,
2793            );
2794        }
2795
2796        tokio::select! {
2797            biased;
2798            Some(completion) = control_completion_rx.recv() => {
2799                if let Err(error) = process_route_bind_completion(
2800                    &writer_tx,
2801                    completion,
2802                    &mut routes,
2803                    &mut root_channels,
2804                    &mut session_identity,
2805                    &mut push_buffer,
2806                    &mut live_roots,
2807                    &mut pending_binds,
2808                    &mut installed_route_epochs,
2809                    &executor,
2810                    &shutdown,
2811                    &dispatch_path_metrics,
2812                )
2813                .await
2814                {
2815                    break Err(error);
2816                }
2817                next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2818                next_health_rollup_at = tokio::time::Instant::now();
2819            }
2820            _ = shutdown.notified() => {
2821                log::warn!("subc attach: fatal executor response requested teardown");
2822                break Ok(ModuleLoopExit::SkipSearchFlush);
2823            }
2824            maybe_frame = reader_rx.recv() => {
2825                let frame = match maybe_frame {
2826                    None => {
2827                        log::info!("subc attach: daemon closed connection");
2828                        break Ok(ModuleLoopExit::Graceful);
2829                    }
2830                    Some(Err(error)) => break Err(error),
2831                    Some(Ok(frame)) => frame,
2832                };
2833                let phase_trace = frame.phase_trace;
2834                let frame = frame.frame;
2835
2836                if !ingress_route_should_be_processed(
2837                    &installed_route_epochs,
2838                    &reclaimed_routes,
2839                    &frame,
2840                ) {
2841                    log::debug!(
2842                        "subc attach: silently dropping {:?} for uninstalled route {}@{}",
2843                        frame.header.ty,
2844                        frame.header.channel,
2845                        frame.header.epoch
2846                    );
2847                    continue;
2848                }
2849
2850                match frame.header.ty {
2851                    FrameType::Ping if frame.header.channel == 0 => {
2852                        let pong = match Frame::build_with_version(
2853                            frame.header.ver,
2854                            FrameType::Pong,
2855                            frame.header.flags,
2856                            0,
2857                            0,
2858                            frame.header.corr,
2859                            Vec::new(),
2860                        ) {
2861                            Ok(pong) => pong,
2862                            Err(error) => break Err(SubcError::FrameBuild(error)),
2863                        };
2864                        if let Err(error) = send_frame(&writer_tx, &dispatch_path_metrics, pong).await {
2865                            break Err(error);
2866                        }
2867                    }
2868                    FrameType::Goodbye if frame.header.channel == 0 => {
2869                        log::info!("subc attach: received channel-0 Goodbye");
2870                        break Ok(ModuleLoopExit::Graceful);
2871                    }
2872                    FrameType::Goodbye => {
2873                        let channel = route_key(frame.header.channel, frame.header.epoch);
2874                        if let Err(error) = teardown_installed_route(
2875                            &writer_tx,
2876                            &dispatch_path_metrics,
2877                            &executor,
2878                            channel,
2879                            "Goodbye",
2880                            None,
2881                            &mut installed_route_epochs,
2882                            &mut routes,
2883                            &mut root_channels,
2884                            &mut bg_subs,
2885                            &mut bg_sub_by_session,
2886                            &mut bg_wake_pending,
2887                            &mut pending_bash_asks,
2888                            &mut live_roots,
2889                            &mut route_bash_cancels,
2890                            &active_tool_calls,
2891                            &mut pending_inspects,
2892                            &mut pending_binds,
2893                            &mut retry_buffer,
2894                            &mut push_buffer,
2895                            &shutdown,
2896                            tool_response_body_limit,
2897                        )
2898                        .await
2899                        {
2900                            break Err(error);
2901                        }
2902                    }
2903                    FrameType::Response | FrameType::Error if frame.header.channel != 0 => {
2904                        if let Err(error) = handle_bash_elicitation_reply(
2905                            &writer_tx,
2906                            &frame,
2907                            &mut pending_bash_asks,
2908                            &routes,
2909                            &mut live_roots,
2910                            &executor,
2911                            &shutdown,
2912                            &bash_deferred_tx,
2913                            &bash_poll_touch_tx,
2914                            &dispatch_path_metrics,
2915                            &mut route_bash_cancels,
2916                            dispatch,
2917                        )
2918                        .await
2919                        {
2920                            break Err(error);
2921                        }
2922                    }
2923                    FrameType::Request if frame.header.channel == 0 => {
2924                        if let Err(error) = handle_control_request(
2925                            &writer_tx,
2926                            &frame,
2927                            &shared_app,
2928                            &executor,
2929                            &mut live_roots,
2930                            &mut pending_binds,
2931                            &mut installed_route_epochs,
2932                            &mut routes,
2933                            &mut root_channels,
2934                            &mut bg_subs,
2935                            &mut bg_sub_by_session,
2936                            &mut bg_wake_pending,
2937                            &mut pending_bash_asks,
2938                            &mut route_bash_cancels,
2939                            &active_tool_calls,
2940                            &mut pending_inspects,
2941                            &mut retry_buffer,
2942                            &mut push_buffer,
2943                            &shutdown,
2944                            &control_completion_tx,
2945                            &dispatch_path_metrics,
2946                            &health_rollup_cache,
2947                            &push_senders,
2948                            dispatch,
2949                            user_config_path.as_deref(),
2950                            tool_response_body_limit,
2951                        )
2952                        .await
2953                        {
2954                            break Err(error);
2955                        }
2956                    }
2957                    FrameType::Request => {
2958                        if let Err(error) = handle_tool_call(
2959                            &writer_tx,
2960                            &frame,
2961                            phase_trace,
2962                            &routes,
2963                            &pending_binds,
2964                            &mut live_roots,
2965                            &executor,
2966                            &active_tool_calls,
2967                            &pending_inspect_setups,
2968                            &shutdown,
2969                            &connection_cancel,
2970                            &bash_deferred_tx,
2971                            &bash_poll_touch_tx,
2972                            &dispatch_path_metrics,
2973                            &mut route_bash_cancels,
2974                            &mut pending_bash_asks,
2975                            &mut next_bash_ask_corr,
2976                            &mut bg_subs,
2977                            &mut bg_sub_by_session,
2978                            &mut bg_wake_pending,
2979                            &mut bg_wake_epoch,
2980                            dispatch,
2981                            &inspect_deferred_tx,
2982                            allow_native_passthrough,
2983                            tool_response_body_limit,
2984                        )
2985                        .await
2986                        {
2987                            break Err(error);
2988                        }
2989                    }
2990                    FrameType::Cancel => {
2991                        let channel = route_key(frame.header.channel, frame.header.epoch);
2992                        cancel_active_tool_call(
2993                            &active_tool_calls,
2994                            executor.as_ref(),
2995                            channel,
2996                            frame.header.corr,
2997                            "Cancel frame",
2998                        );
2999                        if bg_subs.contains_key(&channel) {
3000                            if let Err(error) = end_bg_subscription(
3001                                &writer_tx,
3002                                &dispatch_path_metrics,
3003                                &mut bg_subs,
3004                                &mut bg_sub_by_session,
3005                                &mut bg_wake_pending,
3006                                channel,
3007                                routes.get(&channel),
3008                                "cancel",
3009                            )
3010                            .await
3011                            {
3012                                break Err(error);
3013                            }
3014                        }
3015                        if let Err(error) = cancel_pending_bash_ask_for_tool_call(
3016                            &writer_tx,
3017                            &mut pending_bash_asks,
3018                            channel,
3019                            frame.header.corr,
3020                            &routes,
3021                            &mut live_roots,
3022                            &mut route_bash_cancels,
3023                            &shutdown,
3024                            &dispatch_path_metrics,
3025                        )
3026                        .await
3027                        {
3028                            break Err(error);
3029                        }
3030                    }
3031                    // Incoming push messages are ignored here. Cancel frames are
3032                    // handled above for both active inspect jobs and pending bash
3033                    // elicitation requests.
3034                    _ => {}
3035                }
3036            }
3037            Some(pending) = inspect_deferred_rx.recv() => {
3038                if routes.contains_key(&pending.route) {
3039                    pending_inspects.register(pending);
3040                } else {
3041                    if let Some(cancellation) = &pending.pending.cancellation {
3042                        cancellation.request_cancel();
3043                    }
3044                    finish_active_tool_call(&active_tool_calls, pending.route, pending.corr);
3045                }
3046            }
3047            Some((root_id, frame)) = reliable_rx.recv() => {
3048                // Reliable Push frames are FIFO and must-deliver, but draining an
3049                // unbounded burst in one current-thread turn can starve RouteBind
3050                // completions. The budget defers excess frames, never drops them.
3051                let (_, deferred) = push::drain_reliable_push_turn(
3052                    &writer_tx,
3053                    &dispatch_path_metrics,
3054                    &routes,
3055                    &root_channels,
3056                    &session_identity,
3057                    &mut retry_buffer,
3058                    &mut push_buffer,
3059                    &mut completed_tasks,
3060                    &bg_sub_by_session,
3061                    &mut bg_wake_pending,
3062                    &mut bg_wake_epoch,
3063                    &mut reliable_rx,
3064                    Some((root_id, frame)),
3065                );
3066                if deferred {
3067                    tokio::task::yield_now().await;
3068                }
3069            }
3070            Some((order, root_id, frame)) = lossy_rx.recv() => {
3071                // When both push lanes have work, handle a small reliable slice before lossy work.
3072                // That ordering lets completed task ids suppress stale BashLongRunning frames.
3073                // The slice stays bounded so reliable bursts cannot monopolize this loop turn.
3074                let (_, deferred) = push::drain_reliable_push_turn(
3075                    &writer_tx,
3076                    &dispatch_path_metrics,
3077                    &routes,
3078                    &root_channels,
3079                    &session_identity,
3080                    &mut retry_buffer,
3081                    &mut push_buffer,
3082                    &mut completed_tasks,
3083                    &bg_sub_by_session,
3084                    &mut bg_wake_pending,
3085                    &mut bg_wake_epoch,
3086                    &mut reliable_rx,
3087                    None,
3088                );
3089                if deferred {
3090                    tokio::task::yield_now().await;
3091                }
3092
3093                // Drain the currently queued burst in one loop turn so lossy
3094                // status/progress updates can be merged before reaching subc's
3095                // shared egress queue. Each lossy frame gets a sequence number
3096                // before it goes to the channel or overflow buffer, so the
3097                // combined batch is sorted back into producer order before
3098                // coalescing drops stale updates for the same key.
3099                let mut batch = vec![(order, root_id, frame)];
3100                while let Ok(item) = lossy_rx.try_recv() {
3101                    batch.push(item);
3102                }
3103                batch.extend(lossy_overflow.drain());
3104                push::process_lossy_push_envelope_batch(
3105                    &writer_tx,
3106                    &dispatch_path_metrics,
3107                    &routes,
3108                    &root_channels,
3109                    &completed_tasks,
3110                    batch,
3111                );
3112            }
3113            Some(done) = bash_deferred_rx.recv() => {
3114                decrement_counted_channel(&dispatch_path_metrics.bash_deferred_queued);
3115                if let Err(error) = bash::handle_bash_deferred_completion(
3116                    &writer_tx,
3117                    done,
3118                    &routes,
3119                    &mut live_roots,
3120                    &mut route_bash_cancels,
3121                    &shutdown,
3122                    &dispatch_path_metrics,
3123                )
3124                .await
3125                {
3126                    break Err(error);
3127                }
3128            }
3129            Some(root_id) = bash_poll_touch_rx.recv() => {
3130                decrement_counted_channel(&dispatch_path_metrics.bash_poll_touch_queued);
3131                if let Some(meta) = live_roots.get_mut(&root_id) {
3132                    meta.note_activity();
3133                }
3134            }
3135            Some(completion) = maintenance_rx.recv() => {
3136                decrement_counted_channel(&dispatch_path_metrics.maintenance_queued);
3137                let root_id = completion.root_id.clone();
3138                let response = completion.response;
3139                let response_is_fatal = response_is_fatal_panic(&response);
3140                let bind_pending = pending_binds
3141                    .values()
3142                    .any(|pending| pending.bind_root_id == root_id);
3143                let requiesce = if let Some(meta) = live_roots.get_mut(&root_id) {
3144                    let defer_requeue = meta.unbound_quiesced || bind_pending;
3145                    note_maintenance_completion(
3146                        meta,
3147                        completion.requeue_kind,
3148                        response_is_fatal,
3149                        defer_requeue,
3150                    );
3151                    should_requiesce_after_maintenance(meta, completion.kind, bind_pending)
3152                } else {
3153                    false
3154                };
3155                if requiesce {
3156                    quiesce_unbound_root(&root_id, &mut live_roots, &executor);
3157                }
3158                push::clear_stale_bg_wakes_for_empty_sessions(
3159                    &root_id,
3160                    &completion.empty_bg_sessions,
3161                    &bg_sub_by_session,
3162                    &mut bg_wake_pending,
3163                    &bg_wake_epoch,
3164                );
3165                if response_is_fatal {
3166                    if let Some(meta) = live_roots.get_mut(&root_id) {
3167                        meta.maintenance_poisoned = true;
3168                    }
3169                    log::warn!(
3170                        "subc attach: maintenance drain observed a fatal actor; deferring teardown until a route request can receive actor_fatal"
3171                    );
3172                }
3173            }
3174            _ = tokio::time::sleep(PENDING_POLL_INTERVAL), if !pending_inspects.is_empty() => {
3175                // The next loop turn polls detached inspect completions. Keeping
3176                // the timer here lets already-ready control frames run first.
3177            }
3178            _ = tokio::time::sleep_until(next_drain_at) => {
3179                // Wakes an otherwise-idle loop so the pre-turn drain check
3180                // above runs on schedule; the drain work itself lives there.
3181            }
3182            _ = tokio::time::sleep_until(next_maintenance_at) => {
3183                // Delay cache-draining maintenance until any already-ready
3184                // inbound route/control messages and push completions have run,
3185                // so maintenance does not block the actor from handling the
3186                // first request that arrives after a route bind is acknowledged.
3187                crate::logging::maybe_sweep_logs();
3188                let reaped_lsp_children = shared_app
3189                    .lsp_child_registry()
3190                    .reap_children_with_gone_cwd_or_reclaimed_root();
3191                if reaped_lsp_children > 0 {
3192                    log::warn!(
3193                        "subc attach: reaped {reaped_lsp_children} LSP child process group(s) with a deleted cwd or reclaimed root"
3194                    );
3195                }
3196                let reap = reap_idle_roots(
3197                    Instant::now(),
3198                    &mut live_roots,
3199                    &pending_binds,
3200                    &root_channels,
3201                    &executor,
3202                    &dispatch_path_metrics,
3203                );
3204                for root_id in &reap.forgotten_deleted_roots {
3205                    purge_deleted_root_residents(
3206                        root_id,
3207                        &mut routes,
3208                        &mut root_channels,
3209                        &mut installed_route_epochs,
3210                        &mut route_bash_cancels,
3211                        &active_tool_calls,
3212                        executor.as_ref(),
3213                        &mut retry_buffer,
3214                        &mut reclaimed_routes,
3215                        &mut session_identity,
3216                        &mut push_buffer,
3217                        &mut bg_subs,
3218                        &mut bg_sub_by_session,
3219                        &mut bg_wake_pending,
3220                        &mut bg_wake_epoch,
3221                        &mut pending_bash_asks,
3222                        &dispatch_path_metrics,
3223                    );
3224                }
3225                if reap.evicted > 0 {
3226                    log::debug!("subc attach: reaped {} idle root(s)", reap.evicted);
3227                }
3228                submit_due_maintenance_jobs(
3229                    &executor,
3230                    &mut live_roots,
3231                    &pending_binds,
3232                    &bg_sub_by_session,
3233                    &bg_wake_pending,
3234                    &bg_wake_epoch,
3235                    &maintenance_tx,
3236                    &dispatch_path_metrics,
3237                );
3238                // Opportunistic allocator relief, independent of the idle
3239                // sweep: the sweep's whole-process idle gate never opens while
3240                // any session stays active, which let freed warm-up arenas sit
3241                // resident for the process lifetime (5.1 GB RSS over ~600 MB
3242                // live). Slack threshold + spacing live in memory.rs; the pass
3243                // itself runs on a detached thread.
3244                #[cfg(any(target_os = "macos", target_os = "linux"))]
3245                {
3246                    let now_std = std::time::Instant::now();
3247                    if crate::memory::spawn_allocator_slack_relief_if_due(
3248                        last_slack_relief,
3249                        now_std,
3250                    ) {
3251                        last_slack_relief = Some(now_std);
3252                    }
3253                }
3254                next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
3255            }
3256        }
3257    };
3258
3259    shared_app.set_open_route_count(0);
3260
3261    connection_cancel.cancel();
3262    cancel_all_active_tool_calls(&active_tool_calls, executor.as_ref(), "connection teardown");
3263    let setup_drain_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
3264    while pending_inspect_setups.load(Ordering::SeqCst) != 0
3265        && tokio::time::Instant::now() < setup_drain_deadline
3266    {
3267        tokio::select! {
3268            biased;
3269            Some(pending) = inspect_deferred_rx.recv() => pending_inspects.register(pending),
3270            _ = tokio::time::sleep(Duration::from_millis(5)) => {}
3271        }
3272    }
3273    if pending_inspect_setups.load(Ordering::SeqCst) != 0 {
3274        log::warn!(
3275            "subc attach: timed out waiting for deferred inspect setup registration during shutdown"
3276        );
3277    }
3278    while let Ok(pending) = inspect_deferred_rx.try_recv() {
3279        pending_inspects.register(pending);
3280    }
3281    for resolved in pending_inspects.drain_on_shutdown(executor.as_ref()) {
3282        if let Err(error) = deliver_resolved_subc_inspect(
3283            &writer_tx,
3284            resolved,
3285            &routes,
3286            &mut live_roots,
3287            executor.as_ref(),
3288            &active_tool_calls,
3289            &shutdown,
3290            &dispatch_path_metrics,
3291            tool_response_body_limit,
3292        )
3293        .await
3294        {
3295            log::warn!("subc attach: failed to emit inspect shutdown terminal: {error}");
3296        }
3297    }
3298    // Channel-0 Goodbye, EOF, and fatal exits bypass per-route Goodbye. Settle
3299    // their root lifecycle state before loop-owned routing metadata is dropped.
3300    quiesce_connection_roots(
3301        &mut live_roots,
3302        &mut pending_binds,
3303        &mut routes,
3304        &mut root_channels,
3305        &mut installed_route_epochs,
3306        &mut route_bash_cancels,
3307        &active_tool_calls,
3308        &executor,
3309    );
3310
3311    fleet_status_client.set_route_live(false);
3312    fleet_status_task.abort();
3313    let _ = fleet_status_task.await;
3314
3315    let mut loop_result = loop_result;
3316    if !pending_bash_asks.is_empty() {
3317        let no_routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
3318        if let Err(error) = settle_all_pending_bash_asks(
3319            &writer_tx,
3320            &mut pending_bash_asks,
3321            &no_routes,
3322            &mut live_roots,
3323            &mut route_bash_cancels,
3324            &shutdown,
3325            &dispatch_path_metrics,
3326        )
3327        .await
3328        {
3329            loop_result = loop_result.and(Err(error));
3330        }
3331    }
3332
3333    // The reader task may be parked on `read_frame`; abort it (we are done with
3334    // the connection) and flush the writer.
3335    reader_task.abort();
3336    drop(writer_tx);
3337    let writer_result = finish_writer_task(writer_task).await;
3338    loop_result.and_then(|exit| writer_result.map(|_| exit))
3339}
3340
3341fn spawn_writer_task<W>(
3342    mut write: W,
3343    mut rx: mpsc::Receiver<WriterFrame>,
3344    metrics: Arc<DispatchPathMetrics>,
3345) -> JoinHandle<Result<(), subc_transport::FrameIoError>>
3346where
3347    W: AsyncWrite + Unpin + Send + 'static,
3348{
3349    tokio::spawn(async move {
3350        let mut write_buffer = Vec::new();
3351        while let Some(mut queued) = rx.recv().await {
3352            let measure = queued.tool_response_trace.is_some();
3353            let dequeued = measure.then(Instant::now);
3354            metrics.writer_active.store(true, Ordering::Relaxed);
3355            decrement_counted_channel(&metrics.writer_queued);
3356            let write_timing = write_frame_contiguous(
3357                &mut write,
3358                queued.frame(),
3359                queued.body(),
3360                &mut write_buffer,
3361                measure,
3362            )
3363            .await;
3364            metrics.writer_active.store(false, Ordering::Relaxed);
3365            let write_timing = write_timing?;
3366
3367            if let (Some(trace), Some(dequeued), Some(write_timing)) =
3368                (queued.tool_response_trace.take(), dequeued, write_timing)
3369            {
3370                if let Some(completed) = trace.finish(
3371                    dequeued,
3372                    write_timing.write_started,
3373                    write_timing.write_finished,
3374                    write_timing.frame_bytes,
3375                ) {
3376                    log_ctx::with_session(Some(completed.session), || {
3377                        crate::logging::note_tool_call_trace(
3378                            &completed.name,
3379                            &completed.root,
3380                            completed.channel,
3381                            completed.corr,
3382                            completed.phases,
3383                        );
3384                    });
3385                }
3386            }
3387        }
3388        Ok(())
3389    })
3390}
3391
3392struct FrameWriteTiming {
3393    write_started: Instant,
3394    write_finished: Instant,
3395    frame_bytes: usize,
3396}
3397
3398/// Encode one complete frame into the existing reusable buffer and write it
3399/// without interleaving bytes from another channel. Timing is collected only
3400/// for tool responses, so Push and control frames add no clock reads.
3401async fn write_frame_contiguous<W>(
3402    writer: &mut W,
3403    frame: &Frame,
3404    body: &[u8],
3405    buffer: &mut Vec<u8>,
3406    measure: bool,
3407) -> Result<Option<FrameWriteTiming>, subc_transport::FrameIoError>
3408where
3409    W: AsyncWrite + Unpin,
3410{
3411    if frame.header.len as usize != body.len() {
3412        return Err(subc_transport::FrameIoError::BodyLengthMismatch {
3413            header_len: frame.header.len,
3414            body_len: body.len(),
3415        });
3416    }
3417
3418    let header = frame.header.encode();
3419    buffer.clear();
3420    buffer.reserve(header.len() + body.len());
3421    buffer.extend_from_slice(&header);
3422    buffer.extend_from_slice(body);
3423    let write_started = measure.then(Instant::now);
3424    writer
3425        .write_all(buffer)
3426        .await
3427        .map_err(subc_transport::FrameIoError::Io)?;
3428    Ok(write_started.map(|write_started| FrameWriteTiming {
3429        write_started,
3430        write_finished: Instant::now(),
3431        frame_bytes: buffer.len(),
3432    }))
3433}
3434
3435fn spawn_reader_task<R>(
3436    mut read: R,
3437    tx: mpsc::Sender<Result<DecodedFrame, SubcError>>,
3438) -> JoinHandle<()>
3439where
3440    R: AsyncRead + Unpin + Send + 'static,
3441{
3442    tokio::spawn(async move {
3443        loop {
3444            match read_frame(&mut read).await {
3445                Ok(Some(frame)) => {
3446                    let decoded = DecodedFrame {
3447                        frame,
3448                        phase_trace: PhaseTrace::new(Instant::now()),
3449                    };
3450                    if tx.send(Ok(decoded)).await.is_err() {
3451                        return;
3452                    }
3453                }
3454                Ok(None) => {
3455                    // EOF: let the loop observe channel close as "daemon closed".
3456                    return;
3457                }
3458                Err(error) => {
3459                    // A killed daemon surfaces as ConnectionReset (RST) on
3460                    // Windows where Unix delivers a clean EOF (FIN); a
3461                    // mid-teardown daemon can also abort the socket. Both mean
3462                    // "daemon went away", not a wire fault — normalize them to
3463                    // the clean-close path so module exit behavior matches
3464                    // across platforms (same class subc-core fixed in d33d9a71).
3465                    if let subc_transport::FrameIoError::Io(io_error) = &error {
3466                        if matches!(
3467                            io_error.kind(),
3468                            std::io::ErrorKind::ConnectionReset
3469                                | std::io::ErrorKind::ConnectionAborted
3470                        ) {
3471                            log::info!(
3472                                "subc attach: connection reset by daemon; treating as close"
3473                            );
3474                            return;
3475                        }
3476                    }
3477                    let _ = tx.send(Err(SubcError::FrameIo(error))).await;
3478                    return;
3479                }
3480            }
3481        }
3482    })
3483}
3484
3485async fn finish_writer_task(
3486    mut writer_task: JoinHandle<Result<(), subc_transport::FrameIoError>>,
3487) -> Result<(), SubcError> {
3488    match tokio::time::timeout(Duration::from_millis(100), &mut writer_task).await {
3489        Ok(Ok(Ok(()))) => Ok(()),
3490        Ok(Ok(Err(error))) => Err(SubcError::FrameIo(error)),
3491        Ok(Err(error)) => Err(SubcError::WriterJoin(error)),
3492        Err(_) => {
3493            writer_task.abort();
3494            Ok(())
3495        }
3496    }
3497}
3498
3499fn register_actor_for_bind(
3500    shared_app: &Arc<App>,
3501    executor: &Arc<Executor>,
3502    push_senders: &PushSenders,
3503    bind_root_id: &ProjectRootId,
3504    route_channel: u16,
3505    root_was_live: bool,
3506) -> bool {
3507    if executor.actor_registered(bind_root_id) {
3508        log::debug!(
3509            "subc attach: reusing actor for route {} root {}",
3510            route_channel,
3511            bind_root_id.as_path().display()
3512        );
3513        return false;
3514    }
3515
3516    if root_was_live {
3517        log::warn!(
3518            "subc attach: recreating missing actor for live root {} on route {}",
3519            bind_root_id.as_path().display(),
3520            route_channel
3521        );
3522    }
3523
3524    let actor_ctx = Arc::new(AppContext::from_app(
3525        Arc::clone(shared_app),
3526        Config::default(),
3527    ));
3528    install_bash_compressor(&actor_ctx);
3529    actor_ctx.install_fleet_status_client(Some(push_senders.fleet_status_client.clone()));
3530    actor_ctx.set_progress_sender(Some(push::progress_sender_for_root(
3531        push_senders.clone(),
3532        bind_root_id.clone(),
3533    )));
3534    let inserted = executor.register_actor(bind_root_id.clone(), Arc::clone(&actor_ctx));
3535    drop(actor_ctx);
3536    if inserted {
3537        // Do not insert into live_roots until configure succeeds: live_roots
3538        // drives maintenance, and a half-configured new actor must not be
3539        // maintenance-eligible before its route/session identity exists.
3540        log::debug!(
3541            "subc attach: registered actor for route {} root {}",
3542            route_channel,
3543            bind_root_id.as_path().display()
3544        );
3545    } else {
3546        log::debug!(
3547            "subc attach: actor appeared while binding route {} root {}; reusing it",
3548            route_channel,
3549            bind_root_id.as_path().display()
3550        );
3551    }
3552    inserted
3553}
3554
3555fn rollback_pending_bind_actor(
3556    executor: &Arc<Executor>,
3557    live_roots: &HashMap<ProjectRootId, RootMeta>,
3558    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3559    root_id: &ProjectRootId,
3560    inserted_new_actor: bool,
3561) {
3562    if !inserted_new_actor || live_roots.contains_key(root_id) {
3563        return;
3564    }
3565
3566    if let Some((route, pending)) = pending_binds
3567        .iter_mut()
3568        .find(|(_, pending)| &pending.bind_root_id == root_id)
3569    {
3570        pending.inserted_new_actor = true;
3571        log::debug!(
3572            "subc attach: transferred rollback ownership for root {} to pending route {}",
3573            root_id.as_path().display(),
3574            route
3575        );
3576        return;
3577    }
3578
3579    executor.remove_actor(root_id);
3580}
3581
3582fn route_bind_error_code_for_configure_response(response: &Response) -> &'static str {
3583    match response.data.get("code").and_then(|code| code.as_str()) {
3584        // Preserve typed configure rejections across the bind boundary: a
3585        // malformed fed fingerprint means a federation-module bug or
3586        // fingerprint-format drift, and the fed side matches on the code rather
3587        // than parsing prose.
3588        Some("bad_harness_fingerprint") => "bad_harness_fingerprint",
3589        // Cache-key probe failures are transient (fd pressure, git spawn
3590        // contention); the client retries the bind rather than treating the
3591        // root as permanently divergent.
3592        Some("cache_key_probe_failed") => "cache_key_probe_failed",
3593        // Actor lifecycle gaps are transient from the daemon/client viewpoint:
3594        // a fresh bind can create or join a healthy actor, so do not classify
3595        // them as permanent config divergence.
3596        Some("actor_not_registered" | "actor_fatal") => "actor_not_ready",
3597        _ => "config_divergence",
3598    }
3599}
3600
3601fn queue_post_bind_configure_and_completion_maintenance(
3602    root_id: &ProjectRootId,
3603    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3604) {
3605    let Some(meta) = live_roots.get_mut(root_id) else {
3606        return;
3607    };
3608    if meta.maintenance_poisoned || meta.maintenance_pending {
3609        return;
3610    }
3611
3612    meta.maintenance_pending = true;
3613    meta.maintenance_queued_kinds
3614        .push_back(MaintenanceDrainKind::ConfigureTail);
3615    meta.maintenance_queued_kinds
3616        .push_back(MaintenanceDrainKind::CompletionDrains);
3617}
3618
3619#[allow(clippy::too_many_arguments)]
3620async fn handle_route_bind_completion(
3621    tx: &WriterSender,
3622    completion: RouteBindCompletion,
3623    routes: &mut HashMap<RouteChannel, RouteIdentity>,
3624    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
3625    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
3626    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
3627    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3628    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3629    installed_route_epochs: &mut HashMap<u16, u32>,
3630    executor: &Arc<Executor>,
3631    shutdown: &Arc<Notify>,
3632    metrics: &Arc<DispatchPathMetrics>,
3633) -> Result<(), SubcError> {
3634    let route_id = completion.route;
3635    let Some(pending) = pending_binds.remove(&route_id) else {
3636        log::warn!(
3637            "subc attach: dropping RouteBind completion for non-pending route {}",
3638            completion.route
3639        );
3640        rollback_pending_bind_actor(
3641            executor,
3642            live_roots,
3643            pending_binds,
3644            &completion.bind_root_id,
3645            completion.inserted_new_actor,
3646        );
3647        let has_pending_bind = pending_binds
3648            .values()
3649            .any(|pending| pending.bind_root_id == completion.bind_root_id);
3650        if !root_channels
3651            .get(&completion.bind_root_id)
3652            .is_some_and(|channels| !channels.is_empty())
3653            && !has_pending_bind
3654        {
3655            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3656        }
3657        remove_installed_route(installed_route_epochs, route_id);
3658        return Ok(());
3659    };
3660
3661    if pending.bind_root_id != completion.bind_root_id {
3662        log::warn!(
3663            "subc attach: pending RouteBind root mismatch for route {} (pending {} completion {})",
3664            completion.route,
3665            pending.bind_root_id.as_path().display(),
3666            completion.bind_root_id.as_path().display()
3667        );
3668    }
3669
3670    let inserted_new_actor = pending.inserted_new_actor || completion.inserted_new_actor;
3671    if pending.cancelled {
3672        rollback_pending_bind_actor(
3673            executor,
3674            live_roots,
3675            pending_binds,
3676            &completion.bind_root_id,
3677            inserted_new_actor,
3678        );
3679        let has_pending_bind = pending_binds
3680            .values()
3681            .any(|pending| pending.bind_root_id == completion.bind_root_id);
3682        if !root_channels
3683            .get(&completion.bind_root_id)
3684            .is_some_and(|channels| !channels.is_empty())
3685            && !has_pending_bind
3686        {
3687            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3688        }
3689        log::debug!(
3690            "subc attach: discarded completed RouteBind for cancelled route {} root {}",
3691            completion.route,
3692            completion.bind_root_id.as_path().display()
3693        );
3694        remove_installed_route(installed_route_epochs, route_id);
3695        return Ok(());
3696    }
3697
3698    let failure = if !completion.configure_response.success {
3699        Some((
3700            &completion.configure_response,
3701            "configure failed during route bind",
3702        ))
3703    } else {
3704        None
3705    };
3706
3707    if let Some((response, fallback)) = failure {
3708        rollback_pending_bind_actor(
3709            executor,
3710            live_roots,
3711            pending_binds,
3712            &completion.bind_root_id,
3713            inserted_new_actor,
3714        );
3715        let has_pending_bind = pending_binds
3716            .values()
3717            .any(|pending| pending.bind_root_id == completion.bind_root_id);
3718        if !root_channels
3719            .get(&completion.bind_root_id)
3720            .is_some_and(|channels| !channels.is_empty())
3721            && !has_pending_bind
3722        {
3723            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3724        }
3725        let message = response_message(response, fallback);
3726        let fatal = response_is_fatal_panic(response);
3727        let error_code = route_bind_error_code_for_configure_response(response);
3728        send_route_bind_error_parts(
3729            tx,
3730            completion.ver,
3731            completion.corr,
3732            completion.flags,
3733            error_code,
3734            &message,
3735            metrics,
3736        )
3737        .await?;
3738        remove_installed_route(installed_route_epochs, route_id);
3739        if fatal {
3740            signal_fatal_teardown(
3741                tx,
3742                Some(completion.route),
3743                completion.ver,
3744                completion.corr,
3745                shutdown,
3746                metrics,
3747            )
3748            .await;
3749        }
3750        return Ok(());
3751    }
3752
3753    remember_session_identity(session_identity, &completion.identity);
3754    let replay_key = push::ReplayKey::from_identity(&completion.identity);
3755    let bind_trust = completion.identity.trust;
3756    insert_route_channel(routes, root_channels, route_id, completion.identity);
3757    let restore_watcher = live_roots
3758        .get(&completion.bind_root_id)
3759        .is_some_and(|meta| meta.idle_artifacts_evicted || meta.unbound_quiesced);
3760    live_roots
3761        .entry(completion.bind_root_id.clone())
3762        .and_modify(|meta| {
3763            meta.reactivate_bound();
3764            meta.diagnostics_on_edit = completion.diagnostics_on_edit;
3765            meta.maintenance_poisoned = false;
3766        })
3767        .or_insert_with(|| RootMeta::new(Instant::now()));
3768    if let Some(meta) = live_roots.get_mut(&completion.bind_root_id) {
3769        meta.diagnostics_on_edit = completion.diagnostics_on_edit;
3770        meta.maintenance_poisoned = false;
3771    }
3772    if let Some(ctx) = executor.actor_context(&completion.bind_root_id) {
3773        ctx.mark_subc_bound();
3774        if restore_watcher {
3775            crate::commands::configure::ensure_project_watcher(&ctx);
3776        }
3777    }
3778
3779    let ack =
3780        serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).map_err(SubcError::Json)?;
3781    let response = Frame::build_with_version(
3782        completion.ver,
3783        FrameType::Response,
3784        control_flags(),
3785        0,
3786        0,
3787        completion.corr,
3788        ack,
3789    )
3790    .map_err(SubcError::FrameBuild)?;
3791    send_reliable_writer_frame(tx, metrics, response, "RouteBindAck").await?;
3792    queue_post_bind_configure_and_completion_maintenance(&completion.bind_root_id, live_roots);
3793    let replayed = push::replay_buffered_push_frames(
3794        tx,
3795        metrics,
3796        route_id,
3797        push_buffer,
3798        &replay_key,
3799        bind_trust,
3800    );
3801    if replayed > 0 {
3802        log::debug!(
3803            "subc attach: replayed {} buffered Push frame(s) to route {} root {} harness {} session {}",
3804            replayed,
3805            completion.route,
3806            replay_key.root.as_path().display(),
3807            replay_key.harness,
3808            replay_key.session
3809        );
3810    }
3811    log::info!(
3812        "subc attach: route {} bound to root {}",
3813        completion.route,
3814        completion.bind_root_id.as_path().display()
3815    );
3816    Ok(())
3817}
3818
3819async fn expire_overdue_route_binds(
3820    tx: &WriterSender,
3821    executor: &Arc<Executor>,
3822    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3823    installed_route_epochs: &mut HashMap<u16, u32>,
3824    metrics: &DispatchPathMetrics,
3825) -> Result<(), SubcError> {
3826    let now = Instant::now();
3827    let expired: Vec<_> = pending_binds
3828        .iter()
3829        .filter_map(|(route, pending)| {
3830            let age = now.saturating_duration_since(pending.started_at);
3831            (!pending.deadline_reported && age >= ROUTE_BIND_DEADLINE).then(|| {
3832                (
3833                    *route,
3834                    pending.corr,
3835                    pending.ver,
3836                    pending.flags,
3837                    pending.bind_root_id.clone(),
3838                    pending.configure_request_id.clone(),
3839                    age,
3840                )
3841            })
3842        })
3843        .collect();
3844
3845    for (route, corr, ver, flags, root_id, configure_request_id, age) in expired {
3846        if let Some(pending) = pending_binds.get_mut(&route) {
3847            pending.cancelled = true;
3848            pending.deadline_reported = true;
3849            let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
3850            log::debug!(
3851                "subc attach: cancelled overdue RouteBind configure for route {route} ({outcome:?})"
3852            );
3853        }
3854        remove_installed_route(installed_route_epochs, route);
3855        let age_ms = age.as_millis().min(u128::from(u64::MAX)) as u64;
3856        let deadline_ms = ROUTE_BIND_DEADLINE.as_millis();
3857        send_route_bind_error_parts(
3858            tx,
3859            ver,
3860            corr,
3861            flags,
3862            "actor_not_ready",
3863            &format!("route bind deadline exceeded after {age_ms}ms (deadline {deadline_ms}ms)"),
3864            metrics,
3865        )
3866        .await?;
3867        log::warn!(
3868            "subc attach: route {} bind for root {} exceeded {}ms deadline (configure_request_id={})",
3869            route,
3870            root_id.as_path().display(),
3871            deadline_ms,
3872            configure_request_id
3873        );
3874    }
3875
3876    Ok(())
3877}
3878
3879/// channel-0 control requests: RouteBind plus the cached health probe. RouteBind
3880/// still reconciles the route's RootConfig through the executor's Mutating lane
3881/// and resolves completion on a loop-owned control-completion channel so slow
3882/// configure jobs do not block the transport loop.
3883#[allow(clippy::too_many_arguments)]
3884async fn handle_control_request(
3885    tx: &WriterSender,
3886    frame: &Frame,
3887    shared_app: &Arc<App>,
3888    executor: &Arc<Executor>,
3889    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3890    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3891    installed_route_epochs: &mut HashMap<u16, u32>,
3892    routes: &mut HashMap<RouteChannel, RouteIdentity>,
3893    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
3894    bg_subs: &mut HashMap<RouteChannel, BgSub>,
3895    bg_sub_by_session: &mut BgSubsBySession,
3896    bg_wake_pending: &mut HashSet<RouteChannel>,
3897    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
3898    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
3899    active_tool_calls: &ActiveToolCalls,
3900    pending_inspects: &mut PendingSubcInspects,
3901    retry_buffer: &mut RetryBuffer,
3902    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
3903    shutdown: &Arc<Notify>,
3904    control_completion_tx: &mpsc::Sender<RouteBindCompletion>,
3905    metrics: &Arc<DispatchPathMetrics>,
3906    health_rollup_cache: &HealthRollupCache,
3907    push_senders: &PushSenders,
3908    dispatch: DispatchFn,
3909    user_config_path: Option<&Path>,
3910    tool_response_body_limit: usize,
3911) -> Result<(), SubcError> {
3912    let request =
3913        serde_json::from_slice::<ModuleControlRequest>(&frame.body).map_err(SubcError::Json)?;
3914    match request {
3915        ModuleControlRequest::RouteBind {
3916            route_channel,
3917            epoch,
3918            target: _,
3919            identity,
3920            principal,
3921            consumer_capabilities,
3922            admission_facts: _,
3923        } => {
3924            let route_id = route_key(route_channel, epoch);
3925            if epoch == 0 {
3926                return send_route_bind_error(
3927                    tx,
3928                    frame,
3929                    "config_divergence",
3930                    "route bind uses an invalid channel generation",
3931                    metrics,
3932                )
3933                .await;
3934            }
3935            let mut bind_root_id = None;
3936            if let Some(installed_epoch) = installed_route_epochs.get(&route_channel).copied() {
3937                if installed_epoch >= epoch {
3938                    return send_route_bind_error(
3939                        tx,
3940                        frame,
3941                        "config_divergence",
3942                        "route bind generation is not newer than the installed generation",
3943                        metrics,
3944                    )
3945                    .await;
3946                }
3947
3948                let replacement_root = match ProjectRootId::from_path(&identity.project_root) {
3949                    Ok(root_id) => root_id,
3950                    Err(error) => {
3951                        return send_route_bind_error(
3952                            tx,
3953                            frame,
3954                            "config_divergence",
3955                            &format!("invalid route project root: {error}"),
3956                            metrics,
3957                        )
3958                        .await;
3959                    }
3960                };
3961                teardown_installed_route(
3962                    tx,
3963                    metrics,
3964                    executor,
3965                    route_key(route_channel, installed_epoch),
3966                    "higher-epoch RouteBind",
3967                    Some(&replacement_root),
3968                    installed_route_epochs,
3969                    routes,
3970                    root_channels,
3971                    bg_subs,
3972                    bg_sub_by_session,
3973                    bg_wake_pending,
3974                    pending_bash_asks,
3975                    live_roots,
3976                    route_bash_cancels,
3977                    active_tool_calls,
3978                    pending_inspects,
3979                    pending_binds,
3980                    retry_buffer,
3981                    push_buffer,
3982                    shutdown,
3983                    tool_response_body_limit,
3984                )
3985                .await?;
3986                bind_root_id = Some(replacement_root);
3987            }
3988            if pending_binds.contains_key(&route_id) {
3989                return send_route_bind_error(
3990                    tx,
3991                    frame,
3992                    "config_divergence",
3993                    "route bind is already pending for channel",
3994                    metrics,
3995                )
3996                .await;
3997            }
3998            let bind_root_id = match bind_root_id {
3999                Some(root_id) => root_id,
4000                None => match ProjectRootId::from_path(&identity.project_root) {
4001                    Ok(root_id) => root_id,
4002                    Err(error) => {
4003                        return send_route_bind_error(
4004                            tx,
4005                            frame,
4006                            "config_divergence",
4007                            &format!("invalid route project root: {error}"),
4008                            metrics,
4009                        )
4010                        .await;
4011                    }
4012                },
4013            };
4014
4015            // Reconcile RootConfig: build a configure request from the bind
4016            // identity + forwarded config tiers and run it through the executor.
4017            let request_id = format!("subc-bind-{route_channel}");
4018            let bind_project_root = identity.project_root.clone();
4019            let bind_harness = identity.harness.clone();
4020            let bind_session = identity.session.clone();
4021            let bind_trust = trust_for_bind(&bind_harness, &principal);
4022            let bind_principal_id = principal_id(&principal);
4023            // Typed capability declaration from the consumer: the facade stamps it
4024            // from the MCP host's initialize-advertised capabilities. Absent
4025            // means no reverse-request capability — flat deny, fail-closed. A
4026            // consumer over-declaring only earns asks that TTL-deny.
4027            let consumer_elicitation_capable = consumer_capabilities
4028                .as_ref()
4029                .is_some_and(|capabilities| capabilities.iter().any(|c| c == "elicitation"));
4030            log::info!(
4031                "subc attach: route {} harness={} principal={} trust={} elicitation={}",
4032                route_channel,
4033                bind_harness,
4034                principal_label(&principal),
4035                bind_trust.label(),
4036                consumer_elicitation_capable
4037            );
4038
4039            // Config is single-per-project, read by AFT directly from the
4040            // CortexKit config files (user: ~/.config/cortexkit/aft.jsonc,
4041            // project: <root>/.cortexkit/aft.jsonc). Wire-relayed config tiers are
4042            // IGNORED entirely: a front (runner, mcp:*, or fed:*) cannot push config over
4043            // the wire. This is what makes config harness-INDEPENDENT — every
4044            // harness binding a project gets the identical on-disk config, so two
4045            // trust domains sharing the per-root actor can never diverge or
4046            // inherit each other's capabilities (the cross-bind escalation class).
4047            // Wire-relayed config tiers (if the protocol still carries them) are
4048            // ignored entirely; the per-tier trust boundary (user trusted, project
4049            // privileged-dropped) is applied to the FILE tiers in handle_configure.
4050            let local_tiers = crate::subc_config::read_local_cortexkit_config_tiers(
4051                user_config_path,
4052                Path::new(&bind_project_root),
4053            );
4054            let config_tiers: Vec<Value> = local_tiers
4055                .iter()
4056                .map(|t| json!({ "tier": t.tier, "source": t.source, "doc": t.doc }))
4057                .collect();
4058            let diagnostics_on_edit = diagnostics_on_edit_from_tiers(&local_tiers);
4059            let configure_json = json!({
4060                "id": request_id,
4061                "command": "configure",
4062                "project_root": bind_project_root,
4063                "harness": bind_harness,
4064                "session_id": bind_session.clone(),
4065                "config": config_tiers,
4066            });
4067            let configure_req = match serde_json::from_value::<RawRequest>(configure_json) {
4068                Ok(req) => req,
4069                Err(error) => {
4070                    return send_route_bind_error(
4071                        tx,
4072                        frame,
4073                        "config_divergence",
4074                        &format!("failed to build configure request: {error}"),
4075                        metrics,
4076                    )
4077                    .await;
4078                }
4079            };
4080
4081            let route_identity = RouteIdentity(Arc::new(RouteIdentityData {
4082                root: bind_root_id.clone(),
4083                project_root: PathBuf::from(&bind_project_root),
4084                harness: bind_harness.clone(),
4085                session: bind_session.clone(),
4086                trust: bind_trust,
4087                spawn_principal: AuthenticatedPrincipal::RouteBind {
4088                    trust: bind_trust.sandbox_trust(),
4089                    route_channel,
4090                    route_epoch: epoch,
4091                    project_root: PathBuf::from(&bind_project_root),
4092                    harness: bind_harness.clone(),
4093                    session_id: bind_session.clone(),
4094                    principal_id: bind_principal_id,
4095                },
4096                consumer_elicitation_capable,
4097            }));
4098            let configure_session = route_identity.session.clone();
4099            let root_was_live = live_roots.contains_key(&bind_root_id);
4100            let inserted_new_actor = register_actor_for_bind(
4101                shared_app,
4102                executor,
4103                push_senders,
4104                &bind_root_id,
4105                route_channel,
4106                root_was_live,
4107            );
4108
4109            let configure_request_id = configure_req.id.clone();
4110            installed_route_epochs.insert(route_channel, epoch);
4111            if let Some(meta) = live_roots.get_mut(&bind_root_id) {
4112                meta.maintenance_queued_kinds.clear();
4113                meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
4114            }
4115            let (configure_rx, configure_cancellation) = executor.submit_cancellable_async(
4116                bind_root_id.clone(),
4117                Lane::Mutating,
4118                configure_request_id.clone(),
4119                Box::new(move |ctx| {
4120                    log_ctx::with_session(Some(configure_session.clone()), || {
4121                        dispatch(configure_req, ctx)
4122                    })
4123                }),
4124            );
4125            pending_binds.insert(
4126                route_id,
4127                PendingBind {
4128                    bind_root_id: bind_root_id.clone(),
4129                    inserted_new_actor,
4130                    cancelled: false,
4131                    configure_request_id: configure_request_id.clone(),
4132                    started_at: Instant::now(),
4133                    warned_half_deadline: false,
4134                    deadline_reported: false,
4135                    corr: frame.header.corr,
4136                    ver: frame.header.ver,
4137                    flags: frame.header.flags,
4138                    cancellation: configure_cancellation,
4139                },
4140            );
4141
4142            let completion_tx = control_completion_tx.clone();
4143            let completion_identity = route_identity;
4144            let completion_root = bind_root_id.clone();
4145            let completion_route_channel = route_channel;
4146            let completion_ver = frame.header.ver;
4147            let completion_corr = frame.header.corr;
4148            let completion_flags = frame.header.flags;
4149            let completion_metrics = Arc::clone(metrics);
4150            tokio::spawn(async move {
4151                let _response_task = ResponseTaskGuard::new(&completion_metrics);
4152                let configure_response =
4153                    await_executor_response(configure_rx, configure_request_id.clone()).await;
4154                // Send the route-bind acknowledgment as soon as configure succeeds.
4155                // Installing completed search or callgraph builds only refreshes cached
4156                // read data, so a later maintenance pass can do it without delaying the
4157                // daemon's confirmation that the route is usable.
4158                let completion = RouteBindCompletion {
4159                    route: route_key(completion_route_channel, epoch),
4160                    identity: completion_identity,
4161                    bind_root_id: completion_root,
4162                    inserted_new_actor,
4163                    configure_response,
4164                    diagnostics_on_edit,
4165                    ver: completion_ver,
4166                    corr: completion_corr,
4167                    flags: completion_flags,
4168                };
4169                if send_counted_channel(
4170                    &completion_tx,
4171                    &completion_metrics.control_completion_queued,
4172                    completion,
4173                )
4174                .await
4175                .is_err()
4176                {
4177                    log::debug!(
4178                        "subc attach: dropped RouteBind completion for route {} after loop exit",
4179                        completion_route_channel
4180                    );
4181                }
4182            });
4183
4184            health_rollup_cache.refresh(executor, shared_app);
4185            Ok(())
4186        }
4187        ModuleControlRequest::HealthCheck {} => {
4188            metrics.record_bg_runtime(bg_subs.len(), bg_wake_pending.len());
4189            let report = build_health_report(
4190                health_rollup_cache,
4191                executor,
4192                pending_binds,
4193                metrics,
4194                shared_app,
4195            );
4196            let body = serde_json::to_vec(&ModuleControlResponse::from(report))
4197                .map_err(SubcError::Json)?;
4198            let response = Frame::build_with_version(
4199                frame.header.ver,
4200                FrameType::Response,
4201                frame.header.flags,
4202                0,
4203                0,
4204                frame.header.corr,
4205                body,
4206            )
4207            .map_err(SubcError::FrameBuild)?;
4208            send_frame(tx, metrics, response).await
4209        }
4210    }
4211}
4212
4213fn install_bash_compressor(ctx: &AppContext) {
4214    // Mirrors main.rs per-actor compressor installation for subc-created actors.
4215    let filter_registry_handle = ctx.shared_filter_registry();
4216    let compress_flag = ctx.bash_compress_flag();
4217    ctx.bash_background().set_compressor_with_exit_code(
4218        move |command: &str, output: String, exit_code: Option<i32>| {
4219            if !compress_flag.load(std::sync::atomic::Ordering::Relaxed) {
4220                return crate::compress::CompressionResult::new(output);
4221            }
4222            let registry_guard = match filter_registry_handle.read() {
4223                Ok(g) => g,
4224                Err(poisoned) => poisoned.into_inner(),
4225            };
4226            crate::compress::compress_with_registry_exit_code(
4227                command,
4228                &output,
4229                exit_code,
4230                &registry_guard,
4231            )
4232        },
4233    );
4234}
4235
4236fn diagnostics_on_edit_from_tiers(tiers: &[ConfigTier]) -> bool {
4237    let mut diagnostics_on_edit = false;
4238    for tier in tiers {
4239        if let Some(value) = diagnostics_on_edit_from_doc(&tier.doc) {
4240            diagnostics_on_edit = value;
4241        }
4242    }
4243    diagnostics_on_edit
4244}
4245
4246fn diagnostics_on_edit_from_doc(doc: &str) -> Option<bool> {
4247    let stripped = strip_jsonc(doc);
4248    let value = serde_json::from_str::<Value>(&stripped).ok()?;
4249    value
4250        .get("lsp")
4251        .and_then(Value::as_object)?
4252        .get("diagnostics_on_edit")
4253        .and_then(Value::as_bool)
4254}
4255
4256async fn send_route_bind_error(
4257    tx: &WriterSender,
4258    frame: &Frame,
4259    code: &str,
4260    message: &str,
4261    metrics: &DispatchPathMetrics,
4262) -> Result<(), SubcError> {
4263    send_route_bind_error_parts(
4264        tx,
4265        frame.header.ver,
4266        frame.header.corr,
4267        frame.header.flags,
4268        code,
4269        message,
4270        metrics,
4271    )
4272    .await
4273}
4274
4275async fn send_route_bind_error_parts(
4276    tx: &WriterSender,
4277    ver: u8,
4278    corr: u64,
4279    flags: Flags,
4280    code: &str,
4281    message: &str,
4282    metrics: &DispatchPathMetrics,
4283) -> Result<(), SubcError> {
4284    let response = build_error_frame(ver, 0, 0, corr, flags, code, message)?;
4285    send_reliable_writer_frame(tx, metrics, response, "RouteBind error").await?;
4286    log_route_bind_rejection(code, message);
4287    Ok(())
4288}
4289
4290/// Per-message rate limit for the bind-rejection warn line. A caller that
4291/// re-attaches a dead root forever turns this line into the entire readable
4292/// tail of the SHARED daemon log (measured 2026-08-09: 2.08M copies, ~40/sec,
4293/// 936MB log — other modules' incident lines pushed out of the tail).
4294/// The line itself stays byte-identical so external counters keep matching;
4295/// repeats inside the window are summarized with a suppressed count on the
4296/// next emission (volume stays diagnosable, per the log-diet convention).
4297fn log_route_bind_rejection(code: &str, message: &str) {
4298    const WINDOW: Duration = Duration::from_secs(60);
4299    static SUPPRESSED: OnceLock<StdMutex<HashMap<String, (Instant, u64)>>> = OnceLock::new();
4300    let map = SUPPRESSED.get_or_init(|| StdMutex::new(HashMap::new()));
4301    let mut map = match map.try_lock() {
4302        Ok(map) => map,
4303        // Contended: log unsuppressed rather than blocking or dropping.
4304        Err(_) => {
4305            log::warn!("subc attach: route bind rejected ({code}): {message}");
4306            return;
4307        }
4308    };
4309    let now = Instant::now();
4310    // Bound the map: dead roots churn, and an unbounded suppression map is
4311    // its own leak. Sweep expired entries once it grows past a fleet-sized
4312    // number of distinct rejection messages.
4313    if map.len() > 512 {
4314        map.retain(|_, (start, _)| now.duration_since(*start) < WINDOW);
4315    }
4316    match map.get_mut(message) {
4317        Some((window_start, suppressed)) if now.duration_since(*window_start) < WINDOW => {
4318            *suppressed += 1;
4319        }
4320        Some((window_start, suppressed)) => {
4321            if *suppressed > 0 {
4322                log::warn!(
4323                    "subc attach: route bind rejected ({code}): {message} (repeated {}x in last 60s)",
4324                    *suppressed
4325                );
4326            } else {
4327                log::warn!("subc attach: route bind rejected ({code}): {message}");
4328            }
4329            *window_start = now;
4330            *suppressed = 0;
4331        }
4332        None => {
4333            log::warn!("subc attach: route bind rejected ({code}): {message}");
4334            map.insert(message.to_string(), (now, 0));
4335        }
4336    }
4337}
4338
4339/// Route-channel tool call: `{name, arguments}` → executor lane → dispatch to
4340/// the sync command core → wrap the structured Response in a CallToolResult
4341/// `{content, isError}`. Tool-result mapping: the whole `{success, ...}` Response
4342/// serialized into ONE text block; `isError` carries `success == false`.
4343async fn handle_tool_call(
4344    tx: &WriterSender,
4345    frame: &Frame,
4346    mut phase_trace: PhaseTrace,
4347    routes: &HashMap<RouteChannel, RouteIdentity>,
4348    pending_binds: &HashMap<RouteChannel, PendingBind>,
4349    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4350    executor: &Arc<Executor>,
4351    active_tool_calls: &ActiveToolCalls,
4352    pending_inspect_setups: &Arc<AtomicUsize>,
4353    shutdown: &Arc<Notify>,
4354    connection_cancel: &PersistentCancelSignal,
4355    bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
4356    bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
4357    metrics: &Arc<DispatchPathMetrics>,
4358    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
4359    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
4360    next_bash_ask_corr: &mut u64,
4361    bg_subs: &mut HashMap<RouteChannel, BgSub>,
4362    bg_sub_by_session: &mut BgSubsBySession,
4363    bg_wake_pending: &mut HashSet<RouteChannel>,
4364    bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
4365    dispatch: DispatchFn,
4366    inspect_deferred_tx: &mpsc::UnboundedSender<PendingSubcInspect>,
4367    allow_native_passthrough: bool,
4368    tool_response_body_limit: usize,
4369) -> Result<(), SubcError> {
4370    let route_id = route_key(frame.header.channel, frame.header.epoch);
4371    if pending_binds.contains_key(&route_id) {
4372        let error = build_error_frame(
4373            frame.header.ver,
4374            frame.header.channel,
4375            frame.header.epoch,
4376            frame.header.corr,
4377            frame.header.flags,
4378            "route_not_bound",
4379            "route is not bound before tool call",
4380        )?;
4381        return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
4382    }
4383
4384    let Some(identity) = routes.get(&route_id).cloned() else {
4385        let error = build_error_frame(
4386            frame.header.ver,
4387            frame.header.channel,
4388            frame.header.epoch,
4389            frame.header.corr,
4390            frame.header.flags,
4391            "route_not_bound",
4392            "route is not bound before tool call",
4393        )?;
4394        return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
4395    };
4396    let restore_watcher = live_roots
4397        .get(&identity.root)
4398        .is_some_and(|meta| meta.idle_artifacts_evicted);
4399    if let Some(meta) = live_roots.get_mut(&identity.root) {
4400        meta.reactivate_bound();
4401    }
4402    if restore_watcher {
4403        if let Some(ctx) = executor.actor_context(&identity.root) {
4404            crate::commands::configure::ensure_project_watcher(&ctx);
4405        }
4406    }
4407
4408    let route_request =
4409        serde_json::from_slice::<RouteRequest>(&frame.body).map_err(SubcError::Json)?;
4410    if matches!(
4411        route_request,
4412        RouteRequest::BgEvents(BgEventsRequest {
4413            op: BgEventsOp::BgEvents
4414        })
4415    ) {
4416        if let Some(old_sub) = bg_subs.get(&route_id).cloned() {
4417            metrics.record_bg_subscription_ended(
4418                &old_sub.root,
4419                &old_sub.session,
4420                route_id,
4421                "resubscribe",
4422            );
4423            push::send_reliable_bg_stream_end(tx, metrics, route_id, &old_sub).await?;
4424        }
4425        if !identity.trust.allows_bash_observation() {
4426            bg_subs.remove(&route_id);
4427            bg_wake_pending.remove(&route_id);
4428            remove_bg_subscription_index(bg_sub_by_session, route_id, Some(&identity));
4429            let denied_sub = BgSub {
4430                corr: frame.header.corr,
4431                ver: frame.header.ver,
4432                flags: frame.header.flags,
4433                root: identity.root.clone(),
4434                session: identity.session.clone(),
4435            };
4436            metrics.record_bg_subscription_ended(
4437                &identity.root,
4438                &identity.session,
4439                route_id,
4440                "subscribe-denied",
4441            );
4442            push::send_reliable_bg_stream_end(tx, metrics, route_id, &denied_sub).await?;
4443            return Ok(());
4444        }
4445        bg_subs.insert(
4446            route_id,
4447            BgSub {
4448                corr: frame.header.corr,
4449                ver: frame.header.ver,
4450                flags: frame.header.flags,
4451                root: identity.root.clone(),
4452                session: identity.session.clone(),
4453            },
4454        );
4455        insert_bg_subscription_index(
4456            bg_sub_by_session,
4457            identity.root.clone(),
4458            identity.session.clone(),
4459            route_id,
4460        );
4461        metrics.record_bg_subscription_installed(&identity.root, &identity.session, route_id);
4462        push::arm_bg_wake(
4463            identity.root.clone(),
4464            identity.session.clone(),
4465            route_id,
4466            bg_wake_pending,
4467            bg_wake_epoch,
4468            metrics,
4469        );
4470        return Ok(());
4471    }
4472
4473    let RouteRequest::ToolCall(call) = route_request else {
4474        unreachable!("background event subscription returned above")
4475    };
4476    let bare_name = call.name;
4477    let arguments = strip_agent_preview_arg_owned(call.arguments);
4478    let format_context = crate::subc_format::FormatContext::from_tool_call(
4479        &bare_name,
4480        &arguments,
4481        identity.project_root.as_path(),
4482    );
4483
4484    let request_id = format!("subc-{}-{}", frame.header.channel, frame.header.corr);
4485    let bind_trust = identity.trust;
4486    let diagnostics_on_edit = live_roots
4487        .get(&identity.root)
4488        .map(|meta| meta.diagnostics_on_edit)
4489        .unwrap_or(false);
4490
4491    let requests_host = bare_name == "bash"
4492        && arguments
4493            .get("sandbox")
4494            .or_else(|| {
4495                arguments
4496                    .get("params")
4497                    .and_then(|params| params.get("sandbox"))
4498            })
4499            .and_then(Value::as_str)
4500            == Some("host");
4501    if matches!(bind_trust, BindTrust::Untrusted) && requests_host {
4502        let response = Response::error(
4503            request_id.clone(),
4504            "sandbox_escalation_denied",
4505            "sandbox host escalation is unavailable to untrusted principals",
4506        );
4507        let text = crate::subc_format::format_response_with_context(
4508            &bare_name,
4509            &response,
4510            &format_context,
4511        );
4512        let result = ToolCallResult { text, response };
4513        let response_frame = build_tool_response_frame_with_limit(
4514            frame.header.ver,
4515            route_id,
4516            frame.header.corr,
4517            frame.header.flags,
4518            &result,
4519            bind_trust,
4520            tool_response_body_limit,
4521        )?;
4522        return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4523    }
4524
4525    if matches!(bind_trust, BindTrust::Untrusted)
4526        && is_bash_family_tool(&bare_name)
4527        && (bare_name != "bash" || !identity.consumer_elicitation_capable)
4528    {
4529        let response = bash::bash_denied_untrusted_response(request_id.clone());
4530        let text = crate::subc_format::format_response_with_context(
4531            &bare_name,
4532            &response,
4533            &format_context,
4534        );
4535        let result = ToolCallResult { text, response };
4536        let response_frame = build_tool_response_frame_with_limit(
4537            frame.header.ver,
4538            route_id,
4539            frame.header.corr,
4540            frame.header.flags,
4541            &result,
4542            bind_trust,
4543            tool_response_body_limit,
4544        )?;
4545        return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4546    }
4547
4548    // A non-core name is NOT in the tool manifest. AFT fails closed and
4549    // does not trust subc to enforce the manifest: rejecting here is the
4550    // defense-in-depth backstop that prevents a forwarded native command
4551    // (e.g. `configure`, which would reach handle_configure and bypass
4552    // the RouteBind config-trust cap) from ever reaching dispatch. Only
4553    // the integration-test harness (run_subc_mode_for_test) opens this to
4554    // drive synthetic native commands through the executor.
4555    if !is_subc_agent_core_tool(&bare_name)
4556        && !is_subc_native_plumbing_tool(&bare_name)
4557        && !allow_native_passthrough
4558    {
4559        log::warn!(
4560            "subc tool call: rejecting non-manifest tool name {:?} on route {} (fail-closed)",
4561            bare_name,
4562            frame.header.channel
4563        );
4564        let response = Response::error(
4565            request_id.clone(),
4566            "unknown_tool",
4567            format!("tool {:?} is not in the AFT tool manifest", bare_name),
4568        );
4569        let text = crate::subc_format::format_response_with_context(
4570            &bare_name,
4571            &response,
4572            &format_context,
4573        );
4574        let result = ToolCallResult { text, response };
4575        let response_frame = build_tool_response_frame_with_limit(
4576            frame.header.ver,
4577            route_id,
4578            frame.header.corr,
4579            frame.header.flags,
4580            &result,
4581            bind_trust,
4582            tool_response_body_limit,
4583        )?;
4584        return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4585    }
4586
4587    if bare_name == "bash" {
4588        if matches!(bind_trust, BindTrust::Untrusted) {
4589            let plan = match bash::prepare_bash_elicitation_plan(
4590                &arguments,
4591                identity.project_root.as_path(),
4592            ) {
4593                Ok(plan) => plan,
4594                Err(error) => {
4595                    let response = Response::error(request_id.clone(), error.code, error.message);
4596                    let text = crate::subc_format::format_response_with_context(
4597                        &bare_name,
4598                        &response,
4599                        &format_context,
4600                    );
4601                    let result = ToolCallResult { text, response };
4602                    let response_frame = build_tool_response_frame_with_limit(
4603                        frame.header.ver,
4604                        route_id,
4605                        frame.header.corr,
4606                        frame.header.flags,
4607                        &result,
4608                        bind_trust,
4609                        tool_response_body_limit,
4610                    )?;
4611                    return send_reliable_writer_frame(
4612                        tx,
4613                        metrics,
4614                        response_frame,
4615                        "tool response",
4616                    )
4617                    .await;
4618                }
4619            };
4620
4621            let reverse_corr =
4622                allocate_reverse_corr(pending_bash_asks, route_id, next_bash_ask_corr);
4623            let ask_frame = build_bash_elicitation_request_frame(
4624                frame.header.ver,
4625                route_id,
4626                reverse_corr,
4627                frame.header.flags,
4628                &plan.command,
4629                &plan.asks,
4630            )?;
4631
4632            let meta = live_roots
4633                .entry(identity.root.clone())
4634                .or_insert_with(|| RootMeta::new(Instant::now()));
4635            meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
4636            meta.reactivate_bound();
4637
4638            let route_cancel =
4639                route_bash_cancels
4640                    .entry(route_id)
4641                    .or_insert_with(|| bash::RouteBashCancel {
4642                        token: PersistentCancelSignal::new(),
4643                        active_waits: 0,
4644                    });
4645            route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
4646            let cancel = bash::BashWaitCancel {
4647                connection: connection_cancel.clone(),
4648                route: route_cancel.token.clone(),
4649            };
4650            pending_bash_asks.insert(
4651                ReverseCorrKey {
4652                    route: route_id,
4653                    corr: reverse_corr,
4654                },
4655                PendingBashAsk {
4656                    route: route_id,
4657                    tool_corr: frame.header.corr,
4658                    tool_flags: frame.header.flags,
4659                    tool_ver: frame.header.ver,
4660                    root: identity.root.clone(),
4661                    project_root: identity.project_root.clone(),
4662                    session_id: identity.session.clone(),
4663                    spawn_principal: identity.spawn_principal.clone(),
4664                    edit_slot_survives: call.edit_slot_survives,
4665                    request_id,
4666                    arguments,
4667                    format_context,
4668                    cancel,
4669                    grants: plan.grants,
4670                    expires_at: Instant::now() + bash_elicitation_timeout(),
4671                },
4672            );
4673            return send_reliable_writer_frame(tx, metrics, ask_frame, "bash elicitation request")
4674                .await;
4675        }
4676
4677        let meta = live_roots
4678            .entry(identity.root.clone())
4679            .or_insert_with(|| RootMeta::new(Instant::now()));
4680        meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
4681        meta.reactivate_bound();
4682
4683        let route_cancel =
4684            route_bash_cancels
4685                .entry(route_id)
4686                .or_insert_with(|| bash::RouteBashCancel {
4687                    token: PersistentCancelSignal::new(),
4688                    active_waits: 0,
4689                });
4690        route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
4691        let cancel = bash::BashWaitCancel {
4692            connection: connection_cancel.clone(),
4693            route: route_cancel.token.clone(),
4694        };
4695
4696        bash::submit_deferred_bash(
4697            executor,
4698            bash_deferred_tx,
4699            bash_poll_touch_tx,
4700            metrics,
4701            dispatch,
4702            identity.root.clone(),
4703            identity.project_root.clone(),
4704            identity.session.clone(),
4705            request_id,
4706            route_id,
4707            frame.header.corr,
4708            frame.header.flags,
4709            frame.header.ver,
4710            arguments,
4711            format_context,
4712            cancel,
4713            bind_trust,
4714            identity.spawn_principal.clone(),
4715            call.edit_slot_survives,
4716            None,
4717        );
4718        return Ok(());
4719    }
4720
4721    let lane = command_lane(&bare_name);
4722    let tool_call_context = ToolCallContext {
4723        project_root: identity.project_root.clone(),
4724        session_id: Some(identity.session.clone()),
4725        request_id: request_id.clone(),
4726        diagnostics_on_edit,
4727        preview: call.preview,
4728        edit_slot_survives: call.edit_slot_survives,
4729        report_registration_downgrade: true,
4730    };
4731
4732    if bare_name == "inspect" {
4733        let Some(inspect_ctx) = executor.actor_context(&identity.root) else {
4734            let response = Response::error(
4735                &request_id,
4736                "actor_not_registered",
4737                "executor actor is not registered",
4738            );
4739            let text = crate::subc_format::format_response_with_context(
4740                "inspect",
4741                &response,
4742                &format_context,
4743            );
4744            let result = ToolCallResult { text, response };
4745            let response_frame = build_tool_response_frame_with_limit(
4746                frame.header.ver,
4747                route_id,
4748                frame.header.corr,
4749                frame.header.flags,
4750                &result,
4751                bind_trust,
4752                tool_response_body_limit,
4753            )?;
4754            return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4755        };
4756        let identity_for_run = identity.clone();
4757        let request_id_for_force = request_id.clone();
4758        let format_context_for_run = format_context.clone();
4759        let (setup_tx, setup_rx) = oneshot::channel::<InspectSetupOutcome>();
4760        phase_trace.mark_executor_submitted();
4761        let job: crate::executor::ExecutorJob = Box::new(move |ctx| {
4762            phase_trace.mark_job_admitted();
4763            log_ctx::with_session(Some(identity_for_run.session.clone()), || {
4764                let run = || match prepare_tool_call(
4765                    "inspect",
4766                    arguments,
4767                    &format_context_for_run,
4768                    &tool_call_context,
4769                    ctx,
4770                    Some(&mut phase_trace),
4771                ) {
4772                    Err(result) => {
4773                        let response = result.response;
4774                        let _ = setup_tx.send(InspectSetupOutcome::Immediate {
4775                            text: result.text,
4776                            phase_trace,
4777                        });
4778                        response
4779                    }
4780                    Ok(prepared) => {
4781                        let DispatchOutcome::Deferred(pending) =
4782                            crate::commands::inspect::handle_inspect_deferred_with_restriction(
4783                                &prepared.request,
4784                                Arc::clone(&inspect_ctx),
4785                                matches!(bind_trust, BindTrust::Untrusted),
4786                            )
4787                        else {
4788                            unreachable!("inspect deferred setup returned an immediate response")
4789                        };
4790                        let _ = setup_tx.send(InspectSetupOutcome::Deferred {
4791                            pending,
4792                            surface_downgraded: prepared.surface_downgraded,
4793                            phase_trace,
4794                        });
4795                        Response::success(
4796                            request_id_for_force.clone(),
4797                            json!({ "inspect_deferred": true }),
4798                        )
4799                    }
4800                };
4801                if matches!(bind_trust, BindTrust::Untrusted) {
4802                    ctx.with_force_restrict(&request_id_for_force, run)
4803                } else {
4804                    run()
4805                }
4806            })
4807        });
4808        let inspect_setup_guard = PendingInspectSetupGuard::new(Arc::clone(pending_inspect_setups));
4809        let (rx, cancellation) =
4810            executor.submit_cancellable_async(identity.root.clone(), lane, request_id.clone(), job);
4811        active_tool_calls
4812            .lock()
4813            .unwrap_or_else(std::sync::PoisonError::into_inner)
4814            .insert(
4815                (route_id, frame.header.corr),
4816                ActiveToolCall {
4817                    root_id: identity.root.clone(),
4818                    cancellation,
4819                },
4820            );
4821
4822        let completion_tx = tx.clone();
4823        let completion_shutdown = Arc::clone(shutdown);
4824        let completion_metrics = Arc::clone(metrics);
4825        let active_tool_calls = Arc::clone(active_tool_calls);
4826        let inspect_deferred_tx = inspect_deferred_tx.clone();
4827        let route = route_id;
4828        let corr = frame.header.corr;
4829        let flags = frame.header.flags;
4830        let ver = frame.header.ver;
4831        let root = identity.root.clone();
4832        let session_id = identity.session.clone();
4833        tokio::spawn(async move {
4834            let _response_task = ResponseTaskGuard::new(&completion_metrics);
4835            let _inspect_setup = inspect_setup_guard;
4836            let response = await_executor_response(rx, request_id.clone()).await;
4837            match setup_rx.await {
4838                Ok(InspectSetupOutcome::Deferred {
4839                    pending,
4840                    surface_downgraded,
4841                    phase_trace,
4842                }) => {
4843                    let pending = PendingSubcInspect {
4844                        route,
4845                        corr,
4846                        flags,
4847                        ver,
4848                        root,
4849                        session_id,
4850                        format_context,
4851                        bind_trust,
4852                        pending,
4853                        surface_downgraded,
4854                        phase_trace,
4855                    };
4856                    if let Err(error) = inspect_deferred_tx.send(pending) {
4857                        if let Some(cancellation) = &error.0.pending.cancellation {
4858                            cancellation.request_cancel();
4859                        }
4860                        finish_active_tool_call(&active_tool_calls, route, corr);
4861                    }
4862                }
4863                Ok(InspectSetupOutcome::Immediate { text, phase_trace }) => {
4864                    finish_active_tool_call(&active_tool_calls, route, corr);
4865                    let result = ToolCallResult { text, response };
4866                    let fatal = response_is_fatal_panic(&result.response);
4867                    match build_tool_response_frame_with_limit(
4868                        ver,
4869                        route,
4870                        corr,
4871                        flags,
4872                        &result,
4873                        bind_trust,
4874                        tool_response_body_limit,
4875                    ) {
4876                        Ok(response_frame) => {
4877                            let trace = ToolResponseWriteTrace::new(
4878                                phase_trace,
4879                                "inspect".to_string(),
4880                                identity.project_root.clone(),
4881                                identity.session.clone(),
4882                                route.channel,
4883                                corr,
4884                            );
4885                            if let Err(error) = send_traced_tool_response_frame(
4886                                &completion_tx,
4887                                &completion_metrics,
4888                                response_frame,
4889                                trace,
4890                            )
4891                            .await
4892                            {
4893                                log::warn!(
4894                                    "subc attach: failed to queue inspect setup response: {error}"
4895                                );
4896                            }
4897                        }
4898                        Err(error) => {
4899                            log::error!(
4900                                "subc attach: failed to build inspect setup response: {error}"
4901                            );
4902                        }
4903                    }
4904                    if fatal {
4905                        signal_fatal_teardown(
4906                            &completion_tx,
4907                            Some(route),
4908                            ver,
4909                            corr,
4910                            &completion_shutdown,
4911                            &completion_metrics,
4912                        )
4913                        .await;
4914                    }
4915                }
4916                Err(_) => {
4917                    finish_active_tool_call(&active_tool_calls, route, corr);
4918                    let text = crate::subc_format::format_response_with_context(
4919                        "inspect",
4920                        &response,
4921                        &format_context,
4922                    );
4923                    let result = ToolCallResult { text, response };
4924                    if let Ok(response_frame) = build_tool_response_frame_with_limit(
4925                        ver,
4926                        route,
4927                        corr,
4928                        flags,
4929                        &result,
4930                        bind_trust,
4931                        tool_response_body_limit,
4932                    ) {
4933                        let _ = send_reliable_writer_frame(
4934                            &completion_tx,
4935                            &completion_metrics,
4936                            response_frame,
4937                            "inspect setup failure",
4938                        )
4939                        .await;
4940                    }
4941                }
4942            }
4943        });
4944        return Ok(());
4945    }
4946
4947    let bare_name_for_frame = bare_name.clone();
4948    let identity_for_run = identity.clone();
4949    let completion_session = identity.session.clone();
4950    let completion_root = identity.project_root.clone();
4951    let request_id_for_force = request_id.clone();
4952    let format_context_for_frame = format_context.clone();
4953    let (tool_call_tx, tool_call_rx) = oneshot::channel::<ToolCallCompletion>();
4954    phase_trace.mark_executor_submitted();
4955    let job: crate::executor::ExecutorJob = Box::new(move |ctx| {
4956        phase_trace.mark_job_admitted();
4957        log_ctx::with_session(Some(identity_for_run.session.clone()), || {
4958            let run = || {
4959                let finalizer = |response: &mut Response| {
4960                    crate::response_finalize::finalize_response_with_bg_completions(
4961                        response,
4962                        ctx,
4963                        &identity_for_run.session,
4964                        &bare_name,
4965                        bind_trust.allows_bash_observation(),
4966                    );
4967                };
4968                match run_tool_call(
4969                    &bare_name,
4970                    arguments,
4971                    &format_context,
4972                    &tool_call_context,
4973                    ctx,
4974                    &dispatch,
4975                    Some(&finalizer),
4976                    Some(&mut phase_trace),
4977                ) {
4978                    ToolCallOutcome::Unary(result) => {
4979                        let response = result.response;
4980                        let _ = tool_call_tx.send(ToolCallCompletion {
4981                            text: result.text,
4982                            phase_trace,
4983                        });
4984                        response
4985                    }
4986                }
4987            };
4988            if matches!(bind_trust, BindTrust::Untrusted) {
4989                ctx.with_force_restrict(&request_id_for_force, run)
4990            } else {
4991                run()
4992            }
4993        })
4994    });
4995    let rx = executor.submit_async(identity.root.clone(), lane, request_id.clone(), job);
4996    let completion_tx = tx.clone();
4997    let completion_shutdown = Arc::clone(shutdown);
4998    let route = route_id;
4999    let corr = frame.header.corr;
5000    let flags = frame.header.flags;
5001    let ver = frame.header.ver;
5002    let completion_metrics = Arc::clone(metrics);
5003    tokio::spawn(async move {
5004        let _response_task = ResponseTaskGuard::new(&completion_metrics);
5005        let response = await_executor_response(rx, request_id.clone()).await;
5006        let (text, phase_trace) = match tool_call_rx.await {
5007            Ok(completion) => (completion.text, Some(completion.phase_trace)),
5008            Err(_) => (
5009                crate::subc_format::format_response_with_context(
5010                    &bare_name_for_frame,
5011                    &response,
5012                    &format_context_for_frame,
5013                ),
5014                None,
5015            ),
5016        };
5017        let result = ToolCallResult { text, response };
5018        let fatal = response_is_fatal_panic(&result.response);
5019        match build_tool_response_frame_with_limit(
5020            ver,
5021            route,
5022            corr,
5023            flags,
5024            &result,
5025            bind_trust,
5026            tool_response_body_limit,
5027        ) {
5028            Ok(response_frame) => {
5029                let send_result = if let Some(phase_trace) = phase_trace {
5030                    let trace = ToolResponseWriteTrace::new(
5031                        phase_trace,
5032                        bare_name_for_frame,
5033                        completion_root,
5034                        completion_session,
5035                        route.channel,
5036                        corr,
5037                    );
5038                    send_traced_tool_response_frame(
5039                        &completion_tx,
5040                        &completion_metrics,
5041                        response_frame,
5042                        trace,
5043                    )
5044                    .await
5045                } else {
5046                    send_reliable_writer_frame(
5047                        &completion_tx,
5048                        &completion_metrics,
5049                        response_frame,
5050                        "tool response",
5051                    )
5052                    .await
5053                };
5054                if let Err(error) = send_result {
5055                    log::warn!("subc attach: failed to queue tool response frame: {error}");
5056                }
5057            }
5058            Err(error) => {
5059                log::error!("subc attach: failed to build tool response frame: {error}");
5060            }
5061        }
5062        if fatal {
5063            signal_fatal_teardown(
5064                &completion_tx,
5065                Some(route),
5066                ver,
5067                corr,
5068                &completion_shutdown,
5069                &completion_metrics,
5070            )
5071            .await;
5072        }
5073    });
5074    Ok(())
5075}
5076
5077fn submit_maintenance_job(
5078    executor: &Arc<Executor>,
5079    root_id: ProjectRootId,
5080    kind: MaintenanceDrainKind,
5081    bg_sessions_to_check: Vec<(String, u64)>,
5082    completion_tx: &mpsc::Sender<MaintenanceCompletion>,
5083    metrics: &Arc<DispatchPathMetrics>,
5084) {
5085    let request_id = format!(
5086        "subc-maintenance-drain-{}-{}",
5087        kind.label(),
5088        root_id.as_path().to_string_lossy()
5089    );
5090    let response_id = request_id.clone();
5091    let completion_root_id = root_id.clone();
5092    let maintenance_generation = executor
5093        .actor_context(&root_id)
5094        .map(|ctx| ctx.configure_generation())
5095        .unwrap_or(0);
5096    let (outcome_tx, outcome_rx) = oneshot::channel::<MaintenanceJobOutcome>();
5097    // ConfigureTail runs deferred configure mutations and needs the actor
5098    // epoch write gate. The other drain kinds only mutate subsystem state
5099    // behind that subsystem's own lock, so they run on MaintenanceCommit and
5100    // overlap interactive reads instead of excluding them.
5101    let lane = match kind {
5102        MaintenanceDrainKind::ConfigureTail => Lane::Mutating,
5103        MaintenanceDrainKind::Watcher
5104        | MaintenanceDrainKind::Lsp
5105        | MaintenanceDrainKind::CompletionDrains => Lane::MaintenanceCommit,
5106    };
5107    let job: crate::executor::ExecutorJob = Box::new(move |ctx: &AppContext| {
5108        let outcome = match kind {
5109            MaintenanceDrainKind::Watcher => {
5110                let drained = runtime_drain::drain_watcher_events_bounded(
5111                    ctx,
5112                    runtime_drain::WATCHER_PATH_DRAIN_BATCH_CAP,
5113                );
5114                MaintenanceJobOutcome {
5115                    empty_bg_sessions: Vec::new(),
5116                    requeue_kind: drained.has_more.then_some(kind),
5117                }
5118            }
5119            MaintenanceDrainKind::Lsp => {
5120                let drained = runtime_drain::drain_lsp_events_bounded(
5121                    ctx,
5122                    runtime_drain::LSP_EVENT_DRAIN_BATCH_CAP,
5123                );
5124                MaintenanceJobOutcome {
5125                    empty_bg_sessions: Vec::new(),
5126                    requeue_kind: drained.has_more.then_some(kind),
5127                }
5128            }
5129            MaintenanceDrainKind::ConfigureTail => {
5130                runtime_drain::drain_deferred_configure_maintenance(ctx);
5131                runtime_drain::drain_configure_warning_events(ctx);
5132                MaintenanceJobOutcome::default()
5133            }
5134            MaintenanceDrainKind::CompletionDrains => {
5135                runtime_drain::drain_search_index_events(ctx);
5136                runtime_drain::drain_callgraph_store_events(ctx);
5137                runtime_drain::drain_semantic_index_events(ctx);
5138                runtime_drain::drain_semantic_refresh_events(ctx);
5139                runtime_drain::drain_inspect_events_for_generation(ctx, maintenance_generation);
5140                let empty_bg_sessions = bg_sessions_to_check
5141                    .into_iter()
5142                    .filter(|(session, _)| {
5143                        !ctx.bash_background()
5144                            .has_completions_for_session(Some(session.as_str()))
5145                    })
5146                    .collect();
5147                MaintenanceJobOutcome {
5148                    empty_bg_sessions,
5149                    requeue_kind: None,
5150                }
5151            }
5152        };
5153        let requeued = outcome.requeue_kind.is_some();
5154        let _ = outcome_tx.send(outcome);
5155        Response::success(
5156            response_id,
5157            json!({ "drained": true, "kind": kind.label(), "requeued": requeued }),
5158        )
5159    });
5160    let rx = match kind {
5161        MaintenanceDrainKind::Watcher => executor.submit_coalescable_maintenance_async(
5162            root_id,
5163            lane,
5164            request_id.clone(),
5165            crate::executor::MaintenanceCoalesceKey::WatcherDrain,
5166            job,
5167        ),
5168        MaintenanceDrainKind::Lsp => executor.submit_coalescable_maintenance_async(
5169            root_id,
5170            lane,
5171            request_id.clone(),
5172            crate::executor::MaintenanceCoalesceKey::LspDrain,
5173            job,
5174        ),
5175        MaintenanceDrainKind::ConfigureTail | MaintenanceDrainKind::CompletionDrains => {
5176            executor.submit_maintenance_async(root_id, lane, request_id.clone(), job)
5177        }
5178    };
5179    let completion_tx = completion_tx.clone();
5180    let completion_metrics = Arc::clone(metrics);
5181    tokio::spawn(async move {
5182        let _response_task = ResponseTaskGuard::new(&completion_metrics);
5183        let response = await_executor_response(rx, request_id).await;
5184        let outcome = outcome_rx.await.unwrap_or_default();
5185        let _ = send_counted_channel(
5186            &completion_tx,
5187            &completion_metrics.maintenance_queued,
5188            MaintenanceCompletion {
5189                root_id: completion_root_id,
5190                kind,
5191                response,
5192                empty_bg_sessions: outcome.empty_bg_sessions,
5193                requeue_kind: outcome.requeue_kind,
5194            },
5195        )
5196        .await;
5197    });
5198}
5199
5200async fn await_executor_response(rx: oneshot::Receiver<Response>, request_id: String) -> Response {
5201    rx.await
5202        .unwrap_or_else(|_| Response::error(request_id, "internal_error", "executor dropped"))
5203}
5204
5205async fn deliver_resolved_subc_inspect(
5206    tx: &WriterSender,
5207    mut resolved: ResolvedSubcInspect,
5208    routes: &HashMap<RouteChannel, RouteIdentity>,
5209    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
5210    executor: &Executor,
5211    active_tool_calls: &ActiveToolCalls,
5212    shutdown: &Arc<Notify>,
5213    metrics: &DispatchPathMetrics,
5214    tool_response_body_limit: usize,
5215) -> Result<(), SubcError> {
5216    let entry = &mut resolved.entry;
5217    finish_active_tool_call(active_tool_calls, entry.route, entry.corr);
5218    if let Some(meta) = live_roots.get_mut(&entry.root) {
5219        meta.note_activity();
5220    }
5221
5222    let Some(identity) = routes.get(&entry.route) else {
5223        log::debug!(
5224            "subc attach: dropping deferred inspect response {} for unbound route {}",
5225            entry.pending.request_id,
5226            entry.route
5227        );
5228        return Ok(());
5229    };
5230    let Some(ctx) = executor.actor_context(&entry.root) else {
5231        return Ok(());
5232    };
5233    entry.phase_trace.mark_execute_done();
5234    let finalizer = |response: &mut Response| {
5235        crate::response_finalize::finalize_response_with_bg_completions(
5236            response,
5237            &ctx,
5238            &entry.session_id,
5239            "inspect",
5240            entry.bind_trust.allows_bash_observation(),
5241        );
5242    };
5243    let result = finish_tool_call_response(
5244        "inspect",
5245        &entry.format_context,
5246        resolved.response,
5247        entry.surface_downgraded,
5248        Some(&finalizer),
5249        Some(&mut entry.phase_trace),
5250    );
5251    let fatal = response_is_fatal_panic(&result.response);
5252    let response_frame = build_tool_response_frame_with_limit(
5253        entry.ver,
5254        entry.route,
5255        entry.corr,
5256        entry.flags,
5257        &result,
5258        identity.trust,
5259        tool_response_body_limit,
5260    )?;
5261    let trace = ToolResponseWriteTrace::new(
5262        std::mem::replace(&mut entry.phase_trace, PhaseTrace::new(Instant::now())),
5263        "inspect".to_string(),
5264        identity.project_root.clone(),
5265        entry.session_id.clone(),
5266        entry.route.channel,
5267        entry.corr,
5268    );
5269    send_traced_tool_response_frame(tx, metrics, response_frame, trace).await?;
5270    if fatal {
5271        signal_fatal_teardown(
5272            tx,
5273            Some(entry.route),
5274            entry.ver,
5275            entry.corr,
5276            shutdown,
5277            metrics,
5278        )
5279        .await;
5280    }
5281    Ok(())
5282}
5283
5284async fn signal_fatal_teardown(
5285    tx: &WriterSender,
5286    route: Option<RouteChannel>,
5287    ver: u8,
5288    corr: u64,
5289    shutdown: &Arc<Notify>,
5290    metrics: &DispatchPathMetrics,
5291) {
5292    if let Some(route) = route {
5293        if let Ok(frame) = build_goodbye_frame(ver, route.channel, route.epoch, corr) {
5294            if let Err(error) = send_frame(tx, metrics, frame).await {
5295                log::warn!(
5296                    "subc attach: failed to queue fatal route Goodbye for route {route}: {error}"
5297                );
5298            }
5299        }
5300    }
5301    if let Ok(frame) = build_goodbye_frame(ver, 0, 0, 0) {
5302        if let Err(error) = send_frame(tx, metrics, frame).await {
5303            log::warn!("subc attach: failed to queue fatal channel-0 Goodbye: {error}");
5304        }
5305    }
5306    shutdown.notify_one();
5307}
5308#[derive(Debug, Deserialize)]
5309#[serde(untagged)]
5310enum RouteRequest {
5311    BgEvents(BgEventsRequest),
5312    ToolCall(ToolCallRequest),
5313}
5314
5315#[derive(Debug, Deserialize)]
5316struct BgEventsRequest {
5317    op: BgEventsOp,
5318}
5319
5320#[derive(Debug, Deserialize)]
5321#[serde(rename_all = "snake_case")]
5322enum BgEventsOp {
5323    BgEvents,
5324}
5325
5326#[derive(Debug, Deserialize)]
5327struct ToolCallRequest {
5328    name: String,
5329    #[serde(default)]
5330    arguments: Value,
5331    /// Host-computed registration fact; kept outside agent-controlled arguments.
5332    #[serde(default)]
5333    edit_slot_survives: Option<bool>,
5334    /// Server-owned preview control (B1c-0): the plugin's mutation flow is
5335    /// preview -> permission ask -> apply. Dropping this field made "preview"
5336    /// calls mutate disk before the permission prompt and the subsequent
5337    /// apply fail with not-found.
5338    #[serde(default)]
5339    preview: bool,
5340}
5341
5342#[cfg(test)]
5343pub(crate) mod test_support {
5344    use super::*;
5345    use crate::bash_background::BgTaskStatus;
5346    use crate::protocol::{
5347        BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, ConfigureWarningsFrame,
5348        ProgressFrame, StatusChangedFrame,
5349    };
5350    use serde_json::json;
5351
5352    pub(super) fn test_root(name: &str) -> (tempfile::TempDir, ProjectRootId) {
5353        let dir = tempfile::Builder::new()
5354            .prefix(name)
5355            .tempdir()
5356            .expect("temp root");
5357        let root = ProjectRootId::from_path(dir.path()).expect("project root id");
5358        (dir, root)
5359    }
5360
5361    pub(super) fn test_ctx() -> Arc<AppContext> {
5362        Arc::new(AppContext::new(
5363            Box::new(crate::parser::TreeSitterProvider::new()),
5364            crate::config::Config::default(),
5365        ))
5366    }
5367
5368    fn inspect_context(root: &Path) -> Arc<AppContext> {
5369        let mut config = crate::config::Config::default();
5370        config.project_root = Some(root.to_path_buf());
5371        let ctx = Arc::new(AppContext::new(
5372            Box::new(crate::parser::TreeSitterProvider::new()),
5373            config,
5374        ));
5375        ctx.set_harness(crate::harness::Harness::Opencode);
5376        ctx
5377    }
5378
5379    fn inspect_request(id: &str) -> RawRequest {
5380        serde_json::from_value(json!({ "id": id, "command": "inspect" })).expect("inspect request")
5381    }
5382
5383    fn submit_deferred_inspect_setup(
5384        executor: &Arc<Executor>,
5385        root: &ProjectRootId,
5386        ctx: &Arc<AppContext>,
5387        request_id: &str,
5388    ) -> (PendingResponse, JobCancellation) {
5389        let (pending_tx, pending_rx) = std::sync::mpsc::sync_channel(1);
5390        let request = inspect_request(request_id);
5391        let inspect_ctx = Arc::clone(ctx);
5392        let (_rx, cancellation) = executor.submit_cancellable_async(
5393            root.clone(),
5394            Lane::SerialLspStatus,
5395            request_id.to_string(),
5396            Box::new(move |_| {
5397                let DispatchOutcome::Deferred(pending) =
5398                    crate::commands::inspect::handle_inspect_deferred_with_restriction(
5399                        &request,
5400                        inspect_ctx,
5401                        true,
5402                    )
5403                else {
5404                    panic!("inspect setup must defer")
5405                };
5406                pending_tx.send(pending).expect("send pending inspect");
5407                Response::success("inspect-setup", json!({}))
5408            }),
5409        );
5410        let pending = pending_rx
5411            .recv_timeout(Duration::from_secs(1))
5412            .expect("inspect setup leaves the executor lane");
5413        let deadline = Instant::now() + Duration::from_secs(1);
5414        while !executor.actor_is_idle(root) {
5415            assert!(
5416                Instant::now() < deadline,
5417                "inspect setup kept lane counters live"
5418            );
5419            std::thread::sleep(Duration::from_millis(5));
5420        }
5421        (pending, cancellation)
5422    }
5423
5424    fn wait_for_inspect_terminal(pending: &mut PendingResponse, ctx: &AppContext) -> Response {
5425        let deadline = Instant::now() + Duration::from_secs(60);
5426        loop {
5427            if let Some(response) = (pending.poll)(ctx) {
5428                return response;
5429            }
5430            assert!(Instant::now() < deadline, "inspect terminal timed out");
5431            std::thread::sleep(Duration::from_millis(5));
5432        }
5433    }
5434
5435    #[test]
5436    fn deferred_inspect_releases_lane_for_bind_and_mutation() {
5437        let _serial = crate::commands::inspect::deferred_inspect_test_lock();
5438        let executor = Arc::new(Executor::new());
5439        let (dir, root) = test_root("deferred-inspect-storm");
5440        std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
5441        let ctx = inspect_context(dir.path());
5442        executor.register_actor(root.clone(), Arc::clone(&ctx));
5443        let (started_rx, release_tx) =
5444            crate::commands::inspect::install_deferred_inspect_stat_gate_for_test();
5445        let (mut pending, _cancellation) =
5446            submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-storm");
5447        started_rx
5448            .recv_timeout(Duration::from_secs(1))
5449            .expect("deferred inspect body starts");
5450
5451        for request_id in ["subc-bind-other-session", "subc-edit-other-session"] {
5452            let response = executor.submit(
5453                root.clone(),
5454                Lane::Mutating,
5455                request_id.to_string(),
5456                Box::new(move |_| Response::success(request_id, json!({ "admitted": true }))),
5457            );
5458            assert!(
5459                response
5460                    .recv_timeout(Duration::from_secs(1))
5461                    .expect("writer admits while inspect remains deferred")
5462                    .success
5463            );
5464        }
5465        assert_eq!(
5466            crate::commands::inspect::deferred_inspect_root_count_for_test(),
5467            1,
5468            "writer admissions must not finish the detached inspect"
5469        );
5470
5471        release_tx.send(()).expect("release inspect body");
5472        let terminal = wait_for_inspect_terminal(&mut pending, &ctx);
5473        assert!(
5474            terminal.data.get("inspect_terminal").is_some(),
5475            "inspect must still produce its terminal: {:?}",
5476            terminal.data
5477        );
5478    }
5479
5480    #[test]
5481    fn same_root_deferred_inspects_are_single_flight() {
5482        let _serial = crate::commands::inspect::deferred_inspect_test_lock();
5483        let executor = Arc::new(Executor::new());
5484        let (dir, root) = test_root("single-flight-deferred-inspect");
5485        std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
5486        let ctx = inspect_context(dir.path());
5487        executor.register_actor(root.clone(), Arc::clone(&ctx));
5488        let (started_rx, release_tx) =
5489            crate::commands::inspect::install_deferred_inspect_stat_gate_for_test();
5490        let (mut first, _first_cancellation) =
5491            submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-first");
5492        started_rx
5493            .recv_timeout(Duration::from_secs(1))
5494            .expect("first inspect owns the root flight");
5495        let (mut second, second_cancellation) =
5496            submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-second");
5497
5498        assert_eq!(
5499            crate::commands::inspect::deferred_inspect_root_count_for_test(),
5500            1,
5501            "only one detached body may run for a root"
5502        );
5503        assert!((second.poll)(&ctx).is_none(), "second inspect must queue");
5504        second_cancellation.request_cancel();
5505        let second_terminal = wait_for_inspect_terminal(&mut second, &ctx);
5506        assert_eq!(second_terminal.data["inspect_terminal"], "interrupted");
5507        assert_eq!(
5508            crate::commands::inspect::deferred_inspect_root_count_for_test(),
5509            1,
5510            "cancelling the queued request must not release the active flight"
5511        );
5512
5513        release_tx.send(()).expect("release first inspect");
5514        let first_terminal = wait_for_inspect_terminal(&mut first, &ctx);
5515        assert_eq!(first_terminal.data["inspect_terminal"], "fresh");
5516        assert_eq!(
5517            crate::commands::inspect::deferred_inspect_root_count_for_test(),
5518            0
5519        );
5520    }
5521
5522    #[test]
5523    fn route_abandonment_cancels_detached_inspect_thread() {
5524        let _serial = crate::commands::inspect::deferred_inspect_test_lock();
5525        let executor = Arc::new(Executor::new());
5526        let (dir, root) = test_root("cancelled-deferred-inspect");
5527        std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
5528        let ctx = inspect_context(dir.path());
5529        executor.register_actor(root.clone(), Arc::clone(&ctx));
5530        let active: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
5531        let route = RouteChannel {
5532            channel: 7,
5533            epoch: 1,
5534        };
5535        let (started_rx, _release_tx) =
5536            crate::commands::inspect::install_deferred_inspect_body_gate_for_test();
5537        let (mut pending, cancellation) =
5538            submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-abandoned");
5539        active.lock().expect("active tool call map").insert(
5540            (route, 41),
5541            ActiveToolCall {
5542                root_id: root.clone(),
5543                cancellation,
5544            },
5545        );
5546        started_rx
5547            .recv_timeout(Duration::from_secs(1))
5548            .expect("detached inspect reaches cancellation gate");
5549        assert!(ctx.request_force_restrict("subc-inspect-abandoned"));
5550
5551        assert!(cancel_active_tool_call(
5552            &active,
5553            executor.as_ref(),
5554            route,
5555            41,
5556            "test route abandonment"
5557        ));
5558        let terminal = wait_for_inspect_terminal(&mut pending, &ctx);
5559        assert_eq!(terminal.data["inspect_terminal"], "interrupted");
5560        assert_eq!(
5561            crate::commands::inspect::deferred_inspect_root_count_for_test(),
5562            0
5563        );
5564        assert!(executor.actor_is_idle(&root));
5565        assert!(active.lock().expect("active tool call map").is_empty());
5566        let restriction_deadline = Instant::now() + Duration::from_secs(1);
5567        while ctx.request_force_restrict("subc-inspect-abandoned") {
5568            assert!(
5569                Instant::now() < restriction_deadline,
5570                "detached force-restrict guard leaked"
5571            );
5572            std::thread::sleep(Duration::from_millis(5));
5573        }
5574    }
5575
5576    #[test]
5577    fn shutdown_drain_emits_terminal_and_clears_pending_inspect() {
5578        let _serial = crate::commands::inspect::deferred_inspect_test_lock();
5579        let executor = Arc::new(Executor::new());
5580        let (dir, root) = test_root("shutdown-deferred-inspect");
5581        std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
5582        let ctx = inspect_context(dir.path());
5583        executor.register_actor(root.clone(), Arc::clone(&ctx));
5584        let (started_rx, _release_tx) =
5585            crate::commands::inspect::install_deferred_inspect_body_gate_for_test();
5586        let (pending, cancellation) =
5587            submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-shutdown");
5588        let route = RouteChannel {
5589            channel: 8,
5590            epoch: 1,
5591        };
5592        started_rx
5593            .recv_timeout(Duration::from_secs(1))
5594            .expect("detached inspect reaches shutdown gate");
5595        let mut registry = PendingSubcInspects::default();
5596        registry.register(PendingSubcInspect {
5597            route,
5598            corr: 42,
5599            flags: Flags::new(false, Priority::Passive, false),
5600            ver: PROTOCOL_VERSION,
5601            root: root.clone(),
5602            session_id: "shutdown-session".to_string(),
5603            format_context: crate::subc_format::FormatContext::from_tool_call(
5604                "inspect",
5605                &json!({}),
5606                dir.path(),
5607            ),
5608            bind_trust: BindTrust::FirstParty,
5609            pending,
5610            surface_downgraded: false,
5611            phase_trace: PhaseTrace::new(Instant::now()),
5612        });
5613        let active: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::from([(
5614            (route, 42),
5615            ActiveToolCall {
5616                root_id: root.clone(),
5617                cancellation,
5618            },
5619        )])));
5620
5621        let resolved = registry.drain_on_shutdown(executor.as_ref());
5622        assert!(registry.is_empty());
5623        assert_eq!(resolved.len(), 1);
5624        assert_eq!(
5625            resolved[0].response.data["failure_reason"],
5626            "daemon_shutdown"
5627        );
5628        finish_active_tool_call(&active, route, 42);
5629        let deadline = Instant::now() + Duration::from_secs(1);
5630        while crate::commands::inspect::deferred_inspect_root_count_for_test() != 0 {
5631            assert!(Instant::now() < deadline, "shutdown cancellation was inert");
5632            std::thread::sleep(Duration::from_millis(5));
5633        }
5634        assert!(active.lock().expect("active calls").is_empty());
5635        assert!(executor.actor_is_idle(&root));
5636    }
5637
5638    pub(super) fn wait_for_watcher_count(ctx: &AppContext, expected: usize) {
5639        let deadline = Instant::now() + Duration::from_secs(30);
5640        loop {
5641            let observed = ctx.watcher_registry_count();
5642            if observed == expected {
5643                return;
5644            }
5645            assert!(
5646                Instant::now() < deadline,
5647                "watcher count did not settle before deadline: expected={expected}, observed={observed}"
5648            );
5649            std::thread::sleep(Duration::from_millis(50));
5650        }
5651    }
5652
5653    /// Sweep until `root` is forgotten, mirroring how production reaps.
5654    ///
5655    /// `reap_idle_roots` probes actor idleness with a try-lock and retains the
5656    /// root when the scheduler holds that lock — correct behavior, since the
5657    /// real caller sweeps on a timer and simply catches the root next tick. A
5658    /// test that asserts a single sweep succeeds is therefore asserting it wins
5659    /// a lock race that nothing in production depends on: `register_actor` wakes
5660    /// the scheduler, which grabs the same lock, and on a loaded runner that
5661    /// window is wide enough to lose. Sweep to the outcome instead.
5662    pub(super) fn reap_until_forgotten(
5663        root: &ProjectRootId,
5664        live_roots: &mut HashMap<ProjectRootId, RootMeta>,
5665        pending_binds: &HashMap<RouteChannel, PendingBind>,
5666        root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
5667        executor: &Arc<Executor>,
5668        metrics: &DispatchPathMetrics,
5669    ) -> IdleReapOutcome {
5670        let deadline = Instant::now() + Duration::from_secs(30);
5671        loop {
5672            let outcome = reap_idle_roots(
5673                Instant::now(),
5674                live_roots,
5675                pending_binds,
5676                root_channels,
5677                executor,
5678                metrics,
5679            );
5680            if outcome.forgotten_deleted_roots.contains(root) {
5681                return outcome;
5682            }
5683            assert!(
5684                Instant::now() < deadline,
5685                "deleted root was never forgotten: {root:?}"
5686            );
5687            std::thread::sleep(Duration::from_millis(10));
5688        }
5689    }
5690
5691    pub(super) fn wait_for_actor_root_count(app: &App, expected: usize) {
5692        let deadline = Instant::now() + Duration::from_secs(30);
5693        loop {
5694            let observed = app.actor_root_count();
5695            if observed == expected {
5696                return;
5697            }
5698            assert!(
5699                Instant::now() < deadline,
5700                "actor root count did not settle before deadline: expected={expected}, observed={observed}"
5701            );
5702            std::thread::sleep(Duration::from_millis(50));
5703        }
5704    }
5705
5706    pub(super) fn status_frame(seq: u64) -> PushFrame {
5707        status_frame_with_session(seq, None)
5708    }
5709
5710    pub(super) fn status_frame_with_session(seq: u64, session_id: Option<&str>) -> PushFrame {
5711        PushFrame::StatusChanged(StatusChangedFrame {
5712            frame_type: "status_changed",
5713            session_id: session_id.map(str::to_string),
5714            snapshot: json!({ "seq": seq }),
5715        })
5716    }
5717
5718    pub(super) fn completion_frame(task_id: &str) -> PushFrame {
5719        completion_frame_with_session(task_id, "session-1")
5720    }
5721
5722    pub(super) fn completion_frame_with_session(task_id: &str, session_id: &str) -> PushFrame {
5723        PushFrame::BashCompleted(BashCompletedFrame {
5724            frame_type: "bash_completed",
5725            task_id: task_id.to_string(),
5726            session_id: session_id.to_string(),
5727            status: BgTaskStatus::Completed,
5728            exit_code: Some(0),
5729            command: format!("echo {task_id}"),
5730            output_preview: String::new(),
5731            output_truncated: false,
5732            original_tokens: None,
5733            compressed_tokens: None,
5734            tokens_skipped: false,
5735            status_reason: None,
5736        })
5737    }
5738
5739    pub(super) fn long_running_frame(task_id: &str, elapsed_ms: u64) -> PushFrame {
5740        long_running_frame_with_session(task_id, "session-1", elapsed_ms)
5741    }
5742
5743    pub(super) fn long_running_frame_with_session(
5744        task_id: &str,
5745        session_id: &str,
5746        elapsed_ms: u64,
5747    ) -> PushFrame {
5748        PushFrame::BashLongRunning(BashLongRunningFrame {
5749            frame_type: "bash_long_running",
5750            task_id: task_id.to_string(),
5751            session_id: session_id.to_string(),
5752            command: format!("sleep {elapsed_ms}"),
5753            elapsed_ms,
5754        })
5755    }
5756
5757    pub(super) fn pattern_match_frame(session_id: &str) -> PushFrame {
5758        PushFrame::BashPatternMatch(BashPatternMatchFrame {
5759            frame_type: "bash_pattern_match",
5760            task_id: "task-pattern".to_string(),
5761            session_id: session_id.to_string(),
5762            watch_id: "watch-1".to_string(),
5763            match_text: "needle".to_string(),
5764            match_offset: 7,
5765            context: "haystack needle".to_string(),
5766            once: true,
5767            reason: "pattern_match",
5768        })
5769    }
5770
5771    pub(super) fn configure_warnings_frame(session_id: Option<&str>) -> PushFrame {
5772        PushFrame::ConfigureWarnings(ConfigureWarningsFrame {
5773            frame_type: "configure_warnings",
5774            session_id: session_id.map(str::to_string),
5775            project_root: "/tmp/subc-test".to_string(),
5776            warnings: Vec::new(),
5777        })
5778    }
5779
5780    pub(super) fn route_identity(root: &ProjectRootId, session_id: &str) -> RouteIdentity {
5781        route_identity_with_trust(root, session_id, BindTrust::FirstParty)
5782    }
5783
5784    pub(super) fn route_identity_with_trust(
5785        root: &ProjectRootId,
5786        session_id: &str,
5787        trust: BindTrust,
5788    ) -> RouteIdentity {
5789        RouteIdentity(Arc::new(RouteIdentityData {
5790            root: root.clone(),
5791            project_root: root.as_path().to_path_buf(),
5792            harness: "opencode".to_string(),
5793            session: session_id.to_string(),
5794            trust,
5795            spawn_principal: AuthenticatedPrincipal::RouteBind {
5796                trust: trust.sandbox_trust(),
5797                route_channel: 0,
5798                route_epoch: 0,
5799                project_root: root.as_path().to_path_buf(),
5800                harness: "opencode".to_string(),
5801                session_id: session_id.to_string(),
5802                principal_id: Some(match trust {
5803                    BindTrust::FirstParty => "direct".to_string(),
5804                    BindTrust::Untrusted => "unverified".to_string(),
5805                }),
5806            },
5807            consumer_elicitation_capable: false,
5808        }))
5809    }
5810
5811    pub(super) fn progress_frame(request_id: &str, kind: ProgressKind, chunk: &str) -> PushFrame {
5812        PushFrame::Progress(ProgressFrame::new(request_id, kind, chunk))
5813    }
5814
5815    pub(super) fn status_seq(frame: &PushFrame) -> Option<u64> {
5816        match frame {
5817            PushFrame::StatusChanged(status) => status.snapshot.get("seq").and_then(|v| v.as_u64()),
5818            _ => None,
5819        }
5820    }
5821
5822    pub(super) fn completion_task(frame: &PushFrame) -> Option<&str> {
5823        match frame {
5824            PushFrame::BashCompleted(completion) => Some(completion.task_id.as_str()),
5825            _ => None,
5826        }
5827    }
5828
5829    pub(super) fn push_frame_task_id(frame: &Frame) -> Option<String> {
5830        let body: serde_json::Value = serde_json::from_slice(&frame.body).expect("push body");
5831        body.get("task_id")
5832            .and_then(serde_json::Value::as_str)
5833            .map(str::to_string)
5834    }
5835}
5836
5837#[cfg(test)]
5838mod tests {
5839    use super::test_support::{
5840        completion_frame, reap_until_forgotten, route_identity, test_ctx, test_root,
5841        wait_for_actor_root_count, wait_for_watcher_count,
5842    };
5843    use super::*;
5844    use crate::bash_background::BgTaskStatus;
5845
5846    fn attach_error(kind: io::ErrorKind) -> SubcError {
5847        SubcError::Connect {
5848            endpoint: "127.0.0.1:1".to_string(),
5849            source: io::Error::new(kind, "constructed attach failure"),
5850        }
5851    }
5852
5853    fn auth_io_error(kind: io::ErrorKind) -> SubcError {
5854        SubcError::Auth {
5855            endpoint: "127.0.0.1:1".to_string(),
5856            source: subc_transport::AuthError::Io {
5857                stage: subc_transport::AuthStage::ServerProof,
5858                source: io::Error::new(kind, "constructed auth failure"),
5859            },
5860        }
5861    }
5862
5863    #[test]
5864    fn initial_attach_error_classifier_distinguishes_transient_and_permanent_failures() {
5865        let transient_errors = vec![
5866            attach_error(io::ErrorKind::ConnectionRefused),
5867            attach_error(io::ErrorKind::TimedOut),
5868            attach_error(io::ErrorKind::ConnectionReset),
5869            auth_io_error(io::ErrorKind::ConnectionAborted),
5870            auth_io_error(io::ErrorKind::BrokenPipe),
5871            SubcError::Auth {
5872                endpoint: "127.0.0.1:1".to_string(),
5873                source: subc_transport::AuthError::UnexpectedEof {
5874                    stage: subc_transport::AuthStage::ServerProof,
5875                    expected: 4,
5876                    actual: 0,
5877                },
5878            },
5879            SubcError::Auth {
5880                endpoint: "127.0.0.1:1".to_string(),
5881                source: subc_transport::AuthError::Timeout {
5882                    stage: subc_transport::AuthStage::ServerProof,
5883                    deadline: AUTH_DEADLINE,
5884                },
5885            },
5886        ];
5887        for error in &transient_errors {
5888            assert_eq!(
5889                classify_attach_error(error),
5890                AttachErrorClass::Transient,
5891                "expected transient: {error}"
5892            );
5893        }
5894
5895        let permanent_errors = vec![
5896            attach_error(io::ErrorKind::PermissionDenied),
5897            auth_io_error(io::ErrorKind::InvalidData),
5898            SubcError::Auth {
5899                endpoint: "127.0.0.1:1".to_string(),
5900                source: subc_transport::AuthError::InvalidServerProof,
5901            },
5902            SubcError::Auth {
5903                endpoint: "127.0.0.1:1".to_string(),
5904                source: subc_transport::AuthError::DaemonIdMismatch,
5905            },
5906            SubcError::ConnectionFile {
5907                path: PathBuf::from("subc-connection.json"),
5908                source: subc_transport::ConnectionFileError::Invalid {
5909                    reason: "constructed invalid file".to_string(),
5910                },
5911            },
5912            SubcError::NoEndpoint {
5913                path: PathBuf::from("subc-connection.json"),
5914            },
5915            SubcError::InvalidEndpoint {
5916                path: PathBuf::from("subc-connection.json"),
5917                endpoint: "not-an-ip:1234".to_string(),
5918            },
5919        ];
5920        for error in &permanent_errors {
5921            assert_eq!(
5922                classify_attach_error(error),
5923                AttachErrorClass::Permanent,
5924                "expected permanent: {error}"
5925            );
5926        }
5927    }
5928
5929    #[test]
5930    fn incompatible_wire_version_is_rejected_before_tcp_connect() {
5931        let conn_dir = tempfile::tempdir().expect("connection tempdir");
5932        let conn_path = conn_dir.path().join("subc-connection.json");
5933        let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind listener");
5934        listener
5935            .set_nonblocking(true)
5936            .expect("set listener nonblocking");
5937        let port = listener.local_addr().expect("listener addr").port();
5938        connection_file::write_atomic(
5939            &conn_path,
5940            &connection_file::ConnectionInfo {
5941                schema: connection_file::SCHEMA_VERSION,
5942                wire_version: Some(PROTOCOL_VERSION.wrapping_add(1)),
5943                endpoints: vec![connection_file::Endpoint {
5944                    host: "127.0.0.1".to_string(),
5945                    port,
5946                }],
5947                key: vec![0x42; subc_transport::KEY_LEN],
5948                daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
5949                pid: std::process::id(),
5950                daemon_ver: "subc-test".to_string(),
5951            },
5952        )
5953        .expect("write connection file");
5954
5955        let runtime = tokio::runtime::Builder::new_current_thread()
5956            .enable_all()
5957            .build()
5958            .expect("test runtime");
5959        let result = runtime.block_on(connect_and_authenticate_with_policy(
5960            &conn_path,
5961            AttachRetryPolicy {
5962                budget: Duration::from_secs(1),
5963                initial_backoff: Duration::from_millis(5),
5964                max_backoff: Duration::from_millis(10),
5965                jitter_percent: 0,
5966            },
5967        ));
5968        assert!(matches!(
5969            result,
5970            Err(SubcError::ConnectionFile {
5971                source: connection_file::ConnectionFileError::WireVersionMismatch { .. },
5972                ..
5973            })
5974        ));
5975        assert!(matches!(
5976            listener.accept(),
5977            Err(error) if error.kind() == std::io::ErrorKind::WouldBlock
5978        ));
5979    }
5980
5981    #[test]
5982    fn initial_attach_unreachable_endpoint_retries_until_budget_then_fails_loud() {
5983        let conn_dir = tempfile::tempdir().expect("connection tempdir");
5984        let conn_path = conn_dir.path().join("subc-connection.json");
5985        let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve port");
5986        let port = listener.local_addr().expect("reserved addr").port();
5987        drop(listener);
5988        connection_file::write_atomic(
5989            &conn_path,
5990            &connection_file::ConnectionInfo {
5991                schema: connection_file::SCHEMA_VERSION,
5992                wire_version: Some(PROTOCOL_VERSION),
5993                endpoints: vec![connection_file::Endpoint {
5994                    host: "127.0.0.1".to_string(),
5995                    port,
5996                }],
5997                key: vec![0x42; subc_transport::KEY_LEN],
5998                daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
5999                pid: std::process::id(),
6000                daemon_ver: "subc-test".to_string(),
6001            },
6002        )
6003        .expect("write connection file");
6004
6005        let policy = AttachRetryPolicy {
6006            budget: Duration::from_millis(40),
6007            initial_backoff: Duration::from_millis(5),
6008            max_backoff: Duration::from_millis(10),
6009            jitter_percent: 0,
6010        };
6011        let runtime = tokio::runtime::Builder::new_current_thread()
6012            .enable_all()
6013            .build()
6014            .expect("test runtime");
6015        let started_at = Instant::now();
6016        let result = runtime.block_on(connect_and_authenticate_with_policy(&conn_path, policy));
6017        let elapsed = started_at.elapsed();
6018        let error = match result {
6019            Ok(_) => panic!("unreachable endpoint unexpectedly attached"),
6020            Err(error) => error,
6021        };
6022
6023        assert!(matches!(error, SubcError::Connect { .. }), "{error}");
6024        assert!(
6025            elapsed >= Duration::from_millis(35),
6026            "retry budget ended too early: {elapsed:?}"
6027        );
6028        assert!(
6029            elapsed < Duration::from_secs(1),
6030            "retry budget was not bounded: {elapsed:?}"
6031        );
6032    }
6033
6034    fn due_maintenance_jobs_without_actor_context(
6035        live_roots: &mut HashMap<ProjectRootId, RootMeta>,
6036        budget: usize,
6037        pending_bind_roots: &HashSet<ProjectRootId>,
6038    ) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
6039        due_maintenance_jobs(
6040            live_roots,
6041            None,
6042            &HashMap::new(),
6043            &HashSet::new(),
6044            budget,
6045            pending_bind_roots,
6046        )
6047    }
6048
6049    fn actor_ctx_with_dirty_search_index(
6050        root: &Path,
6051        storage: &Path,
6052        file_name: &str,
6053        old_contents: &str,
6054        new_contents: &str,
6055    ) -> (Arc<AppContext>, PathBuf, PathBuf) {
6056        let file = root.join(file_name);
6057        std::fs::write(&file, old_contents).expect("write source");
6058        let canonical_root = std::fs::canonicalize(root).expect("canonical root");
6059        let ctx = Arc::new(AppContext::new(
6060            Box::new(crate::parser::TreeSitterProvider::new()),
6061            Config {
6062                project_root: Some(root.to_path_buf()),
6063                storage_dir: Some(storage.to_path_buf()),
6064                ..Config::default()
6065            },
6066        ));
6067        ctx.set_canonical_cache_root(canonical_root.clone());
6068
6069        let cache_dir = crate::search_index::resolve_cache_dir(&canonical_root, Some(storage));
6070        let mut index = crate::search_index::SearchIndex::build(&canonical_root);
6071        let git_head = index.stored_git_head().map(str::to_owned);
6072        index.write_to_disk(&cache_dir, git_head.as_deref());
6073
6074        std::fs::write(&file, new_contents).expect("edit source");
6075        index.update_file(&file);
6076        *ctx.search_index()
6077            .write()
6078            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
6079        (ctx, canonical_root, cache_dir)
6080    }
6081
6082    #[test]
6083    fn graceful_shutdown_flushes_every_actor_search_index() {
6084        let storage = tempfile::tempdir().expect("storage tempdir");
6085        let (root1_dir, root1) = test_root("shutdown-flush-root-1");
6086        let (root2_dir, root2) = test_root("shutdown-flush-root-2");
6087        let (ctx1, canonical_root1, cache_dir1) = actor_ctx_with_dirty_search_index(
6088            root1_dir.path(),
6089            storage.path(),
6090            "alpha.txt",
6091            "old actor one token\n",
6092            "new actor one token\n",
6093        );
6094        let (ctx2, canonical_root2, cache_dir2) = actor_ctx_with_dirty_search_index(
6095            root2_dir.path(),
6096            storage.path(),
6097            "beta.txt",
6098            "old actor two token\n",
6099            "new actor two token\n",
6100        );
6101
6102        let executor = Executor::new();
6103        assert!(executor.register_actor(root1.clone(), Arc::clone(&ctx1)));
6104        assert!(executor.register_actor(root2.clone(), Arc::clone(&ctx2)));
6105
6106        flush_actor_indexes_on_graceful_shutdown(&executor.actor_contexts());
6107
6108        let mut restored1 =
6109            crate::search_index::SearchIndex::read_from_disk(&cache_dir1, &canonical_root1)
6110                .expect("load flushed root one index");
6111        restored1.ready = true;
6112        assert_eq!(
6113            restored1
6114                .grep("new actor one token", true, &[], &[], &canonical_root1, 10)
6115                .matches
6116                .len(),
6117            1,
6118            "graceful subc shutdown should flush the first root's trigram delta"
6119        );
6120
6121        let mut restored2 =
6122            crate::search_index::SearchIndex::read_from_disk(&cache_dir2, &canonical_root2)
6123                .expect("load flushed root two index");
6124        restored2.ready = true;
6125        assert_eq!(
6126            restored2
6127                .grep("new actor two token", true, &[], &[], &canonical_root2, 10)
6128                .matches
6129                .len(),
6130            1,
6131            "graceful subc shutdown should flush every registered root"
6132        );
6133    }
6134
6135    #[test]
6136    fn idle_root_reaper_closes_artifacts_and_stops_watcher() {
6137        let _ = env_logger::builder().is_test(true).try_init();
6138        let (root_dir, root) = test_root("idle-root-reaper");
6139        let storage = tempfile::tempdir().expect("storage tempdir");
6140        std::fs::write(
6141            root_dir.path().join("main.rs"),
6142            "fn entry() { leaf(); }\nfn leaf() {}\n",
6143        )
6144        .expect("source file");
6145        let canonical_root = std::fs::canonicalize(root_dir.path()).expect("canonical root");
6146        let app = App::default_shared();
6147        let ctx = Arc::new(AppContext::from_app(
6148            Arc::clone(&app),
6149            Config {
6150                project_root: Some(canonical_root.clone()),
6151                storage_dir: Some(storage.path().to_path_buf()),
6152                callgraph_store: true,
6153                search_index: true,
6154                ..Config::default()
6155            },
6156        ));
6157        ctx.set_canonical_cache_root(canonical_root.clone());
6158        let project_key = crate::search_index::artifact_cache_key(&canonical_root);
6159        crate::root_cache::configure_artifact_access(&canonical_root, &project_key, false);
6160        assert!(ctx
6161            .ensure_callgraph_store()
6162            .expect("build callgraph store")
6163            .is_some());
6164
6165        let cache_dir =
6166            crate::search_index::resolve_cache_dir(&canonical_root, Some(storage.path()));
6167        let mut index = crate::search_index::SearchIndex::build(&canonical_root);
6168        let git_head = index.stored_git_head().map(str::to_owned);
6169        index.write_to_disk(&cache_dir, git_head.as_deref());
6170        *ctx.search_index()
6171            .write()
6172            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
6173        // Seed a completed warm verification so the test can prove eviction
6174        // downgrades it to Strict rather than passing vacuously.
6175        let seeded_generation =
6176            crate::cache_freshness::artifact_generation(&cache_dir.join("cache.bin"))
6177                .expect("seeded artifact generation");
6178        crate::cache_freshness::record_verify_completed(
6179            &canonical_root,
6180            crate::cache_freshness::VerifyArtifact::Search,
6181            Some(seeded_generation),
6182        );
6183        assert!(
6184            matches!(
6185                crate::cache_freshness::warm_verify_plan(
6186                    canonical_root.as_path(),
6187                    crate::cache_freshness::VerifyArtifact::Search,
6188                    Some(seeded_generation),
6189                ),
6190                crate::cache_freshness::WarmVerifyPlan::Skip
6191            ),
6192            "memo must be warm before eviction for the downgrade assertion to bite"
6193        );
6194
6195        let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
6196        let _dispatch_tx = dispatch_tx;
6197        let shutdown = Arc::new(AtomicBool::new(false));
6198        let thread_shutdown = Arc::clone(&shutdown);
6199        let join = std::thread::spawn(move || {
6200            while !thread_shutdown.load(Ordering::SeqCst) {
6201                std::thread::yield_now();
6202            }
6203        });
6204        ctx.install_watcher_runtime(
6205            dispatch_rx,
6206            crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
6207        );
6208        wait_for_watcher_count(&ctx, 1);
6209
6210        let executor = Arc::new(Executor::new());
6211        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6212        ctx.mark_subc_unbound();
6213        let mut live_roots = HashMap::new();
6214        let mut meta = RootMeta::new(Instant::now());
6215        meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
6216        meta.unbound_quiesced = true;
6217        live_roots.insert(root.clone(), meta);
6218
6219        let message = idle_root_eviction_message(&root, &ctx.memory_root_snapshot(), None);
6220        assert!(message.contains("evicted idle root"));
6221        assert!(message.contains("freed ~"));
6222        assert!(message.contains("semantic"));
6223        assert!(!message.contains("semantic not estimated retained"));
6224        assert!(message.contains("trigram"));
6225        assert!(message.contains("retained: bash"));
6226        assert!(message.contains("parser_pool"));
6227
6228        assert_eq!(
6229            reap_idle_roots(
6230                Instant::now(),
6231                &mut live_roots,
6232                &HashMap::new(),
6233                &HashMap::new(),
6234                &executor,
6235                &DispatchPathMetrics::new(),
6236            )
6237            .evicted,
6238            1
6239        );
6240        assert!(ctx.search_index().read().unwrap().is_none());
6241        wait_for_watcher_count(&ctx, 0);
6242        // The watcher stopped with the eviction, so the idle interval is
6243        // unobserved: the pre-seeded warm-verify memo (Skip) must fall back to
6244        // strict content verification (stat-first would miss same-size,
6245        // preserved-mtime edits made while nobody was watching).
6246        assert!(
6247            matches!(
6248                crate::cache_freshness::warm_verify_plan(
6249                    canonical_root.as_path(),
6250                    crate::cache_freshness::VerifyArtifact::Search,
6251                    Some(seeded_generation),
6252                ),
6253                crate::cache_freshness::WarmVerifyPlan::Strict
6254            ),
6255            "idle eviction must force strict re-verification"
6256        );
6257        assert!(
6258            crate::search_index::SearchIndex::read_from_disk(&cache_dir, &canonical_root).is_some()
6259        );
6260        ctx.mark_subc_bound();
6261        assert!(ctx
6262            .ensure_callgraph_store()
6263            .expect("reopen callgraph store")
6264            .is_some());
6265        assert!(live_roots[&root].idle_artifacts_evicted);
6266    }
6267
6268    #[test]
6269    fn idle_root_reaper_applies_ttl_to_unbound_roots() {
6270        let (_root_dir, root) = test_root("idle-root-ttl-gate");
6271        let ctx = test_ctx();
6272        let executor = Arc::new(Executor::new());
6273        assert!(executor.register_actor(root.clone(), ctx));
6274        let ctx = executor.actor_context(&root).expect("actor context");
6275        ctx.mark_subc_unbound();
6276        let now = Instant::now();
6277        let mut meta = RootMeta::new(now);
6278        meta.unbound_quiesced = true;
6279        let mut live_roots = HashMap::from([(root.clone(), meta)]);
6280
6281        // A recently-unbound root stays warm: a transient unbind (host
6282        // restart) must not pay the strict-verify + forced-rebuild teardown
6283        // on the next maintenance sweep.
6284        assert_eq!(
6285            reap_idle_roots(
6286                now,
6287                &mut live_roots,
6288                &HashMap::new(),
6289                &HashMap::new(),
6290                &executor,
6291                &DispatchPathMetrics::new(),
6292            )
6293            .evicted,
6294            0
6295        );
6296        assert!(!live_roots[&root].idle_artifacts_evicted);
6297
6298        // Past the TTL the same unbound root pays the full teardown. The
6299        // pending reconciliation paths retained across the transient-unbind
6300        // window would block eviction forever through
6301        // `artifact_eviction_blocked`; the reaper disposes them because the
6302        // strict gap invalidation subsumes their purpose.
6303        ctx.add_pending_search_index_paths([root.as_path().join("retained.rs")]);
6304        assert_eq!(
6305            reap_idle_roots(
6306                now + IDLE_ROOT_TTL,
6307                &mut live_roots,
6308                &HashMap::new(),
6309                &HashMap::new(),
6310                &executor,
6311                &DispatchPathMetrics::new(),
6312            )
6313            .evicted,
6314            1
6315        );
6316        assert!(live_roots[&root].idle_artifacts_evicted);
6317        assert!(
6318            ctx.take_pending_search_index_paths().is_empty(),
6319            "TTL eviction must dispose retained pending reconciliation paths"
6320        );
6321    }
6322
6323    #[test]
6324    fn blocked_ttl_eviction_restores_taken_pending_reconciliation_state() {
6325        let (_root_dir, root) = test_root("ttl-eviction-blocked-restore");
6326        let ctx = test_ctx();
6327        let executor = Arc::new(Executor::new());
6328        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6329        ctx.mark_subc_unbound();
6330
6331        // Retained pending path from the transient-unbind window plus a
6332        // SECONDARY eviction blocker (a non-ready resident search index, the
6333        // dirty-index blocker in artifact_eviction_blocked). Disposal must be
6334        // transactional: the blocked eviction may be followed by a rebind, and
6335        // the path is the only repair record for its consumed watcher event.
6336        let pending = root.as_path().join("edited-while-unbound.rs");
6337        ctx.add_pending_search_index_paths([pending.clone()]);
6338        let dirty_source = root.as_path().join("dirty.rs");
6339        std::fs::write(&dirty_source, "fn dirty() {}\n").expect("dirty source");
6340        let mut dirty = crate::search_index::SearchIndex::new();
6341        dirty.ready = true;
6342        dirty.update_file(&dirty_source);
6343        *ctx.search_index()
6344            .write()
6345            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(dirty);
6346        assert!(ctx.artifact_eviction_blocked());
6347
6348        let mut live_roots = HashMap::new();
6349        let mut meta = RootMeta::new(Instant::now());
6350        meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
6351        meta.unbound_quiesced = true;
6352        live_roots.insert(root.clone(), meta);
6353
6354        assert_eq!(
6355            reap_idle_roots(
6356                Instant::now(),
6357                &mut live_roots,
6358                &HashMap::new(),
6359                &HashMap::new(),
6360                &executor,
6361                &DispatchPathMetrics::new(),
6362            )
6363            .evicted,
6364            0,
6365            "the dirty index must still block this eviction"
6366        );
6367        assert_eq!(
6368            ctx.take_pending_search_index_paths(),
6369            vec![pending],
6370            "a blocked eviction must restore the taken pending paths"
6371        );
6372    }
6373
6374    #[test]
6375    fn idle_reap_with_bound_route_keeps_watcher_running() {
6376        let (_root_dir, root) = test_root("bound-root-reap-gate");
6377        let ctx = test_ctx();
6378        let executor = Arc::new(Executor::new());
6379        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6380
6381        let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
6382        let _dispatch_tx = dispatch_tx;
6383        let shutdown = Arc::new(AtomicBool::new(false));
6384        let thread_shutdown = Arc::clone(&shutdown);
6385        let join = std::thread::spawn(move || {
6386            while !thread_shutdown.load(Ordering::SeqCst) {
6387                std::thread::yield_now();
6388            }
6389        });
6390        ctx.install_watcher_runtime(
6391            dispatch_rx,
6392            crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
6393        );
6394
6395        let mut meta = RootMeta::new(Instant::now());
6396        meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
6397        let mut live_roots = HashMap::from([(root.clone(), meta)]);
6398        let bound = HashMap::from([(root, HashSet::from([route_key(7, 1)]))]);
6399        assert_eq!(
6400            reap_idle_roots(
6401                Instant::now(),
6402                &mut live_roots,
6403                &HashMap::new(),
6404                &bound,
6405                &executor,
6406                &DispatchPathMetrics::new(),
6407            )
6408            .evicted,
6409            0
6410        );
6411        wait_for_watcher_count(&ctx, 1);
6412        ctx.stop_watcher_runtime_in_background();
6413        wait_for_watcher_count(&ctx, 0);
6414    }
6415
6416    #[test]
6417    fn deleted_root_with_bound_route_is_reclaimed_after_confirmation_and_routes_are_purged() {
6418        let (root_dir, root) = test_root("deleted-bound-root-reap");
6419        let executor = Arc::new(Executor::new());
6420        assert!(executor.register_actor(root.clone(), test_ctx()));
6421        root_dir.close().expect("delete project root");
6422
6423        let route = route_key(19, 3);
6424        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
6425        let cancel_signal = PersistentCancelSignal::new();
6426        let mut routes = HashMap::from([(route, route_identity(&root, "deleted-route"))]);
6427        let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
6428        let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
6429        let mut route_bash_cancels = HashMap::from([(
6430            route,
6431            bash::RouteBashCancel {
6432                token: cancel_signal.clone(),
6433                active_waits: 0,
6434            },
6435        )]);
6436        let metrics = DispatchPathMetrics::new();
6437
6438        let first = reap_idle_roots(
6439            Instant::now(),
6440            &mut live_roots,
6441            &HashMap::new(),
6442            &root_channels,
6443            &executor,
6444            &metrics,
6445        );
6446        assert!(first.forgotten_deleted_roots.is_empty());
6447        assert!(executor.actor_registered(&root));
6448
6449        let mut forgotten = Vec::new();
6450        for _ in 0..100 {
6451            let outcome = reap_idle_roots(
6452                Instant::now(),
6453                &mut live_roots,
6454                &HashMap::new(),
6455                &root_channels,
6456                &executor,
6457                &metrics,
6458            );
6459            if !outcome.forgotten_deleted_roots.is_empty() {
6460                forgotten = outcome.forgotten_deleted_roots;
6461                break;
6462            }
6463            std::thread::sleep(Duration::from_millis(10));
6464        }
6465        assert_eq!(forgotten, vec![root.clone()]);
6466        assert!(!executor.actor_registered(&root));
6467
6468        let mut retry_buffer = HashMap::new();
6469        let mut reclaimed_routes = ReclaimedRoutes::default();
6470        let mut session_identity = HashMap::new();
6471        let mut push_buffer = HashMap::new();
6472        let mut bg_subs = HashMap::from([(
6473            route,
6474            BgSub {
6475                corr: 77,
6476                ver: PROTOCOL_VERSION,
6477                flags: control_flags(),
6478                root: root.clone(),
6479                session: "deleted-route".to_string(),
6480            },
6481        )]);
6482        let mut bg_sub_by_session = HashMap::from([(
6483            (root.clone(), "deleted-route".to_string()),
6484            HashSet::from([route]),
6485        )]);
6486        let mut bg_wake_pending = HashSet::from([route]);
6487        let mut bg_wake_epoch = HashMap::new();
6488        let mut pending_bash_asks = HashMap::new();
6489        let active_tool_calls: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
6490        health::take_bg_observability_logs_for_test();
6491        purge_deleted_root_residents(
6492            &root,
6493            &mut routes,
6494            &mut root_channels,
6495            &mut installed_route_epochs,
6496            &mut route_bash_cancels,
6497            &active_tool_calls,
6498            executor.as_ref(),
6499            &mut retry_buffer,
6500            &mut reclaimed_routes,
6501            &mut session_identity,
6502            &mut push_buffer,
6503            &mut bg_subs,
6504            &mut bg_sub_by_session,
6505            &mut bg_wake_pending,
6506            &mut bg_wake_epoch,
6507            &mut pending_bash_asks,
6508            &metrics,
6509        );
6510
6511        assert!(routes.is_empty());
6512        assert!(root_channels.is_empty());
6513        assert!(installed_route_epochs.is_empty());
6514        assert!(route_bash_cancels.is_empty());
6515        assert!(reclaimed_routes.contains(route));
6516        assert!(cancel_signal.is_cancelled());
6517        assert_eq!(
6518            health::take_bg_observability_logs_for_test(),
6519            vec![format!(
6520                "subc bg subscription: ended root={} session=deleted-route channel=19@3 cause=root-reclaim suppressed=0",
6521                root.as_path().display()
6522            )]
6523        );
6524    }
6525
6526    /// The control for the deleted-root reclamation above: a root whose
6527    /// directory still EXISTS must stay retained while it holds a bound route,
6528    /// even with every other reap precondition satisfied. Without this, the
6529    /// suite cannot tell "reclaim roots that are provably gone" apart from
6530    /// "reap any root that looks idle" — the second would tear down live
6531    /// sessions, and both satisfy the deleted-root tests.
6532    #[test]
6533    fn live_root_with_bound_route_is_never_reclaimed() {
6534        let (_root_dir, root) = test_root("live-bound-root-retained");
6535        let ctx = test_ctx();
6536        ctx.mark_subc_unbound();
6537        let executor = Arc::new(Executor::new());
6538        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6539
6540        let route = RouteChannel {
6541            channel: 7,
6542            epoch: 1,
6543        };
6544        let mut meta = RootMeta::new(Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1));
6545        meta.unbound_quiesced = true;
6546        let mut live_roots = HashMap::from([(root.clone(), meta)]);
6547        let root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
6548
6549        // Sweep three times because reclamation requires the root path to be
6550        // observed missing on two CONSECUTIVE sweeps. A single sweep would pass
6551        // here even if reclamation were wrongly unconditional, since the first
6552        // absence never reclaims on its own.
6553        for _ in 0..3 {
6554            let outcome = reap_idle_roots(
6555                Instant::now(),
6556                &mut live_roots,
6557                &HashMap::new(),
6558                &root_channels,
6559                &executor,
6560                &DispatchPathMetrics::new(),
6561            );
6562            assert!(
6563                outcome.forgotten_deleted_roots.is_empty(),
6564                "a root whose directory exists must never be forgotten"
6565            );
6566        }
6567
6568        assert!(live_roots.contains_key(&root), "live root must be retained");
6569        assert!(
6570            executor.actor_registered(&root),
6571            "live root's actor must survive"
6572        );
6573        assert!(
6574            root.as_path().exists(),
6575            "test vehicle must keep the directory alive; otherwise this control proves nothing"
6576        );
6577    }
6578
6579    #[test]
6580    fn deleted_root_is_not_reclaimed_on_first_absence_observation() {
6581        let (root_dir, root) = test_root("deleted-root-first-observation");
6582        let ctx = test_ctx();
6583        ctx.mark_subc_unbound();
6584        let executor = Arc::new(Executor::new());
6585        assert!(executor.register_actor(root.clone(), ctx));
6586        root_dir.close().expect("delete project root");
6587
6588        let mut meta = RootMeta::new(Instant::now());
6589        meta.unbound_quiesced = true;
6590        let mut live_roots = HashMap::from([(root.clone(), meta)]);
6591        let outcome = reap_idle_roots(
6592            Instant::now(),
6593            &mut live_roots,
6594            &HashMap::new(),
6595            &HashMap::new(),
6596            &executor,
6597            &DispatchPathMetrics::new(),
6598        );
6599
6600        assert!(outcome.forgotten_deleted_roots.is_empty());
6601        assert!(live_roots.contains_key(&root));
6602        assert!(executor.actor_registered(&root));
6603    }
6604
6605    fn spawn_background_for_root(
6606        ctx: &AppContext,
6607        root: &ProjectRootId,
6608        storage: &tempfile::TempDir,
6609        session_id: &str,
6610    ) -> (String, u32) {
6611        // Windows refuses to delete a directory that is a running process's
6612        // cwd (ERROR_SHARING_VIOLATION), so the task must not live inside the
6613        // project root these tests delete. The kill path matches on the task's
6614        // registered project_root, not its cwd, so pointing the cwd at task
6615        // storage keeps the association under test intact.
6616        let command = if cfg!(windows) {
6617            // timeout.exe requires a console; ping is the standard sleep shim.
6618            "ping -n 31 127.0.0.1 > nul"
6619        } else {
6620            "sleep 30"
6621        };
6622        let task_id = ctx
6623            .bash_background()
6624            .spawn(
6625                crate::sandbox_spawn::SpawnPlan::Unsandboxed,
6626                command,
6627                session_id.to_string(),
6628                storage.path().to_path_buf(),
6629                HashMap::new(),
6630                Some(Duration::from_secs(60)),
6631                storage.path().to_path_buf(),
6632                8,
6633                true,
6634                false,
6635                Some(root.as_path().to_path_buf()),
6636            )
6637            .expect("spawn background task");
6638        let snapshot = ctx
6639            .bash_background()
6640            .status(
6641                &task_id,
6642                session_id,
6643                Some(root.as_path()),
6644                Some(storage.path()),
6645                0,
6646            )
6647            .expect("background task status");
6648        (task_id, snapshot.child_pid.expect("background child pid"))
6649    }
6650
6651    fn wait_for_background_exit(pid: u32) {
6652        let deadline = Instant::now() + Duration::from_secs(5);
6653        while crate::bash_background::process::is_process_alive(pid) {
6654            assert!(
6655                Instant::now() < deadline,
6656                "background task process survived kill"
6657            );
6658            std::thread::sleep(Duration::from_millis(20));
6659        }
6660    }
6661
6662    #[test]
6663    fn deleted_root_reclaims_background_task_after_two_absence_sweeps() {
6664        let (root_dir, root) = test_root("deleted-root-background-task");
6665        let storage = tempfile::tempdir().expect("task storage");
6666        let ctx = test_ctx();
6667        let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "reclaim-session");
6668        assert!(crate::bash_background::process::is_process_alive(pid));
6669
6670        let executor = Arc::new(Executor::new());
6671        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6672        assert!(executor.actor_is_idle(&root));
6673        root_dir.close().expect("delete project root");
6674        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
6675        let pending_binds = HashMap::new();
6676        let root_channels = HashMap::new();
6677        let metrics = DispatchPathMetrics::new();
6678
6679        let first = reap_idle_roots(
6680            Instant::now(),
6681            &mut live_roots,
6682            &pending_binds,
6683            &root_channels,
6684            &executor,
6685            &metrics,
6686        );
6687        assert!(first.forgotten_deleted_roots.is_empty());
6688        assert!(crate::bash_background::process::is_process_alive(pid));
6689
6690        let outcome = reap_until_forgotten(
6691            &root,
6692            &mut live_roots,
6693            &pending_binds,
6694            &root_channels,
6695            &executor,
6696            &metrics,
6697        );
6698        assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
6699        wait_for_background_exit(pid);
6700
6701        let snapshot = ctx
6702            .bash_background()
6703            .status(
6704                &task_id,
6705                "reclaim-session",
6706                Some(root.as_path()),
6707                Some(storage.path()),
6708                0,
6709            )
6710            .expect("reclaimed task status");
6711        assert_eq!(snapshot.info.status, BgTaskStatus::Killed);
6712        assert_eq!(
6713            snapshot.info.status_reason.as_deref(),
6714            Some(crate::bash_background::registry::ROOT_RECLAIMED_REASON)
6715        );
6716        assert_eq!(
6717            serde_json::to_value(&snapshot).expect("serialize bash status")["status_reason"],
6718            crate::bash_background::registry::ROOT_RECLAIMED_REASON
6719        );
6720        let completion = ctx
6721            .bash_background()
6722            .drain_completions_for_session(Some("reclaim-session"))
6723            .pop()
6724            .expect("reclaimed task completion");
6725        assert_eq!(
6726            completion.status_reason.as_deref(),
6727            Some(crate::bash_background::registry::ROOT_RECLAIMED_REASON)
6728        );
6729    }
6730
6731    #[test]
6732    fn existing_unbound_root_keeps_background_task_alive_across_sweeps() {
6733        let (root_dir, root) = test_root("existing-root-background-task");
6734        let storage = tempfile::tempdir().expect("task storage");
6735        let ctx = test_ctx();
6736        ctx.mark_subc_unbound();
6737        let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "existing-session");
6738
6739        let executor = Arc::new(Executor::new());
6740        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6741        let mut meta = RootMeta::new(
6742            Instant::now()
6743                .checked_sub(IDLE_ROOT_TTL + Duration::from_secs(1))
6744                .expect("old root timestamp"),
6745        );
6746        meta.unbound_quiesced = true;
6747        let mut live_roots = HashMap::from([(root.clone(), meta)]);
6748        let pending_binds = HashMap::new();
6749        let root_channels = HashMap::new();
6750        let metrics = DispatchPathMetrics::new();
6751
6752        for _ in 0..8 {
6753            reap_idle_roots(
6754                Instant::now(),
6755                &mut live_roots,
6756                &pending_binds,
6757                &root_channels,
6758                &executor,
6759                &metrics,
6760            );
6761            std::thread::sleep(Duration::from_millis(10));
6762        }
6763        assert!(root_dir.path().exists());
6764        assert!(crate::bash_background::process::is_process_alive(pid));
6765        let snapshot = ctx
6766            .bash_background()
6767            .status(
6768                &task_id,
6769                "existing-session",
6770                Some(root.as_path()),
6771                Some(storage.path()),
6772                0,
6773            )
6774            .expect("existing task status");
6775        assert_eq!(snapshot.info.status, BgTaskStatus::Running);
6776        let _ = ctx.bash_background().kill(&task_id, "existing-session");
6777        wait_for_background_exit(pid);
6778    }
6779
6780    #[test]
6781    fn restored_root_between_absence_sweeps_keeps_background_task_alive() {
6782        let (root_dir, root) = test_root("restored-root-background-task");
6783        let storage = tempfile::tempdir().expect("task storage");
6784        let ctx = test_ctx();
6785        let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "restored-session");
6786
6787        let executor = Arc::new(Executor::new());
6788        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6789        root_dir.close().expect("delete project root");
6790        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
6791        let pending_binds = HashMap::new();
6792        let root_channels = HashMap::new();
6793        let metrics = DispatchPathMetrics::new();
6794
6795        let first = reap_idle_roots(
6796            Instant::now(),
6797            &mut live_roots,
6798            &pending_binds,
6799            &root_channels,
6800            &executor,
6801            &metrics,
6802        );
6803        assert!(first.forgotten_deleted_roots.is_empty());
6804        std::fs::create_dir_all(root.as_path()).expect("restore project root");
6805        let second = reap_idle_roots(
6806            Instant::now(),
6807            &mut live_roots,
6808            &pending_binds,
6809            &root_channels,
6810            &executor,
6811            &metrics,
6812        );
6813        assert!(second.forgotten_deleted_roots.is_empty());
6814        assert!(crate::bash_background::process::is_process_alive(pid));
6815        let _ = ctx.bash_background().kill(&task_id, "restored-session");
6816        wait_for_background_exit(pid);
6817    }
6818
6819    #[test]
6820    fn observing_root_again_resets_deleted_sweep_confirmation() {
6821        let (root_dir, root) = test_root("deleted-root-observation-reset");
6822        let ctx = test_ctx();
6823        ctx.mark_subc_unbound();
6824        let executor = Arc::new(Executor::new());
6825        assert!(executor.register_actor(root.clone(), ctx));
6826        root_dir.close().expect("delete project root");
6827
6828        let mut meta = RootMeta::new(Instant::now());
6829        meta.unbound_quiesced = true;
6830        let mut live_roots = HashMap::from([(root.clone(), meta)]);
6831        let pending_binds = HashMap::new();
6832        let root_channels = HashMap::new();
6833        let metrics = DispatchPathMetrics::new();
6834
6835        let first = reap_idle_roots(
6836            Instant::now(),
6837            &mut live_roots,
6838            &pending_binds,
6839            &root_channels,
6840            &executor,
6841            &metrics,
6842        );
6843        assert!(first.forgotten_deleted_roots.is_empty());
6844
6845        std::fs::create_dir_all(root.as_path()).expect("restore project root");
6846        reap_idle_roots(
6847            Instant::now(),
6848            &mut live_roots,
6849            &pending_binds,
6850            &root_channels,
6851            &executor,
6852            &metrics,
6853        );
6854        std::fs::remove_dir_all(root.as_path()).expect("delete project root again");
6855
6856        let after_reset = reap_idle_roots(
6857            Instant::now(),
6858            &mut live_roots,
6859            &pending_binds,
6860            &root_channels,
6861            &executor,
6862            &metrics,
6863        );
6864        assert!(after_reset.forgotten_deleted_roots.is_empty());
6865        assert!(live_roots.contains_key(&root));
6866        assert!(executor.actor_registered(&root));
6867    }
6868
6869    #[test]
6870    fn deleted_idle_root_is_fully_forgotten_and_status_counts_drop() {
6871        let (root_dir, root) = test_root("deleted-root-reap");
6872        let app = App::default_shared();
6873        let ctx = Arc::new(AppContext::from_app(
6874            Arc::clone(&app),
6875            Config {
6876                project_root: Some(root.as_path().to_path_buf()),
6877                ..Config::default()
6878            },
6879        ));
6880        ctx.set_canonical_cache_root(root.as_path().to_path_buf());
6881        ctx.mark_subc_unbound();
6882        let executor = Arc::new(Executor::new());
6883        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6884        assert_eq!(app.actor_root_count(), 1);
6885        drop(ctx);
6886        root_dir.close().expect("delete project root");
6887
6888        let mut meta = RootMeta::new(Instant::now());
6889        meta.unbound_quiesced = true;
6890        let mut live_roots = HashMap::from([(root.clone(), meta)]);
6891        let outcome = reap_until_forgotten(
6892            &root,
6893            &mut live_roots,
6894            &HashMap::new(),
6895            &HashMap::new(),
6896            &executor,
6897            &DispatchPathMetrics::new(),
6898        );
6899        assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
6900        assert!(!executor.actor_registered(&root));
6901        assert!(!live_roots.contains_key(&root));
6902        wait_for_actor_root_count(&app, 0);
6903
6904        let status_ctx = AppContext::from_app(app, Config::default());
6905        let status = status_ctx.build_status_snapshot();
6906        assert_eq!(status["runtime"]["live_actor_roots"], 0);
6907        assert_eq!(status["runtime"]["open_routes"], 0);
6908    }
6909
6910    #[test]
6911    fn deleted_root_reap_blocker_census_is_exposed_in_health_metrics() {
6912        let (root_dir, root) = test_root("deleted-root-reap-census");
6913        let executor = Arc::new(Executor::new());
6914        assert!(executor.register_actor(root.clone(), test_ctx()));
6915        root_dir.close().expect("delete project root");
6916
6917        let mut live_roots = HashMap::from([(root, RootMeta::new(Instant::now()))]);
6918        let metrics = DispatchPathMetrics::new();
6919        let outcome = reap_idle_roots(
6920            Instant::now(),
6921            &mut live_roots,
6922            &HashMap::new(),
6923            &HashMap::new(),
6924            &executor,
6925            &metrics,
6926        );
6927        assert_eq!(outcome.evicted, 0);
6928
6929        let app = crate::context::App::default_shared();
6930        let health_rollup_cache = HealthRollupCache::new();
6931        health_rollup_cache.refresh(&executor, &app);
6932        let report = build_health_report(
6933            &health_rollup_cache,
6934            &executor,
6935            &HashMap::new(),
6936            &metrics,
6937            &app,
6938        );
6939        let reap = report
6940            .metrics
6941            .as_ref()
6942            .and_then(|metrics| metrics.get("reap"))
6943            .expect("reap health metrics");
6944        assert_eq!(reap["deleted_retained"].as_u64(), Some(1));
6945        assert_eq!(reap["blockers"]["absence_unconfirmed"].as_u64(), Some(1));
6946        assert_eq!(reap["blockers"]["unbound_quiesced"].as_u64(), Some(0));
6947        assert_eq!(reap["blockers"]["actor_busy"].as_u64(), Some(0));
6948    }
6949
6950    #[test]
6951    fn connection_exit_quiesces_queued_maintenance_and_deleted_root_is_purged() {
6952        let (root_dir, root) = test_root("connection-exit-deleted-root");
6953        let executor = Arc::new(Executor::new());
6954        assert!(executor.register_actor(root.clone(), test_ctx()));
6955
6956        let route = route_key(11, 1);
6957        let mut meta = RootMeta::new(Instant::now());
6958        meta.maintenance_pending = true;
6959        meta.maintenance_queued_kinds
6960            .push_back(MaintenanceDrainKind::CompletionDrains);
6961        let mut live_roots = HashMap::from([(root.clone(), meta)]);
6962        let mut pending_binds = HashMap::new();
6963        let mut routes = HashMap::from([(route, route_identity(&root, "abandoned"))]);
6964        let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
6965        let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
6966        let mut route_bash_cancels = HashMap::new();
6967        let active_tool_calls: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
6968
6969        quiesce_connection_roots(
6970            &mut live_roots,
6971            &mut pending_binds,
6972            &mut routes,
6973            &mut root_channels,
6974            &mut installed_route_epochs,
6975            &mut route_bash_cancels,
6976            &active_tool_calls,
6977            &executor,
6978        );
6979        assert!(live_roots[&root].unbound_quiesced);
6980        assert!(!live_roots[&root].maintenance_pending);
6981        assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
6982        assert!(routes.is_empty());
6983        assert!(root_channels.is_empty());
6984
6985        root_dir.close().expect("delete project root");
6986        let metrics = DispatchPathMetrics::new();
6987        let outcome = reap_until_forgotten(
6988            &root,
6989            &mut live_roots,
6990            &pending_binds,
6991            &root_channels,
6992            &executor,
6993            &metrics,
6994        );
6995        let mut session_identity = HashMap::new();
6996        let mut push_buffer = HashMap::new();
6997        let mut bg_subs = HashMap::new();
6998        let mut bg_sub_by_session = HashMap::new();
6999        let mut bg_wake_pending = HashSet::new();
7000        let mut bg_wake_epoch = HashMap::new();
7001        let mut pending_bash_asks = HashMap::new();
7002        let mut retry_buffer = HashMap::new();
7003        let mut reclaimed_routes = ReclaimedRoutes::default();
7004        for forgotten in &outcome.forgotten_deleted_roots {
7005            purge_deleted_root_residents(
7006                forgotten,
7007                &mut routes,
7008                &mut root_channels,
7009                &mut installed_route_epochs,
7010                &mut route_bash_cancels,
7011                &active_tool_calls,
7012                executor.as_ref(),
7013                &mut retry_buffer,
7014                &mut reclaimed_routes,
7015                &mut session_identity,
7016                &mut push_buffer,
7017                &mut bg_subs,
7018                &mut bg_sub_by_session,
7019                &mut bg_wake_pending,
7020                &mut bg_wake_epoch,
7021                &mut pending_bash_asks,
7022                &metrics,
7023            );
7024        }
7025
7026        assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
7027        assert!(!executor.actor_registered(&root));
7028        assert!(!live_roots.contains_key(&root));
7029    }
7030
7031    #[test]
7032    fn unbound_root_quiesces_maintenance_without_removing_actor() {
7033        let (_root_dir, root) = test_root("unbound-root-quiesce");
7034        let ctx = test_ctx();
7035        let executor = Arc::new(Executor::new());
7036        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7037        let mut meta = RootMeta::new(Instant::now());
7038        meta.maintenance_pending = true;
7039        meta.maintenance_jobs_in_flight = 1;
7040        meta.maintenance_queued_kinds
7041            .push_back(MaintenanceDrainKind::ConfigureTail);
7042        let mut live_roots = HashMap::from([(root.clone(), meta)]);
7043        // Warm state planted before the unbind: quiesce must keep it.
7044        *ctx.search_index()
7045            .write()
7046            .unwrap_or_else(std::sync::PoisonError::into_inner) =
7047            Some(crate::search_index::SearchIndex::new());
7048        ctx.set_cache_writer_capabilities(true, true);
7049        let pending = root.as_path().join("pending.rs");
7050        ctx.add_pending_search_index_paths([pending.clone()]);
7051        // A warm verify memo must survive the transient unbind: the watcher
7052        // keeps running, so no unobserved window exists and the next warm
7053        // reload must not pay a strict full-corpus re-hash.
7054        let canonical_root = root.as_path().to_path_buf();
7055        let artifact = canonical_root.join("cache.bin");
7056        std::fs::write(&artifact, b"warm-artifact").expect("write artifact");
7057        let seeded_generation = crate::cache_freshness::artifact_generation(&artifact);
7058        crate::cache_freshness::record_verify_completed(
7059            &canonical_root,
7060            crate::cache_freshness::VerifyArtifact::Search,
7061            seeded_generation,
7062        );
7063        assert!(matches!(
7064            crate::cache_freshness::warm_verify_plan(
7065                &canonical_root,
7066                crate::cache_freshness::VerifyArtifact::Search,
7067                seeded_generation,
7068            ),
7069            crate::cache_freshness::WarmVerifyPlan::Skip
7070        ));
7071        // A live watcher runtime must survive quiesce (its events accumulate
7072        // for the rebind replay).
7073        let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
7074        let _dispatch_tx = dispatch_tx;
7075        let shutdown = Arc::new(AtomicBool::new(false));
7076        let thread_shutdown = Arc::clone(&shutdown);
7077        let join = std::thread::spawn(move || {
7078            while !thread_shutdown.load(Ordering::SeqCst) {
7079                std::thread::yield_now();
7080            }
7081        });
7082        ctx.install_watcher_runtime(
7083            dispatch_rx,
7084            crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
7085        );
7086        assert!(ctx.watcher_runtime_active());
7087
7088        quiesce_unbound_root(&root, &mut live_roots, &executor);
7089        let meta = &live_roots[&root];
7090        assert!(meta.unbound_quiesced);
7091        assert!(ctx.subc_unbound_quiesced());
7092        assert!(meta.maintenance_pending);
7093        assert!(meta.maintenance_queued_kinds.is_empty());
7094        assert!(executor.actor_registered(&root));
7095        // Transient unbind keeps the root warm: resident artifacts stay
7096        // resident, no forced callgraph rebuild is planted, and pending
7097        // reconciliation paths survive for the rebind replay.
7098        assert!(
7099            ctx.search_index()
7100                .read()
7101                .unwrap_or_else(std::sync::PoisonError::into_inner)
7102                .is_some(),
7103            "quiesce must not evict resident artifacts"
7104        );
7105        assert_eq!(
7106            ctx.pending_callgraph_store_force_token(),
7107            None,
7108            "quiesce must not force a callgraph rebuild"
7109        );
7110        assert_eq!(
7111            ctx.take_pending_search_index_paths(),
7112            vec![pending],
7113            "quiesce must retain pending watcher-derived paths"
7114        );
7115        assert!(
7116            matches!(
7117                crate::cache_freshness::warm_verify_plan(
7118                    &canonical_root,
7119                    crate::cache_freshness::VerifyArtifact::Search,
7120                    seeded_generation,
7121                ),
7122                crate::cache_freshness::WarmVerifyPlan::Skip
7123            ),
7124            "quiesce must not invalidate the warm verify memo"
7125        );
7126        assert!(
7127            ctx.watcher_runtime_active(),
7128            "quiesce must not stop a running watcher"
7129        );
7130        ctx.stop_watcher_runtime();
7131
7132        let meta = live_roots.get_mut(&root).expect("root metadata");
7133        note_maintenance_completion(
7134            meta,
7135            Some(MaintenanceDrainKind::ConfigureTail),
7136            false,
7137            meta.unbound_quiesced,
7138        );
7139        assert!(!meta.maintenance_pending);
7140        assert!(meta.maintenance_queued_kinds.is_empty());
7141    }
7142
7143    #[test]
7144    fn same_root_higher_epoch_replacement_does_not_quiesce_between_generations() {
7145        let (_dir, root) = test_root("same-root-replacement");
7146        let route = route_key(7, 1);
7147        let installed_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
7148        let root_channels = HashMap::new();
7149
7150        assert!(!route_removal_will_quiesce_root(
7151            &root,
7152            route,
7153            &installed_channels,
7154            false,
7155            Some(&root),
7156        ));
7157        assert!(route_removal_will_quiesce_root(
7158            &root,
7159            route,
7160            &installed_channels,
7161            false,
7162            None,
7163        ));
7164        assert!(!should_quiesce_removed_root(
7165            &root,
7166            &root_channels,
7167            false,
7168            Some(&root),
7169        ));
7170        assert!(should_quiesce_removed_root(
7171            &root,
7172            &root_channels,
7173            false,
7174            None,
7175        ));
7176        assert!(!should_quiesce_removed_root(
7177            &root,
7178            &root_channels,
7179            true,
7180            None,
7181        ));
7182    }
7183
7184    #[test]
7185    fn root_quiesces_only_after_its_last_route_is_removed_and_reactivates_on_bind() {
7186        let (_root_dir, root) = test_root("unbound-root-route-count");
7187        let executor = Arc::new(Executor::new());
7188        assert!(executor.register_actor(root.clone(), test_ctx()));
7189        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
7190        let mut root_channels = HashMap::from([(
7191            root.clone(),
7192            HashSet::from([route_key(7, 1), route_key(8, 1)]),
7193        )]);
7194
7195        remove_root_channel(&mut root_channels, &root, route_key(7, 1));
7196        if !root_channels.contains_key(&root) {
7197            quiesce_unbound_root(&root, &mut live_roots, &executor);
7198        }
7199        assert!(!live_roots[&root].unbound_quiesced);
7200
7201        remove_root_channel(&mut root_channels, &root, route_key(8, 1));
7202        if !root_channels.contains_key(&root) {
7203            quiesce_unbound_root(&root, &mut live_roots, &executor);
7204        }
7205        assert!(live_roots[&root].unbound_quiesced);
7206
7207        live_roots
7208            .get_mut(&root)
7209            .expect("root metadata")
7210            .note_activity();
7211        assert!(
7212            live_roots[&root].unbound_quiesced,
7213            "late asynchronous activity must not reactivate an unbound root"
7214        );
7215
7216        live_roots
7217            .get_mut(&root)
7218            .expect("root metadata")
7219            .reactivate_bound();
7220        assert!(!live_roots[&root].unbound_quiesced);
7221    }
7222
7223    #[test]
7224    fn allocator_pressure_relief_requires_every_root_to_be_idle() {
7225        let (_idle_dir, idle_root) = test_root("allocator-relief-idle");
7226        let (_active_dir, active_root) = test_root("allocator-relief-active");
7227        let now = Instant::now();
7228        let mut live_roots = HashMap::new();
7229        let mut idle = RootMeta::new(now);
7230        idle.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
7231        live_roots.insert(idle_root, idle);
7232        assert!(process_has_been_idle(now, &live_roots));
7233
7234        live_roots.insert(active_root.clone(), RootMeta::new(now));
7235        assert!(!process_has_been_idle(now, &live_roots));
7236
7237        let active = live_roots
7238            .get_mut(&active_root)
7239            .expect("active root metadata");
7240        active.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
7241        active.active_bash_waits = 1;
7242        assert!(!process_has_been_idle(now, &live_roots));
7243    }
7244
7245    #[test]
7246    fn pressure_relief_log_reports_before_and_after_measurements() {
7247        let allocator = crate::memory::AllocatorMemorySnapshot {
7248            status: "measured",
7249            bytes_in_use: Some(8 * 1024 * 1024),
7250            size_allocated: Some(12 * 1024 * 1024),
7251            retained_slack_bytes: Some(4 * 1024 * 1024),
7252            not_estimated: None,
7253        };
7254        let relief = crate::memory::AllocatorPressureRelief {
7255            bytes_released: 3 * 1024 * 1024,
7256            rss_before_bytes: Some(20 * 1024 * 1024),
7257            rss_after_bytes: Some(17 * 1024 * 1024),
7258            allocator_before: allocator.clone(),
7259            allocator_after: crate::memory::AllocatorMemorySnapshot {
7260                size_allocated: Some(9 * 1024 * 1024),
7261                retained_slack_bytes: Some(1024 * 1024),
7262                ..allocator
7263            },
7264        };
7265        let message = pressure_relief_label(&relief);
7266        assert!(message.contains("RSS 20.0 MB -> 17.0 MB"));
7267        assert!(message.contains("allocated 12.0 MB -> 9.0 MB"));
7268        assert!(message.contains("slack 4.0 MB -> 1.0 MB"));
7269        assert!(message.contains("reported 3.0 MB released"));
7270    }
7271
7272    #[test]
7273    fn due_maintenance_jobs_skip_poisoned_roots() {
7274        let (_healthy_dir, healthy_root) = test_root("maintenance-healthy");
7275        let (_poisoned_dir, poisoned_root) = test_root("maintenance-poisoned");
7276        let mut live_roots = HashMap::new();
7277        live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
7278        let mut poisoned_meta = RootMeta::new(Instant::now());
7279        poisoned_meta.maintenance_poisoned = true;
7280        live_roots.insert(poisoned_root.clone(), poisoned_meta);
7281
7282        let (due, deferred) = due_maintenance_jobs_without_actor_context(
7283            &mut live_roots,
7284            MAINTENANCE_SUBMIT_BUDGET,
7285            &HashSet::new(),
7286        );
7287
7288        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
7289        assert!(due.iter().all(|(root, _)| root == &healthy_root));
7290        assert!(!deferred);
7291        assert!(live_roots[&healthy_root].maintenance_pending);
7292        assert_eq!(
7293            live_roots[&healthy_root].maintenance_jobs_in_flight,
7294            INITIAL_MAINTENANCE_JOB_COUNT
7295        );
7296        assert!(!live_roots[&poisoned_root].maintenance_pending);
7297    }
7298
7299    #[test]
7300    fn due_maintenance_jobs_do_not_restart_quiesced_root_work() {
7301        let (_dir, root) = test_root("maintenance-unbound");
7302        let mut meta = RootMeta::new(Instant::now());
7303        meta.unbound_quiesced = true;
7304        let mut live_roots = HashMap::from([(root.clone(), meta)]);
7305
7306        let (due, deferred) = due_maintenance_jobs_without_actor_context(
7307            &mut live_roots,
7308            MAINTENANCE_SUBMIT_BUDGET,
7309            &HashSet::new(),
7310        );
7311
7312        assert!(due.is_empty());
7313        assert!(!deferred);
7314        assert!(!live_roots[&root].maintenance_pending);
7315    }
7316
7317    #[test]
7318    fn idle_bg_subscription_queues_no_jobs_until_a_wake_arrives() {
7319        let (_dir, root) = test_root("maintenance-idle-bg-subscription");
7320        let ctx = test_ctx();
7321        assert!(!ctx.completion_drains_have_work());
7322
7323        let executor = Executor::new();
7324        assert!(executor.register_actor(root.clone(), ctx));
7325        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
7326        let session = "idle-session".to_string();
7327        let channel = route_key(17, 1);
7328        let metrics = DispatchPathMetrics::new();
7329        let bg_sub_by_session =
7330            HashMap::from([((root.clone(), session.clone()), HashSet::from([channel]))]);
7331        let mut bg_wake_pending = HashSet::new();
7332
7333        let (idle_tick_jobs, deferred) = due_maintenance_jobs(
7334            &mut live_roots,
7335            Some(&executor),
7336            &bg_sub_by_session,
7337            &bg_wake_pending,
7338            MAINTENANCE_SUBMIT_BUDGET,
7339            &HashSet::new(),
7340        );
7341        assert!(idle_tick_jobs.is_empty());
7342        assert!(!deferred);
7343        assert!(!live_roots[&root].maintenance_pending);
7344
7345        // A completion can arm its wake after the idle tick's probes. The wake
7346        // remains loop-owned state, so the following tick must observe it.
7347        let mut bg_wake_epoch = HashMap::new();
7348        push::arm_bg_wake(
7349            root.clone(),
7350            session,
7351            channel,
7352            &mut bg_wake_pending,
7353            &mut bg_wake_epoch,
7354            &metrics,
7355        );
7356        let (next_tick_jobs, deferred) = due_maintenance_jobs(
7357            &mut live_roots,
7358            Some(&executor),
7359            &bg_sub_by_session,
7360            &bg_wake_pending,
7361            MAINTENANCE_SUBMIT_BUDGET,
7362            &HashSet::new(),
7363        );
7364        assert_eq!(
7365            next_tick_jobs,
7366            vec![(root, MaintenanceDrainKind::CompletionDrains)]
7367        );
7368        assert!(!deferred);
7369    }
7370
7371    #[tokio::test]
7372    async fn subc_configure_tail_precedes_completed_search_install() {
7373        let root_dir = tempfile::tempdir().unwrap();
7374        let storage = tempfile::tempdir().unwrap();
7375        let root = ProjectRootId::from_path(root_dir.path()).unwrap();
7376        let (ctx, ignored_path) =
7377            runtime_drain::configure_search_order_context_for_test(root_dir.path(), storage.path());
7378        let ctx = Arc::new(ctx);
7379        assert!(!runtime_drain::watcher_path_is_ignored_by_current_matcher(
7380            &ctx,
7381            &ignored_path
7382        ));
7383
7384        let executor = Arc::new(Executor::new());
7385        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7386        let metrics = Arc::new(DispatchPathMetrics::new());
7387        let (completion_tx, mut completion_rx) = mpsc::channel(4);
7388        submit_maintenance_job(
7389            &executor,
7390            root.clone(),
7391            MaintenanceDrainKind::ConfigureTail,
7392            Vec::new(),
7393            &completion_tx,
7394            &metrics,
7395        );
7396        submit_maintenance_job(
7397            &executor,
7398            root,
7399            MaintenanceDrainKind::CompletionDrains,
7400            Vec::new(),
7401            &completion_tx,
7402            &metrics,
7403        );
7404
7405        let first = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
7406            .await
7407            .expect("configure-tail completion timed out")
7408            .expect("configure-tail completion channel closed");
7409        let second = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
7410            .await
7411            .expect("completion-drains completion timed out")
7412            .expect("completion-drains completion channel closed");
7413        assert!(first.response.id.contains("configure-tail"));
7414        assert!(second.response.id.contains("completion-drains"));
7415        assert!(runtime_drain::watcher_path_is_ignored_by_current_matcher(
7416            &ctx,
7417            &ignored_path
7418        ));
7419        assert_eq!(
7420            ctx.search_index()
7421                .read()
7422                .unwrap_or_else(std::sync::PoisonError::into_inner)
7423                .as_ref()
7424                .expect("completed search index installed")
7425                .file_count(),
7426            0,
7427            "configure must install the ignore matcher before pending paths replay"
7428        );
7429        ctx.stop_watcher_runtime();
7430    }
7431
7432    #[test]
7433    fn post_bind_configure_and_completion_jobs_are_queued_in_order() {
7434        let (_dir, root) = test_root("maintenance-post-bind");
7435        let mut live_roots = HashMap::new();
7436        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
7437
7438        queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
7439        queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
7440
7441        let meta = live_roots.get(&root).expect("root metadata");
7442        assert!(meta.maintenance_pending);
7443        assert_eq!(meta.maintenance_jobs_in_flight, 0);
7444        assert_eq!(
7445            meta.maintenance_queued_kinds
7446                .iter()
7447                .copied()
7448                .collect::<Vec<_>>(),
7449            vec![
7450                MaintenanceDrainKind::ConfigureTail,
7451                MaintenanceDrainKind::CompletionDrains,
7452            ]
7453        );
7454
7455        let (due, deferred) = due_maintenance_jobs_without_actor_context(
7456            &mut live_roots,
7457            MAINTENANCE_SUBMIT_BUDGET,
7458            &HashSet::new(),
7459        );
7460
7461        assert_eq!(
7462            due,
7463            vec![
7464                (root.clone(), MaintenanceDrainKind::ConfigureTail),
7465                (root.clone(), MaintenanceDrainKind::CompletionDrains),
7466            ]
7467        );
7468        assert!(!deferred);
7469        assert_eq!(live_roots[&root].maintenance_jobs_in_flight, 2);
7470        assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
7471    }
7472
7473    #[test]
7474    fn due_maintenance_jobs_defers_unsubmitted_roots_without_marking_pending() {
7475        let mut live_roots = HashMap::new();
7476        let mut root_ids = Vec::new();
7477        let mut _dirs = Vec::new();
7478        for index in 0..4 {
7479            let (dir, root_id) = test_root(&format!("maintenance-budget-{index}"));
7480            live_roots.insert(root_id.clone(), RootMeta::new(Instant::now()));
7481            root_ids.push(root_id);
7482            _dirs.push(dir);
7483        }
7484
7485        let small_budget = INITIAL_MAINTENANCE_JOB_COUNT + 1;
7486        let (first_due, first_deferred) = due_maintenance_jobs_without_actor_context(
7487            &mut live_roots,
7488            small_budget,
7489            &HashSet::new(),
7490        );
7491
7492        assert_eq!(first_due.len(), small_budget);
7493        assert!(first_deferred);
7494        let first_due_set: HashSet<_> = first_due.into_iter().map(|(root, _)| root).collect();
7495        assert!(first_due_set
7496            .iter()
7497            .all(|root| live_roots[root].maintenance_pending));
7498        assert!(first_due_set
7499            .iter()
7500            .any(|root| !live_roots[root].maintenance_queued_kinds.is_empty()));
7501
7502        let all_roots: HashSet<_> = root_ids.into_iter().collect();
7503        let deferred_roots: HashSet<_> = all_roots.difference(&first_due_set).cloned().collect();
7504        assert!(deferred_roots
7505            .iter()
7506            .all(|root| !live_roots[root].maintenance_pending));
7507    }
7508
7509    #[test]
7510    fn due_maintenance_jobs_defers_pending_bind_roots() {
7511        let (_bind_dir, bind_root) = test_root("maintenance-pending-bind");
7512        let (_healthy_dir, healthy_root) = test_root("maintenance-no-bind");
7513        let mut live_roots = HashMap::new();
7514        live_roots.insert(bind_root.clone(), RootMeta::new(Instant::now()));
7515        live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
7516        let pending_bind_roots = HashSet::from([bind_root.clone()]);
7517
7518        let (due, deferred) = due_maintenance_jobs_without_actor_context(
7519            &mut live_roots,
7520            usize::MAX,
7521            &pending_bind_roots,
7522        );
7523
7524        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
7525        assert!(due.iter().all(|(root, _)| root == &healthy_root));
7526        assert!(!deferred);
7527        assert!(!live_roots[&bind_root].maintenance_pending);
7528        assert!(live_roots[&bind_root].maintenance_queued_kinds.is_empty());
7529    }
7530
7531    #[test]
7532    fn maintenance_pending_survives_requeue_and_clears_after_final_batch() {
7533        let (_dir, root) = test_root("maintenance-requeue");
7534        let mut live_roots = HashMap::new();
7535        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
7536        let (due, deferred) = due_maintenance_jobs_without_actor_context(
7537            &mut live_roots,
7538            usize::MAX,
7539            &HashSet::new(),
7540        );
7541        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
7542        assert!(due.iter().all(|(due_root, _)| due_root == &root));
7543        assert!(!deferred);
7544
7545        let meta = live_roots.get_mut(&root).unwrap();
7546        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, false);
7547        assert!(meta.maintenance_pending);
7548        assert_eq!(
7549            meta.maintenance_jobs_in_flight,
7550            INITIAL_MAINTENANCE_JOB_COUNT - 1
7551        );
7552        assert_eq!(meta.maintenance_queued_kinds.len(), 1);
7553
7554        let (requeued, deferred) =
7555            due_maintenance_jobs_without_actor_context(&mut live_roots, 1, &HashSet::new());
7556        assert_eq!(
7557            requeued,
7558            vec![(root.clone(), MaintenanceDrainKind::Watcher)]
7559        );
7560        assert!(!deferred);
7561        let meta = live_roots.get_mut(&root).unwrap();
7562        assert_eq!(
7563            meta.maintenance_jobs_in_flight,
7564            INITIAL_MAINTENANCE_JOB_COUNT
7565        );
7566        assert!(meta.maintenance_queued_kinds.is_empty());
7567
7568        for _ in 0..INITIAL_MAINTENANCE_JOB_COUNT {
7569            note_maintenance_completion(meta, None, false, false);
7570        }
7571        assert!(!meta.maintenance_pending);
7572        assert_eq!(meta.maintenance_jobs_in_flight, 0);
7573    }
7574
7575    #[test]
7576    fn maintenance_requeue_drops_while_bind_is_pending() {
7577        let (_dir, root) = test_root("maintenance-bind-requeue");
7578        let mut live_roots = HashMap::new();
7579        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
7580        let (due, _) = due_maintenance_jobs_without_actor_context(
7581            &mut live_roots,
7582            usize::MAX,
7583            &HashSet::new(),
7584        );
7585        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
7586
7587        let meta = live_roots.get_mut(&root).unwrap();
7588        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, true);
7589
7590        assert_eq!(
7591            meta.maintenance_jobs_in_flight,
7592            INITIAL_MAINTENANCE_JOB_COUNT - 1
7593        );
7594        assert!(meta.maintenance_queued_kinds.is_empty());
7595        assert!(meta.maintenance_pending);
7596    }
7597
7598    #[test]
7599    fn parked_lsp_completion_never_requiesces_or_cancels_a_pending_bind() {
7600        let mut meta = RootMeta::new(Instant::now());
7601        meta.unbound_quiesced = true;
7602
7603        assert!(!should_requiesce_after_maintenance(
7604            &meta,
7605            MaintenanceDrainKind::Lsp,
7606            false,
7607        ));
7608        assert!(!should_requiesce_after_maintenance(
7609            &meta,
7610            MaintenanceDrainKind::ConfigureTail,
7611            true,
7612        ));
7613        assert!(should_requiesce_after_maintenance(
7614            &meta,
7615            MaintenanceDrainKind::ConfigureTail,
7616            false,
7617        ));
7618    }
7619
7620    #[test]
7621    fn maintenance_pending_clears_and_poison_stops_requeue_after_fatal() {
7622        let (_dir, root) = test_root("maintenance-fatal");
7623        let mut live_roots = HashMap::new();
7624        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
7625        let (due, _) = due_maintenance_jobs_without_actor_context(
7626            &mut live_roots,
7627            usize::MAX,
7628            &HashSet::new(),
7629        );
7630        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
7631
7632        let meta = live_roots.get_mut(&root).unwrap();
7633        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), true, false);
7634        assert!(meta.maintenance_poisoned);
7635        assert!(meta.maintenance_queued_kinds.is_empty());
7636
7637        for _ in 1..INITIAL_MAINTENANCE_JOB_COUNT {
7638            note_maintenance_completion(meta, None, false, false);
7639        }
7640        assert!(!meta.maintenance_pending);
7641        assert_eq!(meta.maintenance_jobs_in_flight, 0);
7642    }
7643
7644    #[test]
7645    fn trust_for_principal_matrix() {
7646        assert_eq!(
7647            trust_for_principal(&Some(Principal::Direct)),
7648            BindTrust::FirstParty
7649        );
7650        // Every first-party reserved id is asserted BY NAME, in one loop over
7651        // the full set, so two failure classes stay distinguishable: an empty
7652        // or broken allowlist reddens every name at once, while a dropped
7653        // single entry (the rename hazard) reddens exactly the missing name.
7654        // Both halves of each transitional rename pair stay listed until the
7655        // flip settles (see the allowlist comment).
7656        for module_id in [
7657            "llm-runner",
7658            "aft",
7659            "broca",
7660            "alfonso-core",
7661            "prefrontal",
7662            "prefrontal-core",
7663        ] {
7664            assert_eq!(
7665                trust_for_principal(&Some(Principal::Reserved {
7666                    module_id: module_id.to_string(),
7667                })),
7668                BindTrust::FirstParty,
7669                "reserved module id '{module_id}' must resolve to first-party trust"
7670            );
7671        }
7672        assert_eq!(
7673            trust_for_principal(&Some(Principal::Reserved {
7674                module_id: "subc-mcp".to_string(),
7675            })),
7676            BindTrust::Untrusted
7677        );
7678        assert_eq!(
7679            trust_for_principal(&Some(Principal::Reserved {
7680                module_id: "anything-unknown".to_string(),
7681            })),
7682            BindTrust::Untrusted
7683        );
7684        assert_eq!(
7685            trust_for_principal(&Some(Principal::Unverified)),
7686            BindTrust::Untrusted
7687        );
7688        assert_eq!(trust_for_principal(&None), BindTrust::Untrusted);
7689    }
7690
7691    #[test]
7692    fn fed_harness_class_maps_to_untrusted_regardless_of_fingerprint_value() {
7693        let principal = Some(Principal::Direct);
7694        let fingerprint_a = "fed:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
7695        let fingerprint_b = "fed:0123456789abcdef111111111111111111111111111111111111111111111111";
7696
7697        assert_eq!(
7698            trust_for_bind(fingerprint_a, &principal),
7699            BindTrust::Untrusted
7700        );
7701        assert_eq!(
7702            trust_for_bind(fingerprint_b, &principal),
7703            BindTrust::Untrusted
7704        );
7705    }
7706
7707    /// The table above proves `trust_for_principal` maps correctly, and the
7708    /// test above proves the `fed:` harness override wins — but neither
7709    /// exercises the ordinary path, so an implementation that ignored the
7710    /// principal entirely for non-fed harnesses would satisfy both. Pin the
7711    /// delegation itself: on a normal harness the verdict must still come from
7712    /// the principal, in both directions.
7713    #[test]
7714    fn trust_for_bind_delegates_to_the_principal_on_ordinary_harnesses() {
7715        for harness in ["opencode", "pi", "runner", "mcp:claude"] {
7716            assert_eq!(
7717                trust_for_bind(harness, &Some(Principal::Direct)),
7718                BindTrust::FirstParty,
7719                "a direct principal must stay first-party on {harness}"
7720            );
7721            assert_eq!(
7722                trust_for_bind(harness, &Some(Principal::Unverified)),
7723                BindTrust::Untrusted,
7724                "an unverified principal must stay untrusted on {harness}"
7725            );
7726            assert_eq!(
7727                trust_for_bind(harness, &None),
7728                BindTrust::Untrusted,
7729                "an absent principal must fail closed on {harness}"
7730            );
7731            assert_eq!(
7732                trust_for_bind(
7733                    harness,
7734                    &Some(Principal::Reserved {
7735                        module_id: "subc-mcp".to_string(),
7736                    })
7737                ),
7738                BindTrust::Untrusted,
7739                "a non-allowlisted reserved module must stay untrusted on {harness}"
7740            );
7741        }
7742    }
7743
7744    #[tokio::test]
7745    async fn persistent_cancel_resolves_when_fired_before_await() {
7746        // The lost-wakeup guard: cancel() fires exactly once via notify_waiters()
7747        // (no stored permit). A waiter that registers AFTER the cancel must still
7748        // observe it via the flag; a waiter racing the cancel must still be woken.
7749        let signal = PersistentCancelSignal::new();
7750        signal.cancel();
7751        // Fired before we ever call cancelled() — must return immediately, not park.
7752        tokio::time::timeout(Duration::from_secs(1), signal.cancelled())
7753            .await
7754            .expect("cancelled() must resolve when cancel fired beforehand");
7755
7756        // A fresh signal cancelled concurrently with an in-flight cancelled().
7757        let racing = PersistentCancelSignal::new();
7758        let racing_for_task = racing.clone();
7759        let waiter = tokio::spawn(async move { racing_for_task.cancelled().await });
7760        racing.cancel();
7761        tokio::time::timeout(Duration::from_secs(1), waiter)
7762            .await
7763            .expect("cancelled() must resolve when cancel races the await")
7764            .expect("waiter task panicked");
7765    }
7766
7767    #[test]
7768    fn ingress_epoch_validation_rejects_reclaimed_requests_and_drops_other_stale_epochs() {
7769        let installed = HashMap::from([(7, 9)]);
7770        let mut reclaimed = ReclaimedRoutes::default();
7771        reclaimed.insert(route_key(8, 1));
7772        for ty in [
7773            FrameType::Request,
7774            FrameType::Response,
7775            FrameType::Error,
7776            FrameType::Push,
7777            FrameType::Cancel,
7778            FrameType::Goodbye,
7779        ] {
7780            let body = if ty.is_pure_header() {
7781                Vec::new()
7782            } else {
7783                br#"{}"#.to_vec()
7784            };
7785            let stale = Frame::build(ty, control_flags(), 7, 8, 41, body).unwrap();
7786            assert!(
7787                !ingress_route_should_be_processed(&installed, &reclaimed, &stale),
7788                "{ty:?}"
7789            );
7790        }
7791
7792        let reclaimed_request = Frame::build(
7793            FrameType::Request,
7794            control_flags(),
7795            8,
7796            1,
7797            42,
7798            br#"{}"#.to_vec(),
7799        )
7800        .unwrap();
7801        assert!(ingress_route_should_be_processed(
7802            &installed,
7803            &reclaimed,
7804            &reclaimed_request
7805        ));
7806
7807        let never_installed = Frame::build(
7808            FrameType::Request,
7809            control_flags(),
7810            9,
7811            1,
7812            43,
7813            br#"{}"#.to_vec(),
7814        )
7815        .unwrap();
7816        assert!(!ingress_route_should_be_processed(
7817            &installed,
7818            &reclaimed,
7819            &never_installed
7820        ));
7821
7822        let current = Frame::build(
7823            FrameType::Request,
7824            control_flags(),
7825            7,
7826            9,
7827            43,
7828            br#"{}"#.to_vec(),
7829        )
7830        .unwrap();
7831        let control = Frame::build(FrameType::Ping, control_flags(), 0, 0, 44, Vec::new()).unwrap();
7832        assert!(ingress_route_should_be_processed(
7833            &installed, &reclaimed, &current
7834        ));
7835        assert!(ingress_route_should_be_processed(
7836            &installed, &reclaimed, &control
7837        ));
7838        assert_eq!(installed, HashMap::from([(7, 9)]));
7839    }
7840
7841    #[tokio::test]
7842    async fn route_bind_ack_precedes_route_egress_in_writer_queue() {
7843        let (_dir, root) = test_root("route-bind-b2-ordering");
7844        let route = route_key(7, 3);
7845        let identity = RouteIdentity(Arc::new(RouteIdentityData {
7846            root: root.clone(),
7847            project_root: root.as_path().to_path_buf(),
7848            harness: "opencode".to_string(),
7849            session: "b2-session".to_string(),
7850            trust: BindTrust::FirstParty,
7851            spawn_principal: AuthenticatedPrincipal::FirstParty,
7852            consumer_elicitation_capable: false,
7853        }));
7854        let replay_key = push::ReplayKey::from_identity(&identity);
7855        let completion = RouteBindCompletion {
7856            route,
7857            identity,
7858            bind_root_id: root.clone(),
7859            inserted_new_actor: false,
7860            configure_response: Response::success("subc-bind-7", json!({})),
7861            diagnostics_on_edit: false,
7862            ver: PROTOCOL_VERSION,
7863            corr: 91,
7864            flags: control_flags(),
7865        };
7866        let mut pending_binds = HashMap::from([(
7867            route,
7868            PendingBind {
7869                bind_root_id: root,
7870                inserted_new_actor: false,
7871                cancelled: false,
7872                configure_request_id: "subc-bind-7".to_string(),
7873                started_at: Instant::now(),
7874                warned_half_deadline: false,
7875                deadline_reported: false,
7876                corr: 91,
7877                ver: PROTOCOL_VERSION,
7878                flags: control_flags(),
7879                cancellation: crate::executor::JobCancellation::new(),
7880            },
7881        )]);
7882        let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
7883        let mut push_buffer =
7884            HashMap::from([(replay_key, VecDeque::from([completion_frame("b2-replay")]))]);
7885        let (writer_tx, mut writer_rx) = mpsc::channel(8);
7886        let metrics = Arc::new(DispatchPathMetrics::new());
7887
7888        handle_route_bind_completion(
7889            &writer_tx,
7890            completion,
7891            &mut HashMap::new(),
7892            &mut HashMap::new(),
7893            &mut HashMap::new(),
7894            &mut push_buffer,
7895            &mut HashMap::new(),
7896            &mut pending_binds,
7897            &mut installed_route_epochs,
7898            &Arc::new(Executor::new()),
7899            &Arc::new(Notify::new()),
7900            &metrics,
7901        )
7902        .await
7903        .unwrap();
7904
7905        let ack = writer_rx.try_recv().expect("RouteBindAck");
7906        assert_eq!(ack.header.ty, FrameType::Response);
7907        assert_eq!((ack.header.channel, ack.header.epoch), (0, 0));
7908        let route_frame = writer_rx.try_recv().expect("post-ack route frame");
7909        assert_eq!(route_frame.header.ty, FrameType::Push);
7910        assert_eq!(
7911            (route_frame.header.channel, route_frame.header.epoch),
7912            (route.channel, route.epoch)
7913        );
7914    }
7915}