Skip to main content

aft/subc/
mod.rs

1//! subc daemon attach — transport edge.
2//!
3//! When AFT is launched as `aft --subc <connection-file>`, it does NOT run the
4//! standalone NDJSON-over-stdin loop. Instead it connects to a running subc
5//! daemon over loopback TCP, authenticates with the pre-envelope HMAC handshake
6//! (`subc-transport`), then speaks the subc frame protocol (`subc-protocol`):
7//! ModuleHello → HelloAck (register as a tool provider), then a channel-0
8//! control loop (Ping/Pong, RouteBind) plus route-channel tool calls.
9//!
10//! Concurrency: subc routes tool calls through the executor. The tokio
11//! edge never dispatches against `AppContext` inline; per-actor executor lanes
12//! own the reader/mutator epoch, while a writer task serializes outbound frames.
13
14use std::collections::{HashMap, HashSet, VecDeque};
15use std::fmt;
16use std::io;
17use std::net::{IpAddr, SocketAddr};
18use std::ops::Deref;
19use std::path::{Path, PathBuf};
20use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
21use std::sync::{Arc, LazyLock, Mutex as StdMutex, OnceLock};
22use std::time::{Duration, Instant};
23
24use serde::Deserialize;
25use serde_json::{json, Value};
26
27use crate::config::Config;
28use crate::config_resolve::ConfigTier;
29use crate::context::{App, AppContext, ProgressSender, RootHealthSnapshot};
30use crate::executor::{Executor, JobCancellation, Lane};
31use crate::fleet_status::{spawn_fleet_status_dial, FleetStatusClient};
32use crate::jsonc::strip_jsonc;
33use crate::log_ctx;
34use crate::path_identity::ProjectRootId;
35use crate::protocol::{ProgressKind, PushFrame, RawRequest, Response};
36use crate::response_finalize::{DispatchOutcome, PendingResponse};
37use crate::run_tool_call::{
38    finish_tool_call_response, prepare_tool_call, run_tool_call, strip_agent_preview_arg_owned,
39    PhaseTrace, ToolCallContext, ToolCallOutcome, ToolCallResult,
40};
41use crate::runtime_drain;
42use crate::sandbox_spawn::{AuthenticatedPrincipal, PrincipalTrust};
43
44use subc_protocol::manifest::{
45    Bindings, Concurrency, ExecutionMode, IdentityBinding, IdentityScope, ModuleManifest,
46    ProviderRole, StorageBinding, StorageKind, StorageScope, Tool, TrustTier,
47};
48use subc_protocol::session::{
49    HealthReport, HealthStatus, ModuleControlRequest, ModuleControlResponse,
50    MODULE_CONTROL_OP_HEALTH_CHECK,
51};
52use subc_protocol::{
53    ErrorBody, Flags, Frame, FrameType, ModuleHelloBody, Principal, Priority, MAX_FRAME_BODY_LEN,
54    PROTOCOL_VERSION,
55};
56use subc_transport::{authenticate_client, connection_file, read_frame, write_frame};
57use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
58use tokio::net::TcpStream;
59use tokio::sync::{mpsc, oneshot, Notify};
60use tokio::task::JoinHandle;
61
62/// Per-attempt handshake deadline. The initial attach loop has a separate total
63/// budget so a stalled peer cannot consume an unbounded supervisor launch window.
64const AUTH_DEADLINE: Duration = Duration::from_secs(5);
65const ATTACH_RETRY_BUDGET: Duration = Duration::from_secs(60);
66const ATTACH_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(250);
67const ATTACH_RETRY_MAX_BACKOFF: Duration = Duration::from_secs(5);
68const ATTACH_RETRY_JITTER_PERCENT: u64 = 20;
69
70/// Correlation id for the initial ModuleHello (channel 0).
71const HELLO_CORR: u64 = 1;
72
73/// Per-session in-memory replay cap for must-deliver Push frames. This covers
74/// detach/re-attach while AFT stays alive; cross-restart replay is phased later.
75const PUSH_BUFFER_MAX_PER_KEY: usize = 256;
76
77/// Bounded guard for control-frame sends. If the daemon stops reading and the
78/// writer queue stays full, tear the subc edge down instead of stalling the
79/// route loop indefinitely.
80const CONTROL_SEND_TIMEOUT: Duration = Duration::from_millis(250);
81
82/// Cadence for the loop's deadline-driven drain work (retry-buffer flush,
83/// bg-wake emission, maintenance submission). Checked at the top of every
84/// loop turn so busy select arms cannot starve it.
85const DRAIN_TICK_PERIOD: Duration = Duration::from_millis(250);
86
87/// Root-scoped stores and watcher runtimes are reopened lazily after this
88/// period without tool traffic. Keeping the value fixed avoids per-client
89/// eviction policies competing inside the module loop.
90const IDLE_ROOT_TTL: Duration = Duration::from_secs(30 * 60);
91
92const WRITER_QUEUE_CAPACITY: usize = 256;
93
94/// Keep reliable Push bursts from monopolizing the current-thread subc loop;
95/// any remaining must-deliver frames stay queued for the next loop turn.
96const RELIABLE_PUSH_DRAIN_BUDGET: usize = 32;
97
98/// Limit maintenance submissions per tick so background drains cannot delay
99/// control-plane work such as completed RouteBind acknowledgements.
100///
101/// The decomposed maintenance pass charges this budget by Mutating job, not by
102/// root. Size the default burst for one maintenance pass over eight live roots,
103/// while follow-up batches still re-enter the capped queue instead of bypassing
104/// the budget.
105const MAINTENANCE_SUBMIT_BUDGET: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len() * 8;
106const INITIAL_MAINTENANCE_DRAIN_KINDS: [MaintenanceDrainKind; 4] = [
107    MaintenanceDrainKind::Watcher,
108    MaintenanceDrainKind::Lsp,
109    MaintenanceDrainKind::ConfigureTail,
110    MaintenanceDrainKind::CompletionDrains,
111];
112#[cfg(test)]
113const INITIAL_MAINTENANCE_JOB_COUNT: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len();
114
115const RELIABLE_WRITER_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(10);
116const RELIABLE_WRITER_RETRY_MAX_BACKOFF: Duration = Duration::from_millis(250);
117
118const DISPATCH_PATH_BIND_WARN_AFTER: Duration = Duration::from_secs(6);
119const ROUTE_BIND_DEADLINE: Duration = Duration::from_secs(12);
120
121/// Small bounded memory of completed task ids used to suppress stale lossy
122/// long-running reminders that arrive after their reliable completion event.
123const COMPLETED_TASK_SUPPRESSION_MAX: usize = 4096;
124
125/// Bash foreground orchestration polls detached tasks with short read-lane jobs.
126/// The sleep between polls is outside the executor so no read or write worker is
127/// pinned while a foreground command is still running.
128const PENDING_POLL_INTERVAL: Duration = Duration::from_millis(100);
129
130/// Host elicitation asks fail closed if the MCP facade does not answer promptly.
131const BASH_ELICITATION_TIMEOUT: Duration = Duration::from_secs(60);
132const BASH_ELICITATION_CREATE_METHOD: &str = "elicitation/create";
133
134#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
135struct RouteChannel {
136    channel: u16,
137    epoch: u32,
138}
139
140impl fmt::Display for RouteChannel {
141    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
142        write!(f, "{}@{}", self.channel, self.epoch)
143    }
144}
145
146type PushEnvelope = (ProjectRootId, PushFrame);
147type LossyPushEnvelope = (u64, ProjectRootId, PushFrame);
148type RetryBuffer = HashMap<RouteChannel, VecDeque<(push::ReplayKey, PushFrame)>>;
149mod bash;
150mod health;
151mod manifest;
152mod push;
153mod standing;
154mod wire;
155
156use self::health::{
157    build_health_report, warn_slow_pending_binds, warn_slow_running_interactive_jobs,
158    DispatchPathMetrics, HealthRollupCache, ReapBlockerCensus, ResponseTaskGuard,
159    HEALTH_ROLLUP_TTL,
160};
161use self::manifest::{
162    build_manifest, command_lane, control_flags, control_ops, is_bash_family_tool,
163    is_subc_agent_core_tool, is_subc_native_plumbing_tool,
164};
165pub use self::wire::SubcError;
166
167/// Test-only view of the fail-closed tool-call gate: would `name` be admitted
168/// on a bound route (as an agent tool or native plumbing)? Used by the
169/// plugin-send drift guard in `subc_plumbing_drift_test.rs`.
170pub fn is_tool_call_admitted_for_test(name: &str) -> bool {
171    manifest::is_subc_agent_core_tool(name) || manifest::is_subc_native_plumbing_tool(name)
172}
173use self::wire::{
174    build_error_frame, build_goodbye_frame, build_tool_response_frame,
175    build_tool_response_frame_with_limit, decrement_counted_channel, response_is_fatal_panic,
176    response_message, send_counted_channel, send_frame, send_reliable_writer_frame,
177    send_traced_tool_response_frame, ToolResponseWriteTrace, WriterFrame, WriterSender,
178};
179
180struct DecodedFrame {
181    frame: Frame,
182    phase_trace: PhaseTrace,
183}
184
185struct ToolCallCompletion {
186    text: String,
187    phase_trace: PhaseTrace,
188}
189
190#[derive(Clone, Copy, Debug, PartialEq, Eq)]
191enum RouteDetachPolicy {
192    RetainForReplay,
193    CancelOnDetach,
194}
195
196#[derive(Clone)]
197struct ActiveToolCall {
198    root_id: ProjectRootId,
199    cancellation: JobCancellation,
200    detach_policy: RouteDetachPolicy,
201}
202
203type ActiveToolCalls = Arc<StdMutex<HashMap<(RouteChannel, u64), ActiveToolCall>>>;
204
205struct PendingDeferredSetupGuard(Arc<AtomicUsize>);
206
207impl PendingDeferredSetupGuard {
208    fn new(count: Arc<AtomicUsize>) -> Self {
209        count.fetch_add(1, Ordering::SeqCst);
210        Self(count)
211    }
212}
213
214impl Drop for PendingDeferredSetupGuard {
215    fn drop(&mut self) {
216        self.0.fetch_sub(1, Ordering::SeqCst);
217    }
218}
219
220enum DeferredSetupOutcome {
221    Immediate {
222        text: String,
223        phase_trace: PhaseTrace,
224    },
225    Deferred {
226        pending: PendingResponse,
227        surface_downgraded: bool,
228        phase_trace: PhaseTrace,
229    },
230}
231
232struct PendingSubcResponse {
233    route: RouteChannel,
234    corr: u64,
235    flags: Flags,
236    ver: u8,
237    root: ProjectRootId,
238    session_id: String,
239    bare_name: String,
240    format_context: crate::subc_format::FormatContext,
241    bind_trust: BindTrust,
242    pending: PendingResponse,
243    surface_downgraded: bool,
244    phase_trace: PhaseTrace,
245}
246
247struct ResolvedSubcResponse {
248    entry: PendingSubcResponse,
249    response: Response,
250}
251
252#[derive(Default)]
253struct PendingSubcResponses {
254    entries: Vec<PendingSubcResponse>,
255}
256
257impl PendingSubcResponses {
258    fn register(&mut self, pending: PendingSubcResponse) {
259        self.entries.retain(|entry| {
260            let keep = entry.route != pending.route || entry.corr != pending.corr;
261            if !keep {
262                if let Some(cancellation) = &entry.pending.cancellation {
263                    cancellation.request_cancel();
264                }
265            }
266            keep
267        });
268        self.entries.push(pending);
269    }
270
271    fn poll_ready(&mut self, executor: &Executor) -> Vec<ResolvedSubcResponse> {
272        let mut ready = Vec::new();
273        let mut waiting = Vec::with_capacity(self.entries.len());
274        for mut entry in self.entries.drain(..) {
275            let response = executor
276                .actor_context(&entry.root)
277                .and_then(|ctx| (entry.pending.poll)(&ctx));
278            if let Some(response) = response {
279                ready.push(ResolvedSubcResponse { entry, response });
280            } else {
281                waiting.push(entry);
282            }
283        }
284        self.entries = waiting;
285        ready
286    }
287
288    fn cancel_request(&mut self, route: RouteChannel, corr: u64) -> bool {
289        let mut cancelled = false;
290        self.entries.retain(|entry| {
291            let keep = entry.route != route || entry.corr != corr;
292            if !keep {
293                cancelled = true;
294                if let Some(cancellation) = &entry.pending.cancellation {
295                    cancellation.request_cancel();
296                }
297            }
298            keep
299        });
300        cancelled
301    }
302
303    fn drain_route(
304        &mut self,
305        route: RouteChannel,
306        executor: &Executor,
307    ) -> Vec<ResolvedSubcResponse> {
308        self.drain_matching(executor, |entry| entry.route == route)
309    }
310
311    fn drain_on_shutdown(&mut self, executor: &Executor) -> Vec<ResolvedSubcResponse> {
312        self.drain_matching(executor, |_| true)
313    }
314
315    fn drain_matching(
316        &mut self,
317        executor: &Executor,
318        matches: impl Fn(&PendingSubcResponse) -> bool,
319    ) -> Vec<ResolvedSubcResponse> {
320        let mut resolved = Vec::new();
321        let mut waiting = Vec::with_capacity(self.entries.len());
322        for mut entry in self.entries.drain(..) {
323            if !matches(&entry) {
324                waiting.push(entry);
325                continue;
326            }
327            if let Some(cancellation) = &entry.pending.cancellation {
328                cancellation.request_cancel();
329            }
330            if let Some(ctx) = executor.actor_context(&entry.root) {
331                if let Some(on_shutdown) = entry.pending.on_shutdown.as_mut() {
332                    let response = on_shutdown(&ctx);
333                    resolved.push(ResolvedSubcResponse { entry, response });
334                }
335            }
336        }
337        self.entries = waiting;
338        resolved
339    }
340
341    fn is_empty(&self) -> bool {
342        self.entries.is_empty()
343    }
344}
345
346#[derive(Clone)]
347struct PushSenders {
348    lossy_tx: mpsc::Sender<LossyPushEnvelope>,
349    reliable_tx: mpsc::UnboundedSender<PushEnvelope>,
350    lossy_overflow: Arc<push::LossyOverflow>,
351    lossy_seq: Arc<AtomicU64>,
352    fleet_status_client: FleetStatusClient,
353}
354
355#[derive(Clone)]
356struct PersistentCancelSignal {
357    inner: Arc<PersistentCancelInner>,
358}
359
360struct PersistentCancelInner {
361    cancelled: AtomicBool,
362    notify: Notify,
363}
364
365impl PersistentCancelSignal {
366    fn new() -> Self {
367        Self {
368            inner: Arc::new(PersistentCancelInner {
369                cancelled: AtomicBool::new(false),
370                notify: Notify::new(),
371            }),
372        }
373    }
374
375    fn cancel(&self) {
376        if !self.inner.cancelled.swap(true, Ordering::SeqCst) {
377            self.inner.notify.notify_waiters();
378        }
379    }
380
381    fn is_cancelled(&self) -> bool {
382        self.inner.cancelled.load(Ordering::SeqCst)
383    }
384
385    async fn cancelled(&self) {
386        // `enable()` REGISTERS this waiter before we read the flag, closing the
387        // lost-wakeup window: `notify_waiters()` only wakes already-registered
388        // waiters and stores no permit, so without enable() a `cancel()` firing
389        // between the flag read and `.await` would be missed and the future
390        // would park forever (cancel() fires only once). With enable(), a cancel
391        // racing the flag read still wakes the registered waiter. The loop is a
392        // belt-and-suspenders re-check on spurious wakeups.
393        loop {
394            let notified = self.inner.notify.notified();
395            tokio::pin!(notified);
396            notified.as_mut().enable();
397            if self.is_cancelled() {
398                return;
399            }
400            notified.await;
401        }
402    }
403}
404
405fn submit_active_tool_call(
406    executor: &Executor,
407    active: &ActiveToolCalls,
408    route: RouteChannel,
409    corr: u64,
410    root_id: ProjectRootId,
411    lane: Lane,
412    request_id: String,
413    detach_policy: RouteDetachPolicy,
414    job: crate::executor::ExecutorJob,
415) -> oneshot::Receiver<Response> {
416    let (rx, cancellation) =
417        executor.submit_cancellable_async(root_id.clone(), lane, request_id, job);
418    active
419        .lock()
420        .unwrap_or_else(std::sync::PoisonError::into_inner)
421        .insert(
422            (route, corr),
423            ActiveToolCall {
424                root_id,
425                cancellation,
426                detach_policy,
427            },
428        );
429    rx
430}
431
432fn finish_active_tool_call(active: &ActiveToolCalls, route: RouteChannel, corr: u64) -> bool {
433    active
434        .lock()
435        .unwrap_or_else(std::sync::PoisonError::into_inner)
436        .remove(&(route, corr))
437        .is_some()
438}
439
440fn active_tool_call_is_registered(
441    active: &ActiveToolCalls,
442    route: RouteChannel,
443    corr: u64,
444) -> bool {
445    active
446        .lock()
447        .unwrap_or_else(std::sync::PoisonError::into_inner)
448        .contains_key(&(route, corr))
449}
450
451fn cancel_active_tool_call(
452    active: &ActiveToolCalls,
453    executor: &Executor,
454    route: RouteChannel,
455    corr: u64,
456    reason: &str,
457) -> bool {
458    let call = active
459        .lock()
460        .unwrap_or_else(std::sync::PoisonError::into_inner)
461        .remove(&(route, corr));
462    let Some(call) = call else {
463        return false;
464    };
465    let outcome = executor.cancel_job(&call.root_id, &call.cancellation);
466    log::debug!(
467        "subc attach: cancelled active tool call route={route} corr={corr} reason={reason} outcome={outcome:?}"
468    );
469    true
470}
471
472#[derive(Clone, Copy, Debug, PartialEq, Eq)]
473enum RouteWorkDisposition {
474    RetainForReplay,
475    Abandon,
476}
477
478fn apply_route_work_disposition(
479    active: &ActiveToolCalls,
480    executor: &Executor,
481    route: RouteChannel,
482    disposition: RouteWorkDisposition,
483    reason: &str,
484) -> usize {
485    if disposition == RouteWorkDisposition::RetainForReplay {
486        let (retained, cancelled) = {
487            let mut calls = active
488                .lock()
489                .unwrap_or_else(std::sync::PoisonError::into_inner);
490            let mut retained = 0usize;
491            let mut cancelled = Vec::new();
492            calls.retain(|(call_route, _), call| {
493                if *call_route != route {
494                    return true;
495                }
496                match call.detach_policy {
497                    RouteDetachPolicy::RetainForReplay => {
498                        retained += 1;
499                        true
500                    }
501                    RouteDetachPolicy::CancelOnDetach => {
502                        cancelled.push(call.clone());
503                        false
504                    }
505                }
506            });
507            (retained, cancelled)
508        };
509        let cancelled_count = cancelled.len();
510        for call in cancelled {
511            executor.cancel_job(&call.root_id, &call.cancellation);
512        }
513        log::debug!(
514            "subc attach: retained {retained} replayable active tool call(s) and cancelled {cancelled_count} teardown-terminal call(s) route={route} reason={reason}"
515        );
516        return retained;
517    }
518
519    let cancelled = {
520        let mut calls = active
521            .lock()
522            .unwrap_or_else(std::sync::PoisonError::into_inner);
523        let mut cancelled = Vec::new();
524        calls.retain(|(call_route, _), call| {
525            if *call_route == route {
526                cancelled.push(call.clone());
527                false
528            } else {
529                true
530            }
531        });
532        cancelled
533    };
534    for call in &cancelled {
535        let outcome = executor.cancel_job(&call.root_id, &call.cancellation);
536        log::debug!(
537            "subc attach: cancelled active tool call route={route} reason={reason} outcome={outcome:?}"
538        );
539    }
540    cancelled.len()
541}
542
543fn cancel_all_active_tool_calls(
544    active: &ActiveToolCalls,
545    executor: &Executor,
546    reason: &str,
547) -> usize {
548    let cancelled = {
549        let mut calls = active
550            .lock()
551            .unwrap_or_else(std::sync::PoisonError::into_inner);
552        calls.drain().map(|(_, call)| call).collect::<Vec<_>>()
553    };
554    for call in &cancelled {
555        let outcome = executor.cancel_job(&call.root_id, &call.cancellation);
556        log::debug!("subc attach: cancelled active tool call reason={reason} outcome={outcome:?}");
557    }
558    cancelled.len()
559}
560
561#[derive(Debug, Clone, Copy, PartialEq, Eq)]
562pub(crate) enum BindTrust {
563    FirstParty,
564    Untrusted,
565}
566
567impl BindTrust {
568    fn allows_bash_observation(self) -> bool {
569        matches!(self, Self::FirstParty)
570    }
571
572    fn label(self) -> &'static str {
573        match self {
574            Self::FirstParty => "first_party",
575            Self::Untrusted => "untrusted",
576        }
577    }
578
579    fn sandbox_trust(self) -> PrincipalTrust {
580        match self {
581            Self::FirstParty => PrincipalTrust::FirstParty,
582            Self::Untrusted => PrincipalTrust::Untrusted,
583        }
584    }
585}
586
587pub(super) fn trust_for_principal(principal: &Option<Principal>) -> BindTrust {
588    match principal {
589        Some(Principal::Direct) => BindTrust::FirstParty,
590        // Module renames are flag-days: the daemon registry refuses duplicate
591        // active ids, so a renaming module cannot advertise both names during
592        // its transition. This allowlist is DIALLED, not dialling — it must
593        // accept a module's NEW name in a released binary before the module
594        // starts using it, and the old name stays until the flip has settled.
595        // That is why transitional pairs appear here: llm-runner/broca was the
596        // previous rename, alfonso-core/prefrontal is the current one. When
597        // retiring an old name, confirm the fleet no longer spawns it — a
598        // stale entry here is inert, but a missing one silently downgrades a
599        // first-party module to Untrusted and revokes its bash access.
600        Some(Principal::Reserved { module_id })
601            if module_id == "llm-runner"
602                || module_id == "aft"
603                || module_id == "broca"
604                || module_id == "alfonso-core"
605                || module_id == "prefrontal"
606                || module_id == "prefrontal-core" =>
607        {
608            BindTrust::FirstParty
609        }
610        Some(Principal::Reserved { .. }) | Some(Principal::Unverified) | None => {
611            BindTrust::Untrusted
612        }
613    }
614}
615
616fn harness_forces_untrusted(harness: &str) -> bool {
617    harness.starts_with("fed:")
618}
619
620pub(super) fn trust_for_bind(harness: &str, principal: &Option<Principal>) -> BindTrust {
621    if harness_forces_untrusted(harness) {
622        BindTrust::Untrusted
623    } else {
624        trust_for_principal(principal)
625    }
626}
627
628fn principal_id(principal: &Option<Principal>) -> Option<String> {
629    match principal {
630        Some(Principal::Direct) => Some("direct".to_string()),
631        Some(Principal::Reserved { module_id }) => Some(format!("reserved:{module_id}")),
632        Some(Principal::Unverified) => Some("unverified".to_string()),
633        None => None,
634    }
635}
636
637fn principal_label(principal: &Option<Principal>) -> String {
638    principal_id(principal).unwrap_or_else(|| "absent".to_string())
639}
640
641#[derive(Debug)]
642/// Per-root route metadata owned by the subc loop. The `active_bash_waits` field
643/// counts detached bash processes that are still being observed for this root.
644/// Any future logic that evicts roots based on idle time must not evict a root
645/// while this count is greater than zero, because a foreground bash response may
646/// still arrive later.
647struct RootMeta {
648    maintenance_pending: bool,
649    maintenance_jobs_in_flight: usize,
650    maintenance_queued_kinds: VecDeque<MaintenanceDrainKind>,
651    maintenance_last_submitted: Option<Instant>,
652    maintenance_poisoned: bool,
653    last_touched: Instant,
654    diagnostics_on_edit: bool,
655    active_bash_waits: usize,
656    idle_artifacts_evicted: bool,
657    unbound_quiesced: bool,
658    consecutive_missing_sweeps: u8,
659}
660
661#[derive(Debug)]
662struct PendingBind {
663    bind_root_id: ProjectRootId,
664    inserted_new_actor: bool,
665    cancelled: bool,
666    configure_request_id: String,
667    started_at: Instant,
668    warned_half_deadline: bool,
669    deadline_reported: bool,
670    corr: u64,
671    ver: u8,
672    flags: Flags,
673    /// Exact-job cancellation for the submitted configure: Goodbye and
674    /// deadline expiry cancel the executor job operationally (queued jobs are
675    /// removed, running configures return at their next checkpoint) instead of
676    /// only marking bookkeeping.
677    cancellation: crate::executor::JobCancellation,
678}
679
680struct RouteBindCompletion {
681    route: RouteChannel,
682    identity: RouteIdentity,
683    bind_root_id: ProjectRootId,
684    inserted_new_actor: bool,
685    configure_response: Response,
686    diagnostics_on_edit: bool,
687    ver: u8,
688    corr: u64,
689    flags: Flags,
690}
691
692#[derive(Debug, Clone)]
693struct RouteIdentity(Arc<RouteIdentityData>);
694
695#[derive(Debug)]
696struct RouteIdentityData {
697    root: ProjectRootId,
698    project_root: PathBuf,
699    harness: String,
700    session: String,
701    trust: BindTrust,
702    spawn_principal: AuthenticatedPrincipal,
703    consumer_elicitation_capable: bool,
704}
705
706impl Deref for RouteIdentity {
707    type Target = RouteIdentityData;
708
709    fn deref(&self) -> &Self::Target {
710        &self.0
711    }
712}
713
714#[derive(Debug, Clone)]
715struct RetainedSessionIdentity {
716    harness: String,
717    trust: BindTrust,
718}
719
720#[derive(Clone)]
721struct BgSub {
722    corr: u64,
723    ver: u8,
724    flags: Flags,
725    root: ProjectRootId,
726    session: String,
727}
728
729// A session can be observed by multiple long-lived consumer records. Retain
730// every route so each wake uses the correlation captured by that route's BgSub.
731type BgSubsBySession = HashMap<(ProjectRootId, String), HashSet<RouteChannel>>;
732
733struct MaintenanceCompletion {
734    root_id: ProjectRootId,
735    kind: MaintenanceDrainKind,
736    response: Response,
737    empty_bg_sessions: Vec<(String, u64)>,
738    requeue_kind: Option<MaintenanceDrainKind>,
739}
740
741#[derive(Clone, Copy, Debug, PartialEq, Eq)]
742enum MaintenanceDrainKind {
743    Watcher,
744    Lsp,
745    ConfigureTail,
746    CompletionDrains,
747}
748
749impl MaintenanceDrainKind {
750    fn label(self) -> &'static str {
751        match self {
752            Self::Watcher => "watcher",
753            Self::Lsp => "lsp",
754            Self::ConfigureTail => "configure-tail",
755            Self::CompletionDrains => "completion-drains",
756        }
757    }
758}
759
760#[derive(Debug, Default)]
761struct MaintenanceJobOutcome {
762    empty_bg_sessions: Vec<(String, u64)>,
763    requeue_kind: Option<MaintenanceDrainKind>,
764}
765
766#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
767struct ReverseCorrKey {
768    route: RouteChannel,
769    corr: u64,
770}
771
772struct PendingBashAsk {
773    route: RouteChannel,
774    tool_corr: u64,
775    tool_flags: Flags,
776    tool_ver: u8,
777    root: ProjectRootId,
778    project_root: PathBuf,
779    session_id: String,
780    spawn_principal: AuthenticatedPrincipal,
781    edit_slot_survives: Option<bool>,
782    request_id: String,
783    arguments: Value,
784    format_context: crate::subc_format::FormatContext,
785    cancel: bash::BashWaitCancel,
786    grants: Vec<String>,
787    expires_at: Instant,
788}
789
790impl RootMeta {
791    fn new(now: Instant) -> Self {
792        Self {
793            maintenance_pending: false,
794            maintenance_jobs_in_flight: 0,
795            maintenance_queued_kinds: VecDeque::new(),
796            maintenance_last_submitted: None,
797            maintenance_poisoned: false,
798            last_touched: now,
799            diagnostics_on_edit: false,
800            active_bash_waits: 0,
801            idle_artifacts_evicted: false,
802            unbound_quiesced: false,
803            consecutive_missing_sweeps: 0,
804        }
805    }
806
807    fn note_activity(&mut self) {
808        self.last_touched = Instant::now();
809    }
810
811    fn reactivate_bound(&mut self) {
812        self.note_activity();
813        self.idle_artifacts_evicted = false;
814        self.unbound_quiesced = false;
815    }
816}
817
818fn due_maintenance_jobs(
819    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
820    executor: Option<&Executor>,
821    bg_sub_by_session: &BgSubsBySession,
822    bg_wake_pending: &HashSet<RouteChannel>,
823    budget: usize,
824    pending_bind_roots: &HashSet<ProjectRootId>,
825) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
826    let mut jobs = Vec::new();
827    let mut deferred = false;
828    let mut roots = live_roots.keys().cloned().collect::<Vec<_>>();
829    roots.sort_by(|left, right| {
830        let left_last = live_roots
831            .get(left)
832            .and_then(|meta| meta.maintenance_last_submitted);
833        let right_last = live_roots
834            .get(right)
835            .and_then(|meta| meta.maintenance_last_submitted);
836        left_last
837            .cmp(&right_last)
838            .then_with(|| left.as_path().cmp(right.as_path()))
839    });
840
841    for root_id in roots {
842        let Some(meta) = live_roots.get_mut(&root_id) else {
843            continue;
844        };
845        if meta.maintenance_poisoned {
846            continue;
847        }
848
849        if pending_bind_roots.contains(&root_id) {
850            if meta.maintenance_pending || !meta.maintenance_queued_kinds.is_empty() {
851                deferred = true;
852            }
853            continue;
854        }
855
856        if !meta.maintenance_pending {
857            if jobs.len() >= budget {
858                deferred = true;
859                continue;
860            }
861            // Only enqueue kinds with pending work. Probes are cheap and
862            // fail-open (contended sources count as pending), so an idle root
863            // costs four probes per tick instead of four dispatched jobs.
864            let executor_actor_context =
865                executor.and_then(|executor| executor.actor_context(&root_id));
866            let root_has_pending_bg_wake =
867                bg_sub_by_session.iter().any(|((sub_root, _), channels)| {
868                    sub_root == &root_id
869                        && channels
870                            .iter()
871                            .any(|channel| bg_wake_pending.contains(channel))
872                });
873            let kinds_with_work: Vec<MaintenanceDrainKind> = match executor_actor_context {
874                Some(ctx) => INITIAL_MAINTENANCE_DRAIN_KINDS
875                    .into_iter()
876                    .filter(|kind| {
877                        if meta.unbound_quiesced && !matches!(kind, MaintenanceDrainKind::Lsp) {
878                            return false;
879                        }
880                        match kind {
881                            MaintenanceDrainKind::Watcher => ctx.watcher_drain_has_work(),
882                            MaintenanceDrainKind::Lsp => ctx.lsp_drain_has_work(),
883                            MaintenanceDrainKind::ConfigureTail => ctx.configure_tail_has_work(),
884                            // Every CompletionDrains source is visible at this enqueue site:
885                            // AppContext probes completion queues, this loop owns bg wakes,
886                            // and queued continuations bypass probing via maintenance_pending.
887                            // New drain sources must expose a probe here rather than making
888                            // every subscribed root fail open again.
889                            MaintenanceDrainKind::CompletionDrains => {
890                                root_has_pending_bg_wake || ctx.completion_drains_have_work()
891                            }
892                        }
893                    })
894                    .collect(),
895                None if meta.unbound_quiesced => Vec::new(),
896                // No context handle (actor gone mid-tick): enqueue everything.
897                None => INITIAL_MAINTENANCE_DRAIN_KINDS.to_vec(),
898            };
899            if kinds_with_work.is_empty() {
900                continue;
901            }
902            meta.maintenance_pending = true;
903            meta.maintenance_queued_kinds.extend(kinds_with_work);
904        }
905
906        while let Some(kind) = meta.maintenance_queued_kinds.pop_front() {
907            if jobs.len() >= budget {
908                meta.maintenance_queued_kinds.push_front(kind);
909                deferred = true;
910                break;
911            }
912            meta.maintenance_jobs_in_flight += 1;
913            meta.maintenance_last_submitted = Some(Instant::now());
914            jobs.push((root_id.clone(), kind));
915        }
916
917        meta.maintenance_pending =
918            meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
919    }
920
921    (jobs, deferred)
922}
923
924fn eviction_estimate_label(estimate: &crate::memory::MemoryEstimate) -> String {
925    match estimate.estimated_bytes {
926        Some(bytes) => format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
927        None if estimate.status == "busy" => "busy".to_string(),
928        None => "not estimated".to_string(),
929    }
930}
931
932fn optional_memory_label(bytes: Option<u64>) -> String {
933    bytes.map_or_else(
934        || "not estimated".to_string(),
935        |bytes| format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
936    )
937}
938
939fn pressure_relief_label(relief: &crate::memory::AllocatorPressureRelief) -> String {
940    format!(
941        "; allocator pressure relief: RSS {} -> {}, in-use {} -> {}, allocated {} -> {}, slack {} -> {}, allocator reported {:.1} MB released",
942        optional_memory_label(relief.rss_before_bytes),
943        optional_memory_label(relief.rss_after_bytes),
944        optional_memory_label(relief.allocator_before.bytes_in_use),
945        optional_memory_label(relief.allocator_after.bytes_in_use),
946        optional_memory_label(relief.allocator_before.size_allocated),
947        optional_memory_label(relief.allocator_after.size_allocated),
948        optional_memory_label(relief.allocator_before.retained_slack_bytes),
949        optional_memory_label(relief.allocator_after.retained_slack_bytes),
950        relief.bytes_released as f64 / (1024.0 * 1024.0),
951    )
952}
953
954fn idle_root_eviction_message(
955    root_id: &ProjectRootId,
956    memory: &crate::memory::RootMemorySnapshot,
957    pressure_relief: Option<&crate::memory::AllocatorPressureRelief>,
958) -> String {
959    // Bash, LSP, and parser state remain resident. The freed total is deliberately
960    // only the known-byte portion of handles eviction actually drops.
961    let freed_bytes = [
962        &memory.semantic,
963        &memory.trigram,
964        &memory.symbols,
965        &memory.callgraph,
966        &memory.inspect,
967    ]
968    .iter()
969    .filter_map(|estimate| estimate.estimated_bytes)
970    .fold(0u64, u64::saturating_add);
971    let mut message = format!(
972        "evicted idle root {}: freed ~{:.1} MB (semantic {}, trigram {}, symbols {}, callgraph {}, inspect {}; retained: bash {}, lsp {}, parser_pool {})",
973        root_id.as_path().display(),
974        freed_bytes as f64 / (1024.0 * 1024.0),
975        eviction_estimate_label(&memory.semantic),
976        eviction_estimate_label(&memory.trigram),
977        eviction_estimate_label(&memory.symbols),
978        eviction_estimate_label(&memory.callgraph),
979        eviction_estimate_label(&memory.inspect),
980        eviction_estimate_label(&memory.bash),
981        eviction_estimate_label(&memory.lsp),
982        eviction_estimate_label(&memory.parser_pool),
983    );
984    if let Some(pressure_relief) = pressure_relief {
985        message.push_str(&pressure_relief_label(pressure_relief));
986    }
987    message
988}
989
990fn process_has_been_idle(now: Instant, live_roots: &HashMap<ProjectRootId, RootMeta>) -> bool {
991    !live_roots.is_empty()
992        && live_roots.values().all(|meta| {
993            now.saturating_duration_since(meta.last_touched) >= IDLE_ROOT_TTL
994                && meta.active_bash_waits == 0
995                && !meta.maintenance_pending
996                && meta.maintenance_queued_kinds.is_empty()
997        })
998}
999
1000fn allocator_pressure_relief_after_idle_sweep(
1001    now: Instant,
1002    live_roots: &HashMap<ProjectRootId, RootMeta>,
1003    executor: &Executor,
1004) -> Option<crate::memory::AllocatorPressureRelief> {
1005    if !process_has_been_idle(now, live_roots)
1006        || live_roots.keys().any(|root_id| {
1007            executor
1008                .actor_context(root_id)
1009                .is_some_and(|ctx| ctx.artifact_eviction_blocked())
1010        })
1011    {
1012        return None;
1013    }
1014
1015    #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu")))]
1016    {
1017        Some(crate::memory::relieve_allocator_pressure())
1018    }
1019    #[cfg(not(any(target_os = "macos", all(target_os = "linux", target_env = "gnu"))))]
1020    {
1021        None
1022    }
1023}
1024
1025fn quiesce_unbound_root(
1026    root_id: &ProjectRootId,
1027    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1028    executor: &Arc<Executor>,
1029) {
1030    let Some(meta) = live_roots.get_mut(root_id) else {
1031        return;
1032    };
1033
1034    let ctx = executor.actor_context(root_id);
1035    if let Some(ctx) = ctx.as_ref() {
1036        // Close lifecycle admission before touching scheduler queues. A running
1037        // ConfigureTail cannot release gates, install a watcher, or reserve a
1038        // callgraph build after this transition becomes visible.
1039        ctx.mark_subc_unbound();
1040    }
1041    let cancelled = executor.cancel_queued_maintenance(root_id);
1042    // Transient unbind keeps the root WARM: the watcher stays running (its
1043    // events accumulate and replay on rebind, so no unobserved gap exists) and
1044    // resident artifacts stay resident. Host restarts unbind every root and
1045    // rebind seconds later; stopping the watcher here would force strict
1046    // re-verification plus a full callgraph rebuild on every restart. The
1047    // expensive teardown (watcher stop + gap invalidation) belongs to the
1048    // idle-TTL reaper and the root-deleted path.
1049    let discarded = ctx
1050        .map(|ctx| crate::commands::configure::cancel_deferred_configure_maintenance(&ctx))
1051        .unwrap_or(0);
1052    meta.unbound_quiesced = true;
1053    meta.maintenance_queued_kinds.clear();
1054    meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
1055    log::info!(
1056        "subc attach: quiesced unbound root {} (cancelled {} queued maintenance job(s), cancelled {} configure maintenance job(s)); cause=goodbye_unbound",
1057        root_id.as_path().display(),
1058        cancelled,
1059        discarded
1060    );
1061}
1062
1063#[allow(clippy::too_many_arguments)]
1064fn quiesce_connection_roots(
1065    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1066    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
1067    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1068    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1069    installed_route_epochs: &mut HashMap<u16, u32>,
1070    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1071    active_tool_calls: &ActiveToolCalls,
1072    executor: &Arc<Executor>,
1073) {
1074    cancel_all_active_tool_calls(active_tool_calls, executor, "connection teardown");
1075    for cancel in route_bash_cancels.values() {
1076        cancel.token.cancel();
1077    }
1078    route_bash_cancels.clear();
1079
1080    let mut roots = live_roots.keys().cloned().collect::<HashSet<_>>();
1081    for pending in pending_binds.values_mut() {
1082        pending.cancelled = true;
1083        roots.insert(pending.bind_root_id.clone());
1084        let _ = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
1085    }
1086
1087    // A connection exit abandons every installed route at once. Close lifecycle
1088    // admission before cancelling maintenance so no deferred worker can restore
1089    // root activity after the loop-owned route tables disappear.
1090    for root_id in roots {
1091        if live_roots.contains_key(&root_id) {
1092            quiesce_unbound_root(&root_id, live_roots, executor);
1093        } else if let Some(ctx) = executor.actor_context(&root_id) {
1094            ctx.mark_subc_unbound();
1095            executor.cancel_queued_maintenance(&root_id);
1096            crate::commands::configure::cancel_deferred_configure_maintenance(&ctx);
1097        }
1098    }
1099
1100    routes.clear();
1101    root_channels.clear();
1102    installed_route_epochs.clear();
1103}
1104
1105/// Per-channel epoch watermarks for roots reclaimed without a client Goodbye.
1106/// The 16-bit channel space bounds this map, and no root identity or resource
1107/// handle is retained. It exists only so late requests receive a typed error.
1108#[derive(Debug, Default)]
1109struct ReclaimedRoutes {
1110    highest_epoch_by_channel: HashMap<u16, u32>,
1111}
1112
1113impl ReclaimedRoutes {
1114    fn insert(&mut self, route: RouteChannel) {
1115        self.highest_epoch_by_channel
1116            .entry(route.channel)
1117            .and_modify(|epoch| *epoch = (*epoch).max(route.epoch))
1118            .or_insert(route.epoch);
1119    }
1120
1121    fn contains(&self, route: RouteChannel) -> bool {
1122        self.highest_epoch_by_channel
1123            .get(&route.channel)
1124            .is_some_and(|epoch| route.epoch <= *epoch)
1125    }
1126}
1127
1128#[derive(Debug, Default)]
1129struct IdleReapOutcome {
1130    evicted: usize,
1131    forgotten_deleted_roots: Vec<ProjectRootId>,
1132}
1133
1134fn reap_idle_roots(
1135    now: Instant,
1136    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1137    pending_binds: &HashMap<RouteChannel, PendingBind>,
1138    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1139    executor: &Arc<Executor>,
1140    metrics: &DispatchPathMetrics,
1141) -> IdleReapOutcome {
1142    let pending_bind_roots = pending_binds
1143        .values()
1144        .map(|pending| pending.bind_root_id.clone())
1145        .collect::<HashSet<_>>();
1146    let mut census = ReapBlockerCensus::default();
1147    let mut candidates = Vec::new();
1148
1149    for (root_id, meta) in live_roots.iter_mut() {
1150        let deleted = !root_id.as_path().exists();
1151        if deleted {
1152            // A missing directory makes a bound route obsolete, but one failed
1153            // lookup is not enough evidence to tear down a client-visible actor.
1154            // Requiring two maintenance sweeps protects atomic replacement and
1155            // transient filesystem failures; observing the path resets the proof.
1156            // Absence also covers renames: a task's cwd handle can follow the
1157            // moved directory while the registered path disappears. The old
1158            // path is deliberately treated as a retired root identity, so the
1159            // reaper accepts killing such tasks; rename a project only with no
1160            // live tasks rather than relying on cwd-resolution heuristics.
1161            meta.consecutive_missing_sweeps = meta.consecutive_missing_sweeps.saturating_add(1);
1162        } else {
1163            meta.consecutive_missing_sweeps = 0;
1164        }
1165        let deletion_confirmed = meta.consecutive_missing_sweeps >= 2;
1166        let has_bound_route = root_channels
1167            .get(root_id)
1168            .is_some_and(|channels| !channels.is_empty());
1169        let has_pending_bind = pending_bind_roots.contains(root_id);
1170
1171        if deleted {
1172            let mut retained = false;
1173            if !deletion_confirmed {
1174                census.absence_unconfirmed += 1;
1175                retained = true;
1176            }
1177            // Once absence is confirmed, the directory cannot serve this route
1178            // again. Neither a stale route nor the lack of normal unbind cleanup
1179            // justifies retaining the root; purge removes the route after retirement.
1180            if meta.active_bash_waits > 0 {
1181                census.bash_waits += 1;
1182                retained = true;
1183            }
1184            if meta.maintenance_pending {
1185                census.maintenance_pending += 1;
1186                retained = true;
1187            }
1188            if !meta.maintenance_queued_kinds.is_empty() {
1189                census.maintenance_queued += 1;
1190                retained = true;
1191            }
1192            if has_pending_bind {
1193                census.pending_binds += 1;
1194                retained = true;
1195            }
1196            match executor.try_actor_is_idle(root_id) {
1197                Some(true) => {}
1198                Some(false) => {
1199                    census.actor_busy += 1;
1200                    retained = true;
1201                }
1202                None => {
1203                    census.actor_state_busy += 1;
1204                    retained = true;
1205                }
1206            }
1207            if retained {
1208                census.deleted_retained += 1;
1209                continue;
1210            }
1211        } else {
1212            // Route teardown marks the lifecycle admission gate before the last
1213            // channel disappears. Requiring zero bound channels and a quiesced
1214            // lifecycle prevents a still-bound root from losing its watcher.
1215            if has_bound_route
1216                || !meta.unbound_quiesced
1217                || meta.idle_artifacts_evicted
1218                || now.saturating_duration_since(meta.last_touched) < IDLE_ROOT_TTL
1219                || meta.active_bash_waits > 0
1220                || meta.maintenance_pending
1221                || !meta.maintenance_queued_kinds.is_empty()
1222                || has_pending_bind
1223                || !executor.actor_is_idle(root_id)
1224            {
1225                continue;
1226            }
1227        }
1228        candidates.push((root_id.clone(), deleted));
1229    }
1230
1231    let mut reaped = Vec::new();
1232    let mut forgotten_deleted_roots = Vec::new();
1233    for (root_id, deleted) in candidates {
1234        let Some(ctx) = executor.actor_context(&root_id) else {
1235            if deleted {
1236                census.deleted_retained += 1;
1237                census.actor_busy += 1;
1238            }
1239            continue;
1240        };
1241        // A TTL-aged unbound root retained its watcher-derived pending paths
1242        // across the transient-unbind window. Strict gap invalidation subsumes
1243        // them, but every abort path must restore them because a rebind can
1244        // still happen until eviction commits.
1245        //
1246        // After two consecutive directory-absence scans confirm that the
1247        // root is gone, terminate its background task before checking the
1248        // artifact-eviction gate. The task can otherwise keep the root's
1249        // artifacts in use; cleanup first lets confirmed reclamation finish
1250        // without weakening the gate for unrelated active work.
1251        if deleted {
1252            ctx.bash_background()
1253                .kill_running_tasks_for_root(root_id.as_path());
1254        }
1255        let taken_pending = Some(ctx.take_pending_reconciliation_state());
1256        if ctx.artifact_eviction_blocked() {
1257            if let Some(pending) = taken_pending {
1258                ctx.restore_pending_reconciliation_state(pending);
1259            }
1260            if deleted {
1261                census.deleted_retained += 1;
1262                census.artifact_eviction_blocked += 1;
1263            }
1264            continue;
1265        }
1266        let memory_before = ctx.memory_root_snapshot();
1267        if !ctx.evict_idle_artifacts() {
1268            if let Some(pending) = taken_pending {
1269                ctx.restore_pending_reconciliation_state(pending);
1270            }
1271            if deleted {
1272                census.deleted_retained += 1;
1273                census.artifact_eviction_failed += 1;
1274            }
1275            continue;
1276        }
1277        drop(taken_pending);
1278        ctx.stop_watcher_runtime_in_background();
1279        // Edits during watcher downtime are unobserved. Advance publication
1280        // epochs and force strict verification before any later warm reload.
1281        ctx.invalidate_artifacts_after_watcher_gap();
1282
1283        if deleted {
1284            if executor.retire_idle_actor_in_background(&root_id) {
1285                live_roots.remove(&root_id);
1286                forgotten_deleted_roots.push(root_id.clone());
1287            } else {
1288                census.deleted_retained += 1;
1289                census.actor_busy += 1;
1290            }
1291        } else {
1292            if let Some(meta) = live_roots.get_mut(&root_id) {
1293                meta.idle_artifacts_evicted = true;
1294            }
1295            ctx.release_idle_reopenable_resources_in_background();
1296        }
1297        reaped.push((root_id, memory_before));
1298    }
1299
1300    metrics.record_reap(census);
1301    if census.deleted_retained > 0 {
1302        log::info!(
1303            "subc attach: retained {} deleted root(s) during idle reap; blockers={}",
1304            census.deleted_retained,
1305            census.blocker_histogram()
1306        );
1307    }
1308
1309    let pressure_relief = (!reaped.is_empty())
1310        .then(|| allocator_pressure_relief_after_idle_sweep(now, live_roots, executor))
1311        .flatten();
1312    for (root_id, memory_before) in &reaped {
1313        log::info!(
1314            "{}",
1315            idle_root_eviction_message(root_id, memory_before, pressure_relief.as_ref())
1316        );
1317    }
1318    IdleReapOutcome {
1319        evicted: reaped.len(),
1320        forgotten_deleted_roots,
1321    }
1322}
1323
1324#[allow(clippy::too_many_arguments)]
1325fn purge_deleted_root_residents(
1326    root_id: &ProjectRootId,
1327    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1328    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1329    installed_route_epochs: &mut HashMap<u16, u32>,
1330    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1331    active_tool_calls: &ActiveToolCalls,
1332    executor: &Executor,
1333    retry_buffer: &mut RetryBuffer,
1334    reclaimed_routes: &mut ReclaimedRoutes,
1335    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1336    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1337    bg_subs: &mut HashMap<RouteChannel, BgSub>,
1338    bg_sub_by_session: &mut BgSubsBySession,
1339    bg_wake_pending: &mut HashSet<RouteChannel>,
1340    bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
1341    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1342    metrics: &DispatchPathMetrics,
1343) {
1344    let mut stale_routes = root_channels.get(root_id).cloned().unwrap_or_default();
1345    stale_routes.extend(
1346        routes
1347            .iter()
1348            .filter_map(|(route, identity)| (&identity.root == root_id).then_some(*route)),
1349    );
1350    stale_routes.extend(
1351        bg_sub_by_session
1352            .iter()
1353            .filter(|((root, _), _)| root == root_id)
1354            .flat_map(|(_, routes)| routes.iter().copied()),
1355    );
1356    stale_routes.extend(
1357        pending_bash_asks
1358            .values()
1359            .filter_map(|ask| (&ask.root == root_id).then_some(ask.route)),
1360    );
1361
1362    for route in stale_routes {
1363        reclaimed_routes.insert(route);
1364        remove_installed_route(installed_route_epochs, route);
1365        remove_route_channel(routes, root_channels, route);
1366        if let Some(cancel) = route_bash_cancels.remove(&route) {
1367            cancel.token.cancel();
1368        }
1369        apply_route_work_disposition(
1370            active_tool_calls,
1371            executor,
1372            route,
1373            RouteWorkDisposition::Abandon,
1374            "root reclaim",
1375        );
1376        retry_buffer.remove(&route);
1377        if let Some(sub) = bg_subs.remove(&route) {
1378            metrics.record_bg_subscription_ended(&sub.root, &sub.session, route, "root-reclaim");
1379        }
1380        bg_wake_pending.remove(&route);
1381    }
1382    root_channels.remove(root_id);
1383    session_identity.retain(|(root, _), _| root != root_id);
1384    push_buffer.retain(|key, _| &key.root != root_id);
1385    bg_wake_epoch.retain(|(root, _), _| root != root_id);
1386    pending_bash_asks.retain(|_, ask| &ask.root != root_id);
1387    bg_sub_by_session.retain(|(root, _), _| root != root_id);
1388
1389    log::info!(
1390        "subc attach: fully forgot deleted root {}; cause=absence_reclaim",
1391        root_id.as_path().display()
1392    );
1393}
1394
1395#[allow(clippy::too_many_arguments)]
1396fn submit_due_maintenance_jobs(
1397    executor: &Arc<Executor>,
1398    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1399    pending_binds: &HashMap<RouteChannel, PendingBind>,
1400    bg_sub_by_session: &BgSubsBySession,
1401    bg_wake_pending: &HashSet<RouteChannel>,
1402    bg_wake_epoch: &HashMap<(ProjectRootId, String), u64>,
1403    maintenance_tx: &mpsc::Sender<MaintenanceCompletion>,
1404    metrics: &Arc<DispatchPathMetrics>,
1405) {
1406    let pending_bind_roots = pending_binds
1407        .values()
1408        .map(|pending| pending.bind_root_id.clone())
1409        .collect::<HashSet<_>>();
1410    let (due_jobs, deferred_jobs) = due_maintenance_jobs(
1411        live_roots,
1412        Some(executor),
1413        bg_sub_by_session,
1414        bg_wake_pending,
1415        MAINTENANCE_SUBMIT_BUDGET,
1416        &pending_bind_roots,
1417    );
1418    if deferred_jobs {
1419        metrics
1420            .maintenance_budget_deferrals
1421            .fetch_add(1, Ordering::Relaxed);
1422    }
1423    for (root_id, kind) in due_jobs {
1424        let bg_sessions_to_check = if kind == MaintenanceDrainKind::CompletionDrains {
1425            bg_sub_by_session
1426                .iter()
1427                .filter_map(|((root, session), _)| {
1428                    if root == &root_id {
1429                        Some((
1430                            session.clone(),
1431                            bg_wake_epoch
1432                                .get(&(root_id.clone(), session.clone()))
1433                                .copied()
1434                                .unwrap_or(0),
1435                        ))
1436                    } else {
1437                        None
1438                    }
1439                })
1440                .collect()
1441        } else {
1442            Vec::new()
1443        };
1444        submit_maintenance_job(
1445            executor,
1446            root_id,
1447            kind,
1448            bg_sessions_to_check,
1449            maintenance_tx,
1450            metrics,
1451        );
1452    }
1453}
1454
1455fn should_requiesce_after_maintenance(
1456    meta: &RootMeta,
1457    completed_kind: MaintenanceDrainKind,
1458    bind_pending: bool,
1459) -> bool {
1460    meta.unbound_quiesced && completed_kind != MaintenanceDrainKind::Lsp && !bind_pending
1461}
1462
1463fn note_maintenance_completion(
1464    meta: &mut RootMeta,
1465    requeue_kind: Option<MaintenanceDrainKind>,
1466    fatal: bool,
1467    defer_requeue: bool,
1468) {
1469    if fatal {
1470        meta.maintenance_poisoned = true;
1471    }
1472
1473    if let Some(kind) = requeue_kind.filter(|_| !meta.maintenance_poisoned && !defer_requeue) {
1474        meta.maintenance_queued_kinds.push_back(kind);
1475    }
1476
1477    meta.maintenance_jobs_in_flight = meta.maintenance_jobs_in_flight.saturating_sub(1);
1478    meta.maintenance_pending =
1479        meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
1480}
1481
1482fn route_key(channel: u16, epoch: u32) -> RouteChannel {
1483    RouteChannel { channel, epoch }
1484}
1485
1486fn remove_installed_route(installed_epochs: &mut HashMap<u16, u32>, route: RouteChannel) {
1487    if installed_epochs.get(&route.channel).copied() == Some(route.epoch) {
1488        installed_epochs.remove(&route.channel);
1489    }
1490}
1491
1492fn ingress_route_should_be_processed(
1493    installed_epochs: &HashMap<u16, u32>,
1494    reclaimed_routes: &ReclaimedRoutes,
1495    frame: &Frame,
1496) -> bool {
1497    if frame.header.channel == 0
1498        || installed_epochs.get(&frame.header.channel).copied() == Some(frame.header.epoch)
1499    {
1500        return true;
1501    }
1502
1503    // A late request for a reclaimed root reaches the normal unknown-route
1504    // handler, which returns the typed `route_not_bound` error. Other stale or
1505    // never-installed generations remain silent so they cannot affect a newer
1506    // route or change the protocol's rejected-bind behavior.
1507    frame.header.ty == FrameType::Request
1508        && reclaimed_routes.contains(route_key(frame.header.channel, frame.header.epoch))
1509}
1510
1511fn bash_elicitation_timeout() -> Duration {
1512    if cfg!(debug_assertions) {
1513        if let Ok(raw) = std::env::var("AFT_TEST_SUBC_BASH_ELICITATION_TTL_MS") {
1514            if let Ok(ms) = raw.parse::<u64>() {
1515                if ms > 0 {
1516                    return Duration::from_millis(ms);
1517                }
1518            }
1519        }
1520    }
1521    BASH_ELICITATION_TIMEOUT
1522}
1523
1524fn allocate_reverse_corr(
1525    pending_bash_asks: &HashMap<ReverseCorrKey, PendingBashAsk>,
1526    route: RouteChannel,
1527    next_corr: &mut u64,
1528) -> u64 {
1529    loop {
1530        let corr = *next_corr;
1531        *next_corr = (*next_corr).wrapping_add(1).max(1);
1532        if !pending_bash_asks.contains_key(&ReverseCorrKey { route, corr }) {
1533            return corr;
1534        }
1535    }
1536}
1537
1538fn bash_permission_kind_label(kind: &crate::bash_permissions::PermissionKind) -> &'static str {
1539    match kind {
1540        crate::bash_permissions::PermissionKind::ExternalDirectory => "external directory",
1541        crate::bash_permissions::PermissionKind::Bash => "bash",
1542    }
1543}
1544
1545fn bash_elicitation_patterns(asks: &[crate::bash_permissions::PermissionAsk]) -> Vec<String> {
1546    let mut patterns = Vec::new();
1547    let mut seen = HashSet::new();
1548    for ask in asks {
1549        for pattern in ask.patterns.iter().chain(ask.always.iter()) {
1550            if seen.insert(pattern.clone()) {
1551                patterns.push(pattern.clone());
1552            }
1553        }
1554    }
1555    patterns
1556}
1557
1558fn bash_elicitation_message(
1559    command: &str,
1560    asks: &[crate::bash_permissions::PermissionAsk],
1561) -> String {
1562    let command = command.split_whitespace().collect::<Vec<_>>().join(" ");
1563    let patterns = bash_elicitation_patterns(asks);
1564    let pattern_text = if patterns.is_empty() {
1565        "no matched permission patterns".to_string()
1566    } else {
1567        patterns.join(", ")
1568    };
1569    let ask_kinds = asks
1570        .iter()
1571        .map(|ask| bash_permission_kind_label(&ask.kind))
1572        .collect::<HashSet<_>>()
1573        .into_iter()
1574        .collect::<Vec<_>>()
1575        .join(", ");
1576    if ask_kinds.is_empty() {
1577        format!("Allow bash command `{command}`? Matched patterns: {pattern_text}")
1578    } else {
1579        format!("Allow bash command `{command}`? Matched {ask_kinds} patterns: {pattern_text}")
1580    }
1581}
1582
1583fn bash_elicitation_request_body(
1584    command: &str,
1585    asks: &[crate::bash_permissions::PermissionAsk],
1586) -> Value {
1587    json!({
1588        "method": BASH_ELICITATION_CREATE_METHOD,
1589        "params": {
1590            "mode": "form",
1591            "message": bash_elicitation_message(command, asks),
1592            "requestedSchema": {
1593                "type": "object",
1594                "properties": {
1595                    "decision": {
1596                        "type": "string",
1597                        "enum": ["allow", "deny"],
1598                        "description": "Choose allow to run this bash command once, or deny to block it."
1599                    }
1600                },
1601                "required": ["decision"],
1602                "additionalProperties": false
1603            },
1604            "_meta": {
1605                "aft": {
1606                    "tool": "bash",
1607                    "command": command,
1608                    "asks": asks
1609                }
1610            }
1611        }
1612    })
1613}
1614
1615fn build_bash_elicitation_request_frame(
1616    ver: u8,
1617    route: RouteChannel,
1618    corr: u64,
1619    flags: Flags,
1620    command: &str,
1621    asks: &[crate::bash_permissions::PermissionAsk],
1622) -> Result<Frame, SubcError> {
1623    let body = bash_elicitation_request_body(command, asks);
1624    Frame::build_with_version(
1625        ver,
1626        FrameType::Request,
1627        flags,
1628        route.channel,
1629        route.epoch,
1630        corr,
1631        serde_json::to_vec(&body).map_err(SubcError::Json)?,
1632    )
1633    .map_err(SubcError::FrameBuild)
1634}
1635
1636fn bash_elicitation_reply_is_allow(body: &[u8]) -> bool {
1637    let Ok(value) = serde_json::from_slice::<Value>(body) else {
1638        return false;
1639    };
1640    flat_bash_elicitation_reply_is_allow(&value) || mcp_bash_elicitation_reply_is_allow(&value)
1641}
1642
1643fn flat_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1644    let Some(object) = value.as_object() else {
1645        return false;
1646    };
1647    object.len() == 1 && object.get("decision").and_then(Value::as_str) == Some("allow")
1648}
1649
1650fn mcp_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1651    let Some(object) = value.as_object() else {
1652        return false;
1653    };
1654    if object.len() != 2 || object.get("action").and_then(Value::as_str) != Some("accept") {
1655        return false;
1656    }
1657    let Some(content) = object.get("content").and_then(Value::as_object) else {
1658        return false;
1659    };
1660    content.len() == 1 && content.get("decision").and_then(Value::as_str) == Some("allow")
1661}
1662
1663#[allow(clippy::too_many_arguments)]
1664async fn settle_pending_bash_ask_denied(
1665    tx: &WriterSender,
1666    pending: PendingBashAsk,
1667    routes: &HashMap<RouteChannel, RouteIdentity>,
1668    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1669    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1670    shutdown: &Arc<Notify>,
1671    metrics: &DispatchPathMetrics,
1672) -> Result<(), SubcError> {
1673    let completion = bash::bash_denied_untrusted_completion(
1674        pending.route,
1675        pending.tool_corr,
1676        pending.tool_flags,
1677        pending.tool_ver,
1678        pending.root,
1679        pending.request_id,
1680        pending.format_context,
1681    );
1682    bash::handle_bash_deferred_completion(
1683        tx,
1684        completion,
1685        routes,
1686        live_roots,
1687        route_bash_cancels,
1688        shutdown,
1689        metrics,
1690    )
1691    .await
1692}
1693
1694fn take_pending_bash_asks_for_route(
1695    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1696    route: RouteChannel,
1697) -> Vec<PendingBashAsk> {
1698    let keys = pending_bash_asks
1699        .keys()
1700        .copied()
1701        .filter(|key| key.route == route)
1702        .collect::<Vec<_>>();
1703    keys.into_iter()
1704        .filter_map(|key| pending_bash_asks.remove(&key))
1705        .collect()
1706}
1707
1708#[allow(clippy::too_many_arguments)]
1709async fn settle_pending_bash_asks_for_route(
1710    tx: &WriterSender,
1711    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1712    route: RouteChannel,
1713    routes: &HashMap<RouteChannel, RouteIdentity>,
1714    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1715    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1716    shutdown: &Arc<Notify>,
1717    metrics: &DispatchPathMetrics,
1718) -> Result<(), SubcError> {
1719    for pending in take_pending_bash_asks_for_route(pending_bash_asks, route) {
1720        settle_pending_bash_ask_denied(
1721            tx,
1722            pending,
1723            routes,
1724            live_roots,
1725            route_bash_cancels,
1726            shutdown,
1727            metrics,
1728        )
1729        .await?;
1730    }
1731    Ok(())
1732}
1733
1734#[allow(clippy::too_many_arguments)]
1735async fn settle_all_pending_bash_asks(
1736    tx: &WriterSender,
1737    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1738    routes: &HashMap<RouteChannel, RouteIdentity>,
1739    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1740    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1741    shutdown: &Arc<Notify>,
1742    metrics: &DispatchPathMetrics,
1743) -> Result<(), SubcError> {
1744    let pending = pending_bash_asks
1745        .drain()
1746        .map(|(_, pending)| pending)
1747        .collect::<Vec<_>>();
1748    for pending in pending {
1749        settle_pending_bash_ask_denied(
1750            tx,
1751            pending,
1752            routes,
1753            live_roots,
1754            route_bash_cancels,
1755            shutdown,
1756            metrics,
1757        )
1758        .await?;
1759    }
1760    Ok(())
1761}
1762
1763#[allow(clippy::too_many_arguments)]
1764async fn expire_pending_bash_asks(
1765    tx: &WriterSender,
1766    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1767    routes: &HashMap<RouteChannel, RouteIdentity>,
1768    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1769    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1770    shutdown: &Arc<Notify>,
1771    metrics: &DispatchPathMetrics,
1772) -> Result<(), SubcError> {
1773    let now = Instant::now();
1774    let expired = pending_bash_asks
1775        .iter()
1776        .filter_map(|(key, pending)| (pending.expires_at <= now).then_some(*key))
1777        .collect::<Vec<_>>();
1778    for key in expired {
1779        if let Some(pending) = pending_bash_asks.remove(&key) {
1780            log::debug!(
1781                "subc attach: bash elicitation request {} on route {} expired fail-closed",
1782                key.corr,
1783                pending.route
1784            );
1785            settle_pending_bash_ask_denied(
1786                tx,
1787                pending,
1788                routes,
1789                live_roots,
1790                route_bash_cancels,
1791                shutdown,
1792                metrics,
1793            )
1794            .await?;
1795        }
1796    }
1797    Ok(())
1798}
1799
1800#[allow(clippy::too_many_arguments)]
1801async fn handle_bash_elicitation_reply(
1802    tx: &WriterSender,
1803    frame: &Frame,
1804    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1805    routes: &HashMap<RouteChannel, RouteIdentity>,
1806    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1807    executor: &Arc<Executor>,
1808    shutdown: &Arc<Notify>,
1809    bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
1810    bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
1811    metrics: &Arc<DispatchPathMetrics>,
1812    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1813    dispatch: DispatchFn,
1814) -> Result<(), SubcError> {
1815    let key = ReverseCorrKey {
1816        route: route_key(frame.header.channel, frame.header.epoch),
1817        corr: frame.header.corr,
1818    };
1819    let Some(pending) = pending_bash_asks.remove(&key) else {
1820        return Ok(());
1821    };
1822
1823    if frame.header.ty == FrameType::Response && bash_elicitation_reply_is_allow(&frame.body) {
1824        if routes.contains_key(&key.route) {
1825            bash::submit_deferred_bash(
1826                executor,
1827                bash_deferred_tx,
1828                bash_poll_touch_tx,
1829                metrics,
1830                dispatch,
1831                pending.root,
1832                pending.project_root,
1833                pending.session_id,
1834                pending.request_id,
1835                pending.route,
1836                pending.tool_corr,
1837                pending.tool_flags,
1838                pending.tool_ver,
1839                pending.arguments,
1840                pending.format_context,
1841                pending.cancel,
1842                BindTrust::Untrusted,
1843                pending.spawn_principal,
1844                pending.edit_slot_survives,
1845                Some(pending.grants),
1846            );
1847            return Ok(());
1848        }
1849        log::debug!(
1850            "subc attach: dropping allowed bash elicitation reply {} for unbound route {}",
1851            key.corr,
1852            pending.route
1853        );
1854    }
1855
1856    settle_pending_bash_ask_denied(
1857        tx,
1858        pending,
1859        routes,
1860        live_roots,
1861        route_bash_cancels,
1862        shutdown,
1863        metrics,
1864    )
1865    .await
1866}
1867
1868#[allow(clippy::too_many_arguments)]
1869async fn cancel_pending_bash_ask_for_tool_call(
1870    tx: &WriterSender,
1871    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1872    route: RouteChannel,
1873    tool_corr: u64,
1874    routes: &HashMap<RouteChannel, RouteIdentity>,
1875    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1876    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1877    shutdown: &Arc<Notify>,
1878    metrics: &DispatchPathMetrics,
1879) -> Result<(), SubcError> {
1880    let keys = pending_bash_asks
1881        .iter()
1882        .filter_map(|(key, pending)| {
1883            (key.route == route && pending.tool_corr == tool_corr).then_some(*key)
1884        })
1885        .collect::<Vec<_>>();
1886    for key in keys {
1887        if let Some(pending) = pending_bash_asks.remove(&key) {
1888            settle_pending_bash_ask_denied(
1889                tx,
1890                pending,
1891                routes,
1892                live_roots,
1893                route_bash_cancels,
1894                shutdown,
1895                metrics,
1896            )
1897            .await?;
1898        }
1899    }
1900    Ok(())
1901}
1902
1903fn remove_root_channel(
1904    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1905    root: &ProjectRootId,
1906    channel: RouteChannel,
1907) {
1908    let remove_root = if let Some(channels) = root_channels.get_mut(root) {
1909        channels.remove(&channel);
1910        channels.is_empty()
1911    } else {
1912        false
1913    };
1914    if remove_root {
1915        root_channels.remove(root);
1916    }
1917}
1918
1919fn remove_route_channel(
1920    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1921    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1922    channel: RouteChannel,
1923) -> Option<RouteIdentity> {
1924    let removed = routes.remove(&channel);
1925    if let Some(identity) = &removed {
1926        remove_root_channel(root_channels, &identity.root, channel);
1927    }
1928    removed
1929}
1930
1931fn insert_route_channel(
1932    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1933    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1934    channel: RouteChannel,
1935    identity: RouteIdentity,
1936) {
1937    if let Some(previous) = routes.insert(channel, identity.clone()) {
1938        remove_root_channel(root_channels, &previous.root, channel);
1939    }
1940    root_channels
1941        .entry(identity.root.clone())
1942        .or_default()
1943        .insert(channel);
1944}
1945
1946fn insert_bg_subscription_index(
1947    bg_sub_by_session: &mut BgSubsBySession,
1948    root: ProjectRootId,
1949    session: String,
1950    channel: RouteChannel,
1951) {
1952    bg_sub_by_session
1953        .entry((root, session))
1954        .or_default()
1955        .insert(channel);
1956}
1957
1958fn remove_bg_subscription_index(
1959    bg_sub_by_session: &mut BgSubsBySession,
1960    channel: RouteChannel,
1961    identity: Option<&RouteIdentity>,
1962) {
1963    if let Some(identity) = identity {
1964        let key = (identity.root.clone(), identity.session.clone());
1965        let remove_key = bg_sub_by_session.get_mut(&key).is_some_and(|channels| {
1966            channels.remove(&channel);
1967            channels.is_empty()
1968        });
1969        if remove_key {
1970            bg_sub_by_session.remove(&key);
1971        }
1972    } else {
1973        bg_sub_by_session.retain(|_, channels| {
1974            channels.remove(&channel);
1975            !channels.is_empty()
1976        });
1977    }
1978}
1979
1980fn route_removal_will_quiesce_root(
1981    root: &ProjectRootId,
1982    route: RouteChannel,
1983    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1984    has_pending_bind: bool,
1985    replacement_root: Option<&ProjectRootId>,
1986) -> bool {
1987    let removes_last_route = root_channels
1988        .get(root)
1989        .is_some_and(|channels| channels.len() == 1 && channels.contains(&route));
1990    removes_last_route && !has_pending_bind && replacement_root != Some(root)
1991}
1992
1993fn should_quiesce_removed_root(
1994    root: &ProjectRootId,
1995    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1996    has_pending_bind: bool,
1997    replacement_root: Option<&ProjectRootId>,
1998) -> bool {
1999    !root_channels.contains_key(root) && !has_pending_bind && replacement_root != Some(root)
2000}
2001
2002async fn end_bg_subscription(
2003    writer_tx: &WriterSender,
2004    metrics: &DispatchPathMetrics,
2005    bg_subs: &mut HashMap<RouteChannel, BgSub>,
2006    bg_sub_by_session: &mut BgSubsBySession,
2007    bg_wake_pending: &mut HashSet<RouteChannel>,
2008    channel: RouteChannel,
2009    identity: Option<&RouteIdentity>,
2010    cause: &str,
2011) -> Result<(), SubcError> {
2012    if let Some(sub) = bg_subs.remove(&channel) {
2013        bg_wake_pending.remove(&channel);
2014        remove_bg_subscription_index(bg_sub_by_session, channel, identity);
2015        metrics.record_bg_subscription_ended(&sub.root, &sub.session, channel, cause);
2016        push::send_reliable_bg_stream_end(writer_tx, metrics, channel, &sub).await?;
2017    }
2018    Ok(())
2019}
2020
2021#[allow(clippy::too_many_arguments)]
2022async fn teardown_installed_route(
2023    tx: &WriterSender,
2024    metrics: &DispatchPathMetrics,
2025    executor: &Arc<Executor>,
2026    channel: RouteChannel,
2027    cancellation_reason: &str,
2028    replacement_root: Option<&ProjectRootId>,
2029    installed_route_epochs: &mut HashMap<u16, u32>,
2030    routes: &mut HashMap<RouteChannel, RouteIdentity>,
2031    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2032    bg_subs: &mut HashMap<RouteChannel, BgSub>,
2033    bg_sub_by_session: &mut BgSubsBySession,
2034    bg_wake_pending: &mut HashSet<RouteChannel>,
2035    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
2036    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2037    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
2038    active_tool_calls: &ActiveToolCalls,
2039    pending_responses: &mut PendingSubcResponses,
2040    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2041    retry_buffer: &mut RetryBuffer,
2042    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2043    shutdown: &Arc<Notify>,
2044    tool_response_body_limit: usize,
2045) -> Result<(), SubcError> {
2046    remove_installed_route(installed_route_epochs, channel);
2047    let bg_end_cause = match cancellation_reason {
2048        "Goodbye" => "goodbye",
2049        "higher-epoch RouteBind" => "higher-epoch",
2050        other => other,
2051    };
2052    end_bg_subscription(
2053        tx,
2054        metrics,
2055        bg_subs,
2056        bg_sub_by_session,
2057        bg_wake_pending,
2058        channel,
2059        routes.get(&channel),
2060        bg_end_cause,
2061    )
2062    .await?;
2063    settle_pending_bash_asks_for_route(
2064        tx,
2065        pending_bash_asks,
2066        channel,
2067        routes,
2068        live_roots,
2069        route_bash_cancels,
2070        shutdown,
2071        metrics,
2072    )
2073    .await?;
2074    if let Some(cancel) = route_bash_cancels.remove(&channel) {
2075        cancel.token.cancel();
2076    }
2077    for resolved in pending_responses.drain_route(channel, executor) {
2078        deliver_resolved_subc_response(
2079            tx,
2080            resolved,
2081            routes,
2082            live_roots,
2083            executor.as_ref(),
2084            active_tool_calls,
2085            shutdown,
2086            metrics,
2087            tool_response_body_limit,
2088        )
2089        .await?;
2090    }
2091    // Closing a route does not abandon its session: reliable responses are
2092    // retained for detach/rebind replay. Cancellation belongs to explicit
2093    // Cancel frames, whole-connection teardown, or root reclamation.
2094    apply_route_work_disposition(
2095        active_tool_calls,
2096        executor,
2097        channel,
2098        RouteWorkDisposition::RetainForReplay,
2099        cancellation_reason,
2100    );
2101    if let Some(pending) = pending_binds.get_mut(&channel) {
2102        pending.cancelled = true;
2103        let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
2104        log::debug!(
2105            "subc attach: cancelled pending RouteBind for route {} on {cancellation_reason} (configure job: {outcome:?})",
2106            channel.channel
2107        );
2108    }
2109    let migrated = push::migrate_retry_buffer_to_push_buffer(retry_buffer, channel, push_buffer);
2110    if let Some(identity) = routes.get(&channel) {
2111        let has_pending_bind = pending_binds
2112            .values()
2113            .any(|pending| pending.bind_root_id == identity.root);
2114        if route_removal_will_quiesce_root(
2115            &identity.root,
2116            channel,
2117            root_channels,
2118            has_pending_bind,
2119            replacement_root,
2120        ) {
2121            if let Some(ctx) = executor.actor_context(&identity.root) {
2122                // Fence deferred admissions before the final route disappears
2123                // from the loop-owned routing tables.
2124                ctx.mark_subc_unbound();
2125            }
2126        }
2127    }
2128    if let Some(identity) = remove_route_channel(routes, root_channels, channel) {
2129        let session_still_routed = routes
2130            .values()
2131            .any(|route| route.root == identity.root && route.session == identity.session);
2132        if !session_still_routed {
2133            if let Some(ctx) = executor.actor_context(&identity.root) {
2134                ctx.hashline_bindings()
2135                    .teardown(identity.root.as_path(), &identity.session);
2136            }
2137        }
2138        if migrated > 0 {
2139            log::debug!(
2140                "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from route {} into detach replay",
2141                channel.channel
2142            );
2143        }
2144        if let Some(meta) = live_roots.get_mut(&identity.root) {
2145            let idle_for = meta.last_touched.elapsed();
2146            meta.note_activity();
2147            log::debug!(
2148                "subc attach: route {} torn down for root {} harness {} session {} (last touched {:?} ago)",
2149                channel.channel,
2150                identity.root.as_path().display(),
2151                identity.harness,
2152                identity.session,
2153                idle_for
2154            );
2155        } else {
2156            log::debug!(
2157                "subc attach: route {} torn down for root {} harness {} session {}",
2158                channel.channel,
2159                identity.root.as_path().display(),
2160                identity.harness,
2161                identity.session
2162            );
2163        }
2164        let has_pending_bind = pending_binds
2165            .values()
2166            .any(|pending| pending.bind_root_id == identity.root);
2167        if should_quiesce_removed_root(
2168            &identity.root,
2169            root_channels,
2170            has_pending_bind,
2171            replacement_root,
2172        ) {
2173            quiesce_unbound_root(&identity.root, live_roots, executor);
2174        }
2175    } else {
2176        if migrated > 0 {
2177            log::debug!(
2178                "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from unbound route {} into detach replay",
2179                channel.channel
2180            );
2181        }
2182        log::debug!("subc attach: unbound route {} torn down", channel.channel);
2183    }
2184    Ok(())
2185}
2186
2187fn remember_session_identity(
2188    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2189    identity: &RouteIdentity,
2190) {
2191    let key = (identity.root.clone(), identity.session.clone());
2192    if matches!(identity.trust, BindTrust::Untrusted)
2193        && session_identity
2194            .get(&key)
2195            .is_some_and(|retained| matches!(retained.trust, BindTrust::FirstParty))
2196    {
2197        return;
2198    }
2199
2200    // Retained after route Goodbye so reliable session-scoped frames emitted while
2201    // the session is detached can still be keyed by the full (root,harness,session)
2202    // replay triple. Untrusted binds never overwrite a retained first-party
2203    // session identity, because bash completion replay is an observation channel.
2204    session_identity.insert(
2205        key,
2206        RetainedSessionIdentity {
2207            harness: identity.harness.clone(),
2208            trust: identity.trust,
2209        },
2210    );
2211}
2212
2213fn replay_key_for_session(
2214    session_identity: &HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2215    root: &ProjectRootId,
2216    session: &str,
2217) -> Option<(push::ReplayKey, BindTrust)> {
2218    let retained = session_identity.get(&(root.clone(), session.to_string()))?;
2219    Some((
2220        push::ReplayKey {
2221            root: root.clone(),
2222            harness: retained.harness.clone(),
2223            session: session.to_string(),
2224        },
2225        retained.trust,
2226    ))
2227}
2228/// Sync command dispatch, passed in from `main` (the binary owns the command
2229/// table). Invoked only inside executor jobs in subc mode.
2230pub type DispatchFn = fn(RawRequest, &AppContext) -> Response;
2231
2232#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2233enum ModuleLoopExit {
2234    Graceful,
2235    SkipSearchFlush,
2236}
2237
2238/// Entry point for `aft --subc <connection-file>`. Synchronous on the outside;
2239/// owns an isolated current-thread tokio runtime for the async transport.
2240/// Returns `Err` (fail-loud) on any connect/auth/protocol failure — we never
2241/// fall back to the standalone loop, to avoid split-brain index state.
2242pub fn run_subc_mode(
2243    connection_file_path: &Path,
2244    ctx: Arc<AppContext>,
2245    executor: Arc<Executor>,
2246    dispatch: DispatchFn,
2247    user_config_path: Option<PathBuf>,
2248) -> Result<(), SubcError> {
2249    // Production NEVER allows non-manifest tool names on route channels: AFT
2250    // fails closed and does not trust subc to enforce the manifest. The
2251    // test-only harness sets this through `run_subc_mode_for_test`.
2252    run_subc_mode_inner(
2253        connection_file_path,
2254        ctx,
2255        executor,
2256        dispatch,
2257        user_config_path,
2258        false,
2259        MAX_FRAME_BODY_LEN as usize,
2260    )
2261}
2262
2263fn run_subc_mode_inner(
2264    connection_file_path: &Path,
2265    ctx: Arc<AppContext>,
2266    executor: Arc<Executor>,
2267    dispatch: DispatchFn,
2268    user_config_path: Option<PathBuf>,
2269    allow_native_passthrough: bool,
2270    tool_response_body_limit: usize,
2271) -> Result<(), SubcError> {
2272    let runtime = tokio::runtime::Builder::new_current_thread()
2273        .enable_all()
2274        .build()
2275        .map_err(SubcError::Runtime)?;
2276
2277    let executor_for_loop = Arc::clone(&executor);
2278    let loop_result = runtime.block_on(async move {
2279        let shared_app = ctx.app();
2280        drop(ctx);
2281        let stream = connect_and_authenticate(connection_file_path).await?;
2282        log::info!(
2283            "subc attach: authenticated to daemon via {}",
2284            connection_file_path.display()
2285        );
2286        let (read_half, write_half) = tokio::io::split(stream);
2287        run_module_loop(
2288            read_half,
2289            write_half,
2290            connection_file_path,
2291            shared_app,
2292            executor_for_loop,
2293            dispatch,
2294            user_config_path,
2295            allow_native_passthrough,
2296            tool_response_body_limit,
2297        )
2298        .await
2299    });
2300
2301    let actor_contexts = executor.actor_contexts();
2302    if matches!(loop_result, Ok(ModuleLoopExit::Graceful)) {
2303        // EOF/Goodbye teardown flushes each root's index deltas and queued
2304        // callgraph refreshes. Fatal/panic teardown skips this best-effort work.
2305        flush_actor_indexes_on_graceful_shutdown(&actor_contexts);
2306    }
2307    for actor_ctx in &actor_contexts {
2308        actor_ctx.lsp().shutdown_all();
2309        actor_ctx.bash_background().detach();
2310    }
2311
2312    loop_result.map(|_| ())
2313}
2314
2315fn flush_actor_indexes_on_graceful_shutdown(actor_contexts: &[Arc<AppContext>]) {
2316    for actor_ctx in actor_contexts {
2317        let _ = actor_ctx.flush_search_index_on_graceful_shutdown();
2318    }
2319    let _ = crate::callgraph_store::flush_callgraph_store_refreshes_on_graceful_shutdown();
2320}
2321
2322/// Test-only entry that enables the non-manifest native-command passthrough on
2323/// route channels. Integration tests drive synthetic native commands (`glob`,
2324/// `callers`, `subc_test_echo_session`, …) through the executor to exercise
2325/// mechanics; production callers use [`run_subc_mode`], which fails closed.
2326#[doc(hidden)]
2327pub fn run_subc_mode_for_test(
2328    connection_file_path: &Path,
2329    ctx: Arc<AppContext>,
2330    executor: Arc<Executor>,
2331    dispatch: DispatchFn,
2332    user_config_path: Option<PathBuf>,
2333) -> Result<(), SubcError> {
2334    run_subc_mode_inner(
2335        connection_file_path,
2336        ctx,
2337        executor,
2338        dispatch,
2339        user_config_path,
2340        true,
2341        MAX_FRAME_BODY_LEN as usize,
2342    )
2343}
2344
2345/// Test-only entry that lowers the effective tool-response body limit without
2346/// allocating a 64 MiB fixture. The fixed fallback envelope needs 4 KiB of room.
2347#[doc(hidden)]
2348pub fn run_subc_mode_for_test_with_response_body_limit(
2349    connection_file_path: &Path,
2350    ctx: Arc<AppContext>,
2351    executor: Arc<Executor>,
2352    dispatch: DispatchFn,
2353    user_config_path: Option<PathBuf>,
2354    tool_response_body_limit: usize,
2355) -> Result<(), SubcError> {
2356    assert!((4 * 1_024..=MAX_FRAME_BODY_LEN as usize).contains(&tool_response_body_limit));
2357    run_subc_mode_inner(
2358        connection_file_path,
2359        ctx,
2360        executor,
2361        dispatch,
2362        user_config_path,
2363        true,
2364        tool_response_body_limit,
2365    )
2366}
2367
2368#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2369enum AttachErrorClass {
2370    Transient,
2371    Permanent,
2372}
2373
2374impl fmt::Display for AttachErrorClass {
2375    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2376        match self {
2377            Self::Transient => f.write_str("transient"),
2378            Self::Permanent => f.write_str("permanent"),
2379        }
2380    }
2381}
2382
2383#[derive(Clone, Copy)]
2384struct AttachRetryPolicy {
2385    budget: Duration,
2386    initial_backoff: Duration,
2387    max_backoff: Duration,
2388    jitter_percent: u64,
2389}
2390
2391const ATTACH_RETRY_POLICY: AttachRetryPolicy = AttachRetryPolicy {
2392    budget: ATTACH_RETRY_BUDGET,
2393    initial_backoff: ATTACH_RETRY_INITIAL_BACKOFF,
2394    max_backoff: ATTACH_RETRY_MAX_BACKOFF,
2395    jitter_percent: ATTACH_RETRY_JITTER_PERCENT,
2396};
2397
2398/// Retry only failures that can be caused by a daemon bounce or an interrupted
2399/// handshake. Protocol and credential failures are permanent for this process.
2400fn classify_attach_error(error: &SubcError) -> AttachErrorClass {
2401    let transient = match error {
2402        SubcError::Connect { source, .. } => is_transient_attach_io(source.kind()),
2403        SubcError::Auth { source, .. } => match source {
2404            subc_transport::AuthError::Timeout { .. }
2405            | subc_transport::AuthError::UnexpectedEof { .. } => true,
2406            subc_transport::AuthError::Io { source, .. } => is_transient_attach_io(source.kind()),
2407            _ => false,
2408        },
2409        _ => false,
2410    };
2411    if transient {
2412        AttachErrorClass::Transient
2413    } else {
2414        AttachErrorClass::Permanent
2415    }
2416}
2417
2418fn is_transient_attach_io(kind: io::ErrorKind) -> bool {
2419    matches!(
2420        kind,
2421        io::ErrorKind::ConnectionRefused
2422            | io::ErrorKind::TimedOut
2423            | io::ErrorKind::ConnectionReset
2424            | io::ErrorKind::ConnectionAborted
2425            | io::ErrorKind::BrokenPipe
2426            | io::ErrorKind::UnexpectedEof
2427    )
2428}
2429
2430/// Read the connection file → resolve the first endpoint → TCP connect → HMAC
2431/// handshake. Transient initial-attach failures retry on fresh sockets and reread
2432/// the file so a daemon bounce can publish a new endpoint or authentication key.
2433async fn connect_and_authenticate(connection_file_path: &Path) -> Result<TcpStream, SubcError> {
2434    connect_and_authenticate_with_policy(connection_file_path, ATTACH_RETRY_POLICY).await
2435}
2436
2437async fn connect_and_authenticate_with_policy(
2438    connection_file_path: &Path,
2439    policy: AttachRetryPolicy,
2440) -> Result<TcpStream, SubcError> {
2441    let started_at = Instant::now();
2442    let deadline = started_at + policy.budget;
2443    let mut attempt = 0_u32;
2444    let mut backoff = policy.initial_backoff;
2445    let mut history = Vec::new();
2446
2447    loop {
2448        attempt = attempt.saturating_add(1);
2449        let error = match connect_and_authenticate_once(connection_file_path, deadline).await {
2450            Ok(stream) => return Ok(stream),
2451            Err(error) => error,
2452        };
2453        let class = classify_attach_error(&error);
2454        let error_text = error.to_string().lines().collect::<Vec<_>>().join(" ");
2455        history.push(format!("attempt {attempt} [{class}]: {error_text}"));
2456
2457        if class == AttachErrorClass::Permanent {
2458            log_attach_final_failure(started_at.elapsed(), &history);
2459            return Err(error);
2460        }
2461
2462        let remaining = deadline.saturating_duration_since(Instant::now());
2463        if remaining.is_zero() {
2464            log_attach_final_failure(started_at.elapsed(), &history);
2465            return Err(error);
2466        }
2467
2468        let delay = jittered_attach_delay(backoff, policy.jitter_percent, attempt).min(remaining);
2469        log::info!(
2470            "subc attach retry: attempt {attempt} failed; error_class={class}; error={error_text}; next_delay={delay:?}"
2471        );
2472        tokio::time::sleep(delay).await;
2473
2474        if Instant::now() >= deadline {
2475            log_attach_final_failure(started_at.elapsed(), &history);
2476            return Err(error);
2477        }
2478        backoff = backoff.saturating_mul(2).min(policy.max_backoff);
2479    }
2480}
2481
2482fn jittered_attach_delay(base: Duration, jitter_percent: u64, attempt: u32) -> Duration {
2483    let jitter_percent = jitter_percent.min(100);
2484    if jitter_percent == 0 {
2485        return base;
2486    }
2487
2488    let mut random_bytes = [0_u8; 8];
2489    let random = if getrandom::fill(&mut random_bytes).is_ok() {
2490        u64::from_le_bytes(random_bytes)
2491    } else {
2492        let timestamp = std::time::SystemTime::now()
2493            .duration_since(std::time::UNIX_EPOCH)
2494            .unwrap_or_default()
2495            .subsec_nanos();
2496        u64::from(timestamp) ^ u64::from(attempt)
2497    };
2498    let span = jitter_percent.saturating_mul(2).saturating_add(1);
2499    let multiplier_percent = 100 - jitter_percent + random % span;
2500    let base_millis = u64::try_from(base.as_millis()).unwrap_or(u64::MAX);
2501    Duration::from_millis(base_millis.saturating_mul(multiplier_percent) / 100)
2502}
2503
2504fn log_attach_final_failure(elapsed: Duration, history: &[String]) {
2505    log::error!(
2506        "subc initial attach failed after {} attempt(s) in {elapsed:?}; attempt history: {}",
2507        history.len(),
2508        history.join(" | ")
2509    );
2510}
2511
2512async fn connect_and_authenticate_once(
2513    connection_file_path: &Path,
2514    deadline: Instant,
2515) -> Result<TcpStream, SubcError> {
2516    // This read intentionally lives inside the per-attempt function. The daemon
2517    // publishes connection files atomically and may change both port and key.
2518    let conn = connection_file::read_for_client(connection_file_path).map_err(|source| {
2519        SubcError::ConnectionFile {
2520            path: connection_file_path.to_path_buf(),
2521            source,
2522        }
2523    })?;
2524
2525    let endpoint = conn
2526        .endpoints
2527        .first()
2528        .ok_or_else(|| SubcError::NoEndpoint {
2529            path: connection_file_path.to_path_buf(),
2530        })?;
2531    let endpoint_label = format!("{}:{}", endpoint.host, endpoint.port);
2532    let ip = endpoint
2533        .host
2534        .parse::<IpAddr>()
2535        .map_err(|_| SubcError::InvalidEndpoint {
2536            path: connection_file_path.to_path_buf(),
2537            endpoint: endpoint_label.clone(),
2538        })?;
2539    let addr = SocketAddr::new(ip, endpoint.port);
2540
2541    let connect_budget = deadline.saturating_duration_since(Instant::now());
2542    let mut stream = tokio::time::timeout(connect_budget, TcpStream::connect(addr))
2543        .await
2544        .map_err(|_| SubcError::Connect {
2545            endpoint: endpoint_label.clone(),
2546            source: io::Error::new(
2547                io::ErrorKind::TimedOut,
2548                "initial subc attach retry budget elapsed during TCP connect",
2549            ),
2550        })?
2551        .map_err(|source| SubcError::Connect {
2552            endpoint: endpoint_label.clone(),
2553            source,
2554        })?;
2555    stream
2556        .set_nodelay(true)
2557        .map_err(|source| SubcError::Connect {
2558            endpoint: endpoint_label.clone(),
2559            source,
2560        })?;
2561
2562    let auth_budget = AUTH_DEADLINE.min(deadline.saturating_duration_since(Instant::now()));
2563    authenticate_client(&mut stream, &conn, auth_budget)
2564        .await
2565        .map_err(|source| SubcError::Auth {
2566            endpoint: endpoint_label,
2567            source,
2568        })?;
2569
2570    Ok(stream)
2571}
2572
2573#[allow(clippy::too_many_arguments)]
2574async fn process_route_bind_completion(
2575    writer_tx: &WriterSender,
2576    completion: RouteBindCompletion,
2577    routes: &mut HashMap<RouteChannel, RouteIdentity>,
2578    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2579    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2580    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2581    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2582    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2583    installed_route_epochs: &mut HashMap<u16, u32>,
2584    executor: &Arc<Executor>,
2585    standing_actor: &standing::StandingActor,
2586    shutdown: &Arc<Notify>,
2587    metrics: &Arc<DispatchPathMetrics>,
2588) -> Result<(), SubcError> {
2589    decrement_counted_channel(&metrics.control_completion_queued);
2590    handle_route_bind_completion(
2591        writer_tx,
2592        completion,
2593        routes,
2594        root_channels,
2595        session_identity,
2596        push_buffer,
2597        live_roots,
2598        pending_binds,
2599        installed_route_epochs,
2600        executor,
2601        standing_actor,
2602        shutdown,
2603        metrics,
2604    )
2605    .await
2606}
2607
2608#[allow(clippy::too_many_arguments)]
2609async fn drain_pending_route_bind_completions(
2610    control_completion_rx: &mut mpsc::Receiver<RouteBindCompletion>,
2611    writer_tx: &WriterSender,
2612    routes: &mut HashMap<RouteChannel, RouteIdentity>,
2613    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2614    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2615    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2616    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2617    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2618    installed_route_epochs: &mut HashMap<u16, u32>,
2619    executor: &Arc<Executor>,
2620    standing_actor: &standing::StandingActor,
2621    shutdown: &Arc<Notify>,
2622    metrics: &Arc<DispatchPathMetrics>,
2623) -> Result<usize, SubcError> {
2624    let mut drained = 0;
2625    while let Ok(completion) = control_completion_rx.try_recv() {
2626        process_route_bind_completion(
2627            writer_tx,
2628            completion,
2629            routes,
2630            root_channels,
2631            session_identity,
2632            push_buffer,
2633            live_roots,
2634            pending_binds,
2635            installed_route_epochs,
2636            executor,
2637            standing_actor,
2638            shutdown,
2639            metrics,
2640        )
2641        .await?;
2642        drained += 1;
2643    }
2644    Ok(drained)
2645}
2646
2647/// ModuleHello → HelloAck → control/route loop. Runs until the daemon closes
2648/// the connection (EOF), sends channel-0 Goodbye, or a fatal mutating executor
2649/// response requests whole-connection teardown.
2650async fn run_module_loop<R, W>(
2651    mut read: R,
2652    mut write: W,
2653    connection_file_path: &Path,
2654    shared_app: Arc<App>,
2655    executor: Arc<Executor>,
2656    dispatch: DispatchFn,
2657    user_config_path: Option<PathBuf>,
2658    allow_native_passthrough: bool,
2659    tool_response_body_limit: usize,
2660) -> Result<ModuleLoopExit, SubcError>
2661where
2662    R: AsyncRead + Unpin + Send + 'static,
2663    W: AsyncWrite + Unpin + Send + 'static,
2664{
2665    // ModuleHello: register as a tool provider and advertise the supported control-plane operations.
2666    // Echo the one-time launch nonce the daemon injected via SUBC_LAUNCH_NONCE so a
2667    // reserved module_id's HELLO is accepted; absent for non-reserved/self-connect.
2668    let hello = ModuleHelloBody {
2669        manifest: build_manifest(),
2670        protocol_ver: PROTOCOL_VERSION,
2671        control_ops: control_ops(),
2672        launch_nonce: std::env::var("SUBC_LAUNCH_NONCE").ok(),
2673    };
2674    let hello_frame = Frame::build(
2675        FrameType::Hello,
2676        control_flags(),
2677        0,
2678        0,
2679        HELLO_CORR,
2680        serde_json::to_vec(&hello).map_err(SubcError::Json)?,
2681    )
2682    .map_err(SubcError::FrameBuild)?;
2683    write_frame(&mut write, &hello_frame)
2684        .await
2685        .map_err(SubcError::FrameIo)?;
2686
2687    // Expect HelloAck (registered) or a channel-0 Error (manifest/version reject).
2688    match read_frame(&mut read).await.map_err(SubcError::FrameIo)? {
2689        None => return Err(SubcError::ClosedBeforeHelloAck),
2690        Some(frame) => match frame.header.ty {
2691            FrameType::HelloAck => {
2692                log::info!("subc attach: registered (HelloAck received)");
2693            }
2694            FrameType::Error => {
2695                let body = serde_json::from_slice::<ErrorBody>(&frame.body).ok();
2696                return Err(SubcError::HelloRejected { body });
2697            }
2698            other => return Err(SubcError::UnexpectedFrame { ty: other }),
2699        },
2700    }
2701
2702    let dispatch_path_metrics = Arc::new(DispatchPathMetrics::new());
2703    let (writer_tx, writer_rx) = mpsc::channel::<WriterFrame>(WRITER_QUEUE_CAPACITY);
2704    let writer_task = spawn_writer_task(write, writer_rx, Arc::clone(&dispatch_path_metrics));
2705    // `read_frame` is NOT cancellation-safe, so it must never sit directly inside
2706    // the `select!` below: a drain-interval tick (or shutdown) firing while a
2707    // frame is mid-transit would drop the partially-consumed bytes and desync the
2708    // stream (the next read would parse a body byte as a frame header). A
2709    // dedicated reader task owns the socket, reads whole frames sequentially, and
2710    // forwards them over a channel; the loop selects on the cancel-safe `recv()`.
2711    let (reader_tx, mut reader_rx) = mpsc::channel::<Result<DecodedFrame, SubcError>>(256);
2712    let reader_task = spawn_reader_task(read, reader_tx);
2713    let shutdown = Arc::new(Notify::new());
2714    // Drain-tick deadline is tracked manually and checked at the TOP of every
2715    // loop turn rather than as an Interval select arm: the select below is
2716    // `biased` (bind completions first), and biased polling means a saturated
2717    // higher arm (sustained lossy push traffic keeps lossy_rx always-ready)
2718    // would starve every arm below it, including a timer arm — leaving
2719    // backpressured reliable frames parked in the retry buffer past their
2720    // delivery deadline. The pre-turn check cannot be starved by arm order;
2721    // the sleep_until arm below only exists to wake an otherwise-idle loop.
2722    let mut next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2723    let mut next_maintenance_at = next_drain_at;
2724    let standing_actor =
2725        standing::StandingActor::new(Arc::clone(&shared_app), Arc::clone(&executor));
2726    // Startup reconciliation is intentionally direct; subsequent passes use
2727    // this existing maintenance timer arm and never create a standing timer.
2728    standing_actor.reconcile_at_startup();
2729    let mut next_standing_pass_at = tokio::time::Instant::now();
2730    // Rate-limit stamp for opportunistic allocator slack relief (checked on the
2731    // maintenance tick; policy shared with standalone via memory.rs).
2732    #[cfg(any(target_os = "macos", target_os = "linux"))]
2733    let mut last_slack_relief: Option<std::time::Instant> = None;
2734    let (maintenance_tx, mut maintenance_rx) = mpsc::channel::<MaintenanceCompletion>(256);
2735    let (bash_deferred_tx, mut bash_deferred_rx) =
2736        mpsc::channel::<bash::BashDeferredCompletion>(256);
2737    let (deferred_response_tx, mut deferred_response_rx) =
2738        mpsc::unbounded_channel::<PendingSubcResponse>();
2739    let (bash_poll_touch_tx, mut bash_poll_touch_rx) = mpsc::channel::<ProjectRootId>(256);
2740    let (control_completion_tx, mut control_completion_rx) =
2741        mpsc::channel::<RouteBindCompletion>(256);
2742    let (lossy_tx, mut lossy_rx) = mpsc::channel::<LossyPushEnvelope>(1024);
2743    let lossy_overflow = Arc::new(push::LossyOverflow::default());
2744    let lossy_seq = Arc::new(AtomicU64::new(0));
2745    let (reliable_tx, mut reliable_rx) = mpsc::unbounded_channel::<PushEnvelope>();
2746    let (fleet_status_client, fleet_status_task) =
2747        spawn_fleet_status_dial(connection_file_path, 64);
2748    let push_senders = PushSenders {
2749        lossy_tx,
2750        reliable_tx,
2751        lossy_overflow: Arc::clone(&lossy_overflow),
2752        lossy_seq,
2753        fleet_status_client: fleet_status_client.clone(),
2754    };
2755    let connection_cancel = PersistentCancelSignal::new();
2756    let mut installed_route_epochs: HashMap<u16, u32> = HashMap::new();
2757    let mut routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
2758    let mut bg_subs: HashMap<RouteChannel, BgSub> = HashMap::new();
2759    let mut bg_sub_by_session: BgSubsBySession = HashMap::new();
2760    let mut bg_wake_pending: HashSet<RouteChannel> = HashSet::new();
2761    let mut bg_wake_epoch: HashMap<(ProjectRootId, String), u64> = HashMap::new();
2762    let mut root_channels: HashMap<ProjectRootId, HashSet<RouteChannel>> = HashMap::new();
2763    let mut session_identity: HashMap<(ProjectRootId, String), RetainedSessionIdentity> =
2764        HashMap::new();
2765    let mut push_buffer: HashMap<push::ReplayKey, VecDeque<PushFrame>> = HashMap::new();
2766    let mut retry_buffer: RetryBuffer = HashMap::new();
2767    let mut reclaimed_routes = ReclaimedRoutes::default();
2768    let mut completed_tasks = push::CompletedTaskIds::default();
2769    let mut live_roots: HashMap<ProjectRootId, RootMeta> = HashMap::new();
2770    let mut pending_binds: HashMap<RouteChannel, PendingBind> = HashMap::new();
2771    let mut pending_bash_asks: HashMap<ReverseCorrKey, PendingBashAsk> = HashMap::new();
2772    let mut next_bash_ask_corr: u64 = 1;
2773    let mut route_bash_cancels: HashMap<RouteChannel, bash::RouteBashCancel> = HashMap::new();
2774    let active_tool_calls: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
2775    let pending_deferred_setups = Arc::new(AtomicUsize::new(0));
2776    let mut pending_responses = PendingSubcResponses::default();
2777    let health_rollup_cache = HealthRollupCache::new();
2778    health_rollup_cache.refresh(&executor, &shared_app);
2779    let mut next_health_rollup_at = tokio::time::Instant::now() + HEALTH_ROLLUP_TTL;
2780
2781    let loop_result: Result<ModuleLoopExit, SubcError> = 'module_loop: loop {
2782        shared_app.set_open_route_count(routes.len());
2783        if tokio::time::Instant::now() >= next_health_rollup_at {
2784            health_rollup_cache.refresh(&executor, &shared_app);
2785            next_health_rollup_at = tokio::time::Instant::now() + HEALTH_ROLLUP_TTL;
2786        }
2787        crate::logging::perf_tick(Some(&executor));
2788        dispatch_path_metrics.mark_frame_loop_tick();
2789        let ready_inspects = pending_responses.poll_ready(executor.as_ref());
2790        for resolved in ready_inspects {
2791            if let Err(error) = deliver_resolved_subc_response(
2792                &writer_tx,
2793                resolved,
2794                &routes,
2795                &mut live_roots,
2796                executor.as_ref(),
2797                &active_tool_calls,
2798                &shutdown,
2799                &dispatch_path_metrics,
2800                tool_response_body_limit,
2801            )
2802            .await
2803            {
2804                break 'module_loop Err(error);
2805            }
2806        }
2807        if let Err(error) = expire_pending_bash_asks(
2808            &writer_tx,
2809            &mut pending_bash_asks,
2810            &routes,
2811            &mut live_roots,
2812            &mut route_bash_cancels,
2813            &shutdown,
2814            &dispatch_path_metrics,
2815        )
2816        .await
2817        {
2818            break Err(error);
2819        }
2820
2821        // RouteBind completions are control-plane unblockers. Drain any completed
2822        // binds before entering other branch work so Push and maintenance bursts
2823        // can only add one loop-turn of latency.
2824        match drain_pending_route_bind_completions(
2825            &mut control_completion_rx,
2826            &writer_tx,
2827            &mut routes,
2828            &mut root_channels,
2829            &mut session_identity,
2830            &mut push_buffer,
2831            &mut live_roots,
2832            &mut pending_binds,
2833            &mut installed_route_epochs,
2834            &executor,
2835            &standing_actor,
2836            &shutdown,
2837            &dispatch_path_metrics,
2838        )
2839        .await
2840        {
2841            Ok(drained) => {
2842                if drained > 0 {
2843                    next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2844                    health_rollup_cache.refresh(&executor, &shared_app);
2845                    next_health_rollup_at = tokio::time::Instant::now() + HEALTH_ROLLUP_TTL;
2846                }
2847            }
2848            Err(error) => break Err(error),
2849        }
2850
2851        if tokio::time::Instant::now() >= next_drain_at {
2852            push::emit_bg_event_wakes(
2853                &writer_tx,
2854                &dispatch_path_metrics,
2855                &bg_subs,
2856                &mut bg_wake_pending,
2857            );
2858            warn_slow_pending_binds(&mut pending_binds, &executor);
2859            warn_slow_running_interactive_jobs(&executor);
2860            if let Err(error) = expire_overdue_route_binds(
2861                &writer_tx,
2862                &executor,
2863                &mut pending_binds,
2864                &mut installed_route_epochs,
2865                &dispatch_path_metrics,
2866            )
2867            .await
2868            {
2869                break Err(error);
2870            }
2871
2872            let retried = push::drain_retry_buffers_for_bound_routes(
2873                &writer_tx,
2874                &dispatch_path_metrics,
2875                &routes,
2876                &mut retry_buffer,
2877            );
2878            if retried > 0 {
2879                log::debug!(
2880                    "subc attach: retried {retried} reliable Push frame(s) after writer backpressure"
2881                );
2882            }
2883
2884            next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2885        }
2886
2887        // A lossy emitter may place its newest update in the overflow buffer
2888        // when the bounded channel is full, while this receive loop is draining
2889        // the channel. Drain overflow before selecting again so that raced
2890        // update is delivered on the next timer tick instead of waiting for
2891        // another lossy enqueue.
2892        let overflow_batch = lossy_overflow.drain();
2893        if !overflow_batch.is_empty() {
2894            let (_, deferred) = push::drain_reliable_push_turn(
2895                &writer_tx,
2896                &dispatch_path_metrics,
2897                &routes,
2898                &root_channels,
2899                &session_identity,
2900                &mut retry_buffer,
2901                &mut push_buffer,
2902                &mut completed_tasks,
2903                &bg_sub_by_session,
2904                &mut bg_wake_pending,
2905                &mut bg_wake_epoch,
2906                &mut reliable_rx,
2907                None,
2908            );
2909            if deferred {
2910                tokio::task::yield_now().await;
2911            }
2912
2913            let mut batch = Vec::new();
2914            while let Ok(item) = lossy_rx.try_recv() {
2915                batch.push(item);
2916            }
2917            batch.extend(overflow_batch);
2918            push::process_lossy_push_envelope_batch(
2919                &writer_tx,
2920                &dispatch_path_metrics,
2921                &routes,
2922                &root_channels,
2923                &completed_tasks,
2924                batch,
2925            );
2926        }
2927
2928        tokio::select! {
2929            biased;
2930            Some(completion) = control_completion_rx.recv() => {
2931                if let Err(error) = process_route_bind_completion(
2932                    &writer_tx,
2933                    completion,
2934                    &mut routes,
2935                    &mut root_channels,
2936                    &mut session_identity,
2937                    &mut push_buffer,
2938                    &mut live_roots,
2939                    &mut pending_binds,
2940                    &mut installed_route_epochs,
2941                    &executor,
2942                    &standing_actor,
2943                    &shutdown,
2944                    &dispatch_path_metrics,
2945                )
2946                .await
2947                {
2948                    break Err(error);
2949                }
2950                next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2951                next_health_rollup_at = tokio::time::Instant::now();
2952            }
2953            _ = shutdown.notified() => {
2954                log::warn!("subc attach: fatal executor response requested teardown");
2955                break Ok(ModuleLoopExit::SkipSearchFlush);
2956            }
2957            maybe_frame = reader_rx.recv() => {
2958                let frame = match maybe_frame {
2959                    None => {
2960                        log::info!("subc attach: daemon closed connection");
2961                        break Ok(ModuleLoopExit::Graceful);
2962                    }
2963                    Some(Err(error)) => break Err(error),
2964                    Some(Ok(frame)) => frame,
2965                };
2966                let phase_trace = frame.phase_trace;
2967                let frame = frame.frame;
2968
2969                if !ingress_route_should_be_processed(
2970                    &installed_route_epochs,
2971                    &reclaimed_routes,
2972                    &frame,
2973                ) {
2974                    log::debug!(
2975                        "subc attach: silently dropping {:?} for uninstalled route {}@{}",
2976                        frame.header.ty,
2977                        frame.header.channel,
2978                        frame.header.epoch
2979                    );
2980                    continue;
2981                }
2982
2983                match frame.header.ty {
2984                    FrameType::Ping if frame.header.channel == 0 => {
2985                        let pong = match Frame::build_with_version(
2986                            frame.header.ver,
2987                            FrameType::Pong,
2988                            frame.header.flags,
2989                            0,
2990                            0,
2991                            frame.header.corr,
2992                            Vec::new(),
2993                        ) {
2994                            Ok(pong) => pong,
2995                            Err(error) => break Err(SubcError::FrameBuild(error)),
2996                        };
2997                        if let Err(error) = send_frame(&writer_tx, &dispatch_path_metrics, pong).await {
2998                            break Err(error);
2999                        }
3000                    }
3001                    FrameType::Goodbye if frame.header.channel == 0 => {
3002                        log::info!("subc attach: received channel-0 Goodbye");
3003                        break Ok(ModuleLoopExit::Graceful);
3004                    }
3005                    FrameType::Goodbye => {
3006                        let channel = route_key(frame.header.channel, frame.header.epoch);
3007                        if let Err(error) = teardown_installed_route(
3008                            &writer_tx,
3009                            &dispatch_path_metrics,
3010                            &executor,
3011                            channel,
3012                            "Goodbye",
3013                            None,
3014                            &mut installed_route_epochs,
3015                            &mut routes,
3016                            &mut root_channels,
3017                            &mut bg_subs,
3018                            &mut bg_sub_by_session,
3019                            &mut bg_wake_pending,
3020                            &mut pending_bash_asks,
3021                            &mut live_roots,
3022                            &mut route_bash_cancels,
3023                            &active_tool_calls,
3024                            &mut pending_responses,
3025                            &mut pending_binds,
3026                            &mut retry_buffer,
3027                            &mut push_buffer,
3028                            &shutdown,
3029                            tool_response_body_limit,
3030                        )
3031                        .await
3032                        {
3033                            break Err(error);
3034                        }
3035                    }
3036                    FrameType::Response | FrameType::Error if frame.header.channel != 0 => {
3037                        if let Err(error) = handle_bash_elicitation_reply(
3038                            &writer_tx,
3039                            &frame,
3040                            &mut pending_bash_asks,
3041                            &routes,
3042                            &mut live_roots,
3043                            &executor,
3044                            &shutdown,
3045                            &bash_deferred_tx,
3046                            &bash_poll_touch_tx,
3047                            &dispatch_path_metrics,
3048                            &mut route_bash_cancels,
3049                            dispatch,
3050                        )
3051                        .await
3052                        {
3053                            break Err(error);
3054                        }
3055                    }
3056                    FrameType::Request if frame.header.channel == 0 => {
3057                        if let Err(error) = handle_control_request(
3058                            &writer_tx,
3059                            &frame,
3060                            &shared_app,
3061                            &executor,
3062                            &mut live_roots,
3063                            &mut pending_binds,
3064                            &mut installed_route_epochs,
3065                            &mut routes,
3066                            &mut root_channels,
3067                            &mut bg_subs,
3068                            &mut bg_sub_by_session,
3069                            &mut bg_wake_pending,
3070                            &mut pending_bash_asks,
3071                            &mut route_bash_cancels,
3072                            &active_tool_calls,
3073                            &mut pending_responses,
3074                            &mut retry_buffer,
3075                            &mut push_buffer,
3076                            &shutdown,
3077                            &control_completion_tx,
3078                            &dispatch_path_metrics,
3079                            &health_rollup_cache,
3080                            &push_senders,
3081                            dispatch,
3082                            user_config_path.as_deref(),
3083                            tool_response_body_limit,
3084                        )
3085                        .await
3086                        {
3087                            break Err(error);
3088                        }
3089                    }
3090                    FrameType::Request => {
3091                        if let Err(error) = handle_tool_call(
3092                            &writer_tx,
3093                            &frame,
3094                            phase_trace,
3095                            &routes,
3096                            &pending_binds,
3097                            &mut live_roots,
3098                            &executor,
3099                            &active_tool_calls,
3100                            &pending_deferred_setups,
3101                            &shutdown,
3102                            &connection_cancel,
3103                            &bash_deferred_tx,
3104                            &bash_poll_touch_tx,
3105                            &dispatch_path_metrics,
3106                            &mut route_bash_cancels,
3107                            &mut pending_bash_asks,
3108                            &mut next_bash_ask_corr,
3109                            &mut bg_subs,
3110                            &mut bg_sub_by_session,
3111                            &mut bg_wake_pending,
3112                            &mut bg_wake_epoch,
3113                            dispatch,
3114                            &deferred_response_tx,
3115                            allow_native_passthrough,
3116                            tool_response_body_limit,
3117                        )
3118                        .await
3119                        {
3120                            break Err(error);
3121                        }
3122                    }
3123                    FrameType::Cancel => {
3124                        let channel = route_key(frame.header.channel, frame.header.epoch);
3125                        cancel_active_tool_call(
3126                            &active_tool_calls,
3127                            executor.as_ref(),
3128                            channel,
3129                            frame.header.corr,
3130                            "Cancel frame",
3131                        );
3132                        pending_responses.cancel_request(channel, frame.header.corr);
3133                        if bg_subs.contains_key(&channel) {
3134                            if let Err(error) = end_bg_subscription(
3135                                &writer_tx,
3136                                &dispatch_path_metrics,
3137                                &mut bg_subs,
3138                                &mut bg_sub_by_session,
3139                                &mut bg_wake_pending,
3140                                channel,
3141                                routes.get(&channel),
3142                                "cancel",
3143                            )
3144                            .await
3145                            {
3146                                break Err(error);
3147                            }
3148                        }
3149                        if let Err(error) = cancel_pending_bash_ask_for_tool_call(
3150                            &writer_tx,
3151                            &mut pending_bash_asks,
3152                            channel,
3153                            frame.header.corr,
3154                            &routes,
3155                            &mut live_roots,
3156                            &mut route_bash_cancels,
3157                            &shutdown,
3158                            &dispatch_path_metrics,
3159                        )
3160                        .await
3161                        {
3162                            break Err(error);
3163                        }
3164                    }
3165                    // Incoming push messages are ignored here. Cancel frames are
3166                    // handled above for active and deferred tool calls plus pending
3167                    // bash elicitation requests.
3168                    _ => {}
3169                }
3170            }
3171            Some(pending) = deferred_response_rx.recv() => {
3172                if routes.contains_key(&pending.route)
3173                    && active_tool_call_is_registered(
3174                        &active_tool_calls,
3175                        pending.route,
3176                        pending.corr,
3177                    )
3178                {
3179                    pending_responses.register(pending);
3180                } else {
3181                    if let Some(cancellation) = &pending.pending.cancellation {
3182                        cancellation.request_cancel();
3183                    }
3184                    finish_active_tool_call(&active_tool_calls, pending.route, pending.corr);
3185                }
3186            }
3187            Some((root_id, frame)) = reliable_rx.recv() => {
3188                // Reliable Push frames are FIFO and must-deliver, but draining an
3189                // unbounded burst in one current-thread turn can starve RouteBind
3190                // completions. The budget defers excess frames, never drops them.
3191                let (_, deferred) = push::drain_reliable_push_turn(
3192                    &writer_tx,
3193                    &dispatch_path_metrics,
3194                    &routes,
3195                    &root_channels,
3196                    &session_identity,
3197                    &mut retry_buffer,
3198                    &mut push_buffer,
3199                    &mut completed_tasks,
3200                    &bg_sub_by_session,
3201                    &mut bg_wake_pending,
3202                    &mut bg_wake_epoch,
3203                    &mut reliable_rx,
3204                    Some((root_id, frame)),
3205                );
3206                if deferred {
3207                    tokio::task::yield_now().await;
3208                }
3209            }
3210            Some((order, root_id, frame)) = lossy_rx.recv() => {
3211                // When both push lanes have work, handle a small reliable slice before lossy work.
3212                // That ordering lets completed task ids suppress stale BashLongRunning frames.
3213                // The slice stays bounded so reliable bursts cannot monopolize this loop turn.
3214                let (_, deferred) = push::drain_reliable_push_turn(
3215                    &writer_tx,
3216                    &dispatch_path_metrics,
3217                    &routes,
3218                    &root_channels,
3219                    &session_identity,
3220                    &mut retry_buffer,
3221                    &mut push_buffer,
3222                    &mut completed_tasks,
3223                    &bg_sub_by_session,
3224                    &mut bg_wake_pending,
3225                    &mut bg_wake_epoch,
3226                    &mut reliable_rx,
3227                    None,
3228                );
3229                if deferred {
3230                    tokio::task::yield_now().await;
3231                }
3232
3233                // Drain the currently queued burst in one loop turn so lossy
3234                // status/progress updates can be merged before reaching subc's
3235                // shared egress queue. Each lossy frame gets a sequence number
3236                // before it goes to the channel or overflow buffer, so the
3237                // combined batch is sorted back into producer order before
3238                // coalescing drops stale updates for the same key.
3239                let mut batch = vec![(order, root_id, frame)];
3240                while let Ok(item) = lossy_rx.try_recv() {
3241                    batch.push(item);
3242                }
3243                batch.extend(lossy_overflow.drain());
3244                push::process_lossy_push_envelope_batch(
3245                    &writer_tx,
3246                    &dispatch_path_metrics,
3247                    &routes,
3248                    &root_channels,
3249                    &completed_tasks,
3250                    batch,
3251                );
3252            }
3253            Some(done) = bash_deferred_rx.recv() => {
3254                decrement_counted_channel(&dispatch_path_metrics.bash_deferred_queued);
3255                if let Err(error) = bash::handle_bash_deferred_completion(
3256                    &writer_tx,
3257                    done,
3258                    &routes,
3259                    &mut live_roots,
3260                    &mut route_bash_cancels,
3261                    &shutdown,
3262                    &dispatch_path_metrics,
3263                )
3264                .await
3265                {
3266                    break Err(error);
3267                }
3268            }
3269            Some(root_id) = bash_poll_touch_rx.recv() => {
3270                decrement_counted_channel(&dispatch_path_metrics.bash_poll_touch_queued);
3271                if let Some(meta) = live_roots.get_mut(&root_id) {
3272                    meta.note_activity();
3273                }
3274            }
3275            Some(completion) = maintenance_rx.recv() => {
3276                decrement_counted_channel(&dispatch_path_metrics.maintenance_queued);
3277                let root_id = completion.root_id.clone();
3278                let response = completion.response;
3279                let response_is_fatal = response_is_fatal_panic(&response);
3280                let bind_pending = pending_binds
3281                    .values()
3282                    .any(|pending| pending.bind_root_id == root_id);
3283                let requiesce = if let Some(meta) = live_roots.get_mut(&root_id) {
3284                    let defer_requeue = meta.unbound_quiesced || bind_pending;
3285                    note_maintenance_completion(
3286                        meta,
3287                        completion.requeue_kind,
3288                        response_is_fatal,
3289                        defer_requeue,
3290                    );
3291                    should_requiesce_after_maintenance(meta, completion.kind, bind_pending)
3292                } else {
3293                    false
3294                };
3295                if requiesce {
3296                    quiesce_unbound_root(&root_id, &mut live_roots, &executor);
3297                }
3298                push::clear_stale_bg_wakes_for_empty_sessions(
3299                    &root_id,
3300                    &completion.empty_bg_sessions,
3301                    &bg_sub_by_session,
3302                    &mut bg_wake_pending,
3303                    &bg_wake_epoch,
3304                );
3305                if response_is_fatal {
3306                    if let Some(meta) = live_roots.get_mut(&root_id) {
3307                        meta.maintenance_poisoned = true;
3308                    }
3309                    log::warn!(
3310                        "subc attach: maintenance drain observed a fatal actor; deferring teardown until a route request can receive actor_fatal"
3311                    );
3312                }
3313            }
3314            _ = tokio::time::sleep(PENDING_POLL_INTERVAL), if !pending_responses.is_empty() => {
3315                // The next loop turn polls detached inspect completions. Keeping
3316                // the timer here lets already-ready control frames run first.
3317            }
3318            _ = tokio::time::sleep_until(next_drain_at) => {
3319                // Wakes an otherwise-idle loop so the pre-turn drain check
3320                // above runs on schedule; the drain work itself lives there.
3321            }
3322            _ = tokio::time::sleep_until(next_maintenance_at) => {
3323                // Delay cache-draining maintenance until any already-ready
3324                // inbound route/control messages and push completions have run,
3325                // so maintenance does not block the actor from handling the
3326                // first request that arrives after a route bind is acknowledged.
3327                crate::logging::maybe_sweep_logs();
3328                let reaped_lsp_children = shared_app
3329                    .lsp_child_registry()
3330                    .reap_children_with_gone_cwd_or_reclaimed_root();
3331                if reaped_lsp_children > 0 {
3332                    log::warn!(
3333                        "subc attach: reaped {reaped_lsp_children} LSP child process group(s) with a deleted cwd or reclaimed root"
3334                    );
3335                }
3336                let reap = reap_idle_roots(
3337                    Instant::now(),
3338                    &mut live_roots,
3339                    &pending_binds,
3340                    &root_channels,
3341                    &executor,
3342                    &dispatch_path_metrics,
3343                );
3344                for root_id in &reap.forgotten_deleted_roots {
3345                    purge_deleted_root_residents(
3346                        root_id,
3347                        &mut routes,
3348                        &mut root_channels,
3349                        &mut installed_route_epochs,
3350                        &mut route_bash_cancels,
3351                        &active_tool_calls,
3352                        executor.as_ref(),
3353                        &mut retry_buffer,
3354                        &mut reclaimed_routes,
3355                        &mut session_identity,
3356                        &mut push_buffer,
3357                        &mut bg_subs,
3358                        &mut bg_sub_by_session,
3359                        &mut bg_wake_pending,
3360                        &mut bg_wake_epoch,
3361                        &mut pending_bash_asks,
3362                        &dispatch_path_metrics,
3363                    );
3364                }
3365                if reap.evicted > 0 {
3366                    log::debug!("subc attach: reaped {} idle root(s)", reap.evicted);
3367                }
3368                submit_due_maintenance_jobs(
3369                    &executor,
3370                    &mut live_roots,
3371                    &pending_binds,
3372                    &bg_sub_by_session,
3373                    &bg_wake_pending,
3374                    &bg_wake_epoch,
3375                    &maintenance_tx,
3376                    &dispatch_path_metrics,
3377                );
3378                if tokio::time::Instant::now() >= next_standing_pass_at {
3379                    standing_actor.tick();
3380                    next_standing_pass_at = tokio::time::Instant::now()
3381                        + standing::STANDING_MAINTENANCE_INTERVAL;
3382                }
3383                // Opportunistic allocator relief, independent of the idle
3384                // sweep: the sweep's whole-process idle gate never opens while
3385                // any session stays active, which let freed warm-up arenas sit
3386                // resident for the process lifetime (5.1 GB RSS over ~600 MB
3387                // live). Slack threshold + spacing live in memory.rs; the pass
3388                // itself runs on a detached thread.
3389                #[cfg(any(target_os = "macos", target_os = "linux"))]
3390                {
3391                    let now_std = std::time::Instant::now();
3392                    if crate::memory::spawn_allocator_slack_relief_if_due(
3393                        last_slack_relief,
3394                        now_std,
3395                    ) {
3396                        last_slack_relief = Some(now_std);
3397                    }
3398                }
3399                next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
3400            }
3401        }
3402    };
3403
3404    shared_app.set_open_route_count(0);
3405
3406    connection_cancel.cancel();
3407    cancel_all_active_tool_calls(&active_tool_calls, executor.as_ref(), "connection teardown");
3408    let setup_drain_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
3409    while pending_deferred_setups.load(Ordering::SeqCst) != 0
3410        && tokio::time::Instant::now() < setup_drain_deadline
3411    {
3412        tokio::select! {
3413            biased;
3414            Some(pending) = deferred_response_rx.recv() => pending_responses.register(pending),
3415            _ = tokio::time::sleep(Duration::from_millis(5)) => {}
3416        }
3417    }
3418    if pending_deferred_setups.load(Ordering::SeqCst) != 0 {
3419        log::warn!(
3420            "subc attach: timed out waiting for deferred response setup registration during shutdown"
3421        );
3422    }
3423    while let Ok(pending) = deferred_response_rx.try_recv() {
3424        pending_responses.register(pending);
3425    }
3426    for resolved in pending_responses.drain_on_shutdown(executor.as_ref()) {
3427        if let Err(error) = deliver_resolved_subc_response(
3428            &writer_tx,
3429            resolved,
3430            &routes,
3431            &mut live_roots,
3432            executor.as_ref(),
3433            &active_tool_calls,
3434            &shutdown,
3435            &dispatch_path_metrics,
3436            tool_response_body_limit,
3437        )
3438        .await
3439        {
3440            log::warn!("subc attach: failed to emit deferred shutdown terminal: {error}");
3441        }
3442    }
3443    // Channel-0 Goodbye, EOF, and fatal exits bypass per-route Goodbye. Settle
3444    // their root lifecycle state before loop-owned routing metadata is dropped.
3445    quiesce_connection_roots(
3446        &mut live_roots,
3447        &mut pending_binds,
3448        &mut routes,
3449        &mut root_channels,
3450        &mut installed_route_epochs,
3451        &mut route_bash_cancels,
3452        &active_tool_calls,
3453        &executor,
3454    );
3455
3456    fleet_status_client.set_route_live(false);
3457    fleet_status_task.abort();
3458    let _ = fleet_status_task.await;
3459
3460    let mut loop_result = loop_result;
3461    if !pending_bash_asks.is_empty() {
3462        let no_routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
3463        if let Err(error) = settle_all_pending_bash_asks(
3464            &writer_tx,
3465            &mut pending_bash_asks,
3466            &no_routes,
3467            &mut live_roots,
3468            &mut route_bash_cancels,
3469            &shutdown,
3470            &dispatch_path_metrics,
3471        )
3472        .await
3473        {
3474            loop_result = loop_result.and(Err(error));
3475        }
3476    }
3477
3478    // The reader task may be parked on `read_frame`; abort it (we are done with
3479    // the connection) and flush the writer.
3480    reader_task.abort();
3481    drop(writer_tx);
3482    let writer_result = finish_writer_task(writer_task).await;
3483    loop_result.and_then(|exit| writer_result.map(|_| exit))
3484}
3485
3486fn spawn_writer_task<W>(
3487    mut write: W,
3488    mut rx: mpsc::Receiver<WriterFrame>,
3489    metrics: Arc<DispatchPathMetrics>,
3490) -> JoinHandle<Result<(), subc_transport::FrameIoError>>
3491where
3492    W: AsyncWrite + Unpin + Send + 'static,
3493{
3494    tokio::spawn(async move {
3495        let mut write_buffer = Vec::new();
3496        while let Some(mut queued) = rx.recv().await {
3497            let measure = queued.tool_response_trace.is_some();
3498            let dequeued = measure.then(Instant::now);
3499            metrics.writer_active.store(true, Ordering::Relaxed);
3500            decrement_counted_channel(&metrics.writer_queued);
3501            let write_timing = write_frame_contiguous(
3502                &mut write,
3503                queued.frame(),
3504                queued.body(),
3505                &mut write_buffer,
3506                measure,
3507            )
3508            .await;
3509            metrics.writer_active.store(false, Ordering::Relaxed);
3510            let write_timing = write_timing?;
3511
3512            if let (Some(trace), Some(dequeued), Some(write_timing)) =
3513                (queued.tool_response_trace.take(), dequeued, write_timing)
3514            {
3515                if let Some(completed) = trace.finish(
3516                    dequeued,
3517                    write_timing.write_started,
3518                    write_timing.write_finished,
3519                    write_timing.frame_bytes,
3520                ) {
3521                    log_ctx::with_session(Some(completed.session), || {
3522                        crate::logging::note_tool_call_trace(
3523                            &completed.name,
3524                            &completed.root,
3525                            completed.channel,
3526                            completed.corr,
3527                            completed.phases,
3528                        );
3529                    });
3530                }
3531            }
3532        }
3533        Ok(())
3534    })
3535}
3536
3537struct FrameWriteTiming {
3538    write_started: Instant,
3539    write_finished: Instant,
3540    frame_bytes: usize,
3541}
3542
3543/// Encode one complete frame into the existing reusable buffer and write it
3544/// without interleaving bytes from another channel. Timing is collected only
3545/// for tool responses, so Push and control frames add no clock reads.
3546async fn write_frame_contiguous<W>(
3547    writer: &mut W,
3548    frame: &Frame,
3549    body: &[u8],
3550    buffer: &mut Vec<u8>,
3551    measure: bool,
3552) -> Result<Option<FrameWriteTiming>, subc_transport::FrameIoError>
3553where
3554    W: AsyncWrite + Unpin,
3555{
3556    if frame.header.len as usize != body.len() {
3557        return Err(subc_transport::FrameIoError::BodyLengthMismatch {
3558            header_len: frame.header.len,
3559            body_len: body.len(),
3560        });
3561    }
3562
3563    let header = frame.header.encode();
3564    buffer.clear();
3565    buffer.reserve(header.len() + body.len());
3566    buffer.extend_from_slice(&header);
3567    buffer.extend_from_slice(body);
3568    let write_started = measure.then(Instant::now);
3569    writer
3570        .write_all(buffer)
3571        .await
3572        .map_err(subc_transport::FrameIoError::Io)?;
3573    Ok(write_started.map(|write_started| FrameWriteTiming {
3574        write_started,
3575        write_finished: Instant::now(),
3576        frame_bytes: buffer.len(),
3577    }))
3578}
3579
3580fn spawn_reader_task<R>(
3581    mut read: R,
3582    tx: mpsc::Sender<Result<DecodedFrame, SubcError>>,
3583) -> JoinHandle<()>
3584where
3585    R: AsyncRead + Unpin + Send + 'static,
3586{
3587    tokio::spawn(async move {
3588        loop {
3589            match read_frame(&mut read).await {
3590                Ok(Some(frame)) => {
3591                    let decoded = DecodedFrame {
3592                        frame,
3593                        phase_trace: PhaseTrace::new(Instant::now()),
3594                    };
3595                    if tx.send(Ok(decoded)).await.is_err() {
3596                        return;
3597                    }
3598                }
3599                Ok(None) => {
3600                    // EOF: let the loop observe channel close as "daemon closed".
3601                    return;
3602                }
3603                Err(error) => {
3604                    // A killed daemon surfaces as ConnectionReset (RST) on
3605                    // Windows where Unix delivers a clean EOF (FIN); a
3606                    // mid-teardown daemon can also abort the socket. Both mean
3607                    // "daemon went away", not a wire fault — normalize them to
3608                    // the clean-close path so module exit behavior matches
3609                    // across platforms (same class subc-core fixed in d33d9a71).
3610                    if let subc_transport::FrameIoError::Io(io_error) = &error {
3611                        if matches!(
3612                            io_error.kind(),
3613                            std::io::ErrorKind::ConnectionReset
3614                                | std::io::ErrorKind::ConnectionAborted
3615                        ) {
3616                            log::info!(
3617                                "subc attach: connection reset by daemon; treating as close"
3618                            );
3619                            return;
3620                        }
3621                    }
3622                    let _ = tx.send(Err(SubcError::FrameIo(error))).await;
3623                    return;
3624                }
3625            }
3626        }
3627    })
3628}
3629
3630async fn finish_writer_task(
3631    mut writer_task: JoinHandle<Result<(), subc_transport::FrameIoError>>,
3632) -> Result<(), SubcError> {
3633    match tokio::time::timeout(Duration::from_millis(100), &mut writer_task).await {
3634        Ok(Ok(Ok(()))) => Ok(()),
3635        Ok(Ok(Err(error))) => Err(SubcError::FrameIo(error)),
3636        Ok(Err(error)) => Err(SubcError::WriterJoin(error)),
3637        Err(_) => {
3638            writer_task.abort();
3639            Ok(())
3640        }
3641    }
3642}
3643
3644fn register_actor_for_bind(
3645    shared_app: &Arc<App>,
3646    executor: &Arc<Executor>,
3647    push_senders: &PushSenders,
3648    bind_root_id: &ProjectRootId,
3649    route_channel: u16,
3650    root_was_live: bool,
3651) -> bool {
3652    if executor.actor_registered(bind_root_id) {
3653        log::debug!(
3654            "subc attach: reusing actor for route {} root {}",
3655            route_channel,
3656            bind_root_id.as_path().display()
3657        );
3658        return false;
3659    }
3660
3661    if root_was_live {
3662        log::warn!(
3663            "subc attach: recreating missing actor for live root {} on route {}",
3664            bind_root_id.as_path().display(),
3665            route_channel
3666        );
3667    }
3668
3669    let actor_ctx = Arc::new(AppContext::from_app(
3670        Arc::clone(shared_app),
3671        Config::default(),
3672    ));
3673    install_bash_compressor(&actor_ctx);
3674    actor_ctx.install_fleet_status_client(Some(push_senders.fleet_status_client.clone()));
3675    actor_ctx.set_progress_sender(Some(push::progress_sender_for_root(
3676        push_senders.clone(),
3677        bind_root_id.clone(),
3678    )));
3679    let inserted = executor.register_actor(bind_root_id.clone(), Arc::clone(&actor_ctx));
3680    drop(actor_ctx);
3681    if inserted {
3682        // Do not insert into live_roots until configure succeeds: live_roots
3683        // drives maintenance, and a half-configured new actor must not be
3684        // maintenance-eligible before its route/session identity exists.
3685        log::debug!(
3686            "subc attach: registered actor for route {} root {}",
3687            route_channel,
3688            bind_root_id.as_path().display()
3689        );
3690    } else {
3691        log::debug!(
3692            "subc attach: actor appeared while binding route {} root {}; reusing it",
3693            route_channel,
3694            bind_root_id.as_path().display()
3695        );
3696    }
3697    inserted
3698}
3699
3700fn rollback_pending_bind_actor(
3701    executor: &Arc<Executor>,
3702    live_roots: &HashMap<ProjectRootId, RootMeta>,
3703    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3704    root_id: &ProjectRootId,
3705    inserted_new_actor: bool,
3706) {
3707    if !inserted_new_actor || live_roots.contains_key(root_id) {
3708        return;
3709    }
3710
3711    if let Some((route, pending)) = pending_binds
3712        .iter_mut()
3713        .find(|(_, pending)| &pending.bind_root_id == root_id)
3714    {
3715        pending.inserted_new_actor = true;
3716        log::debug!(
3717            "subc attach: transferred rollback ownership for root {} to pending route {}",
3718            root_id.as_path().display(),
3719            route
3720        );
3721        return;
3722    }
3723
3724    executor.remove_actor(root_id);
3725}
3726
3727fn route_bind_error_code_for_configure_response(response: &Response) -> &'static str {
3728    match response.data.get("code").and_then(|code| code.as_str()) {
3729        // Preserve typed configure rejections across the bind boundary: a
3730        // malformed fed fingerprint means a federation-module bug or
3731        // fingerprint-format drift, and the fed side matches on the code rather
3732        // than parsing prose.
3733        Some("bad_harness_fingerprint") => "bad_harness_fingerprint",
3734        // Cache-key probe failures are transient (fd pressure, git spawn
3735        // contention); the client retries the bind rather than treating the
3736        // root as permanently divergent.
3737        Some("cache_key_probe_failed") => "cache_key_probe_failed",
3738        // Actor lifecycle gaps are transient from the daemon/client viewpoint:
3739        // a fresh bind can create or join a healthy actor, so do not classify
3740        // them as permanent config divergence.
3741        Some("actor_not_registered" | "actor_fatal") => "actor_not_ready",
3742        _ => "config_divergence",
3743    }
3744}
3745
3746fn queue_post_bind_configure_and_completion_maintenance(
3747    root_id: &ProjectRootId,
3748    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3749) {
3750    let Some(meta) = live_roots.get_mut(root_id) else {
3751        return;
3752    };
3753    if meta.maintenance_poisoned || meta.maintenance_pending {
3754        return;
3755    }
3756
3757    meta.maintenance_pending = true;
3758    meta.maintenance_queued_kinds
3759        .push_back(MaintenanceDrainKind::ConfigureTail);
3760    meta.maintenance_queued_kinds
3761        .push_back(MaintenanceDrainKind::CompletionDrains);
3762}
3763
3764#[allow(clippy::too_many_arguments)]
3765async fn handle_route_bind_completion(
3766    tx: &WriterSender,
3767    completion: RouteBindCompletion,
3768    routes: &mut HashMap<RouteChannel, RouteIdentity>,
3769    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
3770    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
3771    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
3772    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3773    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3774    installed_route_epochs: &mut HashMap<u16, u32>,
3775    executor: &Arc<Executor>,
3776    standing_actor: &standing::StandingActor,
3777    shutdown: &Arc<Notify>,
3778    metrics: &Arc<DispatchPathMetrics>,
3779) -> Result<(), SubcError> {
3780    let route_id = completion.route;
3781    let Some(pending) = pending_binds.remove(&route_id) else {
3782        log::warn!(
3783            "subc attach: dropping RouteBind completion for non-pending route {}",
3784            completion.route
3785        );
3786        rollback_pending_bind_actor(
3787            executor,
3788            live_roots,
3789            pending_binds,
3790            &completion.bind_root_id,
3791            completion.inserted_new_actor,
3792        );
3793        let has_pending_bind = pending_binds
3794            .values()
3795            .any(|pending| pending.bind_root_id == completion.bind_root_id);
3796        if !root_channels
3797            .get(&completion.bind_root_id)
3798            .is_some_and(|channels| !channels.is_empty())
3799            && !has_pending_bind
3800        {
3801            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3802        }
3803        remove_installed_route(installed_route_epochs, route_id);
3804        return Ok(());
3805    };
3806
3807    if pending.bind_root_id != completion.bind_root_id {
3808        log::warn!(
3809            "subc attach: pending RouteBind root mismatch for route {} (pending {} completion {})",
3810            completion.route,
3811            pending.bind_root_id.as_path().display(),
3812            completion.bind_root_id.as_path().display()
3813        );
3814    }
3815
3816    let inserted_new_actor = pending.inserted_new_actor || completion.inserted_new_actor;
3817    if pending.cancelled {
3818        rollback_pending_bind_actor(
3819            executor,
3820            live_roots,
3821            pending_binds,
3822            &completion.bind_root_id,
3823            inserted_new_actor,
3824        );
3825        let has_pending_bind = pending_binds
3826            .values()
3827            .any(|pending| pending.bind_root_id == completion.bind_root_id);
3828        if !root_channels
3829            .get(&completion.bind_root_id)
3830            .is_some_and(|channels| !channels.is_empty())
3831            && !has_pending_bind
3832        {
3833            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3834        }
3835        log::debug!(
3836            "subc attach: discarded completed RouteBind for cancelled route {} root {}",
3837            completion.route,
3838            completion.bind_root_id.as_path().display()
3839        );
3840        remove_installed_route(installed_route_epochs, route_id);
3841        return Ok(());
3842    }
3843
3844    let failure = if !completion.configure_response.success {
3845        Some((
3846            &completion.configure_response,
3847            "configure failed during route bind",
3848        ))
3849    } else {
3850        None
3851    };
3852
3853    if let Some((response, fallback)) = failure {
3854        rollback_pending_bind_actor(
3855            executor,
3856            live_roots,
3857            pending_binds,
3858            &completion.bind_root_id,
3859            inserted_new_actor,
3860        );
3861        let has_pending_bind = pending_binds
3862            .values()
3863            .any(|pending| pending.bind_root_id == completion.bind_root_id);
3864        if !root_channels
3865            .get(&completion.bind_root_id)
3866            .is_some_and(|channels| !channels.is_empty())
3867            && !has_pending_bind
3868        {
3869            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3870        }
3871        let message = response_message(response, fallback);
3872        let fatal = response_is_fatal_panic(response);
3873        let error_code = route_bind_error_code_for_configure_response(response);
3874        send_route_bind_error_parts(
3875            tx,
3876            completion.ver,
3877            completion.corr,
3878            completion.flags,
3879            error_code,
3880            &message,
3881            metrics,
3882        )
3883        .await?;
3884        remove_installed_route(installed_route_epochs, route_id);
3885        if fatal {
3886            signal_fatal_teardown(
3887                tx,
3888                Some(completion.route),
3889                completion.ver,
3890                completion.corr,
3891                shutdown,
3892                metrics,
3893            )
3894            .await;
3895        }
3896        return Ok(());
3897    }
3898
3899    remember_session_identity(session_identity, &completion.identity);
3900    let replay_key = push::ReplayKey::from_identity(&completion.identity);
3901    let bind_trust = completion.identity.trust;
3902    insert_route_channel(routes, root_channels, route_id, completion.identity);
3903    let restore_watcher = live_roots
3904        .get(&completion.bind_root_id)
3905        .is_some_and(|meta| meta.idle_artifacts_evicted || meta.unbound_quiesced);
3906    live_roots
3907        .entry(completion.bind_root_id.clone())
3908        .and_modify(|meta| {
3909            meta.reactivate_bound();
3910            meta.diagnostics_on_edit = completion.diagnostics_on_edit;
3911            meta.maintenance_poisoned = false;
3912        })
3913        .or_insert_with(|| RootMeta::new(Instant::now()));
3914    if let Some(meta) = live_roots.get_mut(&completion.bind_root_id) {
3915        meta.diagnostics_on_edit = completion.diagnostics_on_edit;
3916        meta.maintenance_poisoned = false;
3917    }
3918    if let Some(ctx) = executor.actor_context(&completion.bind_root_id) {
3919        // The bind transition revokes any matching unbound standing admission
3920        // before this session can select the shared artifact family.
3921        standing_actor.begin_session_bind(&ctx);
3922        ctx.mark_subc_bound();
3923        if restore_watcher {
3924            crate::commands::configure::ensure_project_watcher(&ctx);
3925        }
3926    }
3927
3928    let ack =
3929        serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).map_err(SubcError::Json)?;
3930    let response = Frame::build_with_version(
3931        completion.ver,
3932        FrameType::Response,
3933        control_flags(),
3934        0,
3935        0,
3936        completion.corr,
3937        ack,
3938    )
3939    .map_err(SubcError::FrameBuild)?;
3940    send_reliable_writer_frame(tx, metrics, response, "RouteBindAck").await?;
3941    queue_post_bind_configure_and_completion_maintenance(&completion.bind_root_id, live_roots);
3942    let replayed = push::replay_buffered_push_frames(
3943        tx,
3944        metrics,
3945        route_id,
3946        push_buffer,
3947        &replay_key,
3948        bind_trust,
3949    );
3950    if replayed > 0 {
3951        log::debug!(
3952            "subc attach: replayed {} buffered Push frame(s) to route {} root {} harness {} session {}",
3953            replayed,
3954            completion.route,
3955            replay_key.root.as_path().display(),
3956            replay_key.harness,
3957            replay_key.session
3958        );
3959    }
3960    log::info!(
3961        "subc attach: route {} bound to root {}",
3962        completion.route,
3963        completion.bind_root_id.as_path().display()
3964    );
3965    Ok(())
3966}
3967
3968async fn expire_overdue_route_binds(
3969    tx: &WriterSender,
3970    executor: &Arc<Executor>,
3971    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3972    installed_route_epochs: &mut HashMap<u16, u32>,
3973    metrics: &DispatchPathMetrics,
3974) -> Result<(), SubcError> {
3975    let now = Instant::now();
3976    let expired: Vec<_> = pending_binds
3977        .iter()
3978        .filter_map(|(route, pending)| {
3979            let age = now.saturating_duration_since(pending.started_at);
3980            (!pending.deadline_reported && age >= ROUTE_BIND_DEADLINE).then(|| {
3981                (
3982                    *route,
3983                    pending.corr,
3984                    pending.ver,
3985                    pending.flags,
3986                    pending.bind_root_id.clone(),
3987                    pending.configure_request_id.clone(),
3988                    age,
3989                )
3990            })
3991        })
3992        .collect();
3993
3994    for (route, corr, ver, flags, root_id, configure_request_id, age) in expired {
3995        if let Some(pending) = pending_binds.get_mut(&route) {
3996            pending.cancelled = true;
3997            pending.deadline_reported = true;
3998            let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
3999            log::debug!(
4000                "subc attach: cancelled overdue RouteBind configure for route {route} ({outcome:?})"
4001            );
4002        }
4003        remove_installed_route(installed_route_epochs, route);
4004        let age_ms = age.as_millis().min(u128::from(u64::MAX)) as u64;
4005        let deadline_ms = ROUTE_BIND_DEADLINE.as_millis();
4006        send_route_bind_error_parts(
4007            tx,
4008            ver,
4009            corr,
4010            flags,
4011            "actor_not_ready",
4012            &format!("route bind deadline exceeded after {age_ms}ms (deadline {deadline_ms}ms)"),
4013            metrics,
4014        )
4015        .await?;
4016        log::warn!(
4017            "subc attach: route {} bind for root {} exceeded {}ms deadline (configure_request_id={})",
4018            route,
4019            root_id.as_path().display(),
4020            deadline_ms,
4021            configure_request_id
4022        );
4023    }
4024
4025    Ok(())
4026}
4027
4028/// channel-0 control requests: RouteBind plus the cached health probe. RouteBind
4029/// still reconciles the route's RootConfig through the executor's Mutating lane
4030/// and resolves completion on a loop-owned control-completion channel so slow
4031/// configure jobs do not block the transport loop.
4032#[allow(clippy::too_many_arguments)]
4033async fn handle_control_request(
4034    tx: &WriterSender,
4035    frame: &Frame,
4036    shared_app: &Arc<App>,
4037    executor: &Arc<Executor>,
4038    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4039    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
4040    installed_route_epochs: &mut HashMap<u16, u32>,
4041    routes: &mut HashMap<RouteChannel, RouteIdentity>,
4042    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
4043    bg_subs: &mut HashMap<RouteChannel, BgSub>,
4044    bg_sub_by_session: &mut BgSubsBySession,
4045    bg_wake_pending: &mut HashSet<RouteChannel>,
4046    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
4047    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
4048    active_tool_calls: &ActiveToolCalls,
4049    pending_responses: &mut PendingSubcResponses,
4050    retry_buffer: &mut RetryBuffer,
4051    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
4052    shutdown: &Arc<Notify>,
4053    control_completion_tx: &mpsc::Sender<RouteBindCompletion>,
4054    metrics: &Arc<DispatchPathMetrics>,
4055    health_rollup_cache: &HealthRollupCache,
4056    push_senders: &PushSenders,
4057    dispatch: DispatchFn,
4058    user_config_path: Option<&Path>,
4059    tool_response_body_limit: usize,
4060) -> Result<(), SubcError> {
4061    let request =
4062        serde_json::from_slice::<ModuleControlRequest>(&frame.body).map_err(SubcError::Json)?;
4063    match request {
4064        ModuleControlRequest::RouteBind {
4065            route_channel,
4066            epoch,
4067            target: _,
4068            identity,
4069            principal,
4070            consumer_capabilities,
4071            admission_facts: _,
4072        } => {
4073            let route_id = route_key(route_channel, epoch);
4074            if epoch == 0 {
4075                return send_route_bind_error(
4076                    tx,
4077                    frame,
4078                    "config_divergence",
4079                    "route bind uses an invalid channel generation",
4080                    metrics,
4081                )
4082                .await;
4083            }
4084            let mut bind_root_id = None;
4085            if let Some(installed_epoch) = installed_route_epochs.get(&route_channel).copied() {
4086                if installed_epoch >= epoch {
4087                    return send_route_bind_error(
4088                        tx,
4089                        frame,
4090                        "config_divergence",
4091                        "route bind generation is not newer than the installed generation",
4092                        metrics,
4093                    )
4094                    .await;
4095                }
4096
4097                let replacement_root = match ProjectRootId::from_path(&identity.project_root) {
4098                    Ok(root_id) => root_id,
4099                    Err(error) => {
4100                        return send_route_bind_error(
4101                            tx,
4102                            frame,
4103                            "config_divergence",
4104                            &format!("invalid route project root: {error}"),
4105                            metrics,
4106                        )
4107                        .await;
4108                    }
4109                };
4110                teardown_installed_route(
4111                    tx,
4112                    metrics,
4113                    executor,
4114                    route_key(route_channel, installed_epoch),
4115                    "higher-epoch RouteBind",
4116                    Some(&replacement_root),
4117                    installed_route_epochs,
4118                    routes,
4119                    root_channels,
4120                    bg_subs,
4121                    bg_sub_by_session,
4122                    bg_wake_pending,
4123                    pending_bash_asks,
4124                    live_roots,
4125                    route_bash_cancels,
4126                    active_tool_calls,
4127                    pending_responses,
4128                    pending_binds,
4129                    retry_buffer,
4130                    push_buffer,
4131                    shutdown,
4132                    tool_response_body_limit,
4133                )
4134                .await?;
4135                bind_root_id = Some(replacement_root);
4136            }
4137            if pending_binds.contains_key(&route_id) {
4138                return send_route_bind_error(
4139                    tx,
4140                    frame,
4141                    "config_divergence",
4142                    "route bind is already pending for channel",
4143                    metrics,
4144                )
4145                .await;
4146            }
4147            let bind_root_id = match bind_root_id {
4148                Some(root_id) => root_id,
4149                None => match ProjectRootId::from_path(&identity.project_root) {
4150                    Ok(root_id) => root_id,
4151                    Err(error) => {
4152                        return send_route_bind_error(
4153                            tx,
4154                            frame,
4155                            "config_divergence",
4156                            &format!("invalid route project root: {error}"),
4157                            metrics,
4158                        )
4159                        .await;
4160                    }
4161                },
4162            };
4163
4164            // Reconcile RootConfig: build a configure request from the bind
4165            // identity + forwarded config tiers and run it through the executor.
4166            let request_id = format!("subc-bind-{route_channel}");
4167            let bind_project_root = identity.project_root.clone();
4168            let bind_harness = identity.harness.clone();
4169            let bind_session = identity.session.clone();
4170            let bind_trust = trust_for_bind(&bind_harness, &principal);
4171            let bind_principal_id = principal_id(&principal);
4172            // Typed capability declaration from the consumer: the facade stamps it
4173            // from the MCP host's initialize-advertised capabilities. Absent
4174            // means no reverse-request capability — flat deny, fail-closed. A
4175            // consumer over-declaring only earns asks that TTL-deny.
4176            let consumer_elicitation_capable = consumer_capabilities
4177                .as_ref()
4178                .is_some_and(|capabilities| capabilities.iter().any(|c| c == "elicitation"));
4179            log::info!(
4180                "subc attach: route {} harness={} principal={} trust={} elicitation={}",
4181                route_channel,
4182                bind_harness,
4183                principal_label(&principal),
4184                bind_trust.label(),
4185                consumer_elicitation_capable
4186            );
4187
4188            // Config is single-per-project, read by AFT directly from the
4189            // CortexKit config files (user: ~/.config/cortexkit/aft.jsonc,
4190            // project: <root>/.cortexkit/aft.jsonc). Wire-relayed config tiers are
4191            // IGNORED entirely: a front (runner, mcp:*, or fed:*) cannot push config over
4192            // the wire. This is what makes config harness-INDEPENDENT — every
4193            // harness binding a project gets the identical on-disk config, so two
4194            // trust domains sharing the per-root actor can never diverge or
4195            // inherit each other's capabilities (the cross-bind escalation class).
4196            // Wire-relayed config tiers (if the protocol still carries them) are
4197            // ignored entirely; the per-tier trust boundary (user trusted, project
4198            // privileged-dropped) is applied to the FILE tiers in handle_configure.
4199            let local_tiers = crate::subc_config::read_local_cortexkit_config_tiers(
4200                user_config_path,
4201                Path::new(&bind_project_root),
4202            );
4203            let config_tiers: Vec<Value> = local_tiers
4204                .iter()
4205                .map(|t| json!({ "tier": t.tier, "source": t.source, "doc": t.doc }))
4206                .collect();
4207            let diagnostics_on_edit = diagnostics_on_edit_from_tiers(&local_tiers);
4208            let configure_json = json!({
4209                "id": request_id,
4210                "command": "configure",
4211                "project_root": bind_project_root,
4212                "harness": bind_harness,
4213                "session_id": bind_session.clone(),
4214                "config": config_tiers,
4215            });
4216            let configure_req = match serde_json::from_value::<RawRequest>(configure_json) {
4217                Ok(req) => req,
4218                Err(error) => {
4219                    return send_route_bind_error(
4220                        tx,
4221                        frame,
4222                        "config_divergence",
4223                        &format!("failed to build configure request: {error}"),
4224                        metrics,
4225                    )
4226                    .await;
4227                }
4228            };
4229
4230            let route_identity = RouteIdentity(Arc::new(RouteIdentityData {
4231                root: bind_root_id.clone(),
4232                project_root: PathBuf::from(&bind_project_root),
4233                harness: bind_harness.clone(),
4234                session: bind_session.clone(),
4235                trust: bind_trust,
4236                spawn_principal: AuthenticatedPrincipal::RouteBind {
4237                    trust: bind_trust.sandbox_trust(),
4238                    route_channel,
4239                    route_epoch: epoch,
4240                    project_root: PathBuf::from(&bind_project_root),
4241                    harness: bind_harness.clone(),
4242                    session_id: bind_session.clone(),
4243                    principal_id: bind_principal_id,
4244                },
4245                consumer_elicitation_capable,
4246            }));
4247            let configure_session = route_identity.session.clone();
4248            let root_was_live = live_roots.contains_key(&bind_root_id);
4249            let inserted_new_actor = register_actor_for_bind(
4250                shared_app,
4251                executor,
4252                push_senders,
4253                &bind_root_id,
4254                route_channel,
4255                root_was_live,
4256            );
4257
4258            let configure_request_id = configure_req.id.clone();
4259            installed_route_epochs.insert(route_channel, epoch);
4260            if let Some(meta) = live_roots.get_mut(&bind_root_id) {
4261                meta.maintenance_queued_kinds.clear();
4262                meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
4263            }
4264            let (configure_rx, configure_cancellation) = executor.submit_cancellable_async(
4265                bind_root_id.clone(),
4266                Lane::Mutating,
4267                configure_request_id.clone(),
4268                Box::new(move |ctx| {
4269                    log_ctx::with_session(Some(configure_session.clone()), || {
4270                        dispatch(configure_req, ctx)
4271                    })
4272                }),
4273            );
4274            pending_binds.insert(
4275                route_id,
4276                PendingBind {
4277                    bind_root_id: bind_root_id.clone(),
4278                    inserted_new_actor,
4279                    cancelled: false,
4280                    configure_request_id: configure_request_id.clone(),
4281                    started_at: Instant::now(),
4282                    warned_half_deadline: false,
4283                    deadline_reported: false,
4284                    corr: frame.header.corr,
4285                    ver: frame.header.ver,
4286                    flags: frame.header.flags,
4287                    cancellation: configure_cancellation,
4288                },
4289            );
4290
4291            let completion_tx = control_completion_tx.clone();
4292            let completion_identity = route_identity;
4293            let completion_root = bind_root_id.clone();
4294            let completion_route_channel = route_channel;
4295            let completion_ver = frame.header.ver;
4296            let completion_corr = frame.header.corr;
4297            let completion_flags = frame.header.flags;
4298            let completion_metrics = Arc::clone(metrics);
4299            tokio::spawn(async move {
4300                let _response_task = ResponseTaskGuard::new(&completion_metrics);
4301                let configure_response =
4302                    await_executor_response(configure_rx, configure_request_id.clone()).await;
4303                // Send the route-bind acknowledgment as soon as configure succeeds.
4304                // Installing completed search or callgraph builds only refreshes cached
4305                // read data, so a later maintenance pass can do it without delaying the
4306                // daemon's confirmation that the route is usable.
4307                let completion = RouteBindCompletion {
4308                    route: route_key(completion_route_channel, epoch),
4309                    identity: completion_identity,
4310                    bind_root_id: completion_root,
4311                    inserted_new_actor,
4312                    configure_response,
4313                    diagnostics_on_edit,
4314                    ver: completion_ver,
4315                    corr: completion_corr,
4316                    flags: completion_flags,
4317                };
4318                if send_counted_channel(
4319                    &completion_tx,
4320                    &completion_metrics.control_completion_queued,
4321                    completion,
4322                )
4323                .await
4324                .is_err()
4325                {
4326                    log::debug!(
4327                        "subc attach: dropped RouteBind completion for route {} after loop exit",
4328                        completion_route_channel
4329                    );
4330                }
4331            });
4332
4333            health_rollup_cache.refresh(executor, shared_app);
4334            Ok(())
4335        }
4336        ModuleControlRequest::HealthCheck {} => {
4337            metrics.record_bg_runtime(bg_subs.len(), bg_wake_pending.len());
4338            let report = build_health_report(
4339                health_rollup_cache,
4340                executor,
4341                pending_binds,
4342                metrics,
4343                shared_app,
4344            );
4345            let body = serde_json::to_vec(&ModuleControlResponse::from(report))
4346                .map_err(SubcError::Json)?;
4347            let response = Frame::build_with_version(
4348                frame.header.ver,
4349                FrameType::Response,
4350                frame.header.flags,
4351                0,
4352                0,
4353                frame.header.corr,
4354                body,
4355            )
4356            .map_err(SubcError::FrameBuild)?;
4357            send_frame(tx, metrics, response).await
4358        }
4359    }
4360}
4361
4362fn install_bash_compressor(ctx: &AppContext) {
4363    // Mirrors main.rs per-actor compressor installation for subc-created actors.
4364    let filter_registry_handle = ctx.shared_filter_registry();
4365    let compress_flag = ctx.bash_compress_flag();
4366    ctx.bash_background().set_compressor_with_exit_code(
4367        move |command: &str, output: String, exit_code: Option<i32>| {
4368            if !compress_flag.load(std::sync::atomic::Ordering::Relaxed) {
4369                return crate::compress::CompressionResult::new(output);
4370            }
4371            let registry_guard = match filter_registry_handle.read() {
4372                Ok(g) => g,
4373                Err(poisoned) => poisoned.into_inner(),
4374            };
4375            crate::compress::compress_with_registry_exit_code(
4376                command,
4377                &output,
4378                exit_code,
4379                &registry_guard,
4380            )
4381        },
4382    );
4383}
4384
4385fn diagnostics_on_edit_from_tiers(tiers: &[ConfigTier]) -> bool {
4386    let mut diagnostics_on_edit = false;
4387    for tier in tiers {
4388        if let Some(value) = diagnostics_on_edit_from_doc(&tier.doc) {
4389            diagnostics_on_edit = value;
4390        }
4391    }
4392    diagnostics_on_edit
4393}
4394
4395fn diagnostics_on_edit_from_doc(doc: &str) -> Option<bool> {
4396    let stripped = strip_jsonc(doc);
4397    let value = serde_json::from_str::<Value>(&stripped).ok()?;
4398    value
4399        .get("lsp")
4400        .and_then(Value::as_object)?
4401        .get("diagnostics_on_edit")
4402        .and_then(Value::as_bool)
4403}
4404
4405async fn send_route_bind_error(
4406    tx: &WriterSender,
4407    frame: &Frame,
4408    code: &str,
4409    message: &str,
4410    metrics: &DispatchPathMetrics,
4411) -> Result<(), SubcError> {
4412    send_route_bind_error_parts(
4413        tx,
4414        frame.header.ver,
4415        frame.header.corr,
4416        frame.header.flags,
4417        code,
4418        message,
4419        metrics,
4420    )
4421    .await
4422}
4423
4424async fn send_route_bind_error_parts(
4425    tx: &WriterSender,
4426    ver: u8,
4427    corr: u64,
4428    flags: Flags,
4429    code: &str,
4430    message: &str,
4431    metrics: &DispatchPathMetrics,
4432) -> Result<(), SubcError> {
4433    let response = build_error_frame(ver, 0, 0, corr, flags, code, message)?;
4434    send_reliable_writer_frame(tx, metrics, response, "RouteBind error").await?;
4435    log_route_bind_rejection(code, message);
4436    Ok(())
4437}
4438
4439/// Per-message rate limit for the bind-rejection warn line. A caller that
4440/// re-attaches a dead root forever turns this line into the entire readable
4441/// tail of the SHARED daemon log (measured 2026-08-09: 2.08M copies, ~40/sec,
4442/// 936MB log — other modules' incident lines pushed out of the tail).
4443/// The line itself stays byte-identical so external counters keep matching;
4444/// repeats inside the window are summarized with a suppressed count on the
4445/// next emission (volume stays diagnosable, per the log-diet convention).
4446fn log_route_bind_rejection(code: &str, message: &str) {
4447    const WINDOW: Duration = Duration::from_secs(60);
4448    static SUPPRESSED: OnceLock<StdMutex<HashMap<String, (Instant, u64)>>> = OnceLock::new();
4449    let map = SUPPRESSED.get_or_init(|| StdMutex::new(HashMap::new()));
4450    let mut map = match map.try_lock() {
4451        Ok(map) => map,
4452        // Contended: log unsuppressed rather than blocking or dropping.
4453        Err(_) => {
4454            log::warn!("subc attach: route bind rejected ({code}): {message}");
4455            return;
4456        }
4457    };
4458    let now = Instant::now();
4459    // Bound the map: dead roots churn, and an unbounded suppression map is
4460    // its own leak. Sweep expired entries once it grows past a fleet-sized
4461    // number of distinct rejection messages.
4462    if map.len() > 512 {
4463        map.retain(|_, (start, _)| now.duration_since(*start) < WINDOW);
4464    }
4465    match map.get_mut(message) {
4466        Some((window_start, suppressed)) if now.duration_since(*window_start) < WINDOW => {
4467            *suppressed += 1;
4468        }
4469        Some((window_start, suppressed)) => {
4470            if *suppressed > 0 {
4471                log::warn!(
4472                    "subc attach: route bind rejected ({code}): {message} (repeated {}x in last 60s)",
4473                    *suppressed
4474                );
4475            } else {
4476                log::warn!("subc attach: route bind rejected ({code}): {message}");
4477            }
4478            *window_start = now;
4479            *suppressed = 0;
4480        }
4481        None => {
4482            log::warn!("subc attach: route bind rejected ({code}): {message}");
4483            map.insert(message.to_string(), (now, 0));
4484        }
4485    }
4486}
4487
4488/// Route-channel tool call: `{name, arguments}` → executor lane → dispatch to
4489/// the sync command core → wrap the structured Response in a CallToolResult
4490/// `{content, isError}`. Tool-result mapping: the whole `{success, ...}` Response
4491/// serialized into ONE text block; `isError` carries `success == false`.
4492async fn handle_tool_call(
4493    tx: &WriterSender,
4494    frame: &Frame,
4495    mut phase_trace: PhaseTrace,
4496    routes: &HashMap<RouteChannel, RouteIdentity>,
4497    pending_binds: &HashMap<RouteChannel, PendingBind>,
4498    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4499    executor: &Arc<Executor>,
4500    active_tool_calls: &ActiveToolCalls,
4501    pending_deferred_setups: &Arc<AtomicUsize>,
4502    shutdown: &Arc<Notify>,
4503    connection_cancel: &PersistentCancelSignal,
4504    bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
4505    bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
4506    metrics: &Arc<DispatchPathMetrics>,
4507    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
4508    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
4509    next_bash_ask_corr: &mut u64,
4510    bg_subs: &mut HashMap<RouteChannel, BgSub>,
4511    bg_sub_by_session: &mut BgSubsBySession,
4512    bg_wake_pending: &mut HashSet<RouteChannel>,
4513    bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
4514    dispatch: DispatchFn,
4515    deferred_response_tx: &mpsc::UnboundedSender<PendingSubcResponse>,
4516    allow_native_passthrough: bool,
4517    tool_response_body_limit: usize,
4518) -> Result<(), SubcError> {
4519    let route_id = route_key(frame.header.channel, frame.header.epoch);
4520    if pending_binds.contains_key(&route_id) {
4521        let error = build_error_frame(
4522            frame.header.ver,
4523            frame.header.channel,
4524            frame.header.epoch,
4525            frame.header.corr,
4526            frame.header.flags,
4527            "route_not_bound",
4528            "route is not bound before tool call",
4529        )?;
4530        return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
4531    }
4532
4533    let Some(identity) = routes.get(&route_id).cloned() else {
4534        let error = build_error_frame(
4535            frame.header.ver,
4536            frame.header.channel,
4537            frame.header.epoch,
4538            frame.header.corr,
4539            frame.header.flags,
4540            "route_not_bound",
4541            "route is not bound before tool call",
4542        )?;
4543        return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
4544    };
4545    let restore_watcher = live_roots
4546        .get(&identity.root)
4547        .is_some_and(|meta| meta.idle_artifacts_evicted);
4548    if let Some(meta) = live_roots.get_mut(&identity.root) {
4549        meta.reactivate_bound();
4550    }
4551    if restore_watcher {
4552        if let Some(ctx) = executor.actor_context(&identity.root) {
4553            crate::commands::configure::ensure_project_watcher(&ctx);
4554        }
4555    }
4556
4557    let route_request =
4558        serde_json::from_slice::<RouteRequest>(&frame.body).map_err(SubcError::Json)?;
4559    if matches!(
4560        route_request,
4561        RouteRequest::BgEvents(BgEventsRequest {
4562            op: BgEventsOp::BgEvents
4563        })
4564    ) {
4565        if let Some(old_sub) = bg_subs.get(&route_id).cloned() {
4566            metrics.record_bg_subscription_ended(
4567                &old_sub.root,
4568                &old_sub.session,
4569                route_id,
4570                "resubscribe",
4571            );
4572            push::send_reliable_bg_stream_end(tx, metrics, route_id, &old_sub).await?;
4573        }
4574        if !identity.trust.allows_bash_observation() {
4575            bg_subs.remove(&route_id);
4576            bg_wake_pending.remove(&route_id);
4577            remove_bg_subscription_index(bg_sub_by_session, route_id, Some(&identity));
4578            let denied_sub = BgSub {
4579                corr: frame.header.corr,
4580                ver: frame.header.ver,
4581                flags: frame.header.flags,
4582                root: identity.root.clone(),
4583                session: identity.session.clone(),
4584            };
4585            metrics.record_bg_subscription_ended(
4586                &identity.root,
4587                &identity.session,
4588                route_id,
4589                "subscribe-denied",
4590            );
4591            push::send_reliable_bg_stream_end(tx, metrics, route_id, &denied_sub).await?;
4592            return Ok(());
4593        }
4594        bg_subs.insert(
4595            route_id,
4596            BgSub {
4597                corr: frame.header.corr,
4598                ver: frame.header.ver,
4599                flags: frame.header.flags,
4600                root: identity.root.clone(),
4601                session: identity.session.clone(),
4602            },
4603        );
4604        insert_bg_subscription_index(
4605            bg_sub_by_session,
4606            identity.root.clone(),
4607            identity.session.clone(),
4608            route_id,
4609        );
4610        metrics.record_bg_subscription_installed(&identity.root, &identity.session, route_id);
4611        push::arm_bg_wake(
4612            identity.root.clone(),
4613            identity.session.clone(),
4614            route_id,
4615            bg_wake_pending,
4616            bg_wake_epoch,
4617            metrics,
4618        );
4619        return Ok(());
4620    }
4621
4622    let RouteRequest::ToolCall(call) = route_request else {
4623        unreachable!("background event subscription returned above")
4624    };
4625    let bare_name = call.name;
4626    let arguments = strip_agent_preview_arg_owned(call.arguments);
4627    let format_context = crate::subc_format::FormatContext::from_tool_call(
4628        &bare_name,
4629        &arguments,
4630        identity.project_root.as_path(),
4631    );
4632
4633    let request_id = format!("subc-{}-{}", frame.header.channel, frame.header.corr);
4634    let bind_trust = identity.trust;
4635    let diagnostics_on_edit = live_roots
4636        .get(&identity.root)
4637        .map(|meta| meta.diagnostics_on_edit)
4638        .unwrap_or(false);
4639
4640    let requests_host = bare_name == "bash"
4641        && arguments
4642            .get("sandbox")
4643            .or_else(|| {
4644                arguments
4645                    .get("params")
4646                    .and_then(|params| params.get("sandbox"))
4647            })
4648            .and_then(Value::as_str)
4649            == Some("host");
4650    if matches!(bind_trust, BindTrust::Untrusted) && requests_host {
4651        let response = Response::error(
4652            request_id.clone(),
4653            "sandbox_escalation_denied",
4654            "sandbox host escalation is unavailable to untrusted principals",
4655        );
4656        let text = crate::subc_format::format_response_with_context(
4657            &bare_name,
4658            &response,
4659            &format_context,
4660        );
4661        let result = ToolCallResult { text, response };
4662        let response_frame = build_tool_response_frame_with_limit(
4663            frame.header.ver,
4664            route_id,
4665            frame.header.corr,
4666            frame.header.flags,
4667            &result,
4668            bind_trust,
4669            tool_response_body_limit,
4670        )?;
4671        return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4672    }
4673
4674    if matches!(bind_trust, BindTrust::Untrusted)
4675        && is_bash_family_tool(&bare_name)
4676        && (bare_name != "bash" || !identity.consumer_elicitation_capable)
4677    {
4678        let response = bash::bash_denied_untrusted_response(request_id.clone());
4679        let text = crate::subc_format::format_response_with_context(
4680            &bare_name,
4681            &response,
4682            &format_context,
4683        );
4684        let result = ToolCallResult { text, response };
4685        let response_frame = build_tool_response_frame_with_limit(
4686            frame.header.ver,
4687            route_id,
4688            frame.header.corr,
4689            frame.header.flags,
4690            &result,
4691            bind_trust,
4692            tool_response_body_limit,
4693        )?;
4694        return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4695    }
4696
4697    // A non-core name is NOT in the tool manifest. AFT fails closed and
4698    // does not trust subc to enforce the manifest: rejecting here is the
4699    // defense-in-depth backstop that prevents a forwarded native command
4700    // (e.g. `configure`, which would reach handle_configure and bypass
4701    // the RouteBind config-trust cap) from ever reaching dispatch. Only
4702    // the integration-test harness (run_subc_mode_for_test) opens this to
4703    // drive synthetic native commands through the executor.
4704    if !is_subc_agent_core_tool(&bare_name)
4705        && !is_subc_native_plumbing_tool(&bare_name)
4706        && !allow_native_passthrough
4707    {
4708        log::warn!(
4709            "subc tool call: rejecting non-manifest tool name {:?} on route {} (fail-closed)",
4710            bare_name,
4711            frame.header.channel
4712        );
4713        let response = Response::error(
4714            request_id.clone(),
4715            "unknown_tool",
4716            format!("tool {:?} is not in the AFT tool manifest", bare_name),
4717        );
4718        let text = crate::subc_format::format_response_with_context(
4719            &bare_name,
4720            &response,
4721            &format_context,
4722        );
4723        let result = ToolCallResult { text, response };
4724        let response_frame = build_tool_response_frame_with_limit(
4725            frame.header.ver,
4726            route_id,
4727            frame.header.corr,
4728            frame.header.flags,
4729            &result,
4730            bind_trust,
4731            tool_response_body_limit,
4732        )?;
4733        return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4734    }
4735
4736    if bare_name == "bash" {
4737        if matches!(bind_trust, BindTrust::Untrusted) {
4738            let plan = match bash::prepare_bash_elicitation_plan(
4739                &arguments,
4740                identity.project_root.as_path(),
4741            ) {
4742                Ok(plan) => plan,
4743                Err(error) => {
4744                    let response = Response::error(request_id.clone(), error.code, error.message);
4745                    let text = crate::subc_format::format_response_with_context(
4746                        &bare_name,
4747                        &response,
4748                        &format_context,
4749                    );
4750                    let result = ToolCallResult { text, response };
4751                    let response_frame = build_tool_response_frame_with_limit(
4752                        frame.header.ver,
4753                        route_id,
4754                        frame.header.corr,
4755                        frame.header.flags,
4756                        &result,
4757                        bind_trust,
4758                        tool_response_body_limit,
4759                    )?;
4760                    return send_reliable_writer_frame(
4761                        tx,
4762                        metrics,
4763                        response_frame,
4764                        "tool response",
4765                    )
4766                    .await;
4767                }
4768            };
4769
4770            let reverse_corr =
4771                allocate_reverse_corr(pending_bash_asks, route_id, next_bash_ask_corr);
4772            let ask_frame = build_bash_elicitation_request_frame(
4773                frame.header.ver,
4774                route_id,
4775                reverse_corr,
4776                frame.header.flags,
4777                &plan.command,
4778                &plan.asks,
4779            )?;
4780
4781            let meta = live_roots
4782                .entry(identity.root.clone())
4783                .or_insert_with(|| RootMeta::new(Instant::now()));
4784            meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
4785            meta.reactivate_bound();
4786
4787            let route_cancel =
4788                route_bash_cancels
4789                    .entry(route_id)
4790                    .or_insert_with(|| bash::RouteBashCancel {
4791                        token: PersistentCancelSignal::new(),
4792                        active_waits: 0,
4793                    });
4794            route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
4795            let cancel = bash::BashWaitCancel {
4796                connection: connection_cancel.clone(),
4797                route: route_cancel.token.clone(),
4798            };
4799            pending_bash_asks.insert(
4800                ReverseCorrKey {
4801                    route: route_id,
4802                    corr: reverse_corr,
4803                },
4804                PendingBashAsk {
4805                    route: route_id,
4806                    tool_corr: frame.header.corr,
4807                    tool_flags: frame.header.flags,
4808                    tool_ver: frame.header.ver,
4809                    root: identity.root.clone(),
4810                    project_root: identity.project_root.clone(),
4811                    session_id: identity.session.clone(),
4812                    spawn_principal: identity.spawn_principal.clone(),
4813                    edit_slot_survives: call.edit_slot_survives,
4814                    request_id,
4815                    arguments,
4816                    format_context,
4817                    cancel,
4818                    grants: plan.grants,
4819                    expires_at: Instant::now() + bash_elicitation_timeout(),
4820                },
4821            );
4822            return send_reliable_writer_frame(tx, metrics, ask_frame, "bash elicitation request")
4823                .await;
4824        }
4825
4826        let meta = live_roots
4827            .entry(identity.root.clone())
4828            .or_insert_with(|| RootMeta::new(Instant::now()));
4829        meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
4830        meta.reactivate_bound();
4831
4832        let route_cancel =
4833            route_bash_cancels
4834                .entry(route_id)
4835                .or_insert_with(|| bash::RouteBashCancel {
4836                    token: PersistentCancelSignal::new(),
4837                    active_waits: 0,
4838                });
4839        route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
4840        let cancel = bash::BashWaitCancel {
4841            connection: connection_cancel.clone(),
4842            route: route_cancel.token.clone(),
4843        };
4844
4845        bash::submit_deferred_bash(
4846            executor,
4847            bash_deferred_tx,
4848            bash_poll_touch_tx,
4849            metrics,
4850            dispatch,
4851            identity.root.clone(),
4852            identity.project_root.clone(),
4853            identity.session.clone(),
4854            request_id,
4855            route_id,
4856            frame.header.corr,
4857            frame.header.flags,
4858            frame.header.ver,
4859            arguments,
4860            format_context,
4861            cancel,
4862            bind_trust,
4863            identity.spawn_principal.clone(),
4864            call.edit_slot_survives,
4865            None,
4866        );
4867        return Ok(());
4868    }
4869
4870    let lane = command_lane(&bare_name);
4871    let tool_call_context = ToolCallContext {
4872        project_root: identity.project_root.clone(),
4873        session_id: Some(identity.session.clone()),
4874        request_id: request_id.clone(),
4875        diagnostics_on_edit,
4876        preview: call.preview,
4877        edit_slot_survives: call.edit_slot_survives,
4878        report_registration_downgrade: true,
4879    };
4880
4881    let uses_deferred_response_seam = bare_name == "inspect"
4882        || crate::commands::lsp_navigation::is_lsp_navigation_command(&bare_name);
4883    if uses_deferred_response_seam {
4884        let Some(deferred_ctx) = executor.actor_context(&identity.root) else {
4885            let response = Response::error(
4886                &request_id,
4887                "actor_not_registered",
4888                "executor actor is not registered",
4889            );
4890            let text = crate::subc_format::format_response_with_context(
4891                &bare_name,
4892                &response,
4893                &format_context,
4894            );
4895            let result = ToolCallResult { text, response };
4896            let response_frame = build_tool_response_frame_with_limit(
4897                frame.header.ver,
4898                route_id,
4899                frame.header.corr,
4900                frame.header.flags,
4901                &result,
4902                bind_trust,
4903                tool_response_body_limit,
4904            )?;
4905            return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4906        };
4907        let identity_for_run = identity.clone();
4908        let request_id_for_force = request_id.clone();
4909        let format_context_for_run = format_context.clone();
4910        let bare_name_for_run = bare_name.clone();
4911        let (setup_tx, setup_rx) = oneshot::channel::<DeferredSetupOutcome>();
4912        phase_trace.mark_executor_submitted();
4913        let job: crate::executor::ExecutorJob = Box::new(move |ctx| {
4914            phase_trace.mark_job_admitted();
4915            log_ctx::with_session(Some(identity_for_run.session.clone()), || {
4916                let run = || match prepare_tool_call(
4917                    &bare_name_for_run,
4918                    arguments,
4919                    &format_context_for_run,
4920                    &tool_call_context,
4921                    ctx,
4922                    Some(&mut phase_trace),
4923                ) {
4924                    Err(result) => {
4925                        let response = result.response;
4926                        let _ = setup_tx.send(DeferredSetupOutcome::Immediate {
4927                            text: result.text,
4928                            phase_trace,
4929                        });
4930                        response
4931                    }
4932                    Ok(prepared) => {
4933                        let outcome = if bare_name_for_run == "inspect" {
4934                            crate::commands::inspect::handle_inspect_deferred_with_restriction(
4935                                &prepared.request,
4936                                Arc::clone(&deferred_ctx),
4937                                matches!(bind_trust, BindTrust::Untrusted),
4938                            )
4939                        } else {
4940                            crate::commands::lsp_navigation::handle_lsp_navigation_deferred_with_restriction(
4941                                &prepared.request,
4942                                Arc::clone(&deferred_ctx),
4943                                matches!(bind_trust, BindTrust::Untrusted),
4944                            )
4945                        };
4946                        match outcome {
4947                            DispatchOutcome::Deferred(pending) => {
4948                                let _ = setup_tx.send(DeferredSetupOutcome::Deferred {
4949                                    pending,
4950                                    surface_downgraded: prepared.surface_downgraded,
4951                                    phase_trace,
4952                                });
4953                                Response::success(
4954                                    request_id_for_force.clone(),
4955                                    json!({ "response_deferred": true }),
4956                                )
4957                            }
4958                            DispatchOutcome::Immediate(response) => {
4959                                phase_trace.mark_execute_done();
4960                                let finalizer = |response: &mut Response| {
4961                                    crate::response_finalize::finalize_response_with_bg_completions(
4962                                        response,
4963                                        ctx,
4964                                        &identity_for_run.session,
4965                                        &bare_name_for_run,
4966                                        bind_trust.allows_bash_observation(),
4967                                    );
4968                                };
4969                                let result = finish_tool_call_response(
4970                                    &bare_name_for_run,
4971                                    &format_context_for_run,
4972                                    response,
4973                                    prepared.surface_downgraded,
4974                                    Some(&finalizer),
4975                                    Some(&mut phase_trace),
4976                                );
4977                                let response = result.response;
4978                                let _ = setup_tx.send(DeferredSetupOutcome::Immediate {
4979                                    text: result.text,
4980                                    phase_trace,
4981                                });
4982                                response
4983                            }
4984                        }
4985                    }
4986                };
4987                if matches!(bind_trust, BindTrust::Untrusted) {
4988                    ctx.with_force_restrict(&request_id_for_force, run)
4989                } else {
4990                    run()
4991                }
4992            })
4993        });
4994        let deferred_setup_guard =
4995            PendingDeferredSetupGuard::new(Arc::clone(pending_deferred_setups));
4996        let rx = submit_active_tool_call(
4997            executor.as_ref(),
4998            active_tool_calls,
4999            route_id,
5000            frame.header.corr,
5001            identity.root.clone(),
5002            lane,
5003            request_id.clone(),
5004            RouteDetachPolicy::CancelOnDetach,
5005            job,
5006        );
5007
5008        let completion_tx = tx.clone();
5009        let completion_shutdown = Arc::clone(shutdown);
5010        let completion_metrics = Arc::clone(metrics);
5011        let active_tool_calls = Arc::clone(active_tool_calls);
5012        let deferred_response_tx = deferred_response_tx.clone();
5013        let route = route_id;
5014        let corr = frame.header.corr;
5015        let flags = frame.header.flags;
5016        let ver = frame.header.ver;
5017        let root = identity.root.clone();
5018        let session_id = identity.session.clone();
5019        tokio::spawn(async move {
5020            let _response_task = ResponseTaskGuard::new(&completion_metrics);
5021            let _deferred_setup = deferred_setup_guard;
5022            let response = await_executor_response(rx, request_id.clone()).await;
5023            match setup_rx.await {
5024                Ok(DeferredSetupOutcome::Deferred {
5025                    pending,
5026                    surface_downgraded,
5027                    phase_trace,
5028                }) => {
5029                    let pending = PendingSubcResponse {
5030                        route,
5031                        corr,
5032                        flags,
5033                        ver,
5034                        root,
5035                        session_id,
5036                        bare_name,
5037                        format_context,
5038                        bind_trust,
5039                        pending,
5040                        surface_downgraded,
5041                        phase_trace,
5042                    };
5043                    if let Err(error) = deferred_response_tx.send(pending) {
5044                        if let Some(cancellation) = &error.0.pending.cancellation {
5045                            cancellation.request_cancel();
5046                        }
5047                        finish_active_tool_call(&active_tool_calls, route, corr);
5048                    }
5049                }
5050                Ok(DeferredSetupOutcome::Immediate { text, phase_trace }) => {
5051                    if !finish_active_tool_call(&active_tool_calls, route, corr) {
5052                        return;
5053                    }
5054                    let result = ToolCallResult { text, response };
5055                    let fatal = response_is_fatal_panic(&result.response);
5056                    match build_tool_response_frame_with_limit(
5057                        ver,
5058                        route,
5059                        corr,
5060                        flags,
5061                        &result,
5062                        bind_trust,
5063                        tool_response_body_limit,
5064                    ) {
5065                        Ok(response_frame) => {
5066                            let trace = ToolResponseWriteTrace::new(
5067                                phase_trace,
5068                                bare_name.clone(),
5069                                identity.project_root.clone(),
5070                                identity.session.clone(),
5071                                route.channel,
5072                                corr,
5073                            );
5074                            if let Err(error) = send_traced_tool_response_frame(
5075                                &completion_tx,
5076                                &completion_metrics,
5077                                response_frame,
5078                                trace,
5079                            )
5080                            .await
5081                            {
5082                                log::warn!(
5083                                    "subc attach: failed to queue deferred-seam setup response: {error}"
5084                                );
5085                            }
5086                        }
5087                        Err(error) => {
5088                            log::error!(
5089                                "subc attach: failed to build deferred-seam setup response: {error}"
5090                            );
5091                        }
5092                    }
5093                    if fatal {
5094                        signal_fatal_teardown(
5095                            &completion_tx,
5096                            Some(route),
5097                            ver,
5098                            corr,
5099                            &completion_shutdown,
5100                            &completion_metrics,
5101                        )
5102                        .await;
5103                    }
5104                }
5105                Err(_) => {
5106                    if !finish_active_tool_call(&active_tool_calls, route, corr) {
5107                        return;
5108                    }
5109                    let text = crate::subc_format::format_response_with_context(
5110                        &bare_name,
5111                        &response,
5112                        &format_context,
5113                    );
5114                    let result = ToolCallResult { text, response };
5115                    if let Ok(response_frame) = build_tool_response_frame_with_limit(
5116                        ver,
5117                        route,
5118                        corr,
5119                        flags,
5120                        &result,
5121                        bind_trust,
5122                        tool_response_body_limit,
5123                    ) {
5124                        let _ = send_reliable_writer_frame(
5125                            &completion_tx,
5126                            &completion_metrics,
5127                            response_frame,
5128                            "deferred setup failure",
5129                        )
5130                        .await;
5131                    }
5132                }
5133            }
5134        });
5135        return Ok(());
5136    }
5137
5138    let bare_name_for_frame = bare_name.clone();
5139    let identity_for_run = identity.clone();
5140    let completion_session = identity.session.clone();
5141    let completion_root = identity.project_root.clone();
5142    let request_id_for_force = request_id.clone();
5143    let format_context_for_frame = format_context.clone();
5144    let (tool_call_tx, tool_call_rx) = oneshot::channel::<ToolCallCompletion>();
5145    phase_trace.mark_executor_submitted();
5146    let job: crate::executor::ExecutorJob = Box::new(move |ctx| {
5147        phase_trace.mark_job_admitted();
5148        log_ctx::with_session(Some(identity_for_run.session.clone()), || {
5149            let run = || {
5150                let finalizer = |response: &mut Response| {
5151                    crate::response_finalize::finalize_response_with_bg_completions(
5152                        response,
5153                        ctx,
5154                        &identity_for_run.session,
5155                        &bare_name,
5156                        bind_trust.allows_bash_observation(),
5157                    );
5158                };
5159                match run_tool_call(
5160                    &bare_name,
5161                    arguments,
5162                    &format_context,
5163                    &tool_call_context,
5164                    ctx,
5165                    &dispatch,
5166                    Some(&finalizer),
5167                    Some(&mut phase_trace),
5168                ) {
5169                    ToolCallOutcome::Unary(result) => {
5170                        let response = result.response;
5171                        let _ = tool_call_tx.send(ToolCallCompletion {
5172                            text: result.text,
5173                            phase_trace,
5174                        });
5175                        response
5176                    }
5177                }
5178            };
5179            if matches!(bind_trust, BindTrust::Untrusted) {
5180                ctx.with_force_restrict(&request_id_for_force, run)
5181            } else {
5182                run()
5183            }
5184        })
5185    });
5186    let rx = submit_active_tool_call(
5187        executor.as_ref(),
5188        active_tool_calls,
5189        route_id,
5190        frame.header.corr,
5191        identity.root.clone(),
5192        lane,
5193        request_id.clone(),
5194        RouteDetachPolicy::RetainForReplay,
5195        job,
5196    );
5197    let completion_tx = tx.clone();
5198    let completion_shutdown = Arc::clone(shutdown);
5199    let route = route_id;
5200    let corr = frame.header.corr;
5201    let flags = frame.header.flags;
5202    let ver = frame.header.ver;
5203    let completion_metrics = Arc::clone(metrics);
5204    let active_tool_calls = Arc::clone(active_tool_calls);
5205    tokio::spawn(async move {
5206        let _response_task = ResponseTaskGuard::new(&completion_metrics);
5207        let response = await_executor_response(rx, request_id.clone()).await;
5208        let (text, phase_trace) = match tool_call_rx.await {
5209            Ok(completion) => (completion.text, Some(completion.phase_trace)),
5210            Err(_) => (
5211                crate::subc_format::format_response_with_context(
5212                    &bare_name_for_frame,
5213                    &response,
5214                    &format_context_for_frame,
5215                ),
5216                None,
5217            ),
5218        };
5219        if !finish_active_tool_call(&active_tool_calls, route, corr) {
5220            return;
5221        }
5222        let result = ToolCallResult { text, response };
5223        let fatal = response_is_fatal_panic(&result.response);
5224        match build_tool_response_frame_with_limit(
5225            ver,
5226            route,
5227            corr,
5228            flags,
5229            &result,
5230            bind_trust,
5231            tool_response_body_limit,
5232        ) {
5233            Ok(response_frame) => {
5234                let send_result = if let Some(phase_trace) = phase_trace {
5235                    let trace = ToolResponseWriteTrace::new(
5236                        phase_trace,
5237                        bare_name_for_frame,
5238                        completion_root,
5239                        completion_session,
5240                        route.channel,
5241                        corr,
5242                    );
5243                    send_traced_tool_response_frame(
5244                        &completion_tx,
5245                        &completion_metrics,
5246                        response_frame,
5247                        trace,
5248                    )
5249                    .await
5250                } else {
5251                    send_reliable_writer_frame(
5252                        &completion_tx,
5253                        &completion_metrics,
5254                        response_frame,
5255                        "tool response",
5256                    )
5257                    .await
5258                };
5259                if let Err(error) = send_result {
5260                    log::warn!("subc attach: failed to queue tool response frame: {error}");
5261                }
5262            }
5263            Err(error) => {
5264                log::error!("subc attach: failed to build tool response frame: {error}");
5265            }
5266        }
5267        if fatal {
5268            signal_fatal_teardown(
5269                &completion_tx,
5270                Some(route),
5271                ver,
5272                corr,
5273                &completion_shutdown,
5274                &completion_metrics,
5275            )
5276            .await;
5277        }
5278    });
5279    Ok(())
5280}
5281
5282fn submit_maintenance_job(
5283    executor: &Arc<Executor>,
5284    root_id: ProjectRootId,
5285    kind: MaintenanceDrainKind,
5286    bg_sessions_to_check: Vec<(String, u64)>,
5287    completion_tx: &mpsc::Sender<MaintenanceCompletion>,
5288    metrics: &Arc<DispatchPathMetrics>,
5289) {
5290    let request_id = format!(
5291        "subc-maintenance-drain-{}-{}",
5292        kind.label(),
5293        root_id.as_path().to_string_lossy()
5294    );
5295    let response_id = request_id.clone();
5296    let completion_root_id = root_id.clone();
5297    let maintenance_generation = executor
5298        .actor_context(&root_id)
5299        .map(|ctx| ctx.configure_generation())
5300        .unwrap_or(0);
5301    let (outcome_tx, outcome_rx) = oneshot::channel::<MaintenanceJobOutcome>();
5302    // ConfigureTail runs deferred configure mutations and needs the actor
5303    // epoch write gate. The other drain kinds only mutate subsystem state
5304    // behind that subsystem's own lock, so they run on MaintenanceCommit and
5305    // overlap interactive reads instead of excluding them.
5306    let lane = match kind {
5307        MaintenanceDrainKind::ConfigureTail => Lane::Mutating,
5308        MaintenanceDrainKind::Watcher
5309        | MaintenanceDrainKind::Lsp
5310        | MaintenanceDrainKind::CompletionDrains => Lane::MaintenanceCommit,
5311    };
5312    let job: crate::executor::ExecutorJob = Box::new(move |ctx: &AppContext| {
5313        let outcome = match kind {
5314            MaintenanceDrainKind::Watcher => {
5315                let drained = runtime_drain::drain_watcher_events_bounded(
5316                    ctx,
5317                    runtime_drain::WATCHER_PATH_DRAIN_BATCH_CAP,
5318                );
5319                MaintenanceJobOutcome {
5320                    empty_bg_sessions: Vec::new(),
5321                    requeue_kind: drained.has_more.then_some(kind),
5322                }
5323            }
5324            MaintenanceDrainKind::Lsp => {
5325                let drained = runtime_drain::drain_lsp_events_bounded(
5326                    ctx,
5327                    runtime_drain::LSP_EVENT_DRAIN_BATCH_CAP,
5328                );
5329                MaintenanceJobOutcome {
5330                    empty_bg_sessions: Vec::new(),
5331                    requeue_kind: drained.has_more.then_some(kind),
5332                }
5333            }
5334            MaintenanceDrainKind::ConfigureTail => {
5335                runtime_drain::drain_deferred_configure_maintenance(ctx);
5336                runtime_drain::drain_configure_warning_events(ctx);
5337                MaintenanceJobOutcome::default()
5338            }
5339            MaintenanceDrainKind::CompletionDrains => {
5340                runtime_drain::drain_search_index_events(ctx);
5341                runtime_drain::drain_callgraph_store_events(ctx);
5342                runtime_drain::drain_semantic_index_events(ctx);
5343                runtime_drain::drain_semantic_refresh_events(ctx);
5344                runtime_drain::drain_inspect_events_for_generation(ctx, maintenance_generation);
5345                let empty_bg_sessions = bg_sessions_to_check
5346                    .into_iter()
5347                    .filter(|(session, _)| {
5348                        !ctx.bash_background()
5349                            .has_completions_for_session(Some(session.as_str()))
5350                    })
5351                    .collect();
5352                MaintenanceJobOutcome {
5353                    empty_bg_sessions,
5354                    requeue_kind: None,
5355                }
5356            }
5357        };
5358        let requeued = outcome.requeue_kind.is_some();
5359        let _ = outcome_tx.send(outcome);
5360        Response::success(
5361            response_id,
5362            json!({ "drained": true, "kind": kind.label(), "requeued": requeued }),
5363        )
5364    });
5365    let rx = match kind {
5366        MaintenanceDrainKind::Watcher => executor.submit_coalescable_maintenance_async(
5367            root_id,
5368            lane,
5369            request_id.clone(),
5370            crate::executor::MaintenanceCoalesceKey::WatcherDrain,
5371            job,
5372        ),
5373        MaintenanceDrainKind::Lsp => executor.submit_coalescable_maintenance_async(
5374            root_id,
5375            lane,
5376            request_id.clone(),
5377            crate::executor::MaintenanceCoalesceKey::LspDrain,
5378            job,
5379        ),
5380        MaintenanceDrainKind::ConfigureTail | MaintenanceDrainKind::CompletionDrains => {
5381            executor.submit_maintenance_async(root_id, lane, request_id.clone(), job)
5382        }
5383    };
5384    let completion_tx = completion_tx.clone();
5385    let completion_metrics = Arc::clone(metrics);
5386    tokio::spawn(async move {
5387        let _response_task = ResponseTaskGuard::new(&completion_metrics);
5388        let response = await_executor_response(rx, request_id).await;
5389        let outcome = outcome_rx.await.unwrap_or_default();
5390        let _ = send_counted_channel(
5391            &completion_tx,
5392            &completion_metrics.maintenance_queued,
5393            MaintenanceCompletion {
5394                root_id: completion_root_id,
5395                kind,
5396                response,
5397                empty_bg_sessions: outcome.empty_bg_sessions,
5398                requeue_kind: outcome.requeue_kind,
5399            },
5400        )
5401        .await;
5402    });
5403}
5404
5405async fn await_executor_response(rx: oneshot::Receiver<Response>, request_id: String) -> Response {
5406    rx.await
5407        .unwrap_or_else(|_| Response::error(request_id, "internal_error", "executor dropped"))
5408}
5409
5410async fn deliver_resolved_subc_response(
5411    tx: &WriterSender,
5412    mut resolved: ResolvedSubcResponse,
5413    routes: &HashMap<RouteChannel, RouteIdentity>,
5414    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
5415    executor: &Executor,
5416    active_tool_calls: &ActiveToolCalls,
5417    shutdown: &Arc<Notify>,
5418    metrics: &DispatchPathMetrics,
5419    tool_response_body_limit: usize,
5420) -> Result<(), SubcError> {
5421    let entry = &mut resolved.entry;
5422    finish_active_tool_call(active_tool_calls, entry.route, entry.corr);
5423    if let Some(meta) = live_roots.get_mut(&entry.root) {
5424        meta.note_activity();
5425    }
5426
5427    let Some(identity) = routes.get(&entry.route) else {
5428        log::debug!(
5429            "subc attach: dropping deferred {} response {} for unbound route {}",
5430            entry.bare_name,
5431            entry.pending.request_id,
5432            entry.route
5433        );
5434        return Ok(());
5435    };
5436    let Some(ctx) = executor.actor_context(&entry.root) else {
5437        return Ok(());
5438    };
5439    entry.phase_trace.mark_execute_done();
5440    let finalizer = |response: &mut Response| {
5441        crate::response_finalize::finalize_response_with_bg_completions(
5442            response,
5443            &ctx,
5444            &entry.session_id,
5445            &entry.bare_name,
5446            entry.bind_trust.allows_bash_observation(),
5447        );
5448    };
5449    let result = finish_tool_call_response(
5450        &entry.bare_name,
5451        &entry.format_context,
5452        resolved.response,
5453        entry.surface_downgraded,
5454        Some(&finalizer),
5455        Some(&mut entry.phase_trace),
5456    );
5457    let fatal = response_is_fatal_panic(&result.response);
5458    let response_frame = build_tool_response_frame_with_limit(
5459        entry.ver,
5460        entry.route,
5461        entry.corr,
5462        entry.flags,
5463        &result,
5464        identity.trust,
5465        tool_response_body_limit,
5466    )?;
5467    let trace = ToolResponseWriteTrace::new(
5468        std::mem::replace(&mut entry.phase_trace, PhaseTrace::new(Instant::now())),
5469        entry.bare_name.clone(),
5470        identity.project_root.clone(),
5471        entry.session_id.clone(),
5472        entry.route.channel,
5473        entry.corr,
5474    );
5475    send_traced_tool_response_frame(tx, metrics, response_frame, trace).await?;
5476    if fatal {
5477        signal_fatal_teardown(
5478            tx,
5479            Some(entry.route),
5480            entry.ver,
5481            entry.corr,
5482            shutdown,
5483            metrics,
5484        )
5485        .await;
5486    }
5487    Ok(())
5488}
5489
5490async fn signal_fatal_teardown(
5491    tx: &WriterSender,
5492    route: Option<RouteChannel>,
5493    ver: u8,
5494    corr: u64,
5495    shutdown: &Arc<Notify>,
5496    metrics: &DispatchPathMetrics,
5497) {
5498    if let Some(route) = route {
5499        if let Ok(frame) = build_goodbye_frame(ver, route.channel, route.epoch, corr) {
5500            if let Err(error) = send_frame(tx, metrics, frame).await {
5501                log::warn!(
5502                    "subc attach: failed to queue fatal route Goodbye for route {route}: {error}"
5503                );
5504            }
5505        }
5506    }
5507    if let Ok(frame) = build_goodbye_frame(ver, 0, 0, 0) {
5508        if let Err(error) = send_frame(tx, metrics, frame).await {
5509            log::warn!("subc attach: failed to queue fatal channel-0 Goodbye: {error}");
5510        }
5511    }
5512    shutdown.notify_one();
5513}
5514#[derive(Debug, Deserialize)]
5515#[serde(untagged)]
5516enum RouteRequest {
5517    BgEvents(BgEventsRequest),
5518    ToolCall(ToolCallRequest),
5519}
5520
5521#[derive(Debug, Deserialize)]
5522struct BgEventsRequest {
5523    op: BgEventsOp,
5524}
5525
5526#[derive(Debug, Deserialize)]
5527#[serde(rename_all = "snake_case")]
5528enum BgEventsOp {
5529    BgEvents,
5530}
5531
5532#[derive(Debug, Deserialize)]
5533struct ToolCallRequest {
5534    name: String,
5535    #[serde(default)]
5536    arguments: Value,
5537    /// Host-computed registration fact; kept outside agent-controlled arguments.
5538    #[serde(default)]
5539    edit_slot_survives: Option<bool>,
5540    /// Server-owned preview control (B1c-0): the plugin's mutation flow is
5541    /// preview -> permission ask -> apply. Dropping this field made "preview"
5542    /// calls mutate disk before the permission prompt and the subsequent
5543    /// apply fail with not-found.
5544    #[serde(default)]
5545    preview: bool,
5546}
5547
5548#[cfg(test)]
5549pub(crate) mod test_support {
5550    use super::*;
5551    use crate::bash_background::BgTaskStatus;
5552    use crate::protocol::{
5553        BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, ConfigureWarningsFrame,
5554        ProgressFrame, StatusChangedFrame,
5555    };
5556    use serde_json::json;
5557
5558    pub(super) fn test_root(name: &str) -> (tempfile::TempDir, ProjectRootId) {
5559        let dir = tempfile::Builder::new()
5560            .prefix(name)
5561            .tempdir()
5562            .expect("temp root");
5563        let root = ProjectRootId::from_path(dir.path()).expect("project root id");
5564        (dir, root)
5565    }
5566
5567    pub(super) fn test_ctx() -> Arc<AppContext> {
5568        Arc::new(AppContext::new(
5569            Box::new(crate::parser::TreeSitterProvider::new()),
5570            crate::config::Config::default(),
5571        ))
5572    }
5573
5574    fn inspect_context(root: &Path) -> Arc<AppContext> {
5575        let mut config = crate::config::Config::default();
5576        config.project_root = Some(root.to_path_buf());
5577        let ctx = Arc::new(AppContext::new(
5578            Box::new(crate::parser::TreeSitterProvider::new()),
5579            config,
5580        ));
5581        ctx.set_harness(crate::harness::Harness::Opencode);
5582        ctx
5583    }
5584
5585    fn inspect_request(id: &str) -> RawRequest {
5586        serde_json::from_value(json!({ "id": id, "command": "inspect" })).expect("inspect request")
5587    }
5588
5589    fn submit_deferred_inspect_setup(
5590        executor: &Arc<Executor>,
5591        root: &ProjectRootId,
5592        ctx: &Arc<AppContext>,
5593        request_id: &str,
5594    ) -> (PendingResponse, JobCancellation) {
5595        let (pending_tx, pending_rx) = std::sync::mpsc::sync_channel(1);
5596        let request = inspect_request(request_id);
5597        let inspect_ctx = Arc::clone(ctx);
5598        let (_rx, cancellation) = executor.submit_cancellable_async(
5599            root.clone(),
5600            Lane::SerialLspStatus,
5601            request_id.to_string(),
5602            Box::new(move |_| {
5603                let DispatchOutcome::Deferred(pending) =
5604                    crate::commands::inspect::handle_inspect_deferred_with_restriction(
5605                        &request,
5606                        inspect_ctx,
5607                        true,
5608                    )
5609                else {
5610                    panic!("inspect setup must defer")
5611                };
5612                pending_tx.send(pending).expect("send pending inspect");
5613                Response::success("inspect-setup", json!({}))
5614            }),
5615        );
5616        let pending = pending_rx
5617            .recv_timeout(Duration::from_secs(1))
5618            .expect("inspect setup leaves the executor lane");
5619        let deadline = Instant::now() + Duration::from_secs(1);
5620        while !executor.actor_is_idle(root) {
5621            assert!(
5622                Instant::now() < deadline,
5623                "inspect setup kept lane counters live"
5624            );
5625            std::thread::sleep(Duration::from_millis(5));
5626        }
5627        (pending, cancellation)
5628    }
5629
5630    fn wait_for_inspect_terminal(pending: &mut PendingResponse, ctx: &AppContext) -> Response {
5631        let deadline = Instant::now() + Duration::from_secs(60);
5632        loop {
5633            if let Some(response) = (pending.poll)(ctx) {
5634                return response;
5635            }
5636            assert!(Instant::now() < deadline, "inspect terminal timed out");
5637            std::thread::sleep(Duration::from_millis(5));
5638        }
5639    }
5640
5641    fn cold_navigation_context(root: &Path) -> (Arc<AppContext>, PathBuf) {
5642        let source_dir = root.join("src");
5643        std::fs::create_dir_all(&source_dir).expect("create source dir");
5644        std::fs::write(root.join("Cargo.toml"), "[package]\nname = \"fixture\"\n")
5645            .expect("write Cargo manifest");
5646        let source = source_dir.join("main.rs");
5647        std::fs::write(&source, "fn main() {}\n").expect("write source");
5648        let binary = root.join("cold-navigation-server");
5649        std::fs::write(&binary, b"fixture").expect("write server placeholder");
5650
5651        let ctx = inspect_context(root);
5652        ctx.lsp()
5653            .override_binary(crate::lsp::registry::ServerKind::Rust, binary);
5654        (ctx, source)
5655    }
5656
5657    fn navigation_request(id: &str, source: &Path) -> RawRequest {
5658        serde_json::from_value(json!({
5659            "id": id,
5660            "command": "lsp_hover",
5661            "file": source.display().to_string(),
5662            "line": 1,
5663            "character": 1,
5664        }))
5665        .expect("navigation request")
5666    }
5667
5668    fn submit_deferred_navigation_setup(
5669        executor: &Arc<Executor>,
5670        root: &ProjectRootId,
5671        ctx: &Arc<AppContext>,
5672        source: &Path,
5673        request_id: &str,
5674    ) -> (PendingResponse, JobCancellation) {
5675        let (pending_tx, pending_rx) = std::sync::mpsc::sync_channel(1);
5676        let request = navigation_request(request_id, source);
5677        let navigation_ctx = Arc::clone(ctx);
5678        let (_rx, cancellation) = executor.submit_cancellable_async(
5679            root.clone(),
5680            Lane::SerialLspStatus,
5681            request_id.to_string(),
5682            Box::new(move |_| {
5683                let DispatchOutcome::Deferred(pending) = crate::commands::lsp_navigation::
5684                    handle_lsp_navigation_deferred_with_restriction(
5685                        &request,
5686                        navigation_ctx,
5687                        true,
5688                    )
5689                else {
5690                    panic!("cold navigation setup must defer")
5691                };
5692                pending_tx.send(pending).expect("send pending navigation");
5693                Response::success("navigation-setup", json!({}))
5694            }),
5695        );
5696        let pending = pending_rx
5697            .recv_timeout(Duration::from_secs(1))
5698            .expect("navigation setup leaves the executor lane");
5699        let deadline = Instant::now() + Duration::from_secs(1);
5700        while !executor.actor_is_idle(root) {
5701            assert!(
5702                Instant::now() < deadline,
5703                "navigation setup kept lane counters live"
5704            );
5705            std::thread::sleep(Duration::from_millis(5));
5706        }
5707        (pending, cancellation)
5708    }
5709
5710    fn wait_for_navigation_terminal(pending: &mut PendingResponse, ctx: &AppContext) -> Response {
5711        let deadline = Instant::now() + Duration::from_secs(1);
5712        loop {
5713            if let Some(response) = (pending.poll)(ctx) {
5714                return response;
5715            }
5716            assert!(Instant::now() < deadline, "navigation terminal timed out");
5717            std::thread::sleep(Duration::from_millis(5));
5718        }
5719    }
5720
5721    #[test]
5722    fn deferred_inspect_releases_lane_for_bind_and_mutation() {
5723        let _serial = crate::commands::inspect::deferred_inspect_test_lock();
5724        let executor = Arc::new(Executor::new());
5725        let (dir, root) = test_root("deferred-inspect-storm");
5726        std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
5727        let ctx = inspect_context(dir.path());
5728        executor.register_actor(root.clone(), Arc::clone(&ctx));
5729        let (started_rx, release_tx) =
5730            crate::commands::inspect::install_deferred_inspect_stat_gate_for_test();
5731        let (mut pending, _cancellation) =
5732            submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-storm");
5733        started_rx
5734            .recv_timeout(Duration::from_secs(1))
5735            .expect("deferred inspect body starts");
5736
5737        for request_id in ["subc-bind-other-session", "subc-edit-other-session"] {
5738            let response = executor.submit(
5739                root.clone(),
5740                Lane::Mutating,
5741                request_id.to_string(),
5742                Box::new(move |_| Response::success(request_id, json!({ "admitted": true }))),
5743            );
5744            assert!(
5745                response
5746                    .recv_timeout(Duration::from_secs(1))
5747                    .expect("writer admits while inspect remains deferred")
5748                    .success
5749            );
5750        }
5751        assert_eq!(
5752            crate::commands::inspect::deferred_inspect_root_count_for_test(),
5753            1,
5754            "writer admissions must not finish the detached inspect"
5755        );
5756
5757        release_tx.send(()).expect("release inspect body");
5758        let terminal = wait_for_inspect_terminal(&mut pending, &ctx);
5759        assert!(
5760            terminal.data.get("inspect_terminal").is_some(),
5761            "inspect must still produce its terminal: {:?}",
5762            terminal.data
5763        );
5764    }
5765
5766    #[test]
5767    fn deferred_cold_navigation_releases_lsp_lane_for_reads_and_mutation() {
5768        let _serial = crate::commands::lsp_navigation::deferred_navigation_test_lock();
5769        let executor = Arc::new(Executor::new());
5770        let (dir, root) = test_root("deferred-cold-navigation");
5771        let (ctx, source) = cold_navigation_context(dir.path());
5772        executor.register_actor(root.clone(), Arc::clone(&ctx));
5773        let (started_rx, _release_tx) =
5774            crate::commands::lsp_navigation::install_deferred_navigation_gate_for_test();
5775        let (mut pending, cancellation) = submit_deferred_navigation_setup(
5776            &executor,
5777            &root,
5778            &ctx,
5779            &source,
5780            "subc-cold-navigation",
5781        );
5782        started_rx
5783            .recv_timeout(Duration::from_secs(1))
5784            .expect("detached navigation reaches its cold-start gate");
5785
5786        for (lane, request_id) in [
5787            (Lane::PureRead, "subc-navigation-read"),
5788            (Lane::Mutating, "subc-navigation-write"),
5789        ] {
5790            let response = executor.submit(
5791                root.clone(),
5792                lane,
5793                request_id.to_string(),
5794                Box::new(move |_| Response::success(request_id, json!({ "admitted": true }))),
5795            );
5796            assert!(
5797                response
5798                    .recv_timeout(Duration::from_secs(1))
5799                    .expect("unrelated work admits while navigation remains deferred")
5800                    .success
5801            );
5802        }
5803        assert_eq!(
5804            crate::commands::lsp_navigation::deferred_navigation_worker_count_for_test(),
5805            1,
5806            "lane admissions must not finish the detached navigation"
5807        );
5808
5809        cancellation.request_cancel();
5810        let terminal = wait_for_navigation_terminal(&mut pending, &ctx);
5811        assert!(!terminal.success);
5812        let deadline = Instant::now() + Duration::from_secs(1);
5813        while crate::commands::lsp_navigation::deferred_navigation_worker_count_for_test() != 0 {
5814            assert!(
5815                Instant::now() < deadline,
5816                "cancelled navigation worker did not settle"
5817            );
5818            std::thread::sleep(Duration::from_millis(5));
5819        }
5820        assert!(executor.actor_is_idle(&root));
5821    }
5822
5823    #[test]
5824    fn cancelling_pending_navigation_removes_entry_without_reply() {
5825        let _serial = crate::commands::lsp_navigation::deferred_navigation_test_lock();
5826        let executor = Arc::new(Executor::new());
5827        let (dir, root) = test_root("cancelled-pending-navigation");
5828        let (ctx, source) = cold_navigation_context(dir.path());
5829        executor.register_actor(root.clone(), Arc::clone(&ctx));
5830        let (started_rx, _release_tx) =
5831            crate::commands::lsp_navigation::install_deferred_navigation_gate_for_test();
5832        let (pending, _cancellation) = submit_deferred_navigation_setup(
5833            &executor,
5834            &root,
5835            &ctx,
5836            &source,
5837            "subc-cancel-navigation",
5838        );
5839        started_rx
5840            .recv_timeout(Duration::from_secs(1))
5841            .expect("detached navigation reaches its cancellation gate");
5842
5843        let route = RouteChannel {
5844            channel: 17,
5845            epoch: 1,
5846        };
5847        let mut registry = PendingSubcResponses::default();
5848        registry.register(PendingSubcResponse {
5849            route,
5850            corr: 71,
5851            flags: Flags::new(false, Priority::Passive, false),
5852            ver: PROTOCOL_VERSION,
5853            root: root.clone(),
5854            session_id: "navigation-cancel-session".to_string(),
5855            bare_name: "lsp_hover".to_string(),
5856            format_context: crate::subc_format::FormatContext::from_tool_call(
5857                "lsp_hover",
5858                &json!({}),
5859                dir.path(),
5860            ),
5861            bind_trust: BindTrust::FirstParty,
5862            pending,
5863            surface_downgraded: false,
5864            phase_trace: PhaseTrace::new(Instant::now()),
5865        });
5866
5867        assert!(registry.cancel_request(route, 71));
5868        assert!(registry.is_empty());
5869        let deadline = Instant::now() + Duration::from_secs(1);
5870        while crate::commands::lsp_navigation::deferred_navigation_worker_count_for_test() != 0 {
5871            assert!(
5872                Instant::now() < deadline,
5873                "pending cancellation did not settle the detached worker"
5874            );
5875            std::thread::sleep(Duration::from_millis(5));
5876        }
5877        assert!(
5878            registry.poll_ready(executor.as_ref()).is_empty(),
5879            "a cancelled navigation must not leak a reply"
5880        );
5881    }
5882
5883    #[test]
5884    fn same_root_deferred_inspects_are_single_flight() {
5885        let _serial = crate::commands::inspect::deferred_inspect_test_lock();
5886        let executor = Arc::new(Executor::new());
5887        let (dir, root) = test_root("single-flight-deferred-inspect");
5888        std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
5889        let ctx = inspect_context(dir.path());
5890        executor.register_actor(root.clone(), Arc::clone(&ctx));
5891        let (started_rx, release_tx) =
5892            crate::commands::inspect::install_deferred_inspect_stat_gate_for_test();
5893        let (mut first, _first_cancellation) =
5894            submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-first");
5895        started_rx
5896            .recv_timeout(Duration::from_secs(1))
5897            .expect("first inspect owns the root flight");
5898        let (mut second, second_cancellation) =
5899            submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-second");
5900
5901        assert_eq!(
5902            crate::commands::inspect::deferred_inspect_root_count_for_test(),
5903            1,
5904            "only one detached body may run for a root"
5905        );
5906        assert!((second.poll)(&ctx).is_none(), "second inspect must queue");
5907        second_cancellation.request_cancel();
5908        let second_terminal = wait_for_inspect_terminal(&mut second, &ctx);
5909        assert_eq!(second_terminal.data["inspect_terminal"], "interrupted");
5910        assert_eq!(
5911            crate::commands::inspect::deferred_inspect_root_count_for_test(),
5912            1,
5913            "cancelling the queued request must not release the active flight"
5914        );
5915
5916        release_tx.send(()).expect("release first inspect");
5917        let first_terminal = wait_for_inspect_terminal(&mut first, &ctx);
5918        assert_eq!(first_terminal.data["inspect_terminal"], "fresh");
5919        assert_eq!(
5920            crate::commands::inspect::deferred_inspect_root_count_for_test(),
5921            0
5922        );
5923    }
5924
5925    #[test]
5926    fn route_abandonment_cancels_detached_inspect_thread() {
5927        let _serial = crate::commands::inspect::deferred_inspect_test_lock();
5928        let executor = Arc::new(Executor::new());
5929        let (dir, root) = test_root("cancelled-deferred-inspect");
5930        std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
5931        let ctx = inspect_context(dir.path());
5932        executor.register_actor(root.clone(), Arc::clone(&ctx));
5933        let active: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
5934        let route = RouteChannel {
5935            channel: 7,
5936            epoch: 1,
5937        };
5938        let (started_rx, _release_tx) =
5939            crate::commands::inspect::install_deferred_inspect_body_gate_for_test();
5940        let (mut pending, cancellation) =
5941            submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-abandoned");
5942        active.lock().expect("active tool call map").insert(
5943            (route, 41),
5944            ActiveToolCall {
5945                root_id: root.clone(),
5946                cancellation,
5947                detach_policy: RouteDetachPolicy::CancelOnDetach,
5948            },
5949        );
5950        started_rx
5951            .recv_timeout(Duration::from_secs(1))
5952            .expect("detached inspect reaches cancellation gate");
5953        assert!(ctx.request_force_restrict("subc-inspect-abandoned"));
5954
5955        assert!(cancel_active_tool_call(
5956            &active,
5957            executor.as_ref(),
5958            route,
5959            41,
5960            "test route abandonment"
5961        ));
5962        let terminal = wait_for_inspect_terminal(&mut pending, &ctx);
5963        assert_eq!(terminal.data["inspect_terminal"], "interrupted");
5964        assert_eq!(
5965            crate::commands::inspect::deferred_inspect_root_count_for_test(),
5966            0
5967        );
5968        assert!(executor.actor_is_idle(&root));
5969        assert!(active.lock().expect("active tool call map").is_empty());
5970        let restriction_deadline = Instant::now() + Duration::from_secs(1);
5971        while ctx.request_force_restrict("subc-inspect-abandoned") {
5972            assert!(
5973                Instant::now() < restriction_deadline,
5974                "detached force-restrict guard leaked"
5975            );
5976            std::thread::sleep(Duration::from_millis(5));
5977        }
5978    }
5979
5980    #[test]
5981    fn true_abandonment_cancels_but_route_detach_retains_interactive_search() {
5982        // This scenario requires the replayable and terminal calls to run at the
5983        // same time. Pin two read slots instead of deriving the topology from
5984        // the host, where a small runner can queue the terminal call forever
5985        // behind the intentionally retained call.
5986        let executor = Arc::new(Executor::with_config(crate::executor::ExecutorConfig {
5987            pool_size: 3,
5988            read_cap: 2,
5989            actor_cap: 2,
5990            heavy_permits: 1,
5991            drr_quantum: 1,
5992        }));
5993        let (_dir, root) = test_root("cancelled-interactive-search");
5994        executor.register_actor(root.clone(), test_ctx());
5995        let active: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
5996        let route = RouteChannel {
5997            channel: 9,
5998            epoch: 1,
5999        };
6000
6001        let disabled_iterations = Arc::new(AtomicUsize::new(0));
6002        let disabled_probe = Arc::clone(&disabled_iterations);
6003        let (disabled_started_tx, disabled_started_rx) = std::sync::mpsc::sync_channel(1);
6004        let (disabled_rx, disabled_cancellation) = executor.submit_cancellable_async(
6005            root.clone(),
6006            Lane::PureRead,
6007            "untracked-search".to_string(),
6008            Box::new(move |_| {
6009                disabled_started_tx
6010                    .send(())
6011                    .expect("signal untracked search");
6012                let deadline = Instant::now() + Duration::from_secs(5);
6013                while !crate::commands::semantic_search::search_cancellation_requested() {
6014                    if Instant::now() >= deadline {
6015                        return Response::error(
6016                            "untracked-search",
6017                            "test_timeout",
6018                            "untracked search did not receive cancellation",
6019                        );
6020                    }
6021                    disabled_probe.fetch_add(1, Ordering::Relaxed);
6022                    std::thread::yield_now();
6023                }
6024                Response::error(
6025                    "untracked-search",
6026                    "request_cancelled",
6027                    "cancelled at search checkpoint",
6028                )
6029            }),
6030        );
6031        disabled_started_rx
6032            .recv_timeout(Duration::from_secs(1))
6033            .expect("untracked search starts");
6034        assert_eq!(
6035            apply_route_work_disposition(
6036                &active,
6037                executor.as_ref(),
6038                route,
6039                RouteWorkDisposition::Abandon,
6040                "disabled cancellation wiring",
6041            ),
6042            0
6043        );
6044        let iterations_before = disabled_iterations.load(Ordering::Relaxed);
6045        std::thread::sleep(Duration::from_millis(10));
6046        assert!(
6047            disabled_iterations.load(Ordering::Relaxed) > iterations_before,
6048            "without route registration the search keeps computing"
6049        );
6050        disabled_cancellation.request_cancel();
6051        let disabled_response = disabled_rx
6052            .blocking_recv()
6053            .expect("untracked search settles after explicit cleanup");
6054        assert_eq!(disabled_response.data["code"], "request_cancelled");
6055
6056        let tracked_iterations = Arc::new(AtomicUsize::new(0));
6057        let tracked_probe = Arc::clone(&tracked_iterations);
6058        let (tracked_started_tx, tracked_started_rx) = std::sync::mpsc::sync_channel(1);
6059        let tracked_rx = submit_active_tool_call(
6060            executor.as_ref(),
6061            &active,
6062            route,
6063            42,
6064            root.clone(),
6065            Lane::PureRead,
6066            "tracked-search".to_string(),
6067            RouteDetachPolicy::RetainForReplay,
6068            Box::new(move |_| {
6069                tracked_started_tx.send(()).expect("signal tracked search");
6070                let deadline = Instant::now() + Duration::from_secs(5);
6071                while !crate::commands::semantic_search::search_cancellation_requested() {
6072                    if Instant::now() >= deadline {
6073                        return Response::error(
6074                            "tracked-search",
6075                            "test_timeout",
6076                            "tracked search did not receive cancellation",
6077                        );
6078                    }
6079                    tracked_probe.fetch_add(1, Ordering::Relaxed);
6080                    std::thread::yield_now();
6081                }
6082                Response::error(
6083                    "tracked-search",
6084                    "request_cancelled",
6085                    "cancelled at search checkpoint",
6086                )
6087            }),
6088        );
6089        tracked_started_rx
6090            .recv_timeout(Duration::from_secs(1))
6091            .expect("tracked search starts");
6092
6093        let (terminal_started_tx, terminal_started_rx) = std::sync::mpsc::sync_channel(1);
6094        let terminal_rx = submit_active_tool_call(
6095            executor.as_ref(),
6096            &active,
6097            route,
6098            43,
6099            root.clone(),
6100            Lane::PureRead,
6101            "teardown-terminal".to_string(),
6102            RouteDetachPolicy::CancelOnDetach,
6103            Box::new(move |_| {
6104                terminal_started_tx
6105                    .send(())
6106                    .expect("signal teardown-terminal call");
6107                let deadline = Instant::now() + Duration::from_secs(5);
6108                while !crate::executor::current_job_cancelled() {
6109                    if Instant::now() >= deadline {
6110                        return Response::error(
6111                            "teardown-terminal",
6112                            "test_timeout",
6113                            "terminal call did not receive cancellation",
6114                        );
6115                    }
6116                    std::thread::yield_now();
6117                }
6118                Response::error(
6119                    "teardown-terminal",
6120                    "request_cancelled",
6121                    "cancelled for terminal-emitting teardown",
6122                )
6123            }),
6124        );
6125        terminal_started_rx
6126            .recv_timeout(Duration::from_secs(1))
6127            .expect("teardown-terminal call starts");
6128        assert_eq!(
6129            apply_route_work_disposition(
6130                &active,
6131                executor.as_ref(),
6132                route,
6133                RouteWorkDisposition::RetainForReplay,
6134                "test route detach",
6135            ),
6136            1,
6137            "only the replayable search remains active after route detach"
6138        );
6139        let terminal_response = terminal_rx
6140            .blocking_recv()
6141            .expect("teardown-terminal call stops at cancellation checkpoint");
6142        assert_eq!(terminal_response.data["code"], "request_cancelled");
6143        let iterations_before_detach = tracked_iterations.load(Ordering::Relaxed);
6144        std::thread::sleep(Duration::from_millis(10));
6145        assert!(
6146            tracked_iterations.load(Ordering::Relaxed) > iterations_before_detach,
6147            "route detach must retain work whose response can replay after rebind"
6148        );
6149        assert_eq!(
6150            apply_route_work_disposition(
6151                &active,
6152                executor.as_ref(),
6153                route,
6154                RouteWorkDisposition::Abandon,
6155                "test session purge",
6156            ),
6157            1
6158        );
6159        let tracked_response = tracked_rx
6160            .blocking_recv()
6161            .expect("tracked search stops at cancellation checkpoint");
6162        assert_eq!(tracked_response.data["code"], "request_cancelled");
6163        assert!(active.lock().expect("active tool calls").is_empty());
6164
6165        let deadline = Instant::now() + Duration::from_secs(1);
6166        while !executor.actor_is_idle(&root) {
6167            assert!(
6168                Instant::now() < deadline,
6169                "cancelled search must release the PureRead lane"
6170            );
6171            std::thread::sleep(Duration::from_millis(2));
6172        }
6173    }
6174
6175    #[test]
6176    fn shutdown_drain_emits_terminal_and_clears_pending_inspect() {
6177        let _serial = crate::commands::inspect::deferred_inspect_test_lock();
6178        let executor = Arc::new(Executor::new());
6179        let (dir, root) = test_root("shutdown-deferred-inspect");
6180        std::fs::write(dir.path().join("README.md"), "# Fixture\n").expect("fixture");
6181        let ctx = inspect_context(dir.path());
6182        executor.register_actor(root.clone(), Arc::clone(&ctx));
6183        let (started_rx, _release_tx) =
6184            crate::commands::inspect::install_deferred_inspect_body_gate_for_test();
6185        let (pending, cancellation) =
6186            submit_deferred_inspect_setup(&executor, &root, &ctx, "subc-inspect-shutdown");
6187        let route = RouteChannel {
6188            channel: 8,
6189            epoch: 1,
6190        };
6191        started_rx
6192            .recv_timeout(Duration::from_secs(1))
6193            .expect("detached inspect reaches shutdown gate");
6194        let mut registry = PendingSubcResponses::default();
6195        registry.register(PendingSubcResponse {
6196            route,
6197            corr: 42,
6198            flags: Flags::new(false, Priority::Passive, false),
6199            ver: PROTOCOL_VERSION,
6200            root: root.clone(),
6201            session_id: "shutdown-session".to_string(),
6202            bare_name: "inspect".to_string(),
6203            format_context: crate::subc_format::FormatContext::from_tool_call(
6204                "inspect",
6205                &json!({}),
6206                dir.path(),
6207            ),
6208            bind_trust: BindTrust::FirstParty,
6209            pending,
6210            surface_downgraded: false,
6211            phase_trace: PhaseTrace::new(Instant::now()),
6212        });
6213        let active: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::from([(
6214            (route, 42),
6215            ActiveToolCall {
6216                root_id: root.clone(),
6217                cancellation,
6218                detach_policy: RouteDetachPolicy::CancelOnDetach,
6219            },
6220        )])));
6221
6222        let resolved = registry.drain_on_shutdown(executor.as_ref());
6223        assert!(registry.is_empty());
6224        assert_eq!(resolved.len(), 1);
6225        assert_eq!(
6226            resolved[0].response.data["failure_reason"],
6227            "daemon_shutdown"
6228        );
6229        finish_active_tool_call(&active, route, 42);
6230        let deadline = Instant::now() + Duration::from_secs(1);
6231        while crate::commands::inspect::deferred_inspect_root_count_for_test() != 0 {
6232            assert!(Instant::now() < deadline, "shutdown cancellation was inert");
6233            std::thread::sleep(Duration::from_millis(5));
6234        }
6235        assert!(active.lock().expect("active calls").is_empty());
6236        assert!(executor.actor_is_idle(&root));
6237    }
6238
6239    pub(super) fn wait_for_watcher_count(ctx: &AppContext, expected: usize) {
6240        let deadline = Instant::now() + Duration::from_secs(30);
6241        loop {
6242            let observed = ctx.watcher_registry_count();
6243            if observed == expected {
6244                return;
6245            }
6246            assert!(
6247                Instant::now() < deadline,
6248                "watcher count did not settle before deadline: expected={expected}, observed={observed}"
6249            );
6250            std::thread::sleep(Duration::from_millis(50));
6251        }
6252    }
6253
6254    /// Sweep until `root` is forgotten, mirroring how production reaps.
6255    ///
6256    /// `reap_idle_roots` probes actor idleness with a try-lock and retains the
6257    /// root when the scheduler holds that lock — correct behavior, since the
6258    /// real caller sweeps on a timer and simply catches the root next tick. A
6259    /// test that asserts a single sweep succeeds is therefore asserting it wins
6260    /// a lock race that nothing in production depends on: `register_actor` wakes
6261    /// the scheduler, which grabs the same lock, and on a loaded runner that
6262    /// window is wide enough to lose. Sweep to the outcome instead.
6263    pub(super) fn reap_until_forgotten(
6264        root: &ProjectRootId,
6265        live_roots: &mut HashMap<ProjectRootId, RootMeta>,
6266        pending_binds: &HashMap<RouteChannel, PendingBind>,
6267        root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
6268        executor: &Arc<Executor>,
6269        metrics: &DispatchPathMetrics,
6270    ) -> IdleReapOutcome {
6271        let deadline = Instant::now() + Duration::from_secs(30);
6272        loop {
6273            let outcome = reap_idle_roots(
6274                Instant::now(),
6275                live_roots,
6276                pending_binds,
6277                root_channels,
6278                executor,
6279                metrics,
6280            );
6281            if outcome.forgotten_deleted_roots.contains(root) {
6282                return outcome;
6283            }
6284            assert!(
6285                Instant::now() < deadline,
6286                "deleted root was never forgotten: {root:?}"
6287            );
6288            std::thread::sleep(Duration::from_millis(10));
6289        }
6290    }
6291
6292    pub(super) fn wait_for_actor_root_count(app: &App, expected: usize) {
6293        let deadline = Instant::now() + Duration::from_secs(30);
6294        loop {
6295            let observed = app.actor_root_count();
6296            if observed == expected {
6297                return;
6298            }
6299            assert!(
6300                Instant::now() < deadline,
6301                "actor root count did not settle before deadline: expected={expected}, observed={observed}"
6302            );
6303            std::thread::sleep(Duration::from_millis(50));
6304        }
6305    }
6306
6307    pub(super) fn status_frame(seq: u64) -> PushFrame {
6308        status_frame_with_session(seq, None)
6309    }
6310
6311    pub(super) fn status_frame_with_session(seq: u64, session_id: Option<&str>) -> PushFrame {
6312        PushFrame::StatusChanged(StatusChangedFrame {
6313            frame_type: "status_changed",
6314            session_id: session_id.map(str::to_string),
6315            snapshot: json!({ "seq": seq }),
6316        })
6317    }
6318
6319    pub(super) fn completion_frame(task_id: &str) -> PushFrame {
6320        completion_frame_with_session(task_id, "session-1")
6321    }
6322
6323    pub(super) fn completion_frame_with_session(task_id: &str, session_id: &str) -> PushFrame {
6324        PushFrame::BashCompleted(BashCompletedFrame {
6325            frame_type: "bash_completed",
6326            task_id: task_id.to_string(),
6327            session_id: session_id.to_string(),
6328            status: BgTaskStatus::Completed,
6329            exit_code: Some(0),
6330            command: format!("echo {task_id}"),
6331            output_preview: String::new(),
6332            output_truncated: false,
6333            original_tokens: None,
6334            compressed_tokens: None,
6335            tokens_skipped: false,
6336            status_reason: None,
6337        })
6338    }
6339
6340    pub(super) fn long_running_frame(task_id: &str, elapsed_ms: u64) -> PushFrame {
6341        long_running_frame_with_session(task_id, "session-1", elapsed_ms)
6342    }
6343
6344    pub(super) fn long_running_frame_with_session(
6345        task_id: &str,
6346        session_id: &str,
6347        elapsed_ms: u64,
6348    ) -> PushFrame {
6349        PushFrame::BashLongRunning(BashLongRunningFrame {
6350            frame_type: "bash_long_running",
6351            task_id: task_id.to_string(),
6352            session_id: session_id.to_string(),
6353            command: format!("sleep {elapsed_ms}"),
6354            elapsed_ms,
6355        })
6356    }
6357
6358    pub(super) fn pattern_match_frame(session_id: &str) -> PushFrame {
6359        PushFrame::BashPatternMatch(BashPatternMatchFrame {
6360            frame_type: "bash_pattern_match",
6361            task_id: "task-pattern".to_string(),
6362            session_id: session_id.to_string(),
6363            watch_id: "watch-1".to_string(),
6364            match_text: "needle".to_string(),
6365            match_offset: 7,
6366            context: "haystack needle".to_string(),
6367            once: true,
6368            reason: "pattern_match",
6369        })
6370    }
6371
6372    pub(super) fn configure_warnings_frame(session_id: Option<&str>) -> PushFrame {
6373        PushFrame::ConfigureWarnings(ConfigureWarningsFrame {
6374            frame_type: "configure_warnings",
6375            session_id: session_id.map(str::to_string),
6376            project_root: "/tmp/subc-test".to_string(),
6377            warnings: Vec::new(),
6378        })
6379    }
6380
6381    pub(super) fn route_identity(root: &ProjectRootId, session_id: &str) -> RouteIdentity {
6382        route_identity_with_trust(root, session_id, BindTrust::FirstParty)
6383    }
6384
6385    pub(super) fn route_identity_with_trust(
6386        root: &ProjectRootId,
6387        session_id: &str,
6388        trust: BindTrust,
6389    ) -> RouteIdentity {
6390        RouteIdentity(Arc::new(RouteIdentityData {
6391            root: root.clone(),
6392            project_root: root.as_path().to_path_buf(),
6393            harness: "opencode".to_string(),
6394            session: session_id.to_string(),
6395            trust,
6396            spawn_principal: AuthenticatedPrincipal::RouteBind {
6397                trust: trust.sandbox_trust(),
6398                route_channel: 0,
6399                route_epoch: 0,
6400                project_root: root.as_path().to_path_buf(),
6401                harness: "opencode".to_string(),
6402                session_id: session_id.to_string(),
6403                principal_id: Some(match trust {
6404                    BindTrust::FirstParty => "direct".to_string(),
6405                    BindTrust::Untrusted => "unverified".to_string(),
6406                }),
6407            },
6408            consumer_elicitation_capable: false,
6409        }))
6410    }
6411
6412    pub(super) fn progress_frame(request_id: &str, kind: ProgressKind, chunk: &str) -> PushFrame {
6413        PushFrame::Progress(ProgressFrame::new(request_id, kind, chunk))
6414    }
6415
6416    pub(super) fn status_seq(frame: &PushFrame) -> Option<u64> {
6417        match frame {
6418            PushFrame::StatusChanged(status) => status.snapshot.get("seq").and_then(|v| v.as_u64()),
6419            _ => None,
6420        }
6421    }
6422
6423    pub(super) fn completion_task(frame: &PushFrame) -> Option<&str> {
6424        match frame {
6425            PushFrame::BashCompleted(completion) => Some(completion.task_id.as_str()),
6426            _ => None,
6427        }
6428    }
6429
6430    pub(super) fn push_frame_task_id(frame: &Frame) -> Option<String> {
6431        let body: serde_json::Value = serde_json::from_slice(&frame.body).expect("push body");
6432        body.get("task_id")
6433            .and_then(serde_json::Value::as_str)
6434            .map(str::to_string)
6435    }
6436}
6437
6438#[cfg(test)]
6439mod tests {
6440    use super::test_support::{
6441        completion_frame, reap_until_forgotten, route_identity, test_ctx, test_root,
6442        wait_for_actor_root_count, wait_for_watcher_count,
6443    };
6444    use super::*;
6445    use crate::bash_background::BgTaskStatus;
6446
6447    fn attach_error(kind: io::ErrorKind) -> SubcError {
6448        SubcError::Connect {
6449            endpoint: "127.0.0.1:1".to_string(),
6450            source: io::Error::new(kind, "constructed attach failure"),
6451        }
6452    }
6453
6454    fn auth_io_error(kind: io::ErrorKind) -> SubcError {
6455        SubcError::Auth {
6456            endpoint: "127.0.0.1:1".to_string(),
6457            source: subc_transport::AuthError::Io {
6458                stage: subc_transport::AuthStage::ServerProof,
6459                source: io::Error::new(kind, "constructed auth failure"),
6460            },
6461        }
6462    }
6463
6464    #[test]
6465    fn initial_attach_error_classifier_distinguishes_transient_and_permanent_failures() {
6466        let transient_errors = vec![
6467            attach_error(io::ErrorKind::ConnectionRefused),
6468            attach_error(io::ErrorKind::TimedOut),
6469            attach_error(io::ErrorKind::ConnectionReset),
6470            auth_io_error(io::ErrorKind::ConnectionAborted),
6471            auth_io_error(io::ErrorKind::BrokenPipe),
6472            SubcError::Auth {
6473                endpoint: "127.0.0.1:1".to_string(),
6474                source: subc_transport::AuthError::UnexpectedEof {
6475                    stage: subc_transport::AuthStage::ServerProof,
6476                    expected: 4,
6477                    actual: 0,
6478                },
6479            },
6480            SubcError::Auth {
6481                endpoint: "127.0.0.1:1".to_string(),
6482                source: subc_transport::AuthError::Timeout {
6483                    stage: subc_transport::AuthStage::ServerProof,
6484                    deadline: AUTH_DEADLINE,
6485                },
6486            },
6487        ];
6488        for error in &transient_errors {
6489            assert_eq!(
6490                classify_attach_error(error),
6491                AttachErrorClass::Transient,
6492                "expected transient: {error}"
6493            );
6494        }
6495
6496        let permanent_errors = vec![
6497            attach_error(io::ErrorKind::PermissionDenied),
6498            auth_io_error(io::ErrorKind::InvalidData),
6499            SubcError::Auth {
6500                endpoint: "127.0.0.1:1".to_string(),
6501                source: subc_transport::AuthError::InvalidServerProof,
6502            },
6503            SubcError::Auth {
6504                endpoint: "127.0.0.1:1".to_string(),
6505                source: subc_transport::AuthError::DaemonIdMismatch,
6506            },
6507            SubcError::ConnectionFile {
6508                path: PathBuf::from("subc-connection.json"),
6509                source: subc_transport::ConnectionFileError::Invalid {
6510                    reason: "constructed invalid file".to_string(),
6511                },
6512            },
6513            SubcError::NoEndpoint {
6514                path: PathBuf::from("subc-connection.json"),
6515            },
6516            SubcError::InvalidEndpoint {
6517                path: PathBuf::from("subc-connection.json"),
6518                endpoint: "not-an-ip:1234".to_string(),
6519            },
6520        ];
6521        for error in &permanent_errors {
6522            assert_eq!(
6523                classify_attach_error(error),
6524                AttachErrorClass::Permanent,
6525                "expected permanent: {error}"
6526            );
6527        }
6528    }
6529
6530    #[test]
6531    fn incompatible_wire_version_is_rejected_before_tcp_connect() {
6532        let conn_dir = tempfile::tempdir().expect("connection tempdir");
6533        let conn_path = conn_dir.path().join("subc-connection.json");
6534        let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind listener");
6535        listener
6536            .set_nonblocking(true)
6537            .expect("set listener nonblocking");
6538        let port = listener.local_addr().expect("listener addr").port();
6539        connection_file::write_atomic(
6540            &conn_path,
6541            &connection_file::ConnectionInfo {
6542                schema: connection_file::SCHEMA_VERSION,
6543                wire_version: Some(PROTOCOL_VERSION.wrapping_add(1)),
6544                endpoints: vec![connection_file::Endpoint {
6545                    host: "127.0.0.1".to_string(),
6546                    port,
6547                }],
6548                key: vec![0x42; subc_transport::KEY_LEN],
6549                daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
6550                pid: std::process::id(),
6551                daemon_ver: "subc-test".to_string(),
6552            },
6553        )
6554        .expect("write connection file");
6555
6556        let runtime = tokio::runtime::Builder::new_current_thread()
6557            .enable_all()
6558            .build()
6559            .expect("test runtime");
6560        let result = runtime.block_on(connect_and_authenticate_with_policy(
6561            &conn_path,
6562            AttachRetryPolicy {
6563                budget: Duration::from_secs(1),
6564                initial_backoff: Duration::from_millis(5),
6565                max_backoff: Duration::from_millis(10),
6566                jitter_percent: 0,
6567            },
6568        ));
6569        assert!(matches!(
6570            result,
6571            Err(SubcError::ConnectionFile {
6572                source: connection_file::ConnectionFileError::WireVersionMismatch { .. },
6573                ..
6574            })
6575        ));
6576        assert!(matches!(
6577            listener.accept(),
6578            Err(error) if error.kind() == std::io::ErrorKind::WouldBlock
6579        ));
6580    }
6581
6582    #[test]
6583    fn initial_attach_unreachable_endpoint_retries_until_budget_then_fails_loud() {
6584        let conn_dir = tempfile::tempdir().expect("connection tempdir");
6585        let conn_path = conn_dir.path().join("subc-connection.json");
6586        let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve port");
6587        let port = listener.local_addr().expect("reserved addr").port();
6588        drop(listener);
6589        connection_file::write_atomic(
6590            &conn_path,
6591            &connection_file::ConnectionInfo {
6592                schema: connection_file::SCHEMA_VERSION,
6593                wire_version: Some(PROTOCOL_VERSION),
6594                endpoints: vec![connection_file::Endpoint {
6595                    host: "127.0.0.1".to_string(),
6596                    port,
6597                }],
6598                key: vec![0x42; subc_transport::KEY_LEN],
6599                daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
6600                pid: std::process::id(),
6601                daemon_ver: "subc-test".to_string(),
6602            },
6603        )
6604        .expect("write connection file");
6605
6606        let policy = AttachRetryPolicy {
6607            budget: Duration::from_millis(40),
6608            initial_backoff: Duration::from_millis(5),
6609            max_backoff: Duration::from_millis(10),
6610            jitter_percent: 0,
6611        };
6612        let runtime = tokio::runtime::Builder::new_current_thread()
6613            .enable_all()
6614            .build()
6615            .expect("test runtime");
6616        let started_at = Instant::now();
6617        let result = runtime.block_on(connect_and_authenticate_with_policy(&conn_path, policy));
6618        let elapsed = started_at.elapsed();
6619        let error = match result {
6620            Ok(_) => panic!("unreachable endpoint unexpectedly attached"),
6621            Err(error) => error,
6622        };
6623
6624        assert!(matches!(error, SubcError::Connect { .. }), "{error}");
6625        assert!(
6626            elapsed >= Duration::from_millis(35),
6627            "retry budget ended too early: {elapsed:?}"
6628        );
6629        assert!(
6630            elapsed < Duration::from_secs(1),
6631            "retry budget was not bounded: {elapsed:?}"
6632        );
6633    }
6634
6635    fn due_maintenance_jobs_without_actor_context(
6636        live_roots: &mut HashMap<ProjectRootId, RootMeta>,
6637        budget: usize,
6638        pending_bind_roots: &HashSet<ProjectRootId>,
6639    ) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
6640        due_maintenance_jobs(
6641            live_roots,
6642            None,
6643            &HashMap::new(),
6644            &HashSet::new(),
6645            budget,
6646            pending_bind_roots,
6647        )
6648    }
6649
6650    fn actor_ctx_with_dirty_search_index(
6651        root: &Path,
6652        storage: &Path,
6653        file_name: &str,
6654        old_contents: &str,
6655        new_contents: &str,
6656    ) -> (Arc<AppContext>, PathBuf, PathBuf) {
6657        let file = root.join(file_name);
6658        std::fs::write(&file, old_contents).expect("write source");
6659        let canonical_root = std::fs::canonicalize(root).expect("canonical root");
6660        let ctx = Arc::new(AppContext::new(
6661            Box::new(crate::parser::TreeSitterProvider::new()),
6662            Config {
6663                project_root: Some(root.to_path_buf()),
6664                storage_dir: Some(storage.to_path_buf()),
6665                ..Config::default()
6666            },
6667        ));
6668        ctx.set_canonical_cache_root(canonical_root.clone());
6669
6670        let cache_dir = crate::search_index::resolve_cache_dir(&canonical_root, Some(storage));
6671        let mut index = crate::search_index::SearchIndex::build(&canonical_root);
6672        let git_head = index.stored_git_head().map(str::to_owned);
6673        index.write_to_disk(&cache_dir, git_head.as_deref());
6674
6675        std::fs::write(&file, new_contents).expect("edit source");
6676        index.update_file(&file);
6677        *ctx.search_index()
6678            .write()
6679            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
6680        (ctx, canonical_root, cache_dir)
6681    }
6682
6683    #[test]
6684    fn graceful_shutdown_flushes_every_actor_search_index() {
6685        let storage = tempfile::tempdir().expect("storage tempdir");
6686        let (root1_dir, root1) = test_root("shutdown-flush-root-1");
6687        let (root2_dir, root2) = test_root("shutdown-flush-root-2");
6688        let (ctx1, canonical_root1, cache_dir1) = actor_ctx_with_dirty_search_index(
6689            root1_dir.path(),
6690            storage.path(),
6691            "alpha.txt",
6692            "old actor one token\n",
6693            "new actor one token\n",
6694        );
6695        let (ctx2, canonical_root2, cache_dir2) = actor_ctx_with_dirty_search_index(
6696            root2_dir.path(),
6697            storage.path(),
6698            "beta.txt",
6699            "old actor two token\n",
6700            "new actor two token\n",
6701        );
6702
6703        let executor = Executor::new();
6704        assert!(executor.register_actor(root1.clone(), Arc::clone(&ctx1)));
6705        assert!(executor.register_actor(root2.clone(), Arc::clone(&ctx2)));
6706
6707        flush_actor_indexes_on_graceful_shutdown(&executor.actor_contexts());
6708
6709        let mut restored1 =
6710            crate::search_index::SearchIndex::read_from_disk(&cache_dir1, &canonical_root1)
6711                .expect("load flushed root one index");
6712        restored1.ready = true;
6713        assert_eq!(
6714            restored1
6715                .grep("new actor one token", true, &[], &[], &canonical_root1, 10)
6716                .matches
6717                .len(),
6718            1,
6719            "graceful subc shutdown should flush the first root's trigram delta"
6720        );
6721
6722        let mut restored2 =
6723            crate::search_index::SearchIndex::read_from_disk(&cache_dir2, &canonical_root2)
6724                .expect("load flushed root two index");
6725        restored2.ready = true;
6726        assert_eq!(
6727            restored2
6728                .grep("new actor two token", true, &[], &[], &canonical_root2, 10)
6729                .matches
6730                .len(),
6731            1,
6732            "graceful subc shutdown should flush every registered root"
6733        );
6734    }
6735
6736    #[test]
6737    fn idle_root_reaper_closes_artifacts_and_stops_watcher() {
6738        let _ = env_logger::builder().is_test(true).try_init();
6739        let (root_dir, root) = test_root("idle-root-reaper");
6740        let storage = tempfile::tempdir().expect("storage tempdir");
6741        std::fs::write(
6742            root_dir.path().join("main.rs"),
6743            "fn entry() { leaf(); }\nfn leaf() {}\n",
6744        )
6745        .expect("source file");
6746        let canonical_root = std::fs::canonicalize(root_dir.path()).expect("canonical root");
6747        let app = App::default_shared();
6748        let ctx = Arc::new(AppContext::from_app(
6749            Arc::clone(&app),
6750            Config {
6751                project_root: Some(canonical_root.clone()),
6752                storage_dir: Some(storage.path().to_path_buf()),
6753                callgraph_store: true,
6754                search_index: true,
6755                ..Config::default()
6756            },
6757        ));
6758        ctx.set_canonical_cache_root(canonical_root.clone());
6759        let project_key = crate::search_index::artifact_cache_key(&canonical_root);
6760        crate::root_cache::configure_artifact_access(&canonical_root, &project_key, false);
6761        assert!(ctx
6762            .ensure_callgraph_store()
6763            .expect("build callgraph store")
6764            .is_some());
6765
6766        let cache_dir =
6767            crate::search_index::resolve_cache_dir(&canonical_root, Some(storage.path()));
6768        let mut index = crate::search_index::SearchIndex::build(&canonical_root);
6769        let git_head = index.stored_git_head().map(str::to_owned);
6770        index.write_to_disk(&cache_dir, git_head.as_deref());
6771        *ctx.search_index()
6772            .write()
6773            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
6774        // Seed a completed warm verification so the test can prove eviction
6775        // downgrades it to Strict rather than passing vacuously.
6776        let seeded_generation =
6777            crate::cache_freshness::artifact_generation(&cache_dir.join("cache.bin"))
6778                .expect("seeded artifact generation");
6779        crate::cache_freshness::record_verify_completed(
6780            &canonical_root,
6781            crate::cache_freshness::VerifyArtifact::Search,
6782            Some(seeded_generation),
6783        );
6784        assert!(
6785            matches!(
6786                crate::cache_freshness::warm_verify_plan(
6787                    canonical_root.as_path(),
6788                    crate::cache_freshness::VerifyArtifact::Search,
6789                    Some(seeded_generation),
6790                ),
6791                crate::cache_freshness::WarmVerifyPlan::Skip
6792            ),
6793            "memo must be warm before eviction for the downgrade assertion to bite"
6794        );
6795
6796        let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
6797        let _dispatch_tx = dispatch_tx;
6798        let shutdown = Arc::new(AtomicBool::new(false));
6799        let thread_shutdown = Arc::clone(&shutdown);
6800        let join = std::thread::spawn(move || {
6801            while !thread_shutdown.load(Ordering::SeqCst) {
6802                std::thread::yield_now();
6803            }
6804        });
6805        ctx.install_watcher_runtime(
6806            dispatch_rx,
6807            crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
6808        );
6809        wait_for_watcher_count(&ctx, 1);
6810
6811        let executor = Arc::new(Executor::new());
6812        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6813        ctx.mark_subc_unbound();
6814        let mut live_roots = HashMap::new();
6815        let mut meta = RootMeta::new(Instant::now());
6816        meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
6817        meta.unbound_quiesced = true;
6818        live_roots.insert(root.clone(), meta);
6819
6820        let message = idle_root_eviction_message(&root, &ctx.memory_root_snapshot(), None);
6821        assert!(message.contains("evicted idle root"));
6822        assert!(message.contains("freed ~"));
6823        assert!(message.contains("semantic"));
6824        assert!(!message.contains("semantic not estimated retained"));
6825        assert!(message.contains("trigram"));
6826        assert!(message.contains("retained: bash"));
6827        assert!(message.contains("parser_pool"));
6828
6829        assert_eq!(
6830            reap_idle_roots(
6831                Instant::now(),
6832                &mut live_roots,
6833                &HashMap::new(),
6834                &HashMap::new(),
6835                &executor,
6836                &DispatchPathMetrics::new(),
6837            )
6838            .evicted,
6839            1
6840        );
6841        assert!(ctx.search_index().read().unwrap().is_none());
6842        wait_for_watcher_count(&ctx, 0);
6843        // The watcher stopped with the eviction, so the idle interval is
6844        // unobserved: the pre-seeded warm-verify memo (Skip) must fall back to
6845        // strict content verification (stat-first would miss same-size,
6846        // preserved-mtime edits made while nobody was watching).
6847        assert!(
6848            matches!(
6849                crate::cache_freshness::warm_verify_plan(
6850                    canonical_root.as_path(),
6851                    crate::cache_freshness::VerifyArtifact::Search,
6852                    Some(seeded_generation),
6853                ),
6854                crate::cache_freshness::WarmVerifyPlan::Strict
6855            ),
6856            "idle eviction must force strict re-verification"
6857        );
6858        assert!(
6859            crate::search_index::SearchIndex::read_from_disk(&cache_dir, &canonical_root).is_some()
6860        );
6861        ctx.mark_subc_bound();
6862        assert!(ctx
6863            .ensure_callgraph_store()
6864            .expect("reopen callgraph store")
6865            .is_some());
6866        assert!(live_roots[&root].idle_artifacts_evicted);
6867    }
6868
6869    #[test]
6870    fn idle_root_reaper_applies_ttl_to_unbound_roots() {
6871        let (_root_dir, root) = test_root("idle-root-ttl-gate");
6872        let ctx = test_ctx();
6873        let executor = Arc::new(Executor::new());
6874        assert!(executor.register_actor(root.clone(), ctx));
6875        let ctx = executor.actor_context(&root).expect("actor context");
6876        ctx.mark_subc_unbound();
6877        let now = Instant::now();
6878        let mut meta = RootMeta::new(now);
6879        meta.unbound_quiesced = true;
6880        let mut live_roots = HashMap::from([(root.clone(), meta)]);
6881
6882        // A recently-unbound root stays warm: a transient unbind (host
6883        // restart) must not pay the strict-verify + forced-rebuild teardown
6884        // on the next maintenance sweep.
6885        assert_eq!(
6886            reap_idle_roots(
6887                now,
6888                &mut live_roots,
6889                &HashMap::new(),
6890                &HashMap::new(),
6891                &executor,
6892                &DispatchPathMetrics::new(),
6893            )
6894            .evicted,
6895            0
6896        );
6897        assert!(!live_roots[&root].idle_artifacts_evicted);
6898
6899        // Past the TTL the same unbound root pays the full teardown. The
6900        // pending reconciliation paths retained across the transient-unbind
6901        // window would block eviction forever through
6902        // `artifact_eviction_blocked`; the reaper disposes them because the
6903        // strict gap invalidation subsumes their purpose.
6904        ctx.add_pending_search_index_paths([root.as_path().join("retained.rs")]);
6905        assert_eq!(
6906            reap_idle_roots(
6907                now + IDLE_ROOT_TTL,
6908                &mut live_roots,
6909                &HashMap::new(),
6910                &HashMap::new(),
6911                &executor,
6912                &DispatchPathMetrics::new(),
6913            )
6914            .evicted,
6915            1
6916        );
6917        assert!(live_roots[&root].idle_artifacts_evicted);
6918        assert!(
6919            ctx.take_pending_search_index_paths().is_empty(),
6920            "TTL eviction must dispose retained pending reconciliation paths"
6921        );
6922    }
6923
6924    #[test]
6925    fn blocked_ttl_eviction_restores_taken_pending_reconciliation_state() {
6926        let (_root_dir, root) = test_root("ttl-eviction-blocked-restore");
6927        let ctx = test_ctx();
6928        let executor = Arc::new(Executor::new());
6929        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6930        ctx.mark_subc_unbound();
6931
6932        // Retained pending path from the transient-unbind window plus a
6933        // SECONDARY eviction blocker (a non-ready resident search index, the
6934        // dirty-index blocker in artifact_eviction_blocked). Disposal must be
6935        // transactional: the blocked eviction may be followed by a rebind, and
6936        // the path is the only repair record for its consumed watcher event.
6937        let pending = root.as_path().join("edited-while-unbound.rs");
6938        ctx.add_pending_search_index_paths([pending.clone()]);
6939        let dirty_source = root.as_path().join("dirty.rs");
6940        std::fs::write(&dirty_source, "fn dirty() {}\n").expect("dirty source");
6941        let mut dirty = crate::search_index::SearchIndex::new();
6942        dirty.ready = true;
6943        dirty.update_file(&dirty_source);
6944        *ctx.search_index()
6945            .write()
6946            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(dirty);
6947        assert!(ctx.artifact_eviction_blocked());
6948
6949        let mut live_roots = HashMap::new();
6950        let mut meta = RootMeta::new(Instant::now());
6951        meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
6952        meta.unbound_quiesced = true;
6953        live_roots.insert(root.clone(), meta);
6954
6955        assert_eq!(
6956            reap_idle_roots(
6957                Instant::now(),
6958                &mut live_roots,
6959                &HashMap::new(),
6960                &HashMap::new(),
6961                &executor,
6962                &DispatchPathMetrics::new(),
6963            )
6964            .evicted,
6965            0,
6966            "the dirty index must still block this eviction"
6967        );
6968        assert_eq!(
6969            ctx.take_pending_search_index_paths(),
6970            vec![pending],
6971            "a blocked eviction must restore the taken pending paths"
6972        );
6973    }
6974
6975    #[test]
6976    fn idle_reap_with_bound_route_keeps_watcher_running() {
6977        let (_root_dir, root) = test_root("bound-root-reap-gate");
6978        let ctx = test_ctx();
6979        let executor = Arc::new(Executor::new());
6980        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6981
6982        let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
6983        let _dispatch_tx = dispatch_tx;
6984        let shutdown = Arc::new(AtomicBool::new(false));
6985        let thread_shutdown = Arc::clone(&shutdown);
6986        let join = std::thread::spawn(move || {
6987            while !thread_shutdown.load(Ordering::SeqCst) {
6988                std::thread::yield_now();
6989            }
6990        });
6991        ctx.install_watcher_runtime(
6992            dispatch_rx,
6993            crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
6994        );
6995
6996        let mut meta = RootMeta::new(Instant::now());
6997        meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
6998        let mut live_roots = HashMap::from([(root.clone(), meta)]);
6999        let bound = HashMap::from([(root, HashSet::from([route_key(7, 1)]))]);
7000        assert_eq!(
7001            reap_idle_roots(
7002                Instant::now(),
7003                &mut live_roots,
7004                &HashMap::new(),
7005                &bound,
7006                &executor,
7007                &DispatchPathMetrics::new(),
7008            )
7009            .evicted,
7010            0
7011        );
7012        wait_for_watcher_count(&ctx, 1);
7013        ctx.stop_watcher_runtime_in_background();
7014        wait_for_watcher_count(&ctx, 0);
7015    }
7016
7017    #[test]
7018    fn deleted_root_with_bound_route_is_reclaimed_after_confirmation_and_routes_are_purged() {
7019        let (root_dir, root) = test_root("deleted-bound-root-reap");
7020        let executor = Arc::new(Executor::new());
7021        assert!(executor.register_actor(root.clone(), test_ctx()));
7022        root_dir.close().expect("delete project root");
7023
7024        let route = route_key(19, 3);
7025        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
7026        let cancel_signal = PersistentCancelSignal::new();
7027        let mut routes = HashMap::from([(route, route_identity(&root, "deleted-route"))]);
7028        let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
7029        let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
7030        let mut route_bash_cancels = HashMap::from([(
7031            route,
7032            bash::RouteBashCancel {
7033                token: cancel_signal.clone(),
7034                active_waits: 0,
7035            },
7036        )]);
7037        let metrics = DispatchPathMetrics::new();
7038
7039        let first = reap_idle_roots(
7040            Instant::now(),
7041            &mut live_roots,
7042            &HashMap::new(),
7043            &root_channels,
7044            &executor,
7045            &metrics,
7046        );
7047        assert!(first.forgotten_deleted_roots.is_empty());
7048        assert!(executor.actor_registered(&root));
7049
7050        let mut forgotten = Vec::new();
7051        for _ in 0..100 {
7052            let outcome = reap_idle_roots(
7053                Instant::now(),
7054                &mut live_roots,
7055                &HashMap::new(),
7056                &root_channels,
7057                &executor,
7058                &metrics,
7059            );
7060            if !outcome.forgotten_deleted_roots.is_empty() {
7061                forgotten = outcome.forgotten_deleted_roots;
7062                break;
7063            }
7064            std::thread::sleep(Duration::from_millis(10));
7065        }
7066        assert_eq!(forgotten, vec![root.clone()]);
7067        assert!(!executor.actor_registered(&root));
7068
7069        let mut retry_buffer = HashMap::new();
7070        let mut reclaimed_routes = ReclaimedRoutes::default();
7071        let mut session_identity = HashMap::new();
7072        let mut push_buffer = HashMap::new();
7073        let mut bg_subs = HashMap::from([(
7074            route,
7075            BgSub {
7076                corr: 77,
7077                ver: PROTOCOL_VERSION,
7078                flags: control_flags(),
7079                root: root.clone(),
7080                session: "deleted-route".to_string(),
7081            },
7082        )]);
7083        let mut bg_sub_by_session = HashMap::from([(
7084            (root.clone(), "deleted-route".to_string()),
7085            HashSet::from([route]),
7086        )]);
7087        let mut bg_wake_pending = HashSet::from([route]);
7088        let mut bg_wake_epoch = HashMap::new();
7089        let mut pending_bash_asks = HashMap::new();
7090        let active_tool_calls: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
7091        health::take_bg_observability_logs_for_test();
7092        purge_deleted_root_residents(
7093            &root,
7094            &mut routes,
7095            &mut root_channels,
7096            &mut installed_route_epochs,
7097            &mut route_bash_cancels,
7098            &active_tool_calls,
7099            executor.as_ref(),
7100            &mut retry_buffer,
7101            &mut reclaimed_routes,
7102            &mut session_identity,
7103            &mut push_buffer,
7104            &mut bg_subs,
7105            &mut bg_sub_by_session,
7106            &mut bg_wake_pending,
7107            &mut bg_wake_epoch,
7108            &mut pending_bash_asks,
7109            &metrics,
7110        );
7111
7112        assert!(routes.is_empty());
7113        assert!(root_channels.is_empty());
7114        assert!(installed_route_epochs.is_empty());
7115        assert!(route_bash_cancels.is_empty());
7116        assert!(reclaimed_routes.contains(route));
7117        assert!(cancel_signal.is_cancelled());
7118        assert_eq!(
7119            health::take_bg_observability_logs_for_test(),
7120            vec![format!(
7121                "subc bg subscription: ended root={} session=deleted-route channel=19@3 cause=root-reclaim suppressed=0",
7122                root.as_path().display()
7123            )]
7124        );
7125    }
7126
7127    /// The control for the deleted-root reclamation above: a root whose
7128    /// directory still EXISTS must stay retained while it holds a bound route,
7129    /// even with every other reap precondition satisfied. Without this, the
7130    /// suite cannot tell "reclaim roots that are provably gone" apart from
7131    /// "reap any root that looks idle" — the second would tear down live
7132    /// sessions, and both satisfy the deleted-root tests.
7133    #[test]
7134    fn live_root_with_bound_route_is_never_reclaimed() {
7135        let (_root_dir, root) = test_root("live-bound-root-retained");
7136        let ctx = test_ctx();
7137        ctx.mark_subc_unbound();
7138        let executor = Arc::new(Executor::new());
7139        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7140
7141        let route = RouteChannel {
7142            channel: 7,
7143            epoch: 1,
7144        };
7145        let mut meta = RootMeta::new(Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1));
7146        meta.unbound_quiesced = true;
7147        let mut live_roots = HashMap::from([(root.clone(), meta)]);
7148        let root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
7149
7150        // Sweep three times because reclamation requires the root path to be
7151        // observed missing on two CONSECUTIVE sweeps. A single sweep would pass
7152        // here even if reclamation were wrongly unconditional, since the first
7153        // absence never reclaims on its own.
7154        for _ in 0..3 {
7155            let outcome = reap_idle_roots(
7156                Instant::now(),
7157                &mut live_roots,
7158                &HashMap::new(),
7159                &root_channels,
7160                &executor,
7161                &DispatchPathMetrics::new(),
7162            );
7163            assert!(
7164                outcome.forgotten_deleted_roots.is_empty(),
7165                "a root whose directory exists must never be forgotten"
7166            );
7167        }
7168
7169        assert!(live_roots.contains_key(&root), "live root must be retained");
7170        assert!(
7171            executor.actor_registered(&root),
7172            "live root's actor must survive"
7173        );
7174        assert!(
7175            root.as_path().exists(),
7176            "test vehicle must keep the directory alive; otherwise this control proves nothing"
7177        );
7178    }
7179
7180    #[test]
7181    fn deleted_root_is_not_reclaimed_on_first_absence_observation() {
7182        let (root_dir, root) = test_root("deleted-root-first-observation");
7183        let ctx = test_ctx();
7184        ctx.mark_subc_unbound();
7185        let executor = Arc::new(Executor::new());
7186        assert!(executor.register_actor(root.clone(), ctx));
7187        root_dir.close().expect("delete project root");
7188
7189        let mut meta = RootMeta::new(Instant::now());
7190        meta.unbound_quiesced = true;
7191        let mut live_roots = HashMap::from([(root.clone(), meta)]);
7192        let outcome = reap_idle_roots(
7193            Instant::now(),
7194            &mut live_roots,
7195            &HashMap::new(),
7196            &HashMap::new(),
7197            &executor,
7198            &DispatchPathMetrics::new(),
7199        );
7200
7201        assert!(outcome.forgotten_deleted_roots.is_empty());
7202        assert!(live_roots.contains_key(&root));
7203        assert!(executor.actor_registered(&root));
7204    }
7205
7206    fn spawn_background_for_root(
7207        ctx: &AppContext,
7208        root: &ProjectRootId,
7209        storage: &tempfile::TempDir,
7210        session_id: &str,
7211    ) -> (String, u32) {
7212        // Windows refuses to delete a directory that is a running process's
7213        // cwd (ERROR_SHARING_VIOLATION), so the task must not live inside the
7214        // project root these tests delete. The kill path matches on the task's
7215        // registered project_root, not its cwd, so pointing the cwd at task
7216        // storage keeps the association under test intact.
7217        let command = if cfg!(windows) {
7218            // timeout.exe requires a console; ping is the standard sleep shim.
7219            "ping -n 31 127.0.0.1 > nul"
7220        } else {
7221            "sleep 30"
7222        };
7223        let task_id = ctx
7224            .bash_background()
7225            .spawn(
7226                crate::sandbox_spawn::SpawnPlan::Unsandboxed,
7227                command,
7228                session_id.to_string(),
7229                storage.path().to_path_buf(),
7230                HashMap::new(),
7231                Some(Duration::from_secs(60)),
7232                storage.path().to_path_buf(),
7233                8,
7234                true,
7235                false,
7236                Some(root.as_path().to_path_buf()),
7237            )
7238            .expect("spawn background task");
7239        let snapshot = ctx
7240            .bash_background()
7241            .status(
7242                &task_id,
7243                session_id,
7244                Some(root.as_path()),
7245                Some(storage.path()),
7246                0,
7247            )
7248            .expect("background task status");
7249        (task_id, snapshot.child_pid.expect("background child pid"))
7250    }
7251
7252    fn wait_for_background_exit(pid: u32) {
7253        let deadline = Instant::now() + Duration::from_secs(5);
7254        while crate::bash_background::process::is_process_alive(pid) {
7255            assert!(
7256                Instant::now() < deadline,
7257                "background task process survived kill"
7258            );
7259            std::thread::sleep(Duration::from_millis(20));
7260        }
7261    }
7262
7263    #[test]
7264    fn deleted_root_reclaims_background_task_after_two_absence_sweeps() {
7265        let (root_dir, root) = test_root("deleted-root-background-task");
7266        let storage = tempfile::tempdir().expect("task storage");
7267        let ctx = test_ctx();
7268        let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "reclaim-session");
7269        assert!(crate::bash_background::process::is_process_alive(pid));
7270
7271        let executor = Arc::new(Executor::new());
7272        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7273        assert!(executor.actor_is_idle(&root));
7274        root_dir.close().expect("delete project root");
7275        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
7276        let pending_binds = HashMap::new();
7277        let root_channels = HashMap::new();
7278        let metrics = DispatchPathMetrics::new();
7279
7280        let first = reap_idle_roots(
7281            Instant::now(),
7282            &mut live_roots,
7283            &pending_binds,
7284            &root_channels,
7285            &executor,
7286            &metrics,
7287        );
7288        assert!(first.forgotten_deleted_roots.is_empty());
7289        assert!(crate::bash_background::process::is_process_alive(pid));
7290
7291        let outcome = reap_until_forgotten(
7292            &root,
7293            &mut live_roots,
7294            &pending_binds,
7295            &root_channels,
7296            &executor,
7297            &metrics,
7298        );
7299        assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
7300        wait_for_background_exit(pid);
7301
7302        let snapshot = ctx
7303            .bash_background()
7304            .status(
7305                &task_id,
7306                "reclaim-session",
7307                Some(root.as_path()),
7308                Some(storage.path()),
7309                0,
7310            )
7311            .expect("reclaimed task status");
7312        assert_eq!(snapshot.info.status, BgTaskStatus::Killed);
7313        assert_eq!(
7314            snapshot.info.status_reason.as_deref(),
7315            Some(crate::bash_background::registry::ROOT_RECLAIMED_REASON)
7316        );
7317        assert_eq!(
7318            serde_json::to_value(&snapshot).expect("serialize bash status")["status_reason"],
7319            crate::bash_background::registry::ROOT_RECLAIMED_REASON
7320        );
7321        let completion = ctx
7322            .bash_background()
7323            .drain_completions_for_session(Some("reclaim-session"))
7324            .pop()
7325            .expect("reclaimed task completion");
7326        assert_eq!(
7327            completion.status_reason.as_deref(),
7328            Some(crate::bash_background::registry::ROOT_RECLAIMED_REASON)
7329        );
7330    }
7331
7332    #[test]
7333    fn existing_unbound_root_keeps_background_task_alive_across_sweeps() {
7334        let (root_dir, root) = test_root("existing-root-background-task");
7335        let storage = tempfile::tempdir().expect("task storage");
7336        let ctx = test_ctx();
7337        ctx.mark_subc_unbound();
7338        let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "existing-session");
7339
7340        let executor = Arc::new(Executor::new());
7341        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7342        let mut meta = RootMeta::new(
7343            Instant::now()
7344                .checked_sub(IDLE_ROOT_TTL + Duration::from_secs(1))
7345                .expect("old root timestamp"),
7346        );
7347        meta.unbound_quiesced = true;
7348        let mut live_roots = HashMap::from([(root.clone(), meta)]);
7349        let pending_binds = HashMap::new();
7350        let root_channels = HashMap::new();
7351        let metrics = DispatchPathMetrics::new();
7352
7353        for _ in 0..8 {
7354            reap_idle_roots(
7355                Instant::now(),
7356                &mut live_roots,
7357                &pending_binds,
7358                &root_channels,
7359                &executor,
7360                &metrics,
7361            );
7362            std::thread::sleep(Duration::from_millis(10));
7363        }
7364        assert!(root_dir.path().exists());
7365        assert!(crate::bash_background::process::is_process_alive(pid));
7366        let snapshot = ctx
7367            .bash_background()
7368            .status(
7369                &task_id,
7370                "existing-session",
7371                Some(root.as_path()),
7372                Some(storage.path()),
7373                0,
7374            )
7375            .expect("existing task status");
7376        assert_eq!(snapshot.info.status, BgTaskStatus::Running);
7377        let _ = ctx.bash_background().kill(&task_id, "existing-session");
7378        wait_for_background_exit(pid);
7379    }
7380
7381    #[test]
7382    fn restored_root_between_absence_sweeps_keeps_background_task_alive() {
7383        let (root_dir, root) = test_root("restored-root-background-task");
7384        let storage = tempfile::tempdir().expect("task storage");
7385        let ctx = test_ctx();
7386        let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "restored-session");
7387
7388        let executor = Arc::new(Executor::new());
7389        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7390        root_dir.close().expect("delete project root");
7391        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
7392        let pending_binds = HashMap::new();
7393        let root_channels = HashMap::new();
7394        let metrics = DispatchPathMetrics::new();
7395
7396        let first = reap_idle_roots(
7397            Instant::now(),
7398            &mut live_roots,
7399            &pending_binds,
7400            &root_channels,
7401            &executor,
7402            &metrics,
7403        );
7404        assert!(first.forgotten_deleted_roots.is_empty());
7405        std::fs::create_dir_all(root.as_path()).expect("restore project root");
7406        let second = reap_idle_roots(
7407            Instant::now(),
7408            &mut live_roots,
7409            &pending_binds,
7410            &root_channels,
7411            &executor,
7412            &metrics,
7413        );
7414        assert!(second.forgotten_deleted_roots.is_empty());
7415        assert!(crate::bash_background::process::is_process_alive(pid));
7416        let _ = ctx.bash_background().kill(&task_id, "restored-session");
7417        wait_for_background_exit(pid);
7418    }
7419
7420    #[test]
7421    fn observing_root_again_resets_deleted_sweep_confirmation() {
7422        let (root_dir, root) = test_root("deleted-root-observation-reset");
7423        let ctx = test_ctx();
7424        ctx.mark_subc_unbound();
7425        let executor = Arc::new(Executor::new());
7426        assert!(executor.register_actor(root.clone(), ctx));
7427        root_dir.close().expect("delete project root");
7428
7429        let mut meta = RootMeta::new(Instant::now());
7430        meta.unbound_quiesced = true;
7431        let mut live_roots = HashMap::from([(root.clone(), meta)]);
7432        let pending_binds = HashMap::new();
7433        let root_channels = HashMap::new();
7434        let metrics = DispatchPathMetrics::new();
7435
7436        let first = reap_idle_roots(
7437            Instant::now(),
7438            &mut live_roots,
7439            &pending_binds,
7440            &root_channels,
7441            &executor,
7442            &metrics,
7443        );
7444        assert!(first.forgotten_deleted_roots.is_empty());
7445
7446        std::fs::create_dir_all(root.as_path()).expect("restore project root");
7447        reap_idle_roots(
7448            Instant::now(),
7449            &mut live_roots,
7450            &pending_binds,
7451            &root_channels,
7452            &executor,
7453            &metrics,
7454        );
7455        std::fs::remove_dir_all(root.as_path()).expect("delete project root again");
7456
7457        let after_reset = reap_idle_roots(
7458            Instant::now(),
7459            &mut live_roots,
7460            &pending_binds,
7461            &root_channels,
7462            &executor,
7463            &metrics,
7464        );
7465        assert!(after_reset.forgotten_deleted_roots.is_empty());
7466        assert!(live_roots.contains_key(&root));
7467        assert!(executor.actor_registered(&root));
7468    }
7469
7470    #[test]
7471    fn deleted_idle_root_is_fully_forgotten_and_status_counts_drop() {
7472        let (root_dir, root) = test_root("deleted-root-reap");
7473        let app = App::default_shared();
7474        let ctx = Arc::new(AppContext::from_app(
7475            Arc::clone(&app),
7476            Config {
7477                project_root: Some(root.as_path().to_path_buf()),
7478                ..Config::default()
7479            },
7480        ));
7481        ctx.set_canonical_cache_root(root.as_path().to_path_buf());
7482        ctx.mark_subc_unbound();
7483        let executor = Arc::new(Executor::new());
7484        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7485        assert_eq!(app.actor_root_count(), 1);
7486        drop(ctx);
7487        root_dir.close().expect("delete project root");
7488
7489        let mut meta = RootMeta::new(Instant::now());
7490        meta.unbound_quiesced = true;
7491        let mut live_roots = HashMap::from([(root.clone(), meta)]);
7492        let outcome = reap_until_forgotten(
7493            &root,
7494            &mut live_roots,
7495            &HashMap::new(),
7496            &HashMap::new(),
7497            &executor,
7498            &DispatchPathMetrics::new(),
7499        );
7500        assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
7501        assert!(!executor.actor_registered(&root));
7502        assert!(!live_roots.contains_key(&root));
7503        wait_for_actor_root_count(&app, 0);
7504
7505        let status_ctx = AppContext::from_app(app, Config::default());
7506        let status = status_ctx.build_status_snapshot();
7507        assert_eq!(status["runtime"]["live_actor_roots"], 0);
7508        assert_eq!(status["runtime"]["open_routes"], 0);
7509    }
7510
7511    #[test]
7512    fn deleted_root_reap_blocker_census_is_exposed_in_health_metrics() {
7513        let (root_dir, root) = test_root("deleted-root-reap-census");
7514        let executor = Arc::new(Executor::new());
7515        assert!(executor.register_actor(root.clone(), test_ctx()));
7516        root_dir.close().expect("delete project root");
7517
7518        let mut live_roots = HashMap::from([(root, RootMeta::new(Instant::now()))]);
7519        let metrics = DispatchPathMetrics::new();
7520        let outcome = reap_idle_roots(
7521            Instant::now(),
7522            &mut live_roots,
7523            &HashMap::new(),
7524            &HashMap::new(),
7525            &executor,
7526            &metrics,
7527        );
7528        assert_eq!(outcome.evicted, 0);
7529
7530        let app = crate::context::App::default_shared();
7531        let health_rollup_cache = HealthRollupCache::new();
7532        health_rollup_cache.refresh(&executor, &app);
7533        let report = build_health_report(
7534            &health_rollup_cache,
7535            &executor,
7536            &HashMap::new(),
7537            &metrics,
7538            &app,
7539        );
7540        let reap = report
7541            .metrics
7542            .as_ref()
7543            .and_then(|metrics| metrics.get("reap"))
7544            .expect("reap health metrics");
7545        assert_eq!(reap["deleted_retained"].as_u64(), Some(1));
7546        assert_eq!(reap["blockers"]["absence_unconfirmed"].as_u64(), Some(1));
7547        assert_eq!(reap["blockers"]["unbound_quiesced"].as_u64(), Some(0));
7548        assert_eq!(reap["blockers"]["actor_busy"].as_u64(), Some(0));
7549    }
7550
7551    #[test]
7552    fn connection_exit_quiesces_queued_maintenance_and_deleted_root_is_purged() {
7553        let (root_dir, root) = test_root("connection-exit-deleted-root");
7554        let executor = Arc::new(Executor::new());
7555        assert!(executor.register_actor(root.clone(), test_ctx()));
7556
7557        let route = route_key(11, 1);
7558        let mut meta = RootMeta::new(Instant::now());
7559        meta.maintenance_pending = true;
7560        meta.maintenance_queued_kinds
7561            .push_back(MaintenanceDrainKind::CompletionDrains);
7562        let mut live_roots = HashMap::from([(root.clone(), meta)]);
7563        let mut pending_binds = HashMap::new();
7564        let mut routes = HashMap::from([(route, route_identity(&root, "abandoned"))]);
7565        let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
7566        let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
7567        let mut route_bash_cancels = HashMap::new();
7568        let active_tool_calls: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
7569
7570        quiesce_connection_roots(
7571            &mut live_roots,
7572            &mut pending_binds,
7573            &mut routes,
7574            &mut root_channels,
7575            &mut installed_route_epochs,
7576            &mut route_bash_cancels,
7577            &active_tool_calls,
7578            &executor,
7579        );
7580        assert!(live_roots[&root].unbound_quiesced);
7581        assert!(!live_roots[&root].maintenance_pending);
7582        assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
7583        assert!(routes.is_empty());
7584        assert!(root_channels.is_empty());
7585
7586        root_dir.close().expect("delete project root");
7587        let metrics = DispatchPathMetrics::new();
7588        let outcome = reap_until_forgotten(
7589            &root,
7590            &mut live_roots,
7591            &pending_binds,
7592            &root_channels,
7593            &executor,
7594            &metrics,
7595        );
7596        let mut session_identity = HashMap::new();
7597        let mut push_buffer = HashMap::new();
7598        let mut bg_subs = HashMap::new();
7599        let mut bg_sub_by_session = HashMap::new();
7600        let mut bg_wake_pending = HashSet::new();
7601        let mut bg_wake_epoch = HashMap::new();
7602        let mut pending_bash_asks = HashMap::new();
7603        let mut retry_buffer = HashMap::new();
7604        let mut reclaimed_routes = ReclaimedRoutes::default();
7605        for forgotten in &outcome.forgotten_deleted_roots {
7606            purge_deleted_root_residents(
7607                forgotten,
7608                &mut routes,
7609                &mut root_channels,
7610                &mut installed_route_epochs,
7611                &mut route_bash_cancels,
7612                &active_tool_calls,
7613                executor.as_ref(),
7614                &mut retry_buffer,
7615                &mut reclaimed_routes,
7616                &mut session_identity,
7617                &mut push_buffer,
7618                &mut bg_subs,
7619                &mut bg_sub_by_session,
7620                &mut bg_wake_pending,
7621                &mut bg_wake_epoch,
7622                &mut pending_bash_asks,
7623                &metrics,
7624            );
7625        }
7626
7627        assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
7628        assert!(!executor.actor_registered(&root));
7629        assert!(!live_roots.contains_key(&root));
7630    }
7631
7632    #[test]
7633    fn unbound_root_quiesces_maintenance_without_removing_actor() {
7634        let (_root_dir, root) = test_root("unbound-root-quiesce");
7635        let ctx = test_ctx();
7636        let executor = Arc::new(Executor::new());
7637        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7638        let mut meta = RootMeta::new(Instant::now());
7639        meta.maintenance_pending = true;
7640        meta.maintenance_jobs_in_flight = 1;
7641        meta.maintenance_queued_kinds
7642            .push_back(MaintenanceDrainKind::ConfigureTail);
7643        let mut live_roots = HashMap::from([(root.clone(), meta)]);
7644        // Warm state planted before the unbind: quiesce must keep it.
7645        *ctx.search_index()
7646            .write()
7647            .unwrap_or_else(std::sync::PoisonError::into_inner) =
7648            Some(crate::search_index::SearchIndex::new());
7649        ctx.set_cache_writer_capabilities(true, true);
7650        let pending = root.as_path().join("pending.rs");
7651        ctx.add_pending_search_index_paths([pending.clone()]);
7652        // A warm verify memo must survive the transient unbind: the watcher
7653        // keeps running, so no unobserved window exists and the next warm
7654        // reload must not pay a strict full-corpus re-hash.
7655        let canonical_root = root.as_path().to_path_buf();
7656        let artifact = canonical_root.join("cache.bin");
7657        std::fs::write(&artifact, b"warm-artifact").expect("write artifact");
7658        let seeded_generation = crate::cache_freshness::artifact_generation(&artifact);
7659        crate::cache_freshness::record_verify_completed(
7660            &canonical_root,
7661            crate::cache_freshness::VerifyArtifact::Search,
7662            seeded_generation,
7663        );
7664        assert!(matches!(
7665            crate::cache_freshness::warm_verify_plan(
7666                &canonical_root,
7667                crate::cache_freshness::VerifyArtifact::Search,
7668                seeded_generation,
7669            ),
7670            crate::cache_freshness::WarmVerifyPlan::Skip
7671        ));
7672        // A live watcher runtime must survive quiesce (its events accumulate
7673        // for the rebind replay).
7674        let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
7675        let _dispatch_tx = dispatch_tx;
7676        let shutdown = Arc::new(AtomicBool::new(false));
7677        let thread_shutdown = Arc::clone(&shutdown);
7678        let join = std::thread::spawn(move || {
7679            while !thread_shutdown.load(Ordering::SeqCst) {
7680                std::thread::yield_now();
7681            }
7682        });
7683        ctx.install_watcher_runtime(
7684            dispatch_rx,
7685            crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
7686        );
7687        assert!(ctx.watcher_runtime_active());
7688
7689        quiesce_unbound_root(&root, &mut live_roots, &executor);
7690        let meta = &live_roots[&root];
7691        assert!(meta.unbound_quiesced);
7692        assert!(ctx.subc_unbound_quiesced());
7693        assert!(meta.maintenance_pending);
7694        assert!(meta.maintenance_queued_kinds.is_empty());
7695        assert!(executor.actor_registered(&root));
7696        // Transient unbind keeps the root warm: resident artifacts stay
7697        // resident, no forced callgraph rebuild is planted, and pending
7698        // reconciliation paths survive for the rebind replay.
7699        assert!(
7700            ctx.search_index()
7701                .read()
7702                .unwrap_or_else(std::sync::PoisonError::into_inner)
7703                .is_some(),
7704            "quiesce must not evict resident artifacts"
7705        );
7706        assert_eq!(
7707            ctx.pending_callgraph_store_force_token(),
7708            None,
7709            "quiesce must not force a callgraph rebuild"
7710        );
7711        assert_eq!(
7712            ctx.take_pending_search_index_paths(),
7713            vec![pending],
7714            "quiesce must retain pending watcher-derived paths"
7715        );
7716        assert!(
7717            matches!(
7718                crate::cache_freshness::warm_verify_plan(
7719                    &canonical_root,
7720                    crate::cache_freshness::VerifyArtifact::Search,
7721                    seeded_generation,
7722                ),
7723                crate::cache_freshness::WarmVerifyPlan::Skip
7724            ),
7725            "quiesce must not invalidate the warm verify memo"
7726        );
7727        assert!(
7728            ctx.watcher_runtime_active(),
7729            "quiesce must not stop a running watcher"
7730        );
7731        ctx.stop_watcher_runtime();
7732
7733        let meta = live_roots.get_mut(&root).expect("root metadata");
7734        note_maintenance_completion(
7735            meta,
7736            Some(MaintenanceDrainKind::ConfigureTail),
7737            false,
7738            meta.unbound_quiesced,
7739        );
7740        assert!(!meta.maintenance_pending);
7741        assert!(meta.maintenance_queued_kinds.is_empty());
7742    }
7743
7744    #[test]
7745    fn same_root_higher_epoch_replacement_does_not_quiesce_between_generations() {
7746        let (_dir, root) = test_root("same-root-replacement");
7747        let route = route_key(7, 1);
7748        let installed_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
7749        let root_channels = HashMap::new();
7750
7751        assert!(!route_removal_will_quiesce_root(
7752            &root,
7753            route,
7754            &installed_channels,
7755            false,
7756            Some(&root),
7757        ));
7758        assert!(route_removal_will_quiesce_root(
7759            &root,
7760            route,
7761            &installed_channels,
7762            false,
7763            None,
7764        ));
7765        assert!(!should_quiesce_removed_root(
7766            &root,
7767            &root_channels,
7768            false,
7769            Some(&root),
7770        ));
7771        assert!(should_quiesce_removed_root(
7772            &root,
7773            &root_channels,
7774            false,
7775            None,
7776        ));
7777        assert!(!should_quiesce_removed_root(
7778            &root,
7779            &root_channels,
7780            true,
7781            None,
7782        ));
7783    }
7784
7785    #[test]
7786    fn root_quiesces_only_after_its_last_route_is_removed_and_reactivates_on_bind() {
7787        let (_root_dir, root) = test_root("unbound-root-route-count");
7788        let executor = Arc::new(Executor::new());
7789        assert!(executor.register_actor(root.clone(), test_ctx()));
7790        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
7791        let mut root_channels = HashMap::from([(
7792            root.clone(),
7793            HashSet::from([route_key(7, 1), route_key(8, 1)]),
7794        )]);
7795
7796        remove_root_channel(&mut root_channels, &root, route_key(7, 1));
7797        if !root_channels.contains_key(&root) {
7798            quiesce_unbound_root(&root, &mut live_roots, &executor);
7799        }
7800        assert!(!live_roots[&root].unbound_quiesced);
7801
7802        remove_root_channel(&mut root_channels, &root, route_key(8, 1));
7803        if !root_channels.contains_key(&root) {
7804            quiesce_unbound_root(&root, &mut live_roots, &executor);
7805        }
7806        assert!(live_roots[&root].unbound_quiesced);
7807
7808        live_roots
7809            .get_mut(&root)
7810            .expect("root metadata")
7811            .note_activity();
7812        assert!(
7813            live_roots[&root].unbound_quiesced,
7814            "late asynchronous activity must not reactivate an unbound root"
7815        );
7816
7817        live_roots
7818            .get_mut(&root)
7819            .expect("root metadata")
7820            .reactivate_bound();
7821        assert!(!live_roots[&root].unbound_quiesced);
7822    }
7823
7824    #[test]
7825    fn allocator_pressure_relief_requires_every_root_to_be_idle() {
7826        let (_idle_dir, idle_root) = test_root("allocator-relief-idle");
7827        let (_active_dir, active_root) = test_root("allocator-relief-active");
7828        let now = Instant::now();
7829        let mut live_roots = HashMap::new();
7830        let mut idle = RootMeta::new(now);
7831        idle.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
7832        live_roots.insert(idle_root, idle);
7833        assert!(process_has_been_idle(now, &live_roots));
7834
7835        live_roots.insert(active_root.clone(), RootMeta::new(now));
7836        assert!(!process_has_been_idle(now, &live_roots));
7837
7838        let active = live_roots
7839            .get_mut(&active_root)
7840            .expect("active root metadata");
7841        active.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
7842        active.active_bash_waits = 1;
7843        assert!(!process_has_been_idle(now, &live_roots));
7844    }
7845
7846    #[test]
7847    fn pressure_relief_log_reports_before_and_after_measurements() {
7848        let allocator = crate::memory::AllocatorMemorySnapshot {
7849            status: "measured",
7850            bytes_in_use: Some(8 * 1024 * 1024),
7851            size_allocated: Some(12 * 1024 * 1024),
7852            retained_slack_bytes: Some(4 * 1024 * 1024),
7853            not_estimated: None,
7854        };
7855        let relief = crate::memory::AllocatorPressureRelief {
7856            bytes_released: 3 * 1024 * 1024,
7857            rss_before_bytes: Some(20 * 1024 * 1024),
7858            rss_after_bytes: Some(17 * 1024 * 1024),
7859            allocator_before: allocator.clone(),
7860            allocator_after: crate::memory::AllocatorMemorySnapshot {
7861                size_allocated: Some(9 * 1024 * 1024),
7862                retained_slack_bytes: Some(1024 * 1024),
7863                ..allocator
7864            },
7865        };
7866        let message = pressure_relief_label(&relief);
7867        assert!(message.contains("RSS 20.0 MB -> 17.0 MB"));
7868        assert!(message.contains("allocated 12.0 MB -> 9.0 MB"));
7869        assert!(message.contains("slack 4.0 MB -> 1.0 MB"));
7870        assert!(message.contains("reported 3.0 MB released"));
7871    }
7872
7873    #[test]
7874    fn due_maintenance_jobs_skip_poisoned_roots() {
7875        let (_healthy_dir, healthy_root) = test_root("maintenance-healthy");
7876        let (_poisoned_dir, poisoned_root) = test_root("maintenance-poisoned");
7877        let mut live_roots = HashMap::new();
7878        live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
7879        let mut poisoned_meta = RootMeta::new(Instant::now());
7880        poisoned_meta.maintenance_poisoned = true;
7881        live_roots.insert(poisoned_root.clone(), poisoned_meta);
7882
7883        let (due, deferred) = due_maintenance_jobs_without_actor_context(
7884            &mut live_roots,
7885            MAINTENANCE_SUBMIT_BUDGET,
7886            &HashSet::new(),
7887        );
7888
7889        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
7890        assert!(due.iter().all(|(root, _)| root == &healthy_root));
7891        assert!(!deferred);
7892        assert!(live_roots[&healthy_root].maintenance_pending);
7893        assert_eq!(
7894            live_roots[&healthy_root].maintenance_jobs_in_flight,
7895            INITIAL_MAINTENANCE_JOB_COUNT
7896        );
7897        assert!(!live_roots[&poisoned_root].maintenance_pending);
7898    }
7899
7900    #[test]
7901    fn due_maintenance_jobs_do_not_restart_quiesced_root_work() {
7902        let (_dir, root) = test_root("maintenance-unbound");
7903        let mut meta = RootMeta::new(Instant::now());
7904        meta.unbound_quiesced = true;
7905        let mut live_roots = HashMap::from([(root.clone(), meta)]);
7906
7907        let (due, deferred) = due_maintenance_jobs_without_actor_context(
7908            &mut live_roots,
7909            MAINTENANCE_SUBMIT_BUDGET,
7910            &HashSet::new(),
7911        );
7912
7913        assert!(due.is_empty());
7914        assert!(!deferred);
7915        assert!(!live_roots[&root].maintenance_pending);
7916    }
7917
7918    #[test]
7919    fn idle_bg_subscription_queues_no_jobs_until_a_wake_arrives() {
7920        let (_dir, root) = test_root("maintenance-idle-bg-subscription");
7921        let ctx = test_ctx();
7922        assert!(!ctx.completion_drains_have_work());
7923
7924        let executor = Executor::new();
7925        assert!(executor.register_actor(root.clone(), ctx));
7926        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
7927        let session = "idle-session".to_string();
7928        let channel = route_key(17, 1);
7929        let metrics = DispatchPathMetrics::new();
7930        let bg_sub_by_session =
7931            HashMap::from([((root.clone(), session.clone()), HashSet::from([channel]))]);
7932        let mut bg_wake_pending = HashSet::new();
7933
7934        let (idle_tick_jobs, deferred) = due_maintenance_jobs(
7935            &mut live_roots,
7936            Some(&executor),
7937            &bg_sub_by_session,
7938            &bg_wake_pending,
7939            MAINTENANCE_SUBMIT_BUDGET,
7940            &HashSet::new(),
7941        );
7942        assert!(idle_tick_jobs.is_empty());
7943        assert!(!deferred);
7944        assert!(!live_roots[&root].maintenance_pending);
7945
7946        // A completion can arm its wake after the idle tick's probes. The wake
7947        // remains loop-owned state, so the following tick must observe it.
7948        let mut bg_wake_epoch = HashMap::new();
7949        push::arm_bg_wake(
7950            root.clone(),
7951            session,
7952            channel,
7953            &mut bg_wake_pending,
7954            &mut bg_wake_epoch,
7955            &metrics,
7956        );
7957        let (next_tick_jobs, deferred) = due_maintenance_jobs(
7958            &mut live_roots,
7959            Some(&executor),
7960            &bg_sub_by_session,
7961            &bg_wake_pending,
7962            MAINTENANCE_SUBMIT_BUDGET,
7963            &HashSet::new(),
7964        );
7965        assert_eq!(
7966            next_tick_jobs,
7967            vec![(root, MaintenanceDrainKind::CompletionDrains)]
7968        );
7969        assert!(!deferred);
7970    }
7971
7972    #[tokio::test]
7973    async fn subc_configure_tail_precedes_completed_search_install() {
7974        let root_dir = tempfile::tempdir().unwrap();
7975        let storage = tempfile::tempdir().unwrap();
7976        let root = ProjectRootId::from_path(root_dir.path()).unwrap();
7977        let (ctx, ignored_path) =
7978            runtime_drain::configure_search_order_context_for_test(root_dir.path(), storage.path());
7979        let ctx = Arc::new(ctx);
7980        assert!(!runtime_drain::watcher_path_is_ignored_by_current_matcher(
7981            &ctx,
7982            &ignored_path
7983        ));
7984
7985        let executor = Arc::new(Executor::new());
7986        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
7987        let metrics = Arc::new(DispatchPathMetrics::new());
7988        let (completion_tx, mut completion_rx) = mpsc::channel(4);
7989        submit_maintenance_job(
7990            &executor,
7991            root.clone(),
7992            MaintenanceDrainKind::ConfigureTail,
7993            Vec::new(),
7994            &completion_tx,
7995            &metrics,
7996        );
7997        submit_maintenance_job(
7998            &executor,
7999            root,
8000            MaintenanceDrainKind::CompletionDrains,
8001            Vec::new(),
8002            &completion_tx,
8003            &metrics,
8004        );
8005
8006        let first = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
8007            .await
8008            .expect("configure-tail completion timed out")
8009            .expect("configure-tail completion channel closed");
8010        let second = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
8011            .await
8012            .expect("completion-drains completion timed out")
8013            .expect("completion-drains completion channel closed");
8014        assert!(first.response.id.contains("configure-tail"));
8015        assert!(second.response.id.contains("completion-drains"));
8016        assert!(runtime_drain::watcher_path_is_ignored_by_current_matcher(
8017            &ctx,
8018            &ignored_path
8019        ));
8020        assert_eq!(
8021            ctx.search_index()
8022                .read()
8023                .unwrap_or_else(std::sync::PoisonError::into_inner)
8024                .as_ref()
8025                .expect("completed search index installed")
8026                .file_count(),
8027            0,
8028            "configure must install the ignore matcher before pending paths replay"
8029        );
8030        ctx.stop_watcher_runtime();
8031    }
8032
8033    #[test]
8034    fn post_bind_configure_and_completion_jobs_are_queued_in_order() {
8035        let (_dir, root) = test_root("maintenance-post-bind");
8036        let mut live_roots = HashMap::new();
8037        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
8038
8039        queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
8040        queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
8041
8042        let meta = live_roots.get(&root).expect("root metadata");
8043        assert!(meta.maintenance_pending);
8044        assert_eq!(meta.maintenance_jobs_in_flight, 0);
8045        assert_eq!(
8046            meta.maintenance_queued_kinds
8047                .iter()
8048                .copied()
8049                .collect::<Vec<_>>(),
8050            vec![
8051                MaintenanceDrainKind::ConfigureTail,
8052                MaintenanceDrainKind::CompletionDrains,
8053            ]
8054        );
8055
8056        let (due, deferred) = due_maintenance_jobs_without_actor_context(
8057            &mut live_roots,
8058            MAINTENANCE_SUBMIT_BUDGET,
8059            &HashSet::new(),
8060        );
8061
8062        assert_eq!(
8063            due,
8064            vec![
8065                (root.clone(), MaintenanceDrainKind::ConfigureTail),
8066                (root.clone(), MaintenanceDrainKind::CompletionDrains),
8067            ]
8068        );
8069        assert!(!deferred);
8070        assert_eq!(live_roots[&root].maintenance_jobs_in_flight, 2);
8071        assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
8072    }
8073
8074    #[test]
8075    fn due_maintenance_jobs_defers_unsubmitted_roots_without_marking_pending() {
8076        let mut live_roots = HashMap::new();
8077        let mut root_ids = Vec::new();
8078        let mut _dirs = Vec::new();
8079        for index in 0..4 {
8080            let (dir, root_id) = test_root(&format!("maintenance-budget-{index}"));
8081            live_roots.insert(root_id.clone(), RootMeta::new(Instant::now()));
8082            root_ids.push(root_id);
8083            _dirs.push(dir);
8084        }
8085
8086        let small_budget = INITIAL_MAINTENANCE_JOB_COUNT + 1;
8087        let (first_due, first_deferred) = due_maintenance_jobs_without_actor_context(
8088            &mut live_roots,
8089            small_budget,
8090            &HashSet::new(),
8091        );
8092
8093        assert_eq!(first_due.len(), small_budget);
8094        assert!(first_deferred);
8095        let first_due_set: HashSet<_> = first_due.into_iter().map(|(root, _)| root).collect();
8096        assert!(first_due_set
8097            .iter()
8098            .all(|root| live_roots[root].maintenance_pending));
8099        assert!(first_due_set
8100            .iter()
8101            .any(|root| !live_roots[root].maintenance_queued_kinds.is_empty()));
8102
8103        let all_roots: HashSet<_> = root_ids.into_iter().collect();
8104        let deferred_roots: HashSet<_> = all_roots.difference(&first_due_set).cloned().collect();
8105        assert!(deferred_roots
8106            .iter()
8107            .all(|root| !live_roots[root].maintenance_pending));
8108    }
8109
8110    #[test]
8111    fn due_maintenance_jobs_defers_pending_bind_roots() {
8112        let (_bind_dir, bind_root) = test_root("maintenance-pending-bind");
8113        let (_healthy_dir, healthy_root) = test_root("maintenance-no-bind");
8114        let mut live_roots = HashMap::new();
8115        live_roots.insert(bind_root.clone(), RootMeta::new(Instant::now()));
8116        live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
8117        let pending_bind_roots = HashSet::from([bind_root.clone()]);
8118
8119        let (due, deferred) = due_maintenance_jobs_without_actor_context(
8120            &mut live_roots,
8121            usize::MAX,
8122            &pending_bind_roots,
8123        );
8124
8125        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
8126        assert!(due.iter().all(|(root, _)| root == &healthy_root));
8127        assert!(!deferred);
8128        assert!(!live_roots[&bind_root].maintenance_pending);
8129        assert!(live_roots[&bind_root].maintenance_queued_kinds.is_empty());
8130    }
8131
8132    #[test]
8133    fn maintenance_pending_survives_requeue_and_clears_after_final_batch() {
8134        let (_dir, root) = test_root("maintenance-requeue");
8135        let mut live_roots = HashMap::new();
8136        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
8137        let (due, deferred) = due_maintenance_jobs_without_actor_context(
8138            &mut live_roots,
8139            usize::MAX,
8140            &HashSet::new(),
8141        );
8142        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
8143        assert!(due.iter().all(|(due_root, _)| due_root == &root));
8144        assert!(!deferred);
8145
8146        let meta = live_roots.get_mut(&root).unwrap();
8147        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, false);
8148        assert!(meta.maintenance_pending);
8149        assert_eq!(
8150            meta.maintenance_jobs_in_flight,
8151            INITIAL_MAINTENANCE_JOB_COUNT - 1
8152        );
8153        assert_eq!(meta.maintenance_queued_kinds.len(), 1);
8154
8155        let (requeued, deferred) =
8156            due_maintenance_jobs_without_actor_context(&mut live_roots, 1, &HashSet::new());
8157        assert_eq!(
8158            requeued,
8159            vec![(root.clone(), MaintenanceDrainKind::Watcher)]
8160        );
8161        assert!(!deferred);
8162        let meta = live_roots.get_mut(&root).unwrap();
8163        assert_eq!(
8164            meta.maintenance_jobs_in_flight,
8165            INITIAL_MAINTENANCE_JOB_COUNT
8166        );
8167        assert!(meta.maintenance_queued_kinds.is_empty());
8168
8169        for _ in 0..INITIAL_MAINTENANCE_JOB_COUNT {
8170            note_maintenance_completion(meta, None, false, false);
8171        }
8172        assert!(!meta.maintenance_pending);
8173        assert_eq!(meta.maintenance_jobs_in_flight, 0);
8174    }
8175
8176    #[test]
8177    fn maintenance_requeue_drops_while_bind_is_pending() {
8178        let (_dir, root) = test_root("maintenance-bind-requeue");
8179        let mut live_roots = HashMap::new();
8180        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
8181        let (due, _) = due_maintenance_jobs_without_actor_context(
8182            &mut live_roots,
8183            usize::MAX,
8184            &HashSet::new(),
8185        );
8186        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
8187
8188        let meta = live_roots.get_mut(&root).unwrap();
8189        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, true);
8190
8191        assert_eq!(
8192            meta.maintenance_jobs_in_flight,
8193            INITIAL_MAINTENANCE_JOB_COUNT - 1
8194        );
8195        assert!(meta.maintenance_queued_kinds.is_empty());
8196        assert!(meta.maintenance_pending);
8197    }
8198
8199    #[test]
8200    fn parked_lsp_completion_never_requiesces_or_cancels_a_pending_bind() {
8201        let mut meta = RootMeta::new(Instant::now());
8202        meta.unbound_quiesced = true;
8203
8204        assert!(!should_requiesce_after_maintenance(
8205            &meta,
8206            MaintenanceDrainKind::Lsp,
8207            false,
8208        ));
8209        assert!(!should_requiesce_after_maintenance(
8210            &meta,
8211            MaintenanceDrainKind::ConfigureTail,
8212            true,
8213        ));
8214        assert!(should_requiesce_after_maintenance(
8215            &meta,
8216            MaintenanceDrainKind::ConfigureTail,
8217            false,
8218        ));
8219    }
8220
8221    #[test]
8222    fn maintenance_pending_clears_and_poison_stops_requeue_after_fatal() {
8223        let (_dir, root) = test_root("maintenance-fatal");
8224        let mut live_roots = HashMap::new();
8225        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
8226        let (due, _) = due_maintenance_jobs_without_actor_context(
8227            &mut live_roots,
8228            usize::MAX,
8229            &HashSet::new(),
8230        );
8231        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
8232
8233        let meta = live_roots.get_mut(&root).unwrap();
8234        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), true, false);
8235        assert!(meta.maintenance_poisoned);
8236        assert!(meta.maintenance_queued_kinds.is_empty());
8237
8238        for _ in 1..INITIAL_MAINTENANCE_JOB_COUNT {
8239            note_maintenance_completion(meta, None, false, false);
8240        }
8241        assert!(!meta.maintenance_pending);
8242        assert_eq!(meta.maintenance_jobs_in_flight, 0);
8243    }
8244
8245    #[test]
8246    fn trust_for_principal_matrix() {
8247        assert_eq!(
8248            trust_for_principal(&Some(Principal::Direct)),
8249            BindTrust::FirstParty
8250        );
8251        // Every first-party reserved id is asserted BY NAME, in one loop over
8252        // the full set, so two failure classes stay distinguishable: an empty
8253        // or broken allowlist reddens every name at once, while a dropped
8254        // single entry (the rename hazard) reddens exactly the missing name.
8255        // Both halves of each transitional rename pair stay listed until the
8256        // flip settles (see the allowlist comment).
8257        for module_id in [
8258            "llm-runner",
8259            "aft",
8260            "broca",
8261            "alfonso-core",
8262            "prefrontal",
8263            "prefrontal-core",
8264        ] {
8265            assert_eq!(
8266                trust_for_principal(&Some(Principal::Reserved {
8267                    module_id: module_id.to_string(),
8268                })),
8269                BindTrust::FirstParty,
8270                "reserved module id '{module_id}' must resolve to first-party trust"
8271            );
8272        }
8273        assert_eq!(
8274            trust_for_principal(&Some(Principal::Reserved {
8275                module_id: "subc-mcp".to_string(),
8276            })),
8277            BindTrust::Untrusted
8278        );
8279        assert_eq!(
8280            trust_for_principal(&Some(Principal::Reserved {
8281                module_id: "anything-unknown".to_string(),
8282            })),
8283            BindTrust::Untrusted
8284        );
8285        assert_eq!(
8286            trust_for_principal(&Some(Principal::Unverified)),
8287            BindTrust::Untrusted
8288        );
8289        assert_eq!(trust_for_principal(&None), BindTrust::Untrusted);
8290    }
8291
8292    #[test]
8293    fn fed_harness_class_maps_to_untrusted_regardless_of_fingerprint_value() {
8294        let principal = Some(Principal::Direct);
8295        let fingerprint_a = "fed:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
8296        let fingerprint_b = "fed:0123456789abcdef111111111111111111111111111111111111111111111111";
8297
8298        assert_eq!(
8299            trust_for_bind(fingerprint_a, &principal),
8300            BindTrust::Untrusted
8301        );
8302        assert_eq!(
8303            trust_for_bind(fingerprint_b, &principal),
8304            BindTrust::Untrusted
8305        );
8306    }
8307
8308    /// The table above proves `trust_for_principal` maps correctly, and the
8309    /// test above proves the `fed:` harness override wins — but neither
8310    /// exercises the ordinary path, so an implementation that ignored the
8311    /// principal entirely for non-fed harnesses would satisfy both. Pin the
8312    /// delegation itself: on a normal harness the verdict must still come from
8313    /// the principal, in both directions.
8314    #[test]
8315    fn trust_for_bind_delegates_to_the_principal_on_ordinary_harnesses() {
8316        for harness in ["opencode", "pi", "runner", "mcp:claude"] {
8317            assert_eq!(
8318                trust_for_bind(harness, &Some(Principal::Direct)),
8319                BindTrust::FirstParty,
8320                "a direct principal must stay first-party on {harness}"
8321            );
8322            assert_eq!(
8323                trust_for_bind(harness, &Some(Principal::Unverified)),
8324                BindTrust::Untrusted,
8325                "an unverified principal must stay untrusted on {harness}"
8326            );
8327            assert_eq!(
8328                trust_for_bind(harness, &None),
8329                BindTrust::Untrusted,
8330                "an absent principal must fail closed on {harness}"
8331            );
8332            assert_eq!(
8333                trust_for_bind(
8334                    harness,
8335                    &Some(Principal::Reserved {
8336                        module_id: "subc-mcp".to_string(),
8337                    })
8338                ),
8339                BindTrust::Untrusted,
8340                "a non-allowlisted reserved module must stay untrusted on {harness}"
8341            );
8342        }
8343    }
8344
8345    #[tokio::test]
8346    async fn persistent_cancel_resolves_when_fired_before_await() {
8347        // The lost-wakeup guard: cancel() fires exactly once via notify_waiters()
8348        // (no stored permit). A waiter that registers AFTER the cancel must still
8349        // observe it via the flag; a waiter racing the cancel must still be woken.
8350        let signal = PersistentCancelSignal::new();
8351        signal.cancel();
8352        // Fired before we ever call cancelled() — must return immediately, not park.
8353        tokio::time::timeout(Duration::from_secs(1), signal.cancelled())
8354            .await
8355            .expect("cancelled() must resolve when cancel fired beforehand");
8356
8357        // A fresh signal cancelled concurrently with an in-flight cancelled().
8358        let racing = PersistentCancelSignal::new();
8359        let racing_for_task = racing.clone();
8360        let waiter = tokio::spawn(async move { racing_for_task.cancelled().await });
8361        racing.cancel();
8362        tokio::time::timeout(Duration::from_secs(1), waiter)
8363            .await
8364            .expect("cancelled() must resolve when cancel races the await")
8365            .expect("waiter task panicked");
8366    }
8367
8368    #[test]
8369    fn ingress_epoch_validation_rejects_reclaimed_requests_and_drops_other_stale_epochs() {
8370        let installed = HashMap::from([(7, 9)]);
8371        let mut reclaimed = ReclaimedRoutes::default();
8372        reclaimed.insert(route_key(8, 1));
8373        for ty in [
8374            FrameType::Request,
8375            FrameType::Response,
8376            FrameType::Error,
8377            FrameType::Push,
8378            FrameType::Cancel,
8379            FrameType::Goodbye,
8380        ] {
8381            let body = if ty.is_pure_header() {
8382                Vec::new()
8383            } else {
8384                br#"{}"#.to_vec()
8385            };
8386            let stale = Frame::build(ty, control_flags(), 7, 8, 41, body).unwrap();
8387            assert!(
8388                !ingress_route_should_be_processed(&installed, &reclaimed, &stale),
8389                "{ty:?}"
8390            );
8391        }
8392
8393        let reclaimed_request = Frame::build(
8394            FrameType::Request,
8395            control_flags(),
8396            8,
8397            1,
8398            42,
8399            br#"{}"#.to_vec(),
8400        )
8401        .unwrap();
8402        assert!(ingress_route_should_be_processed(
8403            &installed,
8404            &reclaimed,
8405            &reclaimed_request
8406        ));
8407
8408        let never_installed = Frame::build(
8409            FrameType::Request,
8410            control_flags(),
8411            9,
8412            1,
8413            43,
8414            br#"{}"#.to_vec(),
8415        )
8416        .unwrap();
8417        assert!(!ingress_route_should_be_processed(
8418            &installed,
8419            &reclaimed,
8420            &never_installed
8421        ));
8422
8423        let current = Frame::build(
8424            FrameType::Request,
8425            control_flags(),
8426            7,
8427            9,
8428            43,
8429            br#"{}"#.to_vec(),
8430        )
8431        .unwrap();
8432        let control = Frame::build(FrameType::Ping, control_flags(), 0, 0, 44, Vec::new()).unwrap();
8433        assert!(ingress_route_should_be_processed(
8434            &installed, &reclaimed, &current
8435        ));
8436        assert!(ingress_route_should_be_processed(
8437            &installed, &reclaimed, &control
8438        ));
8439        assert_eq!(installed, HashMap::from([(7, 9)]));
8440    }
8441
8442    #[tokio::test]
8443    async fn route_bind_ack_precedes_route_egress_in_writer_queue() {
8444        let (_dir, root) = test_root("route-bind-b2-ordering");
8445        let route = route_key(7, 3);
8446        let identity = RouteIdentity(Arc::new(RouteIdentityData {
8447            root: root.clone(),
8448            project_root: root.as_path().to_path_buf(),
8449            harness: "opencode".to_string(),
8450            session: "b2-session".to_string(),
8451            trust: BindTrust::FirstParty,
8452            spawn_principal: AuthenticatedPrincipal::FirstParty,
8453            consumer_elicitation_capable: false,
8454        }));
8455        let replay_key = push::ReplayKey::from_identity(&identity);
8456        let completion = RouteBindCompletion {
8457            route,
8458            identity,
8459            bind_root_id: root.clone(),
8460            inserted_new_actor: false,
8461            configure_response: Response::success("subc-bind-7", json!({})),
8462            diagnostics_on_edit: false,
8463            ver: PROTOCOL_VERSION,
8464            corr: 91,
8465            flags: control_flags(),
8466        };
8467        let mut pending_binds = HashMap::from([(
8468            route,
8469            PendingBind {
8470                bind_root_id: root,
8471                inserted_new_actor: false,
8472                cancelled: false,
8473                configure_request_id: "subc-bind-7".to_string(),
8474                started_at: Instant::now(),
8475                warned_half_deadline: false,
8476                deadline_reported: false,
8477                corr: 91,
8478                ver: PROTOCOL_VERSION,
8479                flags: control_flags(),
8480                cancellation: crate::executor::JobCancellation::new(),
8481            },
8482        )]);
8483        let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
8484        let mut push_buffer =
8485            HashMap::from([(replay_key, VecDeque::from([completion_frame("b2-replay")]))]);
8486        let (writer_tx, mut writer_rx) = mpsc::channel(8);
8487        let metrics = Arc::new(DispatchPathMetrics::new());
8488        let executor = Arc::new(Executor::new());
8489        let standing_actor =
8490            standing::StandingActor::new(App::default_shared(), Arc::clone(&executor));
8491
8492        handle_route_bind_completion(
8493            &writer_tx,
8494            completion,
8495            &mut HashMap::new(),
8496            &mut HashMap::new(),
8497            &mut HashMap::new(),
8498            &mut push_buffer,
8499            &mut HashMap::new(),
8500            &mut pending_binds,
8501            &mut installed_route_epochs,
8502            &executor,
8503            &standing_actor,
8504            &Arc::new(Notify::new()),
8505            &metrics,
8506        )
8507        .await
8508        .unwrap();
8509
8510        let ack = writer_rx.try_recv().expect("RouteBindAck");
8511        assert_eq!(ack.header.ty, FrameType::Response);
8512        assert_eq!((ack.header.channel, ack.header.epoch), (0, 0));
8513        let route_frame = writer_rx.try_recv().expect("post-ack route frame");
8514        assert_eq!(route_frame.header.ty, FrameType::Push);
8515        assert_eq!(
8516            (route_frame.header.channel, route_frame.header.epoch),
8517            (route.channel, route.epoch)
8518        );
8519    }
8520}