Skip to main content

aft/subc/
mod.rs

1//! subc daemon attach — transport edge.
2//!
3//! When AFT is launched as `aft --subc <connection-file>`, it does NOT run the
4//! standalone NDJSON-over-stdin loop. Instead it connects to a running subc
5//! daemon over loopback TCP, authenticates with the pre-envelope HMAC handshake
6//! (`subc-transport`), then speaks the subc frame protocol (`subc-protocol`):
7//! ModuleHello → HelloAck (register as a tool provider), then a channel-0
8//! control loop (Ping/Pong, RouteBind) plus route-channel tool calls.
9//!
10//! Concurrency: subc routes tool calls through the executor. The tokio
11//! edge never dispatches against `AppContext` inline; per-actor executor lanes
12//! own the reader/mutator epoch, while a writer task serializes outbound frames.
13
14use std::collections::{HashMap, HashSet, VecDeque};
15use std::fmt;
16use std::io;
17use std::net::{IpAddr, SocketAddr};
18use std::ops::Deref;
19use std::path::{Path, PathBuf};
20use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
21use std::sync::{Arc, LazyLock, Mutex as StdMutex, OnceLock};
22use std::time::{Duration, Instant};
23
24use serde::Deserialize;
25use serde_json::{json, Value};
26
27use crate::config::Config;
28use crate::config_resolve::ConfigTier;
29use crate::context::{App, AppContext, ProgressSender, RootHealthSnapshot};
30use crate::executor::{Executor, JobCancellation, Lane};
31use crate::fleet_status::{spawn_fleet_status_dial, FleetStatusClient};
32use crate::jsonc::strip_jsonc;
33use crate::log_ctx;
34use crate::path_identity::ProjectRootId;
35use crate::protocol::{ProgressKind, PushFrame, RawRequest, Response};
36use crate::run_tool_call::{
37    run_tool_call, strip_agent_preview_arg_owned, PhaseTrace, ToolCallContext, ToolCallOutcome,
38    ToolCallResult,
39};
40use crate::runtime_drain;
41use crate::sandbox_spawn::{AuthenticatedPrincipal, PrincipalTrust};
42
43use subc_protocol::manifest::{
44    Bindings, Concurrency, ExecutionMode, IdentityBinding, IdentityScope, ModuleManifest,
45    ProviderRole, StorageBinding, StorageKind, StorageScope, Tool, TrustTier,
46};
47use subc_protocol::session::{
48    HealthReport, HealthStatus, ModuleControlRequest, ModuleControlResponse,
49    MODULE_CONTROL_OP_HEALTH_CHECK,
50};
51use subc_protocol::{
52    ErrorBody, Flags, Frame, FrameType, ModuleHelloBody, Principal, Priority, MAX_FRAME_BODY_LEN,
53    PROTOCOL_VERSION,
54};
55use subc_transport::{authenticate_client, connection_file, read_frame, write_frame};
56use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
57use tokio::net::TcpStream;
58use tokio::sync::{mpsc, oneshot, Notify};
59use tokio::task::JoinHandle;
60
61/// Per-attempt handshake deadline. The initial attach loop has a separate total
62/// budget so a stalled peer cannot consume an unbounded supervisor launch window.
63const AUTH_DEADLINE: Duration = Duration::from_secs(5);
64const ATTACH_RETRY_BUDGET: Duration = Duration::from_secs(60);
65const ATTACH_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(250);
66const ATTACH_RETRY_MAX_BACKOFF: Duration = Duration::from_secs(5);
67const ATTACH_RETRY_JITTER_PERCENT: u64 = 20;
68
69/// Correlation id for the initial ModuleHello (channel 0).
70const HELLO_CORR: u64 = 1;
71
72/// Per-session in-memory replay cap for must-deliver Push frames. This covers
73/// detach/re-attach while AFT stays alive; cross-restart replay is phased later.
74const PUSH_BUFFER_MAX_PER_KEY: usize = 256;
75
76/// Bounded guard for control-frame sends. If the daemon stops reading and the
77/// writer queue stays full, tear the subc edge down instead of stalling the
78/// route loop indefinitely.
79const CONTROL_SEND_TIMEOUT: Duration = Duration::from_millis(250);
80
81/// Cadence for the loop's deadline-driven drain work (retry-buffer flush,
82/// bg-wake emission, maintenance submission). Checked at the top of every
83/// loop turn so busy select arms cannot starve it.
84const DRAIN_TICK_PERIOD: Duration = Duration::from_millis(250);
85
86/// Root-scoped stores and watcher runtimes are reopened lazily after this
87/// period without tool traffic. Keeping the value fixed avoids per-client
88/// eviction policies competing inside the module loop.
89const IDLE_ROOT_TTL: Duration = Duration::from_secs(30 * 60);
90
91const WRITER_QUEUE_CAPACITY: usize = 256;
92
93/// Keep reliable Push bursts from monopolizing the current-thread subc loop;
94/// any remaining must-deliver frames stay queued for the next loop turn.
95const RELIABLE_PUSH_DRAIN_BUDGET: usize = 32;
96
97/// Limit maintenance submissions per tick so background drains cannot delay
98/// control-plane work such as completed RouteBind acknowledgements.
99///
100/// The decomposed maintenance pass charges this budget by Mutating job, not by
101/// root. Size the default burst for one maintenance pass over eight live roots,
102/// while follow-up batches still re-enter the capped queue instead of bypassing
103/// the budget.
104const MAINTENANCE_SUBMIT_BUDGET: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len() * 8;
105const INITIAL_MAINTENANCE_DRAIN_KINDS: [MaintenanceDrainKind; 4] = [
106    MaintenanceDrainKind::Watcher,
107    MaintenanceDrainKind::Lsp,
108    MaintenanceDrainKind::ConfigureTail,
109    MaintenanceDrainKind::CompletionDrains,
110];
111#[cfg(test)]
112const INITIAL_MAINTENANCE_JOB_COUNT: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len();
113
114const RELIABLE_WRITER_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(10);
115const RELIABLE_WRITER_RETRY_MAX_BACKOFF: Duration = Duration::from_millis(250);
116
117const DISPATCH_PATH_BIND_WARN_AFTER: Duration = Duration::from_secs(6);
118const ROUTE_BIND_DEADLINE: Duration = Duration::from_secs(12);
119
120/// Small bounded memory of completed task ids used to suppress stale lossy
121/// long-running reminders that arrive after their reliable completion event.
122const COMPLETED_TASK_SUPPRESSION_MAX: usize = 4096;
123
124/// Bash foreground orchestration polls detached tasks with short read-lane jobs.
125/// The sleep between polls is outside the executor so no read or write worker is
126/// pinned while a foreground command is still running.
127const PENDING_POLL_INTERVAL: Duration = Duration::from_millis(100);
128
129/// Host elicitation asks fail closed if the MCP facade does not answer promptly.
130const BASH_ELICITATION_TIMEOUT: Duration = Duration::from_secs(60);
131const BASH_ELICITATION_CREATE_METHOD: &str = "elicitation/create";
132
133#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
134struct RouteChannel {
135    channel: u16,
136    epoch: u32,
137}
138
139impl fmt::Display for RouteChannel {
140    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
141        write!(f, "{}@{}", self.channel, self.epoch)
142    }
143}
144
145type PushEnvelope = (ProjectRootId, PushFrame);
146type LossyPushEnvelope = (u64, ProjectRootId, PushFrame);
147type RetryBuffer = HashMap<RouteChannel, VecDeque<(push::ReplayKey, PushFrame)>>;
148mod bash;
149mod health;
150mod manifest;
151mod push;
152mod wire;
153
154use self::health::{
155    build_health_report, warn_slow_pending_binds, DispatchPathMetrics, HealthRollupCache,
156    ReapBlockerCensus, ResponseTaskGuard, HEALTH_ROLLUP_TTL,
157};
158use self::manifest::{
159    build_manifest, command_lane, control_flags, control_ops, is_bash_family_tool,
160    is_subc_agent_core_tool, is_subc_native_plumbing_tool,
161};
162pub use self::wire::SubcError;
163
164/// Test-only view of the fail-closed tool-call gate: would `name` be admitted
165/// on a bound route (as an agent tool or native plumbing)? Used by the
166/// plugin-send drift guard in `subc_plumbing_drift_test.rs`.
167pub fn is_tool_call_admitted_for_test(name: &str) -> bool {
168    manifest::is_subc_agent_core_tool(name) || manifest::is_subc_native_plumbing_tool(name)
169}
170use self::wire::{
171    build_error_frame, build_goodbye_frame, build_tool_response_frame,
172    build_tool_response_frame_with_limit, decrement_counted_channel, response_is_fatal_panic,
173    response_message, send_counted_channel, send_frame, send_reliable_writer_frame,
174    send_traced_tool_response_frame, ToolResponseWriteTrace, WriterFrame, WriterSender,
175};
176
177struct DecodedFrame {
178    frame: Frame,
179    phase_trace: PhaseTrace,
180}
181
182struct ToolCallCompletion {
183    text: String,
184    phase_trace: PhaseTrace,
185}
186
187#[derive(Clone)]
188struct ActiveToolCall {
189    root_id: ProjectRootId,
190    cancellation: JobCancellation,
191}
192
193type ActiveToolCalls = Arc<StdMutex<HashMap<(RouteChannel, u64), ActiveToolCall>>>;
194
195#[derive(Clone)]
196struct PushSenders {
197    lossy_tx: mpsc::Sender<LossyPushEnvelope>,
198    reliable_tx: mpsc::UnboundedSender<PushEnvelope>,
199    lossy_overflow: Arc<push::LossyOverflow>,
200    lossy_seq: Arc<AtomicU64>,
201    fleet_status_client: FleetStatusClient,
202}
203
204#[derive(Clone)]
205struct PersistentCancelSignal {
206    inner: Arc<PersistentCancelInner>,
207}
208
209struct PersistentCancelInner {
210    cancelled: AtomicBool,
211    notify: Notify,
212}
213
214impl PersistentCancelSignal {
215    fn new() -> Self {
216        Self {
217            inner: Arc::new(PersistentCancelInner {
218                cancelled: AtomicBool::new(false),
219                notify: Notify::new(),
220            }),
221        }
222    }
223
224    fn cancel(&self) {
225        if !self.inner.cancelled.swap(true, Ordering::SeqCst) {
226            self.inner.notify.notify_waiters();
227        }
228    }
229
230    fn is_cancelled(&self) -> bool {
231        self.inner.cancelled.load(Ordering::SeqCst)
232    }
233
234    async fn cancelled(&self) {
235        // `enable()` REGISTERS this waiter before we read the flag, closing the
236        // lost-wakeup window: `notify_waiters()` only wakes already-registered
237        // waiters and stores no permit, so without enable() a `cancel()` firing
238        // between the flag read and `.await` would be missed and the future
239        // would park forever (cancel() fires only once). With enable(), a cancel
240        // racing the flag read still wakes the registered waiter. The loop is a
241        // belt-and-suspenders re-check on spurious wakeups.
242        loop {
243            let notified = self.inner.notify.notified();
244            tokio::pin!(notified);
245            notified.as_mut().enable();
246            if self.is_cancelled() {
247                return;
248            }
249            notified.await;
250        }
251    }
252}
253
254fn finish_active_tool_call(active: &ActiveToolCalls, route: RouteChannel, corr: u64) {
255    active
256        .lock()
257        .unwrap_or_else(std::sync::PoisonError::into_inner)
258        .remove(&(route, corr));
259}
260
261fn cancel_active_tool_call(
262    active: &ActiveToolCalls,
263    executor: &Executor,
264    route: RouteChannel,
265    corr: u64,
266    reason: &str,
267) -> bool {
268    let call = active
269        .lock()
270        .unwrap_or_else(std::sync::PoisonError::into_inner)
271        .remove(&(route, corr));
272    let Some(call) = call else {
273        return false;
274    };
275    let outcome = executor.cancel_job(&call.root_id, &call.cancellation);
276    log::debug!(
277        "subc attach: cancelled active tool call route={route} corr={corr} reason={reason} outcome={outcome:?}"
278    );
279    true
280}
281
282fn cancel_active_tool_calls_for_route(
283    active: &ActiveToolCalls,
284    executor: &Executor,
285    route: RouteChannel,
286    reason: &str,
287) -> usize {
288    let cancelled = {
289        let mut calls = active
290            .lock()
291            .unwrap_or_else(std::sync::PoisonError::into_inner);
292        let mut cancelled = Vec::new();
293        calls.retain(|(call_route, _), call| {
294            if *call_route == route {
295                cancelled.push(call.clone());
296                false
297            } else {
298                true
299            }
300        });
301        cancelled
302    };
303    for call in &cancelled {
304        let outcome = executor.cancel_job(&call.root_id, &call.cancellation);
305        log::debug!(
306            "subc attach: cancelled active tool call route={route} reason={reason} outcome={outcome:?}"
307        );
308    }
309    cancelled.len()
310}
311
312fn cancel_all_active_tool_calls(
313    active: &ActiveToolCalls,
314    executor: &Executor,
315    reason: &str,
316) -> usize {
317    let cancelled = {
318        let mut calls = active
319            .lock()
320            .unwrap_or_else(std::sync::PoisonError::into_inner);
321        calls.drain().map(|(_, call)| call).collect::<Vec<_>>()
322    };
323    for call in &cancelled {
324        let outcome = executor.cancel_job(&call.root_id, &call.cancellation);
325        log::debug!("subc attach: cancelled active tool call reason={reason} outcome={outcome:?}");
326    }
327    cancelled.len()
328}
329
330#[derive(Debug, Clone, Copy, PartialEq, Eq)]
331pub(crate) enum BindTrust {
332    FirstParty,
333    Untrusted,
334}
335
336impl BindTrust {
337    fn allows_bash_observation(self) -> bool {
338        matches!(self, Self::FirstParty)
339    }
340
341    fn label(self) -> &'static str {
342        match self {
343            Self::FirstParty => "first_party",
344            Self::Untrusted => "untrusted",
345        }
346    }
347
348    fn sandbox_trust(self) -> PrincipalTrust {
349        match self {
350            Self::FirstParty => PrincipalTrust::FirstParty,
351            Self::Untrusted => PrincipalTrust::Untrusted,
352        }
353    }
354}
355
356pub(super) fn trust_for_principal(principal: &Option<Principal>) -> BindTrust {
357    match principal {
358        Some(Principal::Direct) => BindTrust::FirstParty,
359        // Module renames are flag-days: the daemon registry refuses duplicate
360        // active ids, so a renaming module cannot advertise both names during
361        // its transition. This allowlist is DIALLED, not dialling — it must
362        // accept a module's NEW name in a released binary before the module
363        // starts using it, and the old name stays until the flip has settled.
364        // That is why transitional pairs appear here: llm-runner/broca was the
365        // previous rename, alfonso-core/prefrontal is the current one. When
366        // retiring an old name, confirm the fleet no longer spawns it — a
367        // stale entry here is inert, but a missing one silently downgrades a
368        // first-party module to Untrusted and revokes its bash access.
369        Some(Principal::Reserved { module_id })
370            if module_id == "llm-runner"
371                || module_id == "aft"
372                || module_id == "broca"
373                || module_id == "alfonso-core"
374                || module_id == "prefrontal"
375                || module_id == "prefrontal-core" =>
376        {
377            BindTrust::FirstParty
378        }
379        Some(Principal::Reserved { .. }) | Some(Principal::Unverified) | None => {
380            BindTrust::Untrusted
381        }
382    }
383}
384
385fn harness_forces_untrusted(harness: &str) -> bool {
386    harness.starts_with("fed:")
387}
388
389pub(super) fn trust_for_bind(harness: &str, principal: &Option<Principal>) -> BindTrust {
390    if harness_forces_untrusted(harness) {
391        BindTrust::Untrusted
392    } else {
393        trust_for_principal(principal)
394    }
395}
396
397fn principal_id(principal: &Option<Principal>) -> Option<String> {
398    match principal {
399        Some(Principal::Direct) => Some("direct".to_string()),
400        Some(Principal::Reserved { module_id }) => Some(format!("reserved:{module_id}")),
401        Some(Principal::Unverified) => Some("unverified".to_string()),
402        None => None,
403    }
404}
405
406fn principal_label(principal: &Option<Principal>) -> String {
407    principal_id(principal).unwrap_or_else(|| "absent".to_string())
408}
409
410#[derive(Debug)]
411/// Per-root route metadata owned by the subc loop. The `active_bash_waits` field
412/// counts detached bash processes that are still being observed for this root.
413/// Any future logic that evicts roots based on idle time must not evict a root
414/// while this count is greater than zero, because a foreground bash response may
415/// still arrive later.
416struct RootMeta {
417    maintenance_pending: bool,
418    maintenance_jobs_in_flight: usize,
419    maintenance_queued_kinds: VecDeque<MaintenanceDrainKind>,
420    maintenance_last_submitted: Option<Instant>,
421    maintenance_poisoned: bool,
422    last_touched: Instant,
423    diagnostics_on_edit: bool,
424    active_bash_waits: usize,
425    idle_artifacts_evicted: bool,
426    unbound_quiesced: bool,
427    consecutive_missing_sweeps: u8,
428}
429
430#[derive(Debug)]
431struct PendingBind {
432    bind_root_id: ProjectRootId,
433    inserted_new_actor: bool,
434    cancelled: bool,
435    configure_request_id: String,
436    started_at: Instant,
437    warned_half_deadline: bool,
438    deadline_reported: bool,
439    corr: u64,
440    ver: u8,
441    flags: Flags,
442    /// Exact-job cancellation for the submitted configure: Goodbye and
443    /// deadline expiry cancel the executor job operationally (queued jobs are
444    /// removed, running configures return at their next checkpoint) instead of
445    /// only marking bookkeeping.
446    cancellation: crate::executor::JobCancellation,
447}
448
449struct RouteBindCompletion {
450    route: RouteChannel,
451    identity: RouteIdentity,
452    bind_root_id: ProjectRootId,
453    inserted_new_actor: bool,
454    configure_response: Response,
455    diagnostics_on_edit: bool,
456    ver: u8,
457    corr: u64,
458    flags: Flags,
459}
460
461#[derive(Debug, Clone)]
462struct RouteIdentity(Arc<RouteIdentityData>);
463
464#[derive(Debug)]
465struct RouteIdentityData {
466    root: ProjectRootId,
467    project_root: PathBuf,
468    harness: String,
469    session: String,
470    trust: BindTrust,
471    spawn_principal: AuthenticatedPrincipal,
472    consumer_elicitation_capable: bool,
473}
474
475impl Deref for RouteIdentity {
476    type Target = RouteIdentityData;
477
478    fn deref(&self) -> &Self::Target {
479        &self.0
480    }
481}
482
483#[derive(Debug, Clone)]
484struct RetainedSessionIdentity {
485    harness: String,
486    trust: BindTrust,
487}
488
489#[derive(Clone)]
490struct BgSub {
491    corr: u64,
492    ver: u8,
493    flags: Flags,
494    root: ProjectRootId,
495    session: String,
496}
497
498// A session can be observed by multiple long-lived consumer records. Retain
499// every route so each wake uses the correlation captured by that route's BgSub.
500type BgSubsBySession = HashMap<(ProjectRootId, String), HashSet<RouteChannel>>;
501
502struct MaintenanceCompletion {
503    root_id: ProjectRootId,
504    kind: MaintenanceDrainKind,
505    response: Response,
506    empty_bg_sessions: Vec<(String, u64)>,
507    requeue_kind: Option<MaintenanceDrainKind>,
508}
509
510#[derive(Clone, Copy, Debug, PartialEq, Eq)]
511enum MaintenanceDrainKind {
512    Watcher,
513    Lsp,
514    ConfigureTail,
515    CompletionDrains,
516}
517
518impl MaintenanceDrainKind {
519    fn label(self) -> &'static str {
520        match self {
521            Self::Watcher => "watcher",
522            Self::Lsp => "lsp",
523            Self::ConfigureTail => "configure-tail",
524            Self::CompletionDrains => "completion-drains",
525        }
526    }
527}
528
529#[derive(Debug, Default)]
530struct MaintenanceJobOutcome {
531    empty_bg_sessions: Vec<(String, u64)>,
532    requeue_kind: Option<MaintenanceDrainKind>,
533}
534
535#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
536struct ReverseCorrKey {
537    route: RouteChannel,
538    corr: u64,
539}
540
541struct PendingBashAsk {
542    route: RouteChannel,
543    tool_corr: u64,
544    tool_flags: Flags,
545    tool_ver: u8,
546    root: ProjectRootId,
547    project_root: PathBuf,
548    session_id: String,
549    spawn_principal: AuthenticatedPrincipal,
550    edit_slot_survives: Option<bool>,
551    request_id: String,
552    arguments: Value,
553    format_context: crate::subc_format::FormatContext,
554    cancel: bash::BashWaitCancel,
555    grants: Vec<String>,
556    expires_at: Instant,
557}
558
559impl RootMeta {
560    fn new(now: Instant) -> Self {
561        Self {
562            maintenance_pending: false,
563            maintenance_jobs_in_flight: 0,
564            maintenance_queued_kinds: VecDeque::new(),
565            maintenance_last_submitted: None,
566            maintenance_poisoned: false,
567            last_touched: now,
568            diagnostics_on_edit: false,
569            active_bash_waits: 0,
570            idle_artifacts_evicted: false,
571            unbound_quiesced: false,
572            consecutive_missing_sweeps: 0,
573        }
574    }
575
576    fn note_activity(&mut self) {
577        self.last_touched = Instant::now();
578    }
579
580    fn reactivate_bound(&mut self) {
581        self.note_activity();
582        self.idle_artifacts_evicted = false;
583        self.unbound_quiesced = false;
584    }
585}
586
587fn due_maintenance_jobs(
588    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
589    executor: Option<&Executor>,
590    bg_sub_by_session: &BgSubsBySession,
591    bg_wake_pending: &HashSet<RouteChannel>,
592    budget: usize,
593    pending_bind_roots: &HashSet<ProjectRootId>,
594) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
595    let mut jobs = Vec::new();
596    let mut deferred = false;
597    let mut roots = live_roots.keys().cloned().collect::<Vec<_>>();
598    roots.sort_by(|left, right| {
599        let left_last = live_roots
600            .get(left)
601            .and_then(|meta| meta.maintenance_last_submitted);
602        let right_last = live_roots
603            .get(right)
604            .and_then(|meta| meta.maintenance_last_submitted);
605        left_last
606            .cmp(&right_last)
607            .then_with(|| left.as_path().cmp(right.as_path()))
608    });
609
610    for root_id in roots {
611        let Some(meta) = live_roots.get_mut(&root_id) else {
612            continue;
613        };
614        if meta.maintenance_poisoned {
615            continue;
616        }
617
618        if pending_bind_roots.contains(&root_id) {
619            if meta.maintenance_pending || !meta.maintenance_queued_kinds.is_empty() {
620                deferred = true;
621            }
622            continue;
623        }
624
625        if !meta.maintenance_pending {
626            if jobs.len() >= budget {
627                deferred = true;
628                continue;
629            }
630            // Only enqueue kinds with pending work. Probes are cheap and
631            // fail-open (contended sources count as pending), so an idle root
632            // costs four probes per tick instead of four dispatched jobs.
633            let executor_actor_context =
634                executor.and_then(|executor| executor.actor_context(&root_id));
635            let root_has_pending_bg_wake =
636                bg_sub_by_session.iter().any(|((sub_root, _), channels)| {
637                    sub_root == &root_id
638                        && channels
639                            .iter()
640                            .any(|channel| bg_wake_pending.contains(channel))
641                });
642            let kinds_with_work: Vec<MaintenanceDrainKind> = match executor_actor_context {
643                Some(ctx) => INITIAL_MAINTENANCE_DRAIN_KINDS
644                    .into_iter()
645                    .filter(|kind| {
646                        if meta.unbound_quiesced && !matches!(kind, MaintenanceDrainKind::Lsp) {
647                            return false;
648                        }
649                        match kind {
650                            MaintenanceDrainKind::Watcher => ctx.watcher_drain_has_work(),
651                            MaintenanceDrainKind::Lsp => ctx.lsp_drain_has_work(),
652                            MaintenanceDrainKind::ConfigureTail => ctx.configure_tail_has_work(),
653                            // Every CompletionDrains source is visible at this enqueue site:
654                            // AppContext probes completion queues, this loop owns bg wakes,
655                            // and queued continuations bypass probing via maintenance_pending.
656                            // New drain sources must expose a probe here rather than making
657                            // every subscribed root fail open again.
658                            MaintenanceDrainKind::CompletionDrains => {
659                                root_has_pending_bg_wake || ctx.completion_drains_have_work()
660                            }
661                        }
662                    })
663                    .collect(),
664                None if meta.unbound_quiesced => Vec::new(),
665                // No context handle (actor gone mid-tick): enqueue everything.
666                None => INITIAL_MAINTENANCE_DRAIN_KINDS.to_vec(),
667            };
668            if kinds_with_work.is_empty() {
669                continue;
670            }
671            meta.maintenance_pending = true;
672            meta.maintenance_queued_kinds.extend(kinds_with_work);
673        }
674
675        while let Some(kind) = meta.maintenance_queued_kinds.pop_front() {
676            if jobs.len() >= budget {
677                meta.maintenance_queued_kinds.push_front(kind);
678                deferred = true;
679                break;
680            }
681            meta.maintenance_jobs_in_flight += 1;
682            meta.maintenance_last_submitted = Some(Instant::now());
683            jobs.push((root_id.clone(), kind));
684        }
685
686        meta.maintenance_pending =
687            meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
688    }
689
690    (jobs, deferred)
691}
692
693fn eviction_estimate_label(estimate: &crate::memory::MemoryEstimate) -> String {
694    match estimate.estimated_bytes {
695        Some(bytes) => format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
696        None if estimate.status == "busy" => "busy".to_string(),
697        None => "not estimated".to_string(),
698    }
699}
700
701fn optional_memory_label(bytes: Option<u64>) -> String {
702    bytes.map_or_else(
703        || "not estimated".to_string(),
704        |bytes| format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
705    )
706}
707
708fn pressure_relief_label(relief: &crate::memory::AllocatorPressureRelief) -> String {
709    format!(
710        "; allocator pressure relief: RSS {} -> {}, in-use {} -> {}, allocated {} -> {}, slack {} -> {}, allocator reported {:.1} MB released",
711        optional_memory_label(relief.rss_before_bytes),
712        optional_memory_label(relief.rss_after_bytes),
713        optional_memory_label(relief.allocator_before.bytes_in_use),
714        optional_memory_label(relief.allocator_after.bytes_in_use),
715        optional_memory_label(relief.allocator_before.size_allocated),
716        optional_memory_label(relief.allocator_after.size_allocated),
717        optional_memory_label(relief.allocator_before.retained_slack_bytes),
718        optional_memory_label(relief.allocator_after.retained_slack_bytes),
719        relief.bytes_released as f64 / (1024.0 * 1024.0),
720    )
721}
722
723fn idle_root_eviction_message(
724    root_id: &ProjectRootId,
725    memory: &crate::memory::RootMemorySnapshot,
726    pressure_relief: Option<&crate::memory::AllocatorPressureRelief>,
727) -> String {
728    // Bash, LSP, and parser state remain resident. The freed total is deliberately
729    // only the known-byte portion of handles eviction actually drops.
730    let freed_bytes = [
731        &memory.semantic,
732        &memory.trigram,
733        &memory.symbols,
734        &memory.callgraph,
735        &memory.inspect,
736    ]
737    .iter()
738    .filter_map(|estimate| estimate.estimated_bytes)
739    .fold(0u64, u64::saturating_add);
740    let mut message = format!(
741        "evicted idle root {}: freed ~{:.1} MB (semantic {}, trigram {}, symbols {}, callgraph {}, inspect {}; retained: bash {}, lsp {}, parser_pool {})",
742        root_id.as_path().display(),
743        freed_bytes as f64 / (1024.0 * 1024.0),
744        eviction_estimate_label(&memory.semantic),
745        eviction_estimate_label(&memory.trigram),
746        eviction_estimate_label(&memory.symbols),
747        eviction_estimate_label(&memory.callgraph),
748        eviction_estimate_label(&memory.inspect),
749        eviction_estimate_label(&memory.bash),
750        eviction_estimate_label(&memory.lsp),
751        eviction_estimate_label(&memory.parser_pool),
752    );
753    if let Some(pressure_relief) = pressure_relief {
754        message.push_str(&pressure_relief_label(pressure_relief));
755    }
756    message
757}
758
759fn process_has_been_idle(now: Instant, live_roots: &HashMap<ProjectRootId, RootMeta>) -> bool {
760    !live_roots.is_empty()
761        && live_roots.values().all(|meta| {
762            now.saturating_duration_since(meta.last_touched) >= IDLE_ROOT_TTL
763                && meta.active_bash_waits == 0
764                && !meta.maintenance_pending
765                && meta.maintenance_queued_kinds.is_empty()
766        })
767}
768
769fn allocator_pressure_relief_after_idle_sweep(
770    now: Instant,
771    live_roots: &HashMap<ProjectRootId, RootMeta>,
772    executor: &Executor,
773) -> Option<crate::memory::AllocatorPressureRelief> {
774    if !process_has_been_idle(now, live_roots)
775        || live_roots.keys().any(|root_id| {
776            executor
777                .actor_context(root_id)
778                .is_some_and(|ctx| ctx.artifact_eviction_blocked())
779        })
780    {
781        return None;
782    }
783
784    #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu")))]
785    {
786        Some(crate::memory::relieve_allocator_pressure())
787    }
788    #[cfg(not(any(target_os = "macos", all(target_os = "linux", target_env = "gnu"))))]
789    {
790        None
791    }
792}
793
794fn quiesce_unbound_root(
795    root_id: &ProjectRootId,
796    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
797    executor: &Arc<Executor>,
798) {
799    let Some(meta) = live_roots.get_mut(root_id) else {
800        return;
801    };
802
803    let ctx = executor.actor_context(root_id);
804    if let Some(ctx) = ctx.as_ref() {
805        // Close lifecycle admission before touching scheduler queues. A running
806        // ConfigureTail cannot release gates, install a watcher, or reserve a
807        // callgraph build after this transition becomes visible.
808        ctx.mark_subc_unbound();
809    }
810    let cancelled = executor.cancel_queued_maintenance(root_id);
811    // Transient unbind keeps the root WARM: the watcher stays running (its
812    // events accumulate and replay on rebind, so no unobserved gap exists) and
813    // resident artifacts stay resident. Host restarts unbind every root and
814    // rebind seconds later; stopping the watcher here would force strict
815    // re-verification plus a full callgraph rebuild on every restart. The
816    // expensive teardown (watcher stop + gap invalidation) belongs to the
817    // idle-TTL reaper and the root-deleted path.
818    let discarded = ctx
819        .map(|ctx| crate::commands::configure::cancel_deferred_configure_maintenance(&ctx))
820        .unwrap_or(0);
821    meta.unbound_quiesced = true;
822    meta.maintenance_queued_kinds.clear();
823    meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
824    log::info!(
825        "subc attach: quiesced unbound root {} (cancelled {} queued maintenance job(s), cancelled {} configure maintenance job(s)); cause=goodbye_unbound",
826        root_id.as_path().display(),
827        cancelled,
828        discarded
829    );
830}
831
832#[allow(clippy::too_many_arguments)]
833fn quiesce_connection_roots(
834    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
835    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
836    routes: &mut HashMap<RouteChannel, RouteIdentity>,
837    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
838    installed_route_epochs: &mut HashMap<u16, u32>,
839    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
840    active_tool_calls: &ActiveToolCalls,
841    executor: &Arc<Executor>,
842) {
843    cancel_all_active_tool_calls(active_tool_calls, executor, "connection teardown");
844    for cancel in route_bash_cancels.values() {
845        cancel.token.cancel();
846    }
847    route_bash_cancels.clear();
848
849    let mut roots = live_roots.keys().cloned().collect::<HashSet<_>>();
850    for pending in pending_binds.values_mut() {
851        pending.cancelled = true;
852        roots.insert(pending.bind_root_id.clone());
853        let _ = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
854    }
855
856    // A connection exit abandons every installed route at once. Close lifecycle
857    // admission before cancelling maintenance so no deferred worker can restore
858    // root activity after the loop-owned route tables disappear.
859    for root_id in roots {
860        if live_roots.contains_key(&root_id) {
861            quiesce_unbound_root(&root_id, live_roots, executor);
862        } else if let Some(ctx) = executor.actor_context(&root_id) {
863            ctx.mark_subc_unbound();
864            executor.cancel_queued_maintenance(&root_id);
865            crate::commands::configure::cancel_deferred_configure_maintenance(&ctx);
866        }
867    }
868
869    routes.clear();
870    root_channels.clear();
871    installed_route_epochs.clear();
872}
873
874/// Per-channel epoch watermarks for roots reclaimed without a client Goodbye.
875/// The 16-bit channel space bounds this map, and no root identity or resource
876/// handle is retained. It exists only so late requests receive a typed error.
877#[derive(Debug, Default)]
878struct ReclaimedRoutes {
879    highest_epoch_by_channel: HashMap<u16, u32>,
880}
881
882impl ReclaimedRoutes {
883    fn insert(&mut self, route: RouteChannel) {
884        self.highest_epoch_by_channel
885            .entry(route.channel)
886            .and_modify(|epoch| *epoch = (*epoch).max(route.epoch))
887            .or_insert(route.epoch);
888    }
889
890    fn contains(&self, route: RouteChannel) -> bool {
891        self.highest_epoch_by_channel
892            .get(&route.channel)
893            .is_some_and(|epoch| route.epoch <= *epoch)
894    }
895}
896
897#[derive(Debug, Default)]
898struct IdleReapOutcome {
899    evicted: usize,
900    forgotten_deleted_roots: Vec<ProjectRootId>,
901}
902
903fn reap_idle_roots(
904    now: Instant,
905    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
906    pending_binds: &HashMap<RouteChannel, PendingBind>,
907    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
908    executor: &Arc<Executor>,
909    metrics: &DispatchPathMetrics,
910) -> IdleReapOutcome {
911    let pending_bind_roots = pending_binds
912        .values()
913        .map(|pending| pending.bind_root_id.clone())
914        .collect::<HashSet<_>>();
915    let mut census = ReapBlockerCensus::default();
916    let mut candidates = Vec::new();
917
918    for (root_id, meta) in live_roots.iter_mut() {
919        let deleted = !root_id.as_path().exists();
920        if deleted {
921            // A missing directory makes a bound route obsolete, but one failed
922            // lookup is not enough evidence to tear down a client-visible actor.
923            // Requiring two maintenance sweeps protects atomic replacement and
924            // transient filesystem failures; observing the path resets the proof.
925            // Absence also covers renames: a task's cwd handle can follow the
926            // moved directory while the registered path disappears. The old
927            // path is deliberately treated as a retired root identity, so the
928            // reaper accepts killing such tasks; rename a project only with no
929            // live tasks rather than relying on cwd-resolution heuristics.
930            meta.consecutive_missing_sweeps = meta.consecutive_missing_sweeps.saturating_add(1);
931        } else {
932            meta.consecutive_missing_sweeps = 0;
933        }
934        let deletion_confirmed = meta.consecutive_missing_sweeps >= 2;
935        let has_bound_route = root_channels
936            .get(root_id)
937            .is_some_and(|channels| !channels.is_empty());
938        let has_pending_bind = pending_bind_roots.contains(root_id);
939
940        if deleted {
941            let mut retained = false;
942            if !deletion_confirmed {
943                census.absence_unconfirmed += 1;
944                retained = true;
945            }
946            // Once absence is confirmed, the directory cannot serve this route
947            // again. Neither a stale route nor the lack of normal unbind cleanup
948            // justifies retaining the root; purge removes the route after retirement.
949            if meta.active_bash_waits > 0 {
950                census.bash_waits += 1;
951                retained = true;
952            }
953            if meta.maintenance_pending {
954                census.maintenance_pending += 1;
955                retained = true;
956            }
957            if !meta.maintenance_queued_kinds.is_empty() {
958                census.maintenance_queued += 1;
959                retained = true;
960            }
961            if has_pending_bind {
962                census.pending_binds += 1;
963                retained = true;
964            }
965            match executor.try_actor_is_idle(root_id) {
966                Some(true) => {}
967                Some(false) => {
968                    census.actor_busy += 1;
969                    retained = true;
970                }
971                None => {
972                    census.actor_state_busy += 1;
973                    retained = true;
974                }
975            }
976            if retained {
977                census.deleted_retained += 1;
978                continue;
979            }
980        } else {
981            // Route teardown marks the lifecycle admission gate before the last
982            // channel disappears. Requiring zero bound channels and a quiesced
983            // lifecycle prevents a still-bound root from losing its watcher.
984            if has_bound_route
985                || !meta.unbound_quiesced
986                || meta.idle_artifacts_evicted
987                || now.saturating_duration_since(meta.last_touched) < IDLE_ROOT_TTL
988                || meta.active_bash_waits > 0
989                || meta.maintenance_pending
990                || !meta.maintenance_queued_kinds.is_empty()
991                || has_pending_bind
992                || !executor.actor_is_idle(root_id)
993            {
994                continue;
995            }
996        }
997        candidates.push((root_id.clone(), deleted));
998    }
999
1000    let mut reaped = Vec::new();
1001    let mut forgotten_deleted_roots = Vec::new();
1002    for (root_id, deleted) in candidates {
1003        let Some(ctx) = executor.actor_context(&root_id) else {
1004            if deleted {
1005                census.deleted_retained += 1;
1006                census.actor_busy += 1;
1007            }
1008            continue;
1009        };
1010        // A TTL-aged unbound root retained its watcher-derived pending paths
1011        // across the transient-unbind window. Strict gap invalidation subsumes
1012        // them, but every abort path must restore them because a rebind can
1013        // still happen until eviction commits.
1014        //
1015        // After two consecutive directory-absence scans confirm that the
1016        // root is gone, terminate its background task before checking the
1017        // artifact-eviction gate. The task can otherwise keep the root's
1018        // artifacts in use; cleanup first lets confirmed reclamation finish
1019        // without weakening the gate for unrelated active work.
1020        if deleted {
1021            ctx.bash_background()
1022                .kill_running_tasks_for_root(root_id.as_path());
1023        }
1024        let taken_pending = Some(ctx.take_pending_reconciliation_state());
1025        if ctx.artifact_eviction_blocked() {
1026            if let Some(pending) = taken_pending {
1027                ctx.restore_pending_reconciliation_state(pending);
1028            }
1029            if deleted {
1030                census.deleted_retained += 1;
1031                census.artifact_eviction_blocked += 1;
1032            }
1033            continue;
1034        }
1035        let memory_before = ctx.memory_root_snapshot();
1036        if !ctx.evict_idle_artifacts() {
1037            if let Some(pending) = taken_pending {
1038                ctx.restore_pending_reconciliation_state(pending);
1039            }
1040            if deleted {
1041                census.deleted_retained += 1;
1042                census.artifact_eviction_failed += 1;
1043            }
1044            continue;
1045        }
1046        drop(taken_pending);
1047        ctx.stop_watcher_runtime_in_background();
1048        // Edits during watcher downtime are unobserved. Advance publication
1049        // epochs and force strict verification before any later warm reload.
1050        ctx.invalidate_artifacts_after_watcher_gap();
1051
1052        if deleted {
1053            if executor.retire_idle_actor_in_background(&root_id) {
1054                live_roots.remove(&root_id);
1055                forgotten_deleted_roots.push(root_id.clone());
1056            } else {
1057                census.deleted_retained += 1;
1058                census.actor_busy += 1;
1059            }
1060        } else {
1061            if let Some(meta) = live_roots.get_mut(&root_id) {
1062                meta.idle_artifacts_evicted = true;
1063            }
1064            ctx.release_idle_reopenable_resources_in_background();
1065        }
1066        reaped.push((root_id, memory_before));
1067    }
1068
1069    metrics.record_reap(census);
1070    if census.deleted_retained > 0 {
1071        log::info!(
1072            "subc attach: retained {} deleted root(s) during idle reap; blockers={}",
1073            census.deleted_retained,
1074            census.blocker_histogram()
1075        );
1076    }
1077
1078    let pressure_relief = (!reaped.is_empty())
1079        .then(|| allocator_pressure_relief_after_idle_sweep(now, live_roots, executor))
1080        .flatten();
1081    for (root_id, memory_before) in &reaped {
1082        log::info!(
1083            "{}",
1084            idle_root_eviction_message(root_id, memory_before, pressure_relief.as_ref())
1085        );
1086    }
1087    IdleReapOutcome {
1088        evicted: reaped.len(),
1089        forgotten_deleted_roots,
1090    }
1091}
1092
1093#[allow(clippy::too_many_arguments)]
1094fn purge_deleted_root_residents(
1095    root_id: &ProjectRootId,
1096    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1097    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1098    installed_route_epochs: &mut HashMap<u16, u32>,
1099    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1100    active_tool_calls: &ActiveToolCalls,
1101    executor: &Executor,
1102    retry_buffer: &mut RetryBuffer,
1103    reclaimed_routes: &mut ReclaimedRoutes,
1104    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1105    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1106    bg_subs: &mut HashMap<RouteChannel, BgSub>,
1107    bg_sub_by_session: &mut BgSubsBySession,
1108    bg_wake_pending: &mut HashSet<RouteChannel>,
1109    bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
1110    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1111    metrics: &DispatchPathMetrics,
1112) {
1113    let mut stale_routes = root_channels.get(root_id).cloned().unwrap_or_default();
1114    stale_routes.extend(
1115        routes
1116            .iter()
1117            .filter_map(|(route, identity)| (&identity.root == root_id).then_some(*route)),
1118    );
1119    stale_routes.extend(
1120        bg_sub_by_session
1121            .iter()
1122            .filter(|((root, _), _)| root == root_id)
1123            .flat_map(|(_, routes)| routes.iter().copied()),
1124    );
1125    stale_routes.extend(
1126        pending_bash_asks
1127            .values()
1128            .filter_map(|ask| (&ask.root == root_id).then_some(ask.route)),
1129    );
1130
1131    for route in stale_routes {
1132        reclaimed_routes.insert(route);
1133        remove_installed_route(installed_route_epochs, route);
1134        remove_route_channel(routes, root_channels, route);
1135        if let Some(cancel) = route_bash_cancels.remove(&route) {
1136            cancel.token.cancel();
1137        }
1138        cancel_active_tool_calls_for_route(active_tool_calls, executor, route, "root reclaim");
1139        retry_buffer.remove(&route);
1140        if let Some(sub) = bg_subs.remove(&route) {
1141            metrics.record_bg_subscription_ended(&sub.root, &sub.session, route, "root-reclaim");
1142        }
1143        bg_wake_pending.remove(&route);
1144    }
1145    root_channels.remove(root_id);
1146    session_identity.retain(|(root, _), _| root != root_id);
1147    push_buffer.retain(|key, _| &key.root != root_id);
1148    bg_wake_epoch.retain(|(root, _), _| root != root_id);
1149    pending_bash_asks.retain(|_, ask| &ask.root != root_id);
1150    bg_sub_by_session.retain(|(root, _), _| root != root_id);
1151
1152    log::info!(
1153        "subc attach: fully forgot deleted root {}; cause=absence_reclaim",
1154        root_id.as_path().display()
1155    );
1156}
1157
1158#[allow(clippy::too_many_arguments)]
1159fn submit_due_maintenance_jobs(
1160    executor: &Arc<Executor>,
1161    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1162    pending_binds: &HashMap<RouteChannel, PendingBind>,
1163    bg_sub_by_session: &BgSubsBySession,
1164    bg_wake_pending: &HashSet<RouteChannel>,
1165    bg_wake_epoch: &HashMap<(ProjectRootId, String), u64>,
1166    maintenance_tx: &mpsc::Sender<MaintenanceCompletion>,
1167    metrics: &Arc<DispatchPathMetrics>,
1168) {
1169    let pending_bind_roots = pending_binds
1170        .values()
1171        .map(|pending| pending.bind_root_id.clone())
1172        .collect::<HashSet<_>>();
1173    let (due_jobs, deferred_jobs) = due_maintenance_jobs(
1174        live_roots,
1175        Some(executor),
1176        bg_sub_by_session,
1177        bg_wake_pending,
1178        MAINTENANCE_SUBMIT_BUDGET,
1179        &pending_bind_roots,
1180    );
1181    if deferred_jobs {
1182        metrics
1183            .maintenance_budget_deferrals
1184            .fetch_add(1, Ordering::Relaxed);
1185    }
1186    for (root_id, kind) in due_jobs {
1187        let bg_sessions_to_check = if kind == MaintenanceDrainKind::CompletionDrains {
1188            bg_sub_by_session
1189                .iter()
1190                .filter_map(|((root, session), _)| {
1191                    if root == &root_id {
1192                        Some((
1193                            session.clone(),
1194                            bg_wake_epoch
1195                                .get(&(root_id.clone(), session.clone()))
1196                                .copied()
1197                                .unwrap_or(0),
1198                        ))
1199                    } else {
1200                        None
1201                    }
1202                })
1203                .collect()
1204        } else {
1205            Vec::new()
1206        };
1207        submit_maintenance_job(
1208            executor,
1209            root_id,
1210            kind,
1211            bg_sessions_to_check,
1212            maintenance_tx,
1213            metrics,
1214        );
1215    }
1216}
1217
1218fn should_requiesce_after_maintenance(
1219    meta: &RootMeta,
1220    completed_kind: MaintenanceDrainKind,
1221    bind_pending: bool,
1222) -> bool {
1223    meta.unbound_quiesced && completed_kind != MaintenanceDrainKind::Lsp && !bind_pending
1224}
1225
1226fn note_maintenance_completion(
1227    meta: &mut RootMeta,
1228    requeue_kind: Option<MaintenanceDrainKind>,
1229    fatal: bool,
1230    defer_requeue: bool,
1231) {
1232    if fatal {
1233        meta.maintenance_poisoned = true;
1234    }
1235
1236    if let Some(kind) = requeue_kind.filter(|_| !meta.maintenance_poisoned && !defer_requeue) {
1237        meta.maintenance_queued_kinds.push_back(kind);
1238    }
1239
1240    meta.maintenance_jobs_in_flight = meta.maintenance_jobs_in_flight.saturating_sub(1);
1241    meta.maintenance_pending =
1242        meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
1243}
1244
1245fn route_key(channel: u16, epoch: u32) -> RouteChannel {
1246    RouteChannel { channel, epoch }
1247}
1248
1249fn remove_installed_route(installed_epochs: &mut HashMap<u16, u32>, route: RouteChannel) {
1250    if installed_epochs.get(&route.channel).copied() == Some(route.epoch) {
1251        installed_epochs.remove(&route.channel);
1252    }
1253}
1254
1255fn ingress_route_should_be_processed(
1256    installed_epochs: &HashMap<u16, u32>,
1257    reclaimed_routes: &ReclaimedRoutes,
1258    frame: &Frame,
1259) -> bool {
1260    if frame.header.channel == 0
1261        || installed_epochs.get(&frame.header.channel).copied() == Some(frame.header.epoch)
1262    {
1263        return true;
1264    }
1265
1266    // A late request for a reclaimed root reaches the normal unknown-route
1267    // handler, which returns the typed `route_not_bound` error. Other stale or
1268    // never-installed generations remain silent so they cannot affect a newer
1269    // route or change the protocol's rejected-bind behavior.
1270    frame.header.ty == FrameType::Request
1271        && reclaimed_routes.contains(route_key(frame.header.channel, frame.header.epoch))
1272}
1273
1274fn bash_elicitation_timeout() -> Duration {
1275    if cfg!(debug_assertions) {
1276        if let Ok(raw) = std::env::var("AFT_TEST_SUBC_BASH_ELICITATION_TTL_MS") {
1277            if let Ok(ms) = raw.parse::<u64>() {
1278                if ms > 0 {
1279                    return Duration::from_millis(ms);
1280                }
1281            }
1282        }
1283    }
1284    BASH_ELICITATION_TIMEOUT
1285}
1286
1287fn allocate_reverse_corr(
1288    pending_bash_asks: &HashMap<ReverseCorrKey, PendingBashAsk>,
1289    route: RouteChannel,
1290    next_corr: &mut u64,
1291) -> u64 {
1292    loop {
1293        let corr = *next_corr;
1294        *next_corr = (*next_corr).wrapping_add(1).max(1);
1295        if !pending_bash_asks.contains_key(&ReverseCorrKey { route, corr }) {
1296            return corr;
1297        }
1298    }
1299}
1300
1301fn bash_permission_kind_label(kind: &crate::bash_permissions::PermissionKind) -> &'static str {
1302    match kind {
1303        crate::bash_permissions::PermissionKind::ExternalDirectory => "external directory",
1304        crate::bash_permissions::PermissionKind::Bash => "bash",
1305    }
1306}
1307
1308fn bash_elicitation_patterns(asks: &[crate::bash_permissions::PermissionAsk]) -> Vec<String> {
1309    let mut patterns = Vec::new();
1310    let mut seen = HashSet::new();
1311    for ask in asks {
1312        for pattern in ask.patterns.iter().chain(ask.always.iter()) {
1313            if seen.insert(pattern.clone()) {
1314                patterns.push(pattern.clone());
1315            }
1316        }
1317    }
1318    patterns
1319}
1320
1321fn bash_elicitation_message(
1322    command: &str,
1323    asks: &[crate::bash_permissions::PermissionAsk],
1324) -> String {
1325    let command = command.split_whitespace().collect::<Vec<_>>().join(" ");
1326    let patterns = bash_elicitation_patterns(asks);
1327    let pattern_text = if patterns.is_empty() {
1328        "no matched permission patterns".to_string()
1329    } else {
1330        patterns.join(", ")
1331    };
1332    let ask_kinds = asks
1333        .iter()
1334        .map(|ask| bash_permission_kind_label(&ask.kind))
1335        .collect::<HashSet<_>>()
1336        .into_iter()
1337        .collect::<Vec<_>>()
1338        .join(", ");
1339    if ask_kinds.is_empty() {
1340        format!("Allow bash command `{command}`? Matched patterns: {pattern_text}")
1341    } else {
1342        format!("Allow bash command `{command}`? Matched {ask_kinds} patterns: {pattern_text}")
1343    }
1344}
1345
1346fn bash_elicitation_request_body(
1347    command: &str,
1348    asks: &[crate::bash_permissions::PermissionAsk],
1349) -> Value {
1350    json!({
1351        "method": BASH_ELICITATION_CREATE_METHOD,
1352        "params": {
1353            "mode": "form",
1354            "message": bash_elicitation_message(command, asks),
1355            "requestedSchema": {
1356                "type": "object",
1357                "properties": {
1358                    "decision": {
1359                        "type": "string",
1360                        "enum": ["allow", "deny"],
1361                        "description": "Choose allow to run this bash command once, or deny to block it."
1362                    }
1363                },
1364                "required": ["decision"],
1365                "additionalProperties": false
1366            },
1367            "_meta": {
1368                "aft": {
1369                    "tool": "bash",
1370                    "command": command,
1371                    "asks": asks
1372                }
1373            }
1374        }
1375    })
1376}
1377
1378fn build_bash_elicitation_request_frame(
1379    ver: u8,
1380    route: RouteChannel,
1381    corr: u64,
1382    flags: Flags,
1383    command: &str,
1384    asks: &[crate::bash_permissions::PermissionAsk],
1385) -> Result<Frame, SubcError> {
1386    let body = bash_elicitation_request_body(command, asks);
1387    Frame::build_with_version(
1388        ver,
1389        FrameType::Request,
1390        flags,
1391        route.channel,
1392        route.epoch,
1393        corr,
1394        serde_json::to_vec(&body).map_err(SubcError::Json)?,
1395    )
1396    .map_err(SubcError::FrameBuild)
1397}
1398
1399fn bash_elicitation_reply_is_allow(body: &[u8]) -> bool {
1400    let Ok(value) = serde_json::from_slice::<Value>(body) else {
1401        return false;
1402    };
1403    flat_bash_elicitation_reply_is_allow(&value) || mcp_bash_elicitation_reply_is_allow(&value)
1404}
1405
1406fn flat_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1407    let Some(object) = value.as_object() else {
1408        return false;
1409    };
1410    object.len() == 1 && object.get("decision").and_then(Value::as_str) == Some("allow")
1411}
1412
1413fn mcp_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1414    let Some(object) = value.as_object() else {
1415        return false;
1416    };
1417    if object.len() != 2 || object.get("action").and_then(Value::as_str) != Some("accept") {
1418        return false;
1419    }
1420    let Some(content) = object.get("content").and_then(Value::as_object) else {
1421        return false;
1422    };
1423    content.len() == 1 && content.get("decision").and_then(Value::as_str) == Some("allow")
1424}
1425
1426#[allow(clippy::too_many_arguments)]
1427async fn settle_pending_bash_ask_denied(
1428    tx: &WriterSender,
1429    pending: PendingBashAsk,
1430    routes: &HashMap<RouteChannel, RouteIdentity>,
1431    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1432    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1433    shutdown: &Arc<Notify>,
1434    metrics: &DispatchPathMetrics,
1435) -> Result<(), SubcError> {
1436    let completion = bash::bash_denied_untrusted_completion(
1437        pending.route,
1438        pending.tool_corr,
1439        pending.tool_flags,
1440        pending.tool_ver,
1441        pending.root,
1442        pending.request_id,
1443        pending.format_context,
1444    );
1445    bash::handle_bash_deferred_completion(
1446        tx,
1447        completion,
1448        routes,
1449        live_roots,
1450        route_bash_cancels,
1451        shutdown,
1452        metrics,
1453    )
1454    .await
1455}
1456
1457fn take_pending_bash_asks_for_route(
1458    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1459    route: RouteChannel,
1460) -> Vec<PendingBashAsk> {
1461    let keys = pending_bash_asks
1462        .keys()
1463        .copied()
1464        .filter(|key| key.route == route)
1465        .collect::<Vec<_>>();
1466    keys.into_iter()
1467        .filter_map(|key| pending_bash_asks.remove(&key))
1468        .collect()
1469}
1470
1471#[allow(clippy::too_many_arguments)]
1472async fn settle_pending_bash_asks_for_route(
1473    tx: &WriterSender,
1474    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1475    route: RouteChannel,
1476    routes: &HashMap<RouteChannel, RouteIdentity>,
1477    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1478    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1479    shutdown: &Arc<Notify>,
1480    metrics: &DispatchPathMetrics,
1481) -> Result<(), SubcError> {
1482    for pending in take_pending_bash_asks_for_route(pending_bash_asks, route) {
1483        settle_pending_bash_ask_denied(
1484            tx,
1485            pending,
1486            routes,
1487            live_roots,
1488            route_bash_cancels,
1489            shutdown,
1490            metrics,
1491        )
1492        .await?;
1493    }
1494    Ok(())
1495}
1496
1497#[allow(clippy::too_many_arguments)]
1498async fn settle_all_pending_bash_asks(
1499    tx: &WriterSender,
1500    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1501    routes: &HashMap<RouteChannel, RouteIdentity>,
1502    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1503    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1504    shutdown: &Arc<Notify>,
1505    metrics: &DispatchPathMetrics,
1506) -> Result<(), SubcError> {
1507    let pending = pending_bash_asks
1508        .drain()
1509        .map(|(_, pending)| pending)
1510        .collect::<Vec<_>>();
1511    for pending in pending {
1512        settle_pending_bash_ask_denied(
1513            tx,
1514            pending,
1515            routes,
1516            live_roots,
1517            route_bash_cancels,
1518            shutdown,
1519            metrics,
1520        )
1521        .await?;
1522    }
1523    Ok(())
1524}
1525
1526#[allow(clippy::too_many_arguments)]
1527async fn expire_pending_bash_asks(
1528    tx: &WriterSender,
1529    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1530    routes: &HashMap<RouteChannel, RouteIdentity>,
1531    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1532    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1533    shutdown: &Arc<Notify>,
1534    metrics: &DispatchPathMetrics,
1535) -> Result<(), SubcError> {
1536    let now = Instant::now();
1537    let expired = pending_bash_asks
1538        .iter()
1539        .filter_map(|(key, pending)| (pending.expires_at <= now).then_some(*key))
1540        .collect::<Vec<_>>();
1541    for key in expired {
1542        if let Some(pending) = pending_bash_asks.remove(&key) {
1543            log::debug!(
1544                "subc attach: bash elicitation request {} on route {} expired fail-closed",
1545                key.corr,
1546                pending.route
1547            );
1548            settle_pending_bash_ask_denied(
1549                tx,
1550                pending,
1551                routes,
1552                live_roots,
1553                route_bash_cancels,
1554                shutdown,
1555                metrics,
1556            )
1557            .await?;
1558        }
1559    }
1560    Ok(())
1561}
1562
1563#[allow(clippy::too_many_arguments)]
1564async fn handle_bash_elicitation_reply(
1565    tx: &WriterSender,
1566    frame: &Frame,
1567    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1568    routes: &HashMap<RouteChannel, RouteIdentity>,
1569    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1570    executor: &Arc<Executor>,
1571    shutdown: &Arc<Notify>,
1572    bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
1573    bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
1574    metrics: &Arc<DispatchPathMetrics>,
1575    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1576    dispatch: DispatchFn,
1577) -> Result<(), SubcError> {
1578    let key = ReverseCorrKey {
1579        route: route_key(frame.header.channel, frame.header.epoch),
1580        corr: frame.header.corr,
1581    };
1582    let Some(pending) = pending_bash_asks.remove(&key) else {
1583        return Ok(());
1584    };
1585
1586    if frame.header.ty == FrameType::Response && bash_elicitation_reply_is_allow(&frame.body) {
1587        if routes.contains_key(&key.route) {
1588            bash::submit_deferred_bash(
1589                executor,
1590                bash_deferred_tx,
1591                bash_poll_touch_tx,
1592                metrics,
1593                dispatch,
1594                pending.root,
1595                pending.project_root,
1596                pending.session_id,
1597                pending.request_id,
1598                pending.route,
1599                pending.tool_corr,
1600                pending.tool_flags,
1601                pending.tool_ver,
1602                pending.arguments,
1603                pending.format_context,
1604                pending.cancel,
1605                BindTrust::Untrusted,
1606                pending.spawn_principal,
1607                pending.edit_slot_survives,
1608                Some(pending.grants),
1609            );
1610            return Ok(());
1611        }
1612        log::debug!(
1613            "subc attach: dropping allowed bash elicitation reply {} for unbound route {}",
1614            key.corr,
1615            pending.route
1616        );
1617    }
1618
1619    settle_pending_bash_ask_denied(
1620        tx,
1621        pending,
1622        routes,
1623        live_roots,
1624        route_bash_cancels,
1625        shutdown,
1626        metrics,
1627    )
1628    .await
1629}
1630
1631#[allow(clippy::too_many_arguments)]
1632async fn cancel_pending_bash_ask_for_tool_call(
1633    tx: &WriterSender,
1634    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1635    route: RouteChannel,
1636    tool_corr: u64,
1637    routes: &HashMap<RouteChannel, RouteIdentity>,
1638    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1639    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1640    shutdown: &Arc<Notify>,
1641    metrics: &DispatchPathMetrics,
1642) -> Result<(), SubcError> {
1643    let keys = pending_bash_asks
1644        .iter()
1645        .filter_map(|(key, pending)| {
1646            (key.route == route && pending.tool_corr == tool_corr).then_some(*key)
1647        })
1648        .collect::<Vec<_>>();
1649    for key in keys {
1650        if let Some(pending) = pending_bash_asks.remove(&key) {
1651            settle_pending_bash_ask_denied(
1652                tx,
1653                pending,
1654                routes,
1655                live_roots,
1656                route_bash_cancels,
1657                shutdown,
1658                metrics,
1659            )
1660            .await?;
1661        }
1662    }
1663    Ok(())
1664}
1665
1666fn remove_root_channel(
1667    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1668    root: &ProjectRootId,
1669    channel: RouteChannel,
1670) {
1671    let remove_root = if let Some(channels) = root_channels.get_mut(root) {
1672        channels.remove(&channel);
1673        channels.is_empty()
1674    } else {
1675        false
1676    };
1677    if remove_root {
1678        root_channels.remove(root);
1679    }
1680}
1681
1682fn remove_route_channel(
1683    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1684    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1685    channel: RouteChannel,
1686) -> Option<RouteIdentity> {
1687    let removed = routes.remove(&channel);
1688    if let Some(identity) = &removed {
1689        remove_root_channel(root_channels, &identity.root, channel);
1690    }
1691    removed
1692}
1693
1694fn insert_route_channel(
1695    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1696    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1697    channel: RouteChannel,
1698    identity: RouteIdentity,
1699) {
1700    if let Some(previous) = routes.insert(channel, identity.clone()) {
1701        remove_root_channel(root_channels, &previous.root, channel);
1702    }
1703    root_channels
1704        .entry(identity.root.clone())
1705        .or_default()
1706        .insert(channel);
1707}
1708
1709fn insert_bg_subscription_index(
1710    bg_sub_by_session: &mut BgSubsBySession,
1711    root: ProjectRootId,
1712    session: String,
1713    channel: RouteChannel,
1714) {
1715    bg_sub_by_session
1716        .entry((root, session))
1717        .or_default()
1718        .insert(channel);
1719}
1720
1721fn remove_bg_subscription_index(
1722    bg_sub_by_session: &mut BgSubsBySession,
1723    channel: RouteChannel,
1724    identity: Option<&RouteIdentity>,
1725) {
1726    if let Some(identity) = identity {
1727        let key = (identity.root.clone(), identity.session.clone());
1728        let remove_key = bg_sub_by_session.get_mut(&key).is_some_and(|channels| {
1729            channels.remove(&channel);
1730            channels.is_empty()
1731        });
1732        if remove_key {
1733            bg_sub_by_session.remove(&key);
1734        }
1735    } else {
1736        bg_sub_by_session.retain(|_, channels| {
1737            channels.remove(&channel);
1738            !channels.is_empty()
1739        });
1740    }
1741}
1742
1743fn route_removal_will_quiesce_root(
1744    root: &ProjectRootId,
1745    route: RouteChannel,
1746    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1747    has_pending_bind: bool,
1748    replacement_root: Option<&ProjectRootId>,
1749) -> bool {
1750    let removes_last_route = root_channels
1751        .get(root)
1752        .is_some_and(|channels| channels.len() == 1 && channels.contains(&route));
1753    removes_last_route && !has_pending_bind && replacement_root != Some(root)
1754}
1755
1756fn should_quiesce_removed_root(
1757    root: &ProjectRootId,
1758    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1759    has_pending_bind: bool,
1760    replacement_root: Option<&ProjectRootId>,
1761) -> bool {
1762    !root_channels.contains_key(root) && !has_pending_bind && replacement_root != Some(root)
1763}
1764
1765async fn end_bg_subscription(
1766    writer_tx: &WriterSender,
1767    metrics: &DispatchPathMetrics,
1768    bg_subs: &mut HashMap<RouteChannel, BgSub>,
1769    bg_sub_by_session: &mut BgSubsBySession,
1770    bg_wake_pending: &mut HashSet<RouteChannel>,
1771    channel: RouteChannel,
1772    identity: Option<&RouteIdentity>,
1773    cause: &str,
1774) -> Result<(), SubcError> {
1775    if let Some(sub) = bg_subs.remove(&channel) {
1776        bg_wake_pending.remove(&channel);
1777        remove_bg_subscription_index(bg_sub_by_session, channel, identity);
1778        metrics.record_bg_subscription_ended(&sub.root, &sub.session, channel, cause);
1779        push::send_reliable_bg_stream_end(writer_tx, metrics, channel, &sub).await?;
1780    }
1781    Ok(())
1782}
1783
1784#[allow(clippy::too_many_arguments)]
1785async fn teardown_installed_route(
1786    tx: &WriterSender,
1787    metrics: &DispatchPathMetrics,
1788    executor: &Arc<Executor>,
1789    channel: RouteChannel,
1790    cancellation_reason: &str,
1791    replacement_root: Option<&ProjectRootId>,
1792    installed_route_epochs: &mut HashMap<u16, u32>,
1793    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1794    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1795    bg_subs: &mut HashMap<RouteChannel, BgSub>,
1796    bg_sub_by_session: &mut BgSubsBySession,
1797    bg_wake_pending: &mut HashSet<RouteChannel>,
1798    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1799    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1800    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1801    active_tool_calls: &ActiveToolCalls,
1802    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
1803    retry_buffer: &mut RetryBuffer,
1804    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1805    shutdown: &Arc<Notify>,
1806) -> Result<(), SubcError> {
1807    remove_installed_route(installed_route_epochs, channel);
1808    let bg_end_cause = match cancellation_reason {
1809        "Goodbye" => "goodbye",
1810        "higher-epoch RouteBind" => "higher-epoch",
1811        other => other,
1812    };
1813    end_bg_subscription(
1814        tx,
1815        metrics,
1816        bg_subs,
1817        bg_sub_by_session,
1818        bg_wake_pending,
1819        channel,
1820        routes.get(&channel),
1821        bg_end_cause,
1822    )
1823    .await?;
1824    settle_pending_bash_asks_for_route(
1825        tx,
1826        pending_bash_asks,
1827        channel,
1828        routes,
1829        live_roots,
1830        route_bash_cancels,
1831        shutdown,
1832        metrics,
1833    )
1834    .await?;
1835    if let Some(cancel) = route_bash_cancels.remove(&channel) {
1836        cancel.token.cancel();
1837    }
1838    cancel_active_tool_calls_for_route(active_tool_calls, executor, channel, cancellation_reason);
1839    if let Some(pending) = pending_binds.get_mut(&channel) {
1840        pending.cancelled = true;
1841        let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
1842        log::debug!(
1843            "subc attach: cancelled pending RouteBind for route {} on {cancellation_reason} (configure job: {outcome:?})",
1844            channel.channel
1845        );
1846    }
1847    let migrated = push::migrate_retry_buffer_to_push_buffer(retry_buffer, channel, push_buffer);
1848    if let Some(identity) = routes.get(&channel) {
1849        let has_pending_bind = pending_binds
1850            .values()
1851            .any(|pending| pending.bind_root_id == identity.root);
1852        if route_removal_will_quiesce_root(
1853            &identity.root,
1854            channel,
1855            root_channels,
1856            has_pending_bind,
1857            replacement_root,
1858        ) {
1859            if let Some(ctx) = executor.actor_context(&identity.root) {
1860                // Fence deferred admissions before the final route disappears
1861                // from the loop-owned routing tables.
1862                ctx.mark_subc_unbound();
1863            }
1864        }
1865    }
1866    if let Some(identity) = remove_route_channel(routes, root_channels, channel) {
1867        let session_still_routed = routes
1868            .values()
1869            .any(|route| route.root == identity.root && route.session == identity.session);
1870        if !session_still_routed {
1871            if let Some(ctx) = executor.actor_context(&identity.root) {
1872                ctx.hashline_bindings()
1873                    .teardown(identity.root.as_path(), &identity.session);
1874            }
1875        }
1876        if migrated > 0 {
1877            log::debug!(
1878                "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from route {} into detach replay",
1879                channel.channel
1880            );
1881        }
1882        if let Some(meta) = live_roots.get_mut(&identity.root) {
1883            let idle_for = meta.last_touched.elapsed();
1884            meta.note_activity();
1885            log::debug!(
1886                "subc attach: route {} torn down for root {} harness {} session {} (last touched {:?} ago)",
1887                channel.channel,
1888                identity.root.as_path().display(),
1889                identity.harness,
1890                identity.session,
1891                idle_for
1892            );
1893        } else {
1894            log::debug!(
1895                "subc attach: route {} torn down for root {} harness {} session {}",
1896                channel.channel,
1897                identity.root.as_path().display(),
1898                identity.harness,
1899                identity.session
1900            );
1901        }
1902        let has_pending_bind = pending_binds
1903            .values()
1904            .any(|pending| pending.bind_root_id == identity.root);
1905        if should_quiesce_removed_root(
1906            &identity.root,
1907            root_channels,
1908            has_pending_bind,
1909            replacement_root,
1910        ) {
1911            quiesce_unbound_root(&identity.root, live_roots, executor);
1912        }
1913    } else {
1914        if migrated > 0 {
1915            log::debug!(
1916                "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from unbound route {} into detach replay",
1917                channel.channel
1918            );
1919        }
1920        log::debug!("subc attach: unbound route {} torn down", channel.channel);
1921    }
1922    Ok(())
1923}
1924
1925fn remember_session_identity(
1926    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1927    identity: &RouteIdentity,
1928) {
1929    let key = (identity.root.clone(), identity.session.clone());
1930    if matches!(identity.trust, BindTrust::Untrusted)
1931        && session_identity
1932            .get(&key)
1933            .is_some_and(|retained| matches!(retained.trust, BindTrust::FirstParty))
1934    {
1935        return;
1936    }
1937
1938    // Retained after route Goodbye so reliable session-scoped frames emitted while
1939    // the session is detached can still be keyed by the full (root,harness,session)
1940    // replay triple. Untrusted binds never overwrite a retained first-party
1941    // session identity, because bash completion replay is an observation channel.
1942    session_identity.insert(
1943        key,
1944        RetainedSessionIdentity {
1945            harness: identity.harness.clone(),
1946            trust: identity.trust,
1947        },
1948    );
1949}
1950
1951fn replay_key_for_session(
1952    session_identity: &HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1953    root: &ProjectRootId,
1954    session: &str,
1955) -> Option<(push::ReplayKey, BindTrust)> {
1956    let retained = session_identity.get(&(root.clone(), session.to_string()))?;
1957    Some((
1958        push::ReplayKey {
1959            root: root.clone(),
1960            harness: retained.harness.clone(),
1961            session: session.to_string(),
1962        },
1963        retained.trust,
1964    ))
1965}
1966/// Sync command dispatch, passed in from `main` (the binary owns the command
1967/// table). Invoked only inside executor jobs in subc mode.
1968pub type DispatchFn = fn(RawRequest, &AppContext) -> Response;
1969
1970#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1971enum ModuleLoopExit {
1972    Graceful,
1973    SkipSearchFlush,
1974}
1975
1976/// Entry point for `aft --subc <connection-file>`. Synchronous on the outside;
1977/// owns an isolated current-thread tokio runtime for the async transport.
1978/// Returns `Err` (fail-loud) on any connect/auth/protocol failure — we never
1979/// fall back to the standalone loop, to avoid split-brain index state.
1980pub fn run_subc_mode(
1981    connection_file_path: &Path,
1982    ctx: Arc<AppContext>,
1983    executor: Arc<Executor>,
1984    dispatch: DispatchFn,
1985    user_config_path: Option<PathBuf>,
1986) -> Result<(), SubcError> {
1987    // Production NEVER allows non-manifest tool names on route channels: AFT
1988    // fails closed and does not trust subc to enforce the manifest. The
1989    // test-only harness sets this through `run_subc_mode_for_test`.
1990    run_subc_mode_inner(
1991        connection_file_path,
1992        ctx,
1993        executor,
1994        dispatch,
1995        user_config_path,
1996        false,
1997        MAX_FRAME_BODY_LEN as usize,
1998    )
1999}
2000
2001fn run_subc_mode_inner(
2002    connection_file_path: &Path,
2003    ctx: Arc<AppContext>,
2004    executor: Arc<Executor>,
2005    dispatch: DispatchFn,
2006    user_config_path: Option<PathBuf>,
2007    allow_native_passthrough: bool,
2008    tool_response_body_limit: usize,
2009) -> Result<(), SubcError> {
2010    let runtime = tokio::runtime::Builder::new_current_thread()
2011        .enable_all()
2012        .build()
2013        .map_err(SubcError::Runtime)?;
2014
2015    let executor_for_loop = Arc::clone(&executor);
2016    let loop_result = runtime.block_on(async move {
2017        let shared_app = ctx.app();
2018        drop(ctx);
2019        let stream = connect_and_authenticate(connection_file_path).await?;
2020        log::info!(
2021            "subc attach: authenticated to daemon via {}",
2022            connection_file_path.display()
2023        );
2024        let (read_half, write_half) = tokio::io::split(stream);
2025        run_module_loop(
2026            read_half,
2027            write_half,
2028            connection_file_path,
2029            shared_app,
2030            executor_for_loop,
2031            dispatch,
2032            user_config_path,
2033            allow_native_passthrough,
2034            tool_response_body_limit,
2035        )
2036        .await
2037    });
2038
2039    let actor_contexts = executor.actor_contexts();
2040    if matches!(loop_result, Ok(ModuleLoopExit::Graceful)) {
2041        // EOF/Goodbye teardown flushes each root's index deltas and queued
2042        // callgraph refreshes. Fatal/panic teardown skips this best-effort work.
2043        flush_actor_indexes_on_graceful_shutdown(&actor_contexts);
2044    }
2045    for actor_ctx in &actor_contexts {
2046        actor_ctx.lsp().shutdown_all();
2047        actor_ctx.bash_background().detach();
2048    }
2049
2050    loop_result.map(|_| ())
2051}
2052
2053fn flush_actor_indexes_on_graceful_shutdown(actor_contexts: &[Arc<AppContext>]) {
2054    for actor_ctx in actor_contexts {
2055        let _ = actor_ctx.flush_search_index_on_graceful_shutdown();
2056    }
2057    let _ = crate::callgraph_store::flush_callgraph_store_refreshes_on_graceful_shutdown();
2058}
2059
2060/// Test-only entry that enables the non-manifest native-command passthrough on
2061/// route channels. Integration tests drive synthetic native commands (`glob`,
2062/// `callers`, `subc_test_echo_session`, …) through the executor to exercise
2063/// mechanics; production callers use [`run_subc_mode`], which fails closed.
2064#[doc(hidden)]
2065pub fn run_subc_mode_for_test(
2066    connection_file_path: &Path,
2067    ctx: Arc<AppContext>,
2068    executor: Arc<Executor>,
2069    dispatch: DispatchFn,
2070    user_config_path: Option<PathBuf>,
2071) -> Result<(), SubcError> {
2072    run_subc_mode_inner(
2073        connection_file_path,
2074        ctx,
2075        executor,
2076        dispatch,
2077        user_config_path,
2078        true,
2079        MAX_FRAME_BODY_LEN as usize,
2080    )
2081}
2082
2083/// Test-only entry that lowers the effective tool-response body limit without
2084/// allocating a 64 MiB fixture. The fixed fallback envelope needs 4 KiB of room.
2085#[doc(hidden)]
2086pub fn run_subc_mode_for_test_with_response_body_limit(
2087    connection_file_path: &Path,
2088    ctx: Arc<AppContext>,
2089    executor: Arc<Executor>,
2090    dispatch: DispatchFn,
2091    user_config_path: Option<PathBuf>,
2092    tool_response_body_limit: usize,
2093) -> Result<(), SubcError> {
2094    assert!((4 * 1_024..=MAX_FRAME_BODY_LEN as usize).contains(&tool_response_body_limit));
2095    run_subc_mode_inner(
2096        connection_file_path,
2097        ctx,
2098        executor,
2099        dispatch,
2100        user_config_path,
2101        true,
2102        tool_response_body_limit,
2103    )
2104}
2105
2106#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2107enum AttachErrorClass {
2108    Transient,
2109    Permanent,
2110}
2111
2112impl fmt::Display for AttachErrorClass {
2113    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2114        match self {
2115            Self::Transient => f.write_str("transient"),
2116            Self::Permanent => f.write_str("permanent"),
2117        }
2118    }
2119}
2120
2121#[derive(Clone, Copy)]
2122struct AttachRetryPolicy {
2123    budget: Duration,
2124    initial_backoff: Duration,
2125    max_backoff: Duration,
2126    jitter_percent: u64,
2127}
2128
2129const ATTACH_RETRY_POLICY: AttachRetryPolicy = AttachRetryPolicy {
2130    budget: ATTACH_RETRY_BUDGET,
2131    initial_backoff: ATTACH_RETRY_INITIAL_BACKOFF,
2132    max_backoff: ATTACH_RETRY_MAX_BACKOFF,
2133    jitter_percent: ATTACH_RETRY_JITTER_PERCENT,
2134};
2135
2136/// Retry only failures that can be caused by a daemon bounce or an interrupted
2137/// handshake. Protocol and credential failures are permanent for this process.
2138fn classify_attach_error(error: &SubcError) -> AttachErrorClass {
2139    let transient = match error {
2140        SubcError::Connect { source, .. } => is_transient_attach_io(source.kind()),
2141        SubcError::Auth { source, .. } => match source {
2142            subc_transport::AuthError::Timeout { .. }
2143            | subc_transport::AuthError::UnexpectedEof { .. } => true,
2144            subc_transport::AuthError::Io { source, .. } => is_transient_attach_io(source.kind()),
2145            _ => false,
2146        },
2147        _ => false,
2148    };
2149    if transient {
2150        AttachErrorClass::Transient
2151    } else {
2152        AttachErrorClass::Permanent
2153    }
2154}
2155
2156fn is_transient_attach_io(kind: io::ErrorKind) -> bool {
2157    matches!(
2158        kind,
2159        io::ErrorKind::ConnectionRefused
2160            | io::ErrorKind::TimedOut
2161            | io::ErrorKind::ConnectionReset
2162            | io::ErrorKind::ConnectionAborted
2163            | io::ErrorKind::BrokenPipe
2164            | io::ErrorKind::UnexpectedEof
2165    )
2166}
2167
2168/// Read the connection file → resolve the first endpoint → TCP connect → HMAC
2169/// handshake. Transient initial-attach failures retry on fresh sockets and reread
2170/// the file so a daemon bounce can publish a new endpoint or authentication key.
2171async fn connect_and_authenticate(connection_file_path: &Path) -> Result<TcpStream, SubcError> {
2172    connect_and_authenticate_with_policy(connection_file_path, ATTACH_RETRY_POLICY).await
2173}
2174
2175async fn connect_and_authenticate_with_policy(
2176    connection_file_path: &Path,
2177    policy: AttachRetryPolicy,
2178) -> Result<TcpStream, SubcError> {
2179    let started_at = Instant::now();
2180    let deadline = started_at + policy.budget;
2181    let mut attempt = 0_u32;
2182    let mut backoff = policy.initial_backoff;
2183    let mut history = Vec::new();
2184
2185    loop {
2186        attempt = attempt.saturating_add(1);
2187        let error = match connect_and_authenticate_once(connection_file_path, deadline).await {
2188            Ok(stream) => return Ok(stream),
2189            Err(error) => error,
2190        };
2191        let class = classify_attach_error(&error);
2192        let error_text = error.to_string().lines().collect::<Vec<_>>().join(" ");
2193        history.push(format!("attempt {attempt} [{class}]: {error_text}"));
2194
2195        if class == AttachErrorClass::Permanent {
2196            log_attach_final_failure(started_at.elapsed(), &history);
2197            return Err(error);
2198        }
2199
2200        let remaining = deadline.saturating_duration_since(Instant::now());
2201        if remaining.is_zero() {
2202            log_attach_final_failure(started_at.elapsed(), &history);
2203            return Err(error);
2204        }
2205
2206        let delay = jittered_attach_delay(backoff, policy.jitter_percent, attempt).min(remaining);
2207        log::info!(
2208            "subc attach retry: attempt {attempt} failed; error_class={class}; error={error_text}; next_delay={delay:?}"
2209        );
2210        tokio::time::sleep(delay).await;
2211
2212        if Instant::now() >= deadline {
2213            log_attach_final_failure(started_at.elapsed(), &history);
2214            return Err(error);
2215        }
2216        backoff = backoff.saturating_mul(2).min(policy.max_backoff);
2217    }
2218}
2219
2220fn jittered_attach_delay(base: Duration, jitter_percent: u64, attempt: u32) -> Duration {
2221    let jitter_percent = jitter_percent.min(100);
2222    if jitter_percent == 0 {
2223        return base;
2224    }
2225
2226    let mut random_bytes = [0_u8; 8];
2227    let random = if getrandom::fill(&mut random_bytes).is_ok() {
2228        u64::from_le_bytes(random_bytes)
2229    } else {
2230        let timestamp = std::time::SystemTime::now()
2231            .duration_since(std::time::UNIX_EPOCH)
2232            .unwrap_or_default()
2233            .subsec_nanos();
2234        u64::from(timestamp) ^ u64::from(attempt)
2235    };
2236    let span = jitter_percent.saturating_mul(2).saturating_add(1);
2237    let multiplier_percent = 100 - jitter_percent + random % span;
2238    let base_millis = u64::try_from(base.as_millis()).unwrap_or(u64::MAX);
2239    Duration::from_millis(base_millis.saturating_mul(multiplier_percent) / 100)
2240}
2241
2242fn log_attach_final_failure(elapsed: Duration, history: &[String]) {
2243    log::error!(
2244        "subc initial attach failed after {} attempt(s) in {elapsed:?}; attempt history: {}",
2245        history.len(),
2246        history.join(" | ")
2247    );
2248}
2249
2250async fn connect_and_authenticate_once(
2251    connection_file_path: &Path,
2252    deadline: Instant,
2253) -> Result<TcpStream, SubcError> {
2254    // This read intentionally lives inside the per-attempt function. The daemon
2255    // publishes connection files atomically and may change both port and key.
2256    let conn = connection_file::read_for_client(connection_file_path).map_err(|source| {
2257        SubcError::ConnectionFile {
2258            path: connection_file_path.to_path_buf(),
2259            source,
2260        }
2261    })?;
2262
2263    let endpoint = conn
2264        .endpoints
2265        .first()
2266        .ok_or_else(|| SubcError::NoEndpoint {
2267            path: connection_file_path.to_path_buf(),
2268        })?;
2269    let endpoint_label = format!("{}:{}", endpoint.host, endpoint.port);
2270    let ip = endpoint
2271        .host
2272        .parse::<IpAddr>()
2273        .map_err(|_| SubcError::InvalidEndpoint {
2274            path: connection_file_path.to_path_buf(),
2275            endpoint: endpoint_label.clone(),
2276        })?;
2277    let addr = SocketAddr::new(ip, endpoint.port);
2278
2279    let connect_budget = deadline.saturating_duration_since(Instant::now());
2280    let mut stream = tokio::time::timeout(connect_budget, TcpStream::connect(addr))
2281        .await
2282        .map_err(|_| SubcError::Connect {
2283            endpoint: endpoint_label.clone(),
2284            source: io::Error::new(
2285                io::ErrorKind::TimedOut,
2286                "initial subc attach retry budget elapsed during TCP connect",
2287            ),
2288        })?
2289        .map_err(|source| SubcError::Connect {
2290            endpoint: endpoint_label.clone(),
2291            source,
2292        })?;
2293    stream
2294        .set_nodelay(true)
2295        .map_err(|source| SubcError::Connect {
2296            endpoint: endpoint_label.clone(),
2297            source,
2298        })?;
2299
2300    let auth_budget = AUTH_DEADLINE.min(deadline.saturating_duration_since(Instant::now()));
2301    authenticate_client(&mut stream, &conn, auth_budget)
2302        .await
2303        .map_err(|source| SubcError::Auth {
2304            endpoint: endpoint_label,
2305            source,
2306        })?;
2307
2308    Ok(stream)
2309}
2310
2311#[allow(clippy::too_many_arguments)]
2312async fn process_route_bind_completion(
2313    writer_tx: &WriterSender,
2314    completion: RouteBindCompletion,
2315    routes: &mut HashMap<RouteChannel, RouteIdentity>,
2316    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2317    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2318    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2319    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2320    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2321    installed_route_epochs: &mut HashMap<u16, u32>,
2322    executor: &Arc<Executor>,
2323    shutdown: &Arc<Notify>,
2324    metrics: &Arc<DispatchPathMetrics>,
2325) -> Result<(), SubcError> {
2326    decrement_counted_channel(&metrics.control_completion_queued);
2327    handle_route_bind_completion(
2328        writer_tx,
2329        completion,
2330        routes,
2331        root_channels,
2332        session_identity,
2333        push_buffer,
2334        live_roots,
2335        pending_binds,
2336        installed_route_epochs,
2337        executor,
2338        shutdown,
2339        metrics,
2340    )
2341    .await
2342}
2343
2344#[allow(clippy::too_many_arguments)]
2345async fn drain_pending_route_bind_completions(
2346    control_completion_rx: &mut mpsc::Receiver<RouteBindCompletion>,
2347    writer_tx: &WriterSender,
2348    routes: &mut HashMap<RouteChannel, RouteIdentity>,
2349    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2350    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2351    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2352    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2353    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2354    installed_route_epochs: &mut HashMap<u16, u32>,
2355    executor: &Arc<Executor>,
2356    shutdown: &Arc<Notify>,
2357    metrics: &Arc<DispatchPathMetrics>,
2358) -> Result<usize, SubcError> {
2359    let mut drained = 0;
2360    while let Ok(completion) = control_completion_rx.try_recv() {
2361        process_route_bind_completion(
2362            writer_tx,
2363            completion,
2364            routes,
2365            root_channels,
2366            session_identity,
2367            push_buffer,
2368            live_roots,
2369            pending_binds,
2370            installed_route_epochs,
2371            executor,
2372            shutdown,
2373            metrics,
2374        )
2375        .await?;
2376        drained += 1;
2377    }
2378    Ok(drained)
2379}
2380
2381/// ModuleHello → HelloAck → control/route loop. Runs until the daemon closes
2382/// the connection (EOF), sends channel-0 Goodbye, or a fatal mutating executor
2383/// response requests whole-connection teardown.
2384async fn run_module_loop<R, W>(
2385    mut read: R,
2386    mut write: W,
2387    connection_file_path: &Path,
2388    shared_app: Arc<App>,
2389    executor: Arc<Executor>,
2390    dispatch: DispatchFn,
2391    user_config_path: Option<PathBuf>,
2392    allow_native_passthrough: bool,
2393    tool_response_body_limit: usize,
2394) -> Result<ModuleLoopExit, SubcError>
2395where
2396    R: AsyncRead + Unpin + Send + 'static,
2397    W: AsyncWrite + Unpin + Send + 'static,
2398{
2399    // ModuleHello: register as a tool provider and advertise the supported control-plane operations.
2400    // Echo the one-time launch nonce the daemon injected via SUBC_LAUNCH_NONCE so a
2401    // reserved module_id's HELLO is accepted; absent for non-reserved/self-connect.
2402    let hello = ModuleHelloBody {
2403        manifest: build_manifest(),
2404        protocol_ver: PROTOCOL_VERSION,
2405        control_ops: control_ops(),
2406        launch_nonce: std::env::var("SUBC_LAUNCH_NONCE").ok(),
2407    };
2408    let hello_frame = Frame::build(
2409        FrameType::Hello,
2410        control_flags(),
2411        0,
2412        0,
2413        HELLO_CORR,
2414        serde_json::to_vec(&hello).map_err(SubcError::Json)?,
2415    )
2416    .map_err(SubcError::FrameBuild)?;
2417    write_frame(&mut write, &hello_frame)
2418        .await
2419        .map_err(SubcError::FrameIo)?;
2420
2421    // Expect HelloAck (registered) or a channel-0 Error (manifest/version reject).
2422    match read_frame(&mut read).await.map_err(SubcError::FrameIo)? {
2423        None => return Err(SubcError::ClosedBeforeHelloAck),
2424        Some(frame) => match frame.header.ty {
2425            FrameType::HelloAck => {
2426                log::info!("subc attach: registered (HelloAck received)");
2427            }
2428            FrameType::Error => {
2429                let body = serde_json::from_slice::<ErrorBody>(&frame.body).ok();
2430                return Err(SubcError::HelloRejected { body });
2431            }
2432            other => return Err(SubcError::UnexpectedFrame { ty: other }),
2433        },
2434    }
2435
2436    let dispatch_path_metrics = Arc::new(DispatchPathMetrics::new());
2437    let (writer_tx, writer_rx) = mpsc::channel::<WriterFrame>(WRITER_QUEUE_CAPACITY);
2438    let writer_task = spawn_writer_task(write, writer_rx, Arc::clone(&dispatch_path_metrics));
2439    // `read_frame` is NOT cancellation-safe, so it must never sit directly inside
2440    // the `select!` below: a drain-interval tick (or shutdown) firing while a
2441    // frame is mid-transit would drop the partially-consumed bytes and desync the
2442    // stream (the next read would parse a body byte as a frame header). A
2443    // dedicated reader task owns the socket, reads whole frames sequentially, and
2444    // forwards them over a channel; the loop selects on the cancel-safe `recv()`.
2445    let (reader_tx, mut reader_rx) = mpsc::channel::<Result<DecodedFrame, SubcError>>(256);
2446    let reader_task = spawn_reader_task(read, reader_tx);
2447    let shutdown = Arc::new(Notify::new());
2448    // Drain-tick deadline is tracked manually and checked at the TOP of every
2449    // loop turn rather than as an Interval select arm: the select below is
2450    // `biased` (bind completions first), and biased polling means a saturated
2451    // higher arm (sustained lossy push traffic keeps lossy_rx always-ready)
2452    // would starve every arm below it, including a timer arm — leaving
2453    // backpressured reliable frames parked in the retry buffer past their
2454    // delivery deadline. The pre-turn check cannot be starved by arm order;
2455    // the sleep_until arm below only exists to wake an otherwise-idle loop.
2456    let mut next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2457    let mut next_maintenance_at = next_drain_at;
2458    // Rate-limit stamp for opportunistic allocator slack relief (checked on the
2459    // maintenance tick; policy shared with standalone via memory.rs).
2460    #[cfg(any(target_os = "macos", target_os = "linux"))]
2461    let mut last_slack_relief: Option<std::time::Instant> = None;
2462    let (maintenance_tx, mut maintenance_rx) = mpsc::channel::<MaintenanceCompletion>(256);
2463    let (bash_deferred_tx, mut bash_deferred_rx) =
2464        mpsc::channel::<bash::BashDeferredCompletion>(256);
2465    let (bash_poll_touch_tx, mut bash_poll_touch_rx) = mpsc::channel::<ProjectRootId>(256);
2466    let (control_completion_tx, mut control_completion_rx) =
2467        mpsc::channel::<RouteBindCompletion>(256);
2468    let (lossy_tx, mut lossy_rx) = mpsc::channel::<LossyPushEnvelope>(1024);
2469    let lossy_overflow = Arc::new(push::LossyOverflow::default());
2470    let lossy_seq = Arc::new(AtomicU64::new(0));
2471    let (reliable_tx, mut reliable_rx) = mpsc::unbounded_channel::<PushEnvelope>();
2472    let (fleet_status_client, fleet_status_task) =
2473        spawn_fleet_status_dial(connection_file_path, 64);
2474    let push_senders = PushSenders {
2475        lossy_tx,
2476        reliable_tx,
2477        lossy_overflow: Arc::clone(&lossy_overflow),
2478        lossy_seq,
2479        fleet_status_client: fleet_status_client.clone(),
2480    };
2481    let connection_cancel = PersistentCancelSignal::new();
2482    let mut installed_route_epochs: HashMap<u16, u32> = HashMap::new();
2483    let mut routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
2484    let mut bg_subs: HashMap<RouteChannel, BgSub> = HashMap::new();
2485    let mut bg_sub_by_session: BgSubsBySession = HashMap::new();
2486    let mut bg_wake_pending: HashSet<RouteChannel> = HashSet::new();
2487    let mut bg_wake_epoch: HashMap<(ProjectRootId, String), u64> = HashMap::new();
2488    let mut root_channels: HashMap<ProjectRootId, HashSet<RouteChannel>> = HashMap::new();
2489    let mut session_identity: HashMap<(ProjectRootId, String), RetainedSessionIdentity> =
2490        HashMap::new();
2491    let mut push_buffer: HashMap<push::ReplayKey, VecDeque<PushFrame>> = HashMap::new();
2492    let mut retry_buffer: RetryBuffer = HashMap::new();
2493    let mut reclaimed_routes = ReclaimedRoutes::default();
2494    let mut completed_tasks = push::CompletedTaskIds::default();
2495    let mut live_roots: HashMap<ProjectRootId, RootMeta> = HashMap::new();
2496    let mut pending_binds: HashMap<RouteChannel, PendingBind> = HashMap::new();
2497    let mut pending_bash_asks: HashMap<ReverseCorrKey, PendingBashAsk> = HashMap::new();
2498    let mut next_bash_ask_corr: u64 = 1;
2499    let mut route_bash_cancels: HashMap<RouteChannel, bash::RouteBashCancel> = HashMap::new();
2500    let active_tool_calls: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
2501    let health_rollup_cache = HealthRollupCache::new();
2502    health_rollup_cache.refresh(&executor, &shared_app);
2503    let mut next_health_rollup_at = tokio::time::Instant::now() + HEALTH_ROLLUP_TTL;
2504
2505    let loop_result: Result<ModuleLoopExit, SubcError> = loop {
2506        shared_app.set_open_route_count(routes.len());
2507        if tokio::time::Instant::now() >= next_health_rollup_at {
2508            health_rollup_cache.refresh(&executor, &shared_app);
2509            next_health_rollup_at = tokio::time::Instant::now() + HEALTH_ROLLUP_TTL;
2510        }
2511        crate::logging::perf_tick(Some(&executor));
2512        dispatch_path_metrics.mark_frame_loop_tick();
2513        if let Err(error) = expire_pending_bash_asks(
2514            &writer_tx,
2515            &mut pending_bash_asks,
2516            &routes,
2517            &mut live_roots,
2518            &mut route_bash_cancels,
2519            &shutdown,
2520            &dispatch_path_metrics,
2521        )
2522        .await
2523        {
2524            break Err(error);
2525        }
2526
2527        // RouteBind completions are control-plane unblockers. Drain any completed
2528        // binds before entering other branch work so Push and maintenance bursts
2529        // can only add one loop-turn of latency.
2530        match drain_pending_route_bind_completions(
2531            &mut control_completion_rx,
2532            &writer_tx,
2533            &mut routes,
2534            &mut root_channels,
2535            &mut session_identity,
2536            &mut push_buffer,
2537            &mut live_roots,
2538            &mut pending_binds,
2539            &mut installed_route_epochs,
2540            &executor,
2541            &shutdown,
2542            &dispatch_path_metrics,
2543        )
2544        .await
2545        {
2546            Ok(drained) => {
2547                if drained > 0 {
2548                    next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2549                    health_rollup_cache.refresh(&executor, &shared_app);
2550                    next_health_rollup_at = tokio::time::Instant::now() + HEALTH_ROLLUP_TTL;
2551                }
2552            }
2553            Err(error) => break Err(error),
2554        }
2555
2556        if tokio::time::Instant::now() >= next_drain_at {
2557            push::emit_bg_event_wakes(
2558                &writer_tx,
2559                &dispatch_path_metrics,
2560                &bg_subs,
2561                &mut bg_wake_pending,
2562            );
2563            warn_slow_pending_binds(&mut pending_binds, &executor);
2564            if let Err(error) = expire_overdue_route_binds(
2565                &writer_tx,
2566                &executor,
2567                &mut pending_binds,
2568                &mut installed_route_epochs,
2569                &dispatch_path_metrics,
2570            )
2571            .await
2572            {
2573                break Err(error);
2574            }
2575
2576            let retried = push::drain_retry_buffers_for_bound_routes(
2577                &writer_tx,
2578                &dispatch_path_metrics,
2579                &routes,
2580                &mut retry_buffer,
2581            );
2582            if retried > 0 {
2583                log::debug!(
2584                    "subc attach: retried {retried} reliable Push frame(s) after writer backpressure"
2585                );
2586            }
2587
2588            next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2589        }
2590
2591        // A lossy emitter may place its newest update in the overflow buffer
2592        // when the bounded channel is full, while this receive loop is draining
2593        // the channel. Drain overflow before selecting again so that raced
2594        // update is delivered on the next timer tick instead of waiting for
2595        // another lossy enqueue.
2596        let overflow_batch = lossy_overflow.drain();
2597        if !overflow_batch.is_empty() {
2598            let (_, deferred) = push::drain_reliable_push_turn(
2599                &writer_tx,
2600                &dispatch_path_metrics,
2601                &routes,
2602                &root_channels,
2603                &session_identity,
2604                &mut retry_buffer,
2605                &mut push_buffer,
2606                &mut completed_tasks,
2607                &bg_sub_by_session,
2608                &mut bg_wake_pending,
2609                &mut bg_wake_epoch,
2610                &mut reliable_rx,
2611                None,
2612            );
2613            if deferred {
2614                tokio::task::yield_now().await;
2615            }
2616
2617            let mut batch = Vec::new();
2618            while let Ok(item) = lossy_rx.try_recv() {
2619                batch.push(item);
2620            }
2621            batch.extend(overflow_batch);
2622            push::process_lossy_push_envelope_batch(
2623                &writer_tx,
2624                &dispatch_path_metrics,
2625                &routes,
2626                &root_channels,
2627                &completed_tasks,
2628                batch,
2629            );
2630        }
2631
2632        tokio::select! {
2633            biased;
2634            Some(completion) = control_completion_rx.recv() => {
2635                if let Err(error) = process_route_bind_completion(
2636                    &writer_tx,
2637                    completion,
2638                    &mut routes,
2639                    &mut root_channels,
2640                    &mut session_identity,
2641                    &mut push_buffer,
2642                    &mut live_roots,
2643                    &mut pending_binds,
2644                    &mut installed_route_epochs,
2645                    &executor,
2646                    &shutdown,
2647                    &dispatch_path_metrics,
2648                )
2649                .await
2650                {
2651                    break Err(error);
2652                }
2653                next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2654                next_health_rollup_at = tokio::time::Instant::now();
2655            }
2656            _ = shutdown.notified() => {
2657                log::warn!("subc attach: fatal executor response requested teardown");
2658                break Ok(ModuleLoopExit::SkipSearchFlush);
2659            }
2660            maybe_frame = reader_rx.recv() => {
2661                let frame = match maybe_frame {
2662                    None => {
2663                        log::info!("subc attach: daemon closed connection");
2664                        break Ok(ModuleLoopExit::Graceful);
2665                    }
2666                    Some(Err(error)) => break Err(error),
2667                    Some(Ok(frame)) => frame,
2668                };
2669                let phase_trace = frame.phase_trace;
2670                let frame = frame.frame;
2671
2672                if !ingress_route_should_be_processed(
2673                    &installed_route_epochs,
2674                    &reclaimed_routes,
2675                    &frame,
2676                ) {
2677                    log::debug!(
2678                        "subc attach: silently dropping {:?} for uninstalled route {}@{}",
2679                        frame.header.ty,
2680                        frame.header.channel,
2681                        frame.header.epoch
2682                    );
2683                    continue;
2684                }
2685
2686                match frame.header.ty {
2687                    FrameType::Ping if frame.header.channel == 0 => {
2688                        let pong = match Frame::build_with_version(
2689                            frame.header.ver,
2690                            FrameType::Pong,
2691                            frame.header.flags,
2692                            0,
2693                            0,
2694                            frame.header.corr,
2695                            Vec::new(),
2696                        ) {
2697                            Ok(pong) => pong,
2698                            Err(error) => break Err(SubcError::FrameBuild(error)),
2699                        };
2700                        if let Err(error) = send_frame(&writer_tx, &dispatch_path_metrics, pong).await {
2701                            break Err(error);
2702                        }
2703                    }
2704                    FrameType::Goodbye if frame.header.channel == 0 => {
2705                        log::info!("subc attach: received channel-0 Goodbye");
2706                        break Ok(ModuleLoopExit::Graceful);
2707                    }
2708                    FrameType::Goodbye => {
2709                        let channel = route_key(frame.header.channel, frame.header.epoch);
2710                        if let Err(error) = teardown_installed_route(
2711                            &writer_tx,
2712                            &dispatch_path_metrics,
2713                            &executor,
2714                            channel,
2715                            "Goodbye",
2716                            None,
2717                            &mut installed_route_epochs,
2718                            &mut routes,
2719                            &mut root_channels,
2720                            &mut bg_subs,
2721                            &mut bg_sub_by_session,
2722                            &mut bg_wake_pending,
2723                            &mut pending_bash_asks,
2724                            &mut live_roots,
2725                            &mut route_bash_cancels,
2726                            &active_tool_calls,
2727                            &mut pending_binds,
2728                            &mut retry_buffer,
2729                            &mut push_buffer,
2730                            &shutdown,
2731                        )
2732                        .await
2733                        {
2734                            break Err(error);
2735                        }
2736                    }
2737                    FrameType::Response | FrameType::Error if frame.header.channel != 0 => {
2738                        if let Err(error) = handle_bash_elicitation_reply(
2739                            &writer_tx,
2740                            &frame,
2741                            &mut pending_bash_asks,
2742                            &routes,
2743                            &mut live_roots,
2744                            &executor,
2745                            &shutdown,
2746                            &bash_deferred_tx,
2747                            &bash_poll_touch_tx,
2748                            &dispatch_path_metrics,
2749                            &mut route_bash_cancels,
2750                            dispatch,
2751                        )
2752                        .await
2753                        {
2754                            break Err(error);
2755                        }
2756                    }
2757                    FrameType::Request if frame.header.channel == 0 => {
2758                        if let Err(error) = handle_control_request(
2759                            &writer_tx,
2760                            &frame,
2761                            &shared_app,
2762                            &executor,
2763                            &mut live_roots,
2764                            &mut pending_binds,
2765                            &mut installed_route_epochs,
2766                            &mut routes,
2767                            &mut root_channels,
2768                            &mut bg_subs,
2769                            &mut bg_sub_by_session,
2770                            &mut bg_wake_pending,
2771                            &mut pending_bash_asks,
2772                            &mut route_bash_cancels,
2773                            &active_tool_calls,
2774                            &mut retry_buffer,
2775                            &mut push_buffer,
2776                            &shutdown,
2777                            &control_completion_tx,
2778                            &dispatch_path_metrics,
2779                            &health_rollup_cache,
2780                            &push_senders,
2781                            dispatch,
2782                            user_config_path.as_deref(),
2783                        )
2784                        .await
2785                        {
2786                            break Err(error);
2787                        }
2788                    }
2789                    FrameType::Request => {
2790                        if let Err(error) = handle_tool_call(
2791                            &writer_tx,
2792                            &frame,
2793                            phase_trace,
2794                            &routes,
2795                            &pending_binds,
2796                            &mut live_roots,
2797                            &executor,
2798                            &active_tool_calls,
2799                            &shutdown,
2800                            &connection_cancel,
2801                            &bash_deferred_tx,
2802                            &bash_poll_touch_tx,
2803                            &dispatch_path_metrics,
2804                            &mut route_bash_cancels,
2805                            &mut pending_bash_asks,
2806                            &mut next_bash_ask_corr,
2807                            &mut bg_subs,
2808                            &mut bg_sub_by_session,
2809                            &mut bg_wake_pending,
2810                            &mut bg_wake_epoch,
2811                            dispatch,
2812                            allow_native_passthrough,
2813                            tool_response_body_limit,
2814                        )
2815                        .await
2816                        {
2817                            break Err(error);
2818                        }
2819                    }
2820                    FrameType::Cancel => {
2821                        let channel = route_key(frame.header.channel, frame.header.epoch);
2822                        cancel_active_tool_call(
2823                            &active_tool_calls,
2824                            executor.as_ref(),
2825                            channel,
2826                            frame.header.corr,
2827                            "Cancel frame",
2828                        );
2829                        if bg_subs.contains_key(&channel) {
2830                            if let Err(error) = end_bg_subscription(
2831                                &writer_tx,
2832                                &dispatch_path_metrics,
2833                                &mut bg_subs,
2834                                &mut bg_sub_by_session,
2835                                &mut bg_wake_pending,
2836                                channel,
2837                                routes.get(&channel),
2838                                "cancel",
2839                            )
2840                            .await
2841                            {
2842                                break Err(error);
2843                            }
2844                        }
2845                        if let Err(error) = cancel_pending_bash_ask_for_tool_call(
2846                            &writer_tx,
2847                            &mut pending_bash_asks,
2848                            channel,
2849                            frame.header.corr,
2850                            &routes,
2851                            &mut live_roots,
2852                            &mut route_bash_cancels,
2853                            &shutdown,
2854                            &dispatch_path_metrics,
2855                        )
2856                        .await
2857                        {
2858                            break Err(error);
2859                        }
2860                    }
2861                    // Incoming push messages are ignored here. Cancel frames are
2862                    // handled above for both active inspect jobs and pending bash
2863                    // elicitation requests.
2864                    _ => {}
2865                }
2866            }
2867            Some((root_id, frame)) = reliable_rx.recv() => {
2868                // Reliable Push frames are FIFO and must-deliver, but draining an
2869                // unbounded burst in one current-thread turn can starve RouteBind
2870                // completions. The budget defers excess frames, never drops them.
2871                let (_, deferred) = push::drain_reliable_push_turn(
2872                    &writer_tx,
2873                    &dispatch_path_metrics,
2874                    &routes,
2875                    &root_channels,
2876                    &session_identity,
2877                    &mut retry_buffer,
2878                    &mut push_buffer,
2879                    &mut completed_tasks,
2880                    &bg_sub_by_session,
2881                    &mut bg_wake_pending,
2882                    &mut bg_wake_epoch,
2883                    &mut reliable_rx,
2884                    Some((root_id, frame)),
2885                );
2886                if deferred {
2887                    tokio::task::yield_now().await;
2888                }
2889            }
2890            Some((order, root_id, frame)) = lossy_rx.recv() => {
2891                // When both push lanes have work, handle a small reliable slice before lossy work.
2892                // That ordering lets completed task ids suppress stale BashLongRunning frames.
2893                // The slice stays bounded so reliable bursts cannot monopolize this loop turn.
2894                let (_, deferred) = push::drain_reliable_push_turn(
2895                    &writer_tx,
2896                    &dispatch_path_metrics,
2897                    &routes,
2898                    &root_channels,
2899                    &session_identity,
2900                    &mut retry_buffer,
2901                    &mut push_buffer,
2902                    &mut completed_tasks,
2903                    &bg_sub_by_session,
2904                    &mut bg_wake_pending,
2905                    &mut bg_wake_epoch,
2906                    &mut reliable_rx,
2907                    None,
2908                );
2909                if deferred {
2910                    tokio::task::yield_now().await;
2911                }
2912
2913                // Drain the currently queued burst in one loop turn so lossy
2914                // status/progress updates can be merged before reaching subc's
2915                // shared egress queue. Each lossy frame gets a sequence number
2916                // before it goes to the channel or overflow buffer, so the
2917                // combined batch is sorted back into producer order before
2918                // coalescing drops stale updates for the same key.
2919                let mut batch = vec![(order, root_id, frame)];
2920                while let Ok(item) = lossy_rx.try_recv() {
2921                    batch.push(item);
2922                }
2923                batch.extend(lossy_overflow.drain());
2924                push::process_lossy_push_envelope_batch(
2925                    &writer_tx,
2926                    &dispatch_path_metrics,
2927                    &routes,
2928                    &root_channels,
2929                    &completed_tasks,
2930                    batch,
2931                );
2932            }
2933            Some(done) = bash_deferred_rx.recv() => {
2934                decrement_counted_channel(&dispatch_path_metrics.bash_deferred_queued);
2935                if let Err(error) = bash::handle_bash_deferred_completion(
2936                    &writer_tx,
2937                    done,
2938                    &routes,
2939                    &mut live_roots,
2940                    &mut route_bash_cancels,
2941                    &shutdown,
2942                    &dispatch_path_metrics,
2943                )
2944                .await
2945                {
2946                    break Err(error);
2947                }
2948            }
2949            Some(root_id) = bash_poll_touch_rx.recv() => {
2950                decrement_counted_channel(&dispatch_path_metrics.bash_poll_touch_queued);
2951                if let Some(meta) = live_roots.get_mut(&root_id) {
2952                    meta.note_activity();
2953                }
2954            }
2955            Some(completion) = maintenance_rx.recv() => {
2956                decrement_counted_channel(&dispatch_path_metrics.maintenance_queued);
2957                let root_id = completion.root_id.clone();
2958                let response = completion.response;
2959                let response_is_fatal = response_is_fatal_panic(&response);
2960                let bind_pending = pending_binds
2961                    .values()
2962                    .any(|pending| pending.bind_root_id == root_id);
2963                let requiesce = if let Some(meta) = live_roots.get_mut(&root_id) {
2964                    let defer_requeue = meta.unbound_quiesced || bind_pending;
2965                    note_maintenance_completion(
2966                        meta,
2967                        completion.requeue_kind,
2968                        response_is_fatal,
2969                        defer_requeue,
2970                    );
2971                    should_requiesce_after_maintenance(meta, completion.kind, bind_pending)
2972                } else {
2973                    false
2974                };
2975                if requiesce {
2976                    quiesce_unbound_root(&root_id, &mut live_roots, &executor);
2977                }
2978                push::clear_stale_bg_wakes_for_empty_sessions(
2979                    &root_id,
2980                    &completion.empty_bg_sessions,
2981                    &bg_sub_by_session,
2982                    &mut bg_wake_pending,
2983                    &bg_wake_epoch,
2984                );
2985                if response_is_fatal {
2986                    if let Some(meta) = live_roots.get_mut(&root_id) {
2987                        meta.maintenance_poisoned = true;
2988                    }
2989                    log::warn!(
2990                        "subc attach: maintenance drain observed a fatal actor; deferring teardown until a route request can receive actor_fatal"
2991                    );
2992                }
2993            }
2994            _ = tokio::time::sleep_until(next_drain_at) => {
2995                // Wakes an otherwise-idle loop so the pre-turn drain check
2996                // above runs on schedule; the drain work itself lives there.
2997            }
2998            _ = tokio::time::sleep_until(next_maintenance_at) => {
2999                // Delay cache-draining maintenance until any already-ready
3000                // inbound route/control messages and push completions have run,
3001                // so maintenance does not block the actor from handling the
3002                // first request that arrives after a route bind is acknowledged.
3003                crate::logging::maybe_sweep_logs();
3004                let reaped_lsp_children = shared_app
3005                    .lsp_child_registry()
3006                    .reap_children_with_gone_cwd_or_reclaimed_root();
3007                if reaped_lsp_children > 0 {
3008                    log::warn!(
3009                        "subc attach: reaped {reaped_lsp_children} LSP child process group(s) with a deleted cwd or reclaimed root"
3010                    );
3011                }
3012                let reap = reap_idle_roots(
3013                    Instant::now(),
3014                    &mut live_roots,
3015                    &pending_binds,
3016                    &root_channels,
3017                    &executor,
3018                    &dispatch_path_metrics,
3019                );
3020                for root_id in &reap.forgotten_deleted_roots {
3021                    purge_deleted_root_residents(
3022                        root_id,
3023                        &mut routes,
3024                        &mut root_channels,
3025                        &mut installed_route_epochs,
3026                        &mut route_bash_cancels,
3027                        &active_tool_calls,
3028                        executor.as_ref(),
3029                        &mut retry_buffer,
3030                        &mut reclaimed_routes,
3031                        &mut session_identity,
3032                        &mut push_buffer,
3033                        &mut bg_subs,
3034                        &mut bg_sub_by_session,
3035                        &mut bg_wake_pending,
3036                        &mut bg_wake_epoch,
3037                        &mut pending_bash_asks,
3038                        &dispatch_path_metrics,
3039                    );
3040                }
3041                if reap.evicted > 0 {
3042                    log::debug!("subc attach: reaped {} idle root(s)", reap.evicted);
3043                }
3044                submit_due_maintenance_jobs(
3045                    &executor,
3046                    &mut live_roots,
3047                    &pending_binds,
3048                    &bg_sub_by_session,
3049                    &bg_wake_pending,
3050                    &bg_wake_epoch,
3051                    &maintenance_tx,
3052                    &dispatch_path_metrics,
3053                );
3054                // Opportunistic allocator relief, independent of the idle
3055                // sweep: the sweep's whole-process idle gate never opens while
3056                // any session stays active, which let freed warm-up arenas sit
3057                // resident for the process lifetime (5.1 GB RSS over ~600 MB
3058                // live). Slack threshold + spacing live in memory.rs; the pass
3059                // itself runs on a detached thread.
3060                #[cfg(any(target_os = "macos", target_os = "linux"))]
3061                {
3062                    let now_std = std::time::Instant::now();
3063                    if crate::memory::spawn_allocator_slack_relief_if_due(
3064                        last_slack_relief,
3065                        now_std,
3066                    ) {
3067                        last_slack_relief = Some(now_std);
3068                    }
3069                }
3070                next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
3071            }
3072        }
3073    };
3074
3075    shared_app.set_open_route_count(0);
3076
3077    connection_cancel.cancel();
3078    // Channel-0 Goodbye, EOF, and fatal exits bypass per-route Goodbye. Settle
3079    // their root lifecycle state before loop-owned routing metadata is dropped.
3080    quiesce_connection_roots(
3081        &mut live_roots,
3082        &mut pending_binds,
3083        &mut routes,
3084        &mut root_channels,
3085        &mut installed_route_epochs,
3086        &mut route_bash_cancels,
3087        &active_tool_calls,
3088        &executor,
3089    );
3090
3091    fleet_status_client.set_route_live(false);
3092    fleet_status_task.abort();
3093    let _ = fleet_status_task.await;
3094
3095    let mut loop_result = loop_result;
3096    if !pending_bash_asks.is_empty() {
3097        let no_routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
3098        if let Err(error) = settle_all_pending_bash_asks(
3099            &writer_tx,
3100            &mut pending_bash_asks,
3101            &no_routes,
3102            &mut live_roots,
3103            &mut route_bash_cancels,
3104            &shutdown,
3105            &dispatch_path_metrics,
3106        )
3107        .await
3108        {
3109            loop_result = loop_result.and(Err(error));
3110        }
3111    }
3112
3113    // The reader task may be parked on `read_frame`; abort it (we are done with
3114    // the connection) and flush the writer.
3115    reader_task.abort();
3116    drop(writer_tx);
3117    let writer_result = finish_writer_task(writer_task).await;
3118    loop_result.and_then(|exit| writer_result.map(|_| exit))
3119}
3120
3121fn spawn_writer_task<W>(
3122    mut write: W,
3123    mut rx: mpsc::Receiver<WriterFrame>,
3124    metrics: Arc<DispatchPathMetrics>,
3125) -> JoinHandle<Result<(), subc_transport::FrameIoError>>
3126where
3127    W: AsyncWrite + Unpin + Send + 'static,
3128{
3129    tokio::spawn(async move {
3130        let mut write_buffer = Vec::new();
3131        while let Some(mut queued) = rx.recv().await {
3132            let measure = queued.tool_response_trace.is_some();
3133            let dequeued = measure.then(Instant::now);
3134            metrics.writer_active.store(true, Ordering::Relaxed);
3135            decrement_counted_channel(&metrics.writer_queued);
3136            let write_timing = write_frame_contiguous(
3137                &mut write,
3138                queued.frame(),
3139                queued.body(),
3140                &mut write_buffer,
3141                measure,
3142            )
3143            .await;
3144            metrics.writer_active.store(false, Ordering::Relaxed);
3145            let write_timing = write_timing?;
3146
3147            if let (Some(trace), Some(dequeued), Some(write_timing)) =
3148                (queued.tool_response_trace.take(), dequeued, write_timing)
3149            {
3150                if let Some(completed) = trace.finish(
3151                    dequeued,
3152                    write_timing.write_started,
3153                    write_timing.write_finished,
3154                    write_timing.frame_bytes,
3155                ) {
3156                    log_ctx::with_session(Some(completed.session), || {
3157                        crate::logging::note_tool_call_trace(
3158                            &completed.name,
3159                            &completed.root,
3160                            completed.channel,
3161                            completed.corr,
3162                            completed.phases,
3163                        );
3164                    });
3165                }
3166            }
3167        }
3168        Ok(())
3169    })
3170}
3171
3172struct FrameWriteTiming {
3173    write_started: Instant,
3174    write_finished: Instant,
3175    frame_bytes: usize,
3176}
3177
3178/// Encode one complete frame into the existing reusable buffer and write it
3179/// without interleaving bytes from another channel. Timing is collected only
3180/// for tool responses, so Push and control frames add no clock reads.
3181async fn write_frame_contiguous<W>(
3182    writer: &mut W,
3183    frame: &Frame,
3184    body: &[u8],
3185    buffer: &mut Vec<u8>,
3186    measure: bool,
3187) -> Result<Option<FrameWriteTiming>, subc_transport::FrameIoError>
3188where
3189    W: AsyncWrite + Unpin,
3190{
3191    if frame.header.len as usize != body.len() {
3192        return Err(subc_transport::FrameIoError::BodyLengthMismatch {
3193            header_len: frame.header.len,
3194            body_len: body.len(),
3195        });
3196    }
3197
3198    let header = frame.header.encode();
3199    buffer.clear();
3200    buffer.reserve(header.len() + body.len());
3201    buffer.extend_from_slice(&header);
3202    buffer.extend_from_slice(body);
3203    let write_started = measure.then(Instant::now);
3204    writer
3205        .write_all(buffer)
3206        .await
3207        .map_err(subc_transport::FrameIoError::Io)?;
3208    Ok(write_started.map(|write_started| FrameWriteTiming {
3209        write_started,
3210        write_finished: Instant::now(),
3211        frame_bytes: buffer.len(),
3212    }))
3213}
3214
3215fn spawn_reader_task<R>(
3216    mut read: R,
3217    tx: mpsc::Sender<Result<DecodedFrame, SubcError>>,
3218) -> JoinHandle<()>
3219where
3220    R: AsyncRead + Unpin + Send + 'static,
3221{
3222    tokio::spawn(async move {
3223        loop {
3224            match read_frame(&mut read).await {
3225                Ok(Some(frame)) => {
3226                    let decoded = DecodedFrame {
3227                        frame,
3228                        phase_trace: PhaseTrace::new(Instant::now()),
3229                    };
3230                    if tx.send(Ok(decoded)).await.is_err() {
3231                        return;
3232                    }
3233                }
3234                Ok(None) => {
3235                    // EOF: let the loop observe channel close as "daemon closed".
3236                    return;
3237                }
3238                Err(error) => {
3239                    // A killed daemon surfaces as ConnectionReset (RST) on
3240                    // Windows where Unix delivers a clean EOF (FIN); a
3241                    // mid-teardown daemon can also abort the socket. Both mean
3242                    // "daemon went away", not a wire fault — normalize them to
3243                    // the clean-close path so module exit behavior matches
3244                    // across platforms (same class subc-core fixed in d33d9a71).
3245                    if let subc_transport::FrameIoError::Io(io_error) = &error {
3246                        if matches!(
3247                            io_error.kind(),
3248                            std::io::ErrorKind::ConnectionReset
3249                                | std::io::ErrorKind::ConnectionAborted
3250                        ) {
3251                            log::info!(
3252                                "subc attach: connection reset by daemon; treating as close"
3253                            );
3254                            return;
3255                        }
3256                    }
3257                    let _ = tx.send(Err(SubcError::FrameIo(error))).await;
3258                    return;
3259                }
3260            }
3261        }
3262    })
3263}
3264
3265async fn finish_writer_task(
3266    mut writer_task: JoinHandle<Result<(), subc_transport::FrameIoError>>,
3267) -> Result<(), SubcError> {
3268    match tokio::time::timeout(Duration::from_millis(100), &mut writer_task).await {
3269        Ok(Ok(Ok(()))) => Ok(()),
3270        Ok(Ok(Err(error))) => Err(SubcError::FrameIo(error)),
3271        Ok(Err(error)) => Err(SubcError::WriterJoin(error)),
3272        Err(_) => {
3273            writer_task.abort();
3274            Ok(())
3275        }
3276    }
3277}
3278
3279fn register_actor_for_bind(
3280    shared_app: &Arc<App>,
3281    executor: &Arc<Executor>,
3282    push_senders: &PushSenders,
3283    bind_root_id: &ProjectRootId,
3284    route_channel: u16,
3285    root_was_live: bool,
3286) -> bool {
3287    if executor.actor_registered(bind_root_id) {
3288        log::debug!(
3289            "subc attach: reusing actor for route {} root {}",
3290            route_channel,
3291            bind_root_id.as_path().display()
3292        );
3293        return false;
3294    }
3295
3296    if root_was_live {
3297        log::warn!(
3298            "subc attach: recreating missing actor for live root {} on route {}",
3299            bind_root_id.as_path().display(),
3300            route_channel
3301        );
3302    }
3303
3304    let actor_ctx = Arc::new(AppContext::from_app(
3305        Arc::clone(shared_app),
3306        Config::default(),
3307    ));
3308    install_bash_compressor(&actor_ctx);
3309    actor_ctx.install_fleet_status_client(Some(push_senders.fleet_status_client.clone()));
3310    actor_ctx.set_progress_sender(Some(push::progress_sender_for_root(
3311        push_senders.clone(),
3312        bind_root_id.clone(),
3313    )));
3314    let inserted = executor.register_actor(bind_root_id.clone(), Arc::clone(&actor_ctx));
3315    drop(actor_ctx);
3316    if inserted {
3317        // Do not insert into live_roots until configure succeeds: live_roots
3318        // drives maintenance, and a half-configured new actor must not be
3319        // maintenance-eligible before its route/session identity exists.
3320        log::debug!(
3321            "subc attach: registered actor for route {} root {}",
3322            route_channel,
3323            bind_root_id.as_path().display()
3324        );
3325    } else {
3326        log::debug!(
3327            "subc attach: actor appeared while binding route {} root {}; reusing it",
3328            route_channel,
3329            bind_root_id.as_path().display()
3330        );
3331    }
3332    inserted
3333}
3334
3335fn rollback_pending_bind_actor(
3336    executor: &Arc<Executor>,
3337    live_roots: &HashMap<ProjectRootId, RootMeta>,
3338    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3339    root_id: &ProjectRootId,
3340    inserted_new_actor: bool,
3341) {
3342    if !inserted_new_actor || live_roots.contains_key(root_id) {
3343        return;
3344    }
3345
3346    if let Some((route, pending)) = pending_binds
3347        .iter_mut()
3348        .find(|(_, pending)| &pending.bind_root_id == root_id)
3349    {
3350        pending.inserted_new_actor = true;
3351        log::debug!(
3352            "subc attach: transferred rollback ownership for root {} to pending route {}",
3353            root_id.as_path().display(),
3354            route
3355        );
3356        return;
3357    }
3358
3359    executor.remove_actor(root_id);
3360}
3361
3362fn route_bind_error_code_for_configure_response(response: &Response) -> &'static str {
3363    match response.data.get("code").and_then(|code| code.as_str()) {
3364        // Preserve typed configure rejections across the bind boundary: a
3365        // malformed fed fingerprint means a federation-module bug or
3366        // fingerprint-format drift, and the fed side matches on the code rather
3367        // than parsing prose.
3368        Some("bad_harness_fingerprint") => "bad_harness_fingerprint",
3369        // Cache-key probe failures are transient (fd pressure, git spawn
3370        // contention); the client retries the bind rather than treating the
3371        // root as permanently divergent.
3372        Some("cache_key_probe_failed") => "cache_key_probe_failed",
3373        // Actor lifecycle gaps are transient from the daemon/client viewpoint:
3374        // a fresh bind can create or join a healthy actor, so do not classify
3375        // them as permanent config divergence.
3376        Some("actor_not_registered" | "actor_fatal") => "actor_not_ready",
3377        _ => "config_divergence",
3378    }
3379}
3380
3381fn queue_post_bind_configure_and_completion_maintenance(
3382    root_id: &ProjectRootId,
3383    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3384) {
3385    let Some(meta) = live_roots.get_mut(root_id) else {
3386        return;
3387    };
3388    if meta.maintenance_poisoned || meta.maintenance_pending {
3389        return;
3390    }
3391
3392    meta.maintenance_pending = true;
3393    meta.maintenance_queued_kinds
3394        .push_back(MaintenanceDrainKind::ConfigureTail);
3395    meta.maintenance_queued_kinds
3396        .push_back(MaintenanceDrainKind::CompletionDrains);
3397}
3398
3399#[allow(clippy::too_many_arguments)]
3400async fn handle_route_bind_completion(
3401    tx: &WriterSender,
3402    completion: RouteBindCompletion,
3403    routes: &mut HashMap<RouteChannel, RouteIdentity>,
3404    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
3405    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
3406    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
3407    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3408    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3409    installed_route_epochs: &mut HashMap<u16, u32>,
3410    executor: &Arc<Executor>,
3411    shutdown: &Arc<Notify>,
3412    metrics: &Arc<DispatchPathMetrics>,
3413) -> Result<(), SubcError> {
3414    let route_id = completion.route;
3415    let Some(pending) = pending_binds.remove(&route_id) else {
3416        log::warn!(
3417            "subc attach: dropping RouteBind completion for non-pending route {}",
3418            completion.route
3419        );
3420        rollback_pending_bind_actor(
3421            executor,
3422            live_roots,
3423            pending_binds,
3424            &completion.bind_root_id,
3425            completion.inserted_new_actor,
3426        );
3427        let has_pending_bind = pending_binds
3428            .values()
3429            .any(|pending| pending.bind_root_id == completion.bind_root_id);
3430        if !root_channels
3431            .get(&completion.bind_root_id)
3432            .is_some_and(|channels| !channels.is_empty())
3433            && !has_pending_bind
3434        {
3435            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3436        }
3437        remove_installed_route(installed_route_epochs, route_id);
3438        return Ok(());
3439    };
3440
3441    if pending.bind_root_id != completion.bind_root_id {
3442        log::warn!(
3443            "subc attach: pending RouteBind root mismatch for route {} (pending {} completion {})",
3444            completion.route,
3445            pending.bind_root_id.as_path().display(),
3446            completion.bind_root_id.as_path().display()
3447        );
3448    }
3449
3450    let inserted_new_actor = pending.inserted_new_actor || completion.inserted_new_actor;
3451    if pending.cancelled {
3452        rollback_pending_bind_actor(
3453            executor,
3454            live_roots,
3455            pending_binds,
3456            &completion.bind_root_id,
3457            inserted_new_actor,
3458        );
3459        let has_pending_bind = pending_binds
3460            .values()
3461            .any(|pending| pending.bind_root_id == completion.bind_root_id);
3462        if !root_channels
3463            .get(&completion.bind_root_id)
3464            .is_some_and(|channels| !channels.is_empty())
3465            && !has_pending_bind
3466        {
3467            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3468        }
3469        log::debug!(
3470            "subc attach: discarded completed RouteBind for cancelled route {} root {}",
3471            completion.route,
3472            completion.bind_root_id.as_path().display()
3473        );
3474        remove_installed_route(installed_route_epochs, route_id);
3475        return Ok(());
3476    }
3477
3478    let failure = if !completion.configure_response.success {
3479        Some((
3480            &completion.configure_response,
3481            "configure failed during route bind",
3482        ))
3483    } else {
3484        None
3485    };
3486
3487    if let Some((response, fallback)) = failure {
3488        rollback_pending_bind_actor(
3489            executor,
3490            live_roots,
3491            pending_binds,
3492            &completion.bind_root_id,
3493            inserted_new_actor,
3494        );
3495        let has_pending_bind = pending_binds
3496            .values()
3497            .any(|pending| pending.bind_root_id == completion.bind_root_id);
3498        if !root_channels
3499            .get(&completion.bind_root_id)
3500            .is_some_and(|channels| !channels.is_empty())
3501            && !has_pending_bind
3502        {
3503            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3504        }
3505        let message = response_message(response, fallback);
3506        let fatal = response_is_fatal_panic(response);
3507        let error_code = route_bind_error_code_for_configure_response(response);
3508        send_route_bind_error_parts(
3509            tx,
3510            completion.ver,
3511            completion.corr,
3512            completion.flags,
3513            error_code,
3514            &message,
3515            metrics,
3516        )
3517        .await?;
3518        remove_installed_route(installed_route_epochs, route_id);
3519        if fatal {
3520            signal_fatal_teardown(
3521                tx,
3522                Some(completion.route),
3523                completion.ver,
3524                completion.corr,
3525                shutdown,
3526                metrics,
3527            )
3528            .await;
3529        }
3530        return Ok(());
3531    }
3532
3533    remember_session_identity(session_identity, &completion.identity);
3534    let replay_key = push::ReplayKey::from_identity(&completion.identity);
3535    let bind_trust = completion.identity.trust;
3536    insert_route_channel(routes, root_channels, route_id, completion.identity);
3537    let restore_watcher = live_roots
3538        .get(&completion.bind_root_id)
3539        .is_some_and(|meta| meta.idle_artifacts_evicted || meta.unbound_quiesced);
3540    live_roots
3541        .entry(completion.bind_root_id.clone())
3542        .and_modify(|meta| {
3543            meta.reactivate_bound();
3544            meta.diagnostics_on_edit = completion.diagnostics_on_edit;
3545            meta.maintenance_poisoned = false;
3546        })
3547        .or_insert_with(|| RootMeta::new(Instant::now()));
3548    if let Some(meta) = live_roots.get_mut(&completion.bind_root_id) {
3549        meta.diagnostics_on_edit = completion.diagnostics_on_edit;
3550        meta.maintenance_poisoned = false;
3551    }
3552    if let Some(ctx) = executor.actor_context(&completion.bind_root_id) {
3553        ctx.mark_subc_bound();
3554        if restore_watcher {
3555            crate::commands::configure::ensure_project_watcher(&ctx);
3556        }
3557    }
3558
3559    let ack =
3560        serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).map_err(SubcError::Json)?;
3561    let response = Frame::build_with_version(
3562        completion.ver,
3563        FrameType::Response,
3564        control_flags(),
3565        0,
3566        0,
3567        completion.corr,
3568        ack,
3569    )
3570    .map_err(SubcError::FrameBuild)?;
3571    send_reliable_writer_frame(tx, metrics, response, "RouteBindAck").await?;
3572    queue_post_bind_configure_and_completion_maintenance(&completion.bind_root_id, live_roots);
3573    let replayed = push::replay_buffered_push_frames(
3574        tx,
3575        metrics,
3576        route_id,
3577        push_buffer,
3578        &replay_key,
3579        bind_trust,
3580    );
3581    if replayed > 0 {
3582        log::debug!(
3583            "subc attach: replayed {} buffered Push frame(s) to route {} root {} harness {} session {}",
3584            replayed,
3585            completion.route,
3586            replay_key.root.as_path().display(),
3587            replay_key.harness,
3588            replay_key.session
3589        );
3590    }
3591    log::info!(
3592        "subc attach: route {} bound to root {}",
3593        completion.route,
3594        completion.bind_root_id.as_path().display()
3595    );
3596    Ok(())
3597}
3598
3599async fn expire_overdue_route_binds(
3600    tx: &WriterSender,
3601    executor: &Arc<Executor>,
3602    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3603    installed_route_epochs: &mut HashMap<u16, u32>,
3604    metrics: &DispatchPathMetrics,
3605) -> Result<(), SubcError> {
3606    let now = Instant::now();
3607    let expired: Vec<_> = pending_binds
3608        .iter()
3609        .filter_map(|(route, pending)| {
3610            let age = now.saturating_duration_since(pending.started_at);
3611            (!pending.deadline_reported && age >= ROUTE_BIND_DEADLINE).then(|| {
3612                (
3613                    *route,
3614                    pending.corr,
3615                    pending.ver,
3616                    pending.flags,
3617                    pending.bind_root_id.clone(),
3618                    pending.configure_request_id.clone(),
3619                    age,
3620                )
3621            })
3622        })
3623        .collect();
3624
3625    for (route, corr, ver, flags, root_id, configure_request_id, age) in expired {
3626        if let Some(pending) = pending_binds.get_mut(&route) {
3627            pending.cancelled = true;
3628            pending.deadline_reported = true;
3629            let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
3630            log::debug!(
3631                "subc attach: cancelled overdue RouteBind configure for route {route} ({outcome:?})"
3632            );
3633        }
3634        remove_installed_route(installed_route_epochs, route);
3635        let age_ms = age.as_millis().min(u128::from(u64::MAX)) as u64;
3636        let deadline_ms = ROUTE_BIND_DEADLINE.as_millis();
3637        send_route_bind_error_parts(
3638            tx,
3639            ver,
3640            corr,
3641            flags,
3642            "actor_not_ready",
3643            &format!("route bind deadline exceeded after {age_ms}ms (deadline {deadline_ms}ms)"),
3644            metrics,
3645        )
3646        .await?;
3647        log::warn!(
3648            "subc attach: route {} bind for root {} exceeded {}ms deadline (configure_request_id={})",
3649            route,
3650            root_id.as_path().display(),
3651            deadline_ms,
3652            configure_request_id
3653        );
3654    }
3655
3656    Ok(())
3657}
3658
3659/// channel-0 control requests: RouteBind plus the cached health probe. RouteBind
3660/// still reconciles the route's RootConfig through the executor's Mutating lane
3661/// and resolves completion on a loop-owned control-completion channel so slow
3662/// configure jobs do not block the transport loop.
3663#[allow(clippy::too_many_arguments)]
3664async fn handle_control_request(
3665    tx: &WriterSender,
3666    frame: &Frame,
3667    shared_app: &Arc<App>,
3668    executor: &Arc<Executor>,
3669    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3670    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3671    installed_route_epochs: &mut HashMap<u16, u32>,
3672    routes: &mut HashMap<RouteChannel, RouteIdentity>,
3673    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
3674    bg_subs: &mut HashMap<RouteChannel, BgSub>,
3675    bg_sub_by_session: &mut BgSubsBySession,
3676    bg_wake_pending: &mut HashSet<RouteChannel>,
3677    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
3678    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
3679    active_tool_calls: &ActiveToolCalls,
3680    retry_buffer: &mut RetryBuffer,
3681    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
3682    shutdown: &Arc<Notify>,
3683    control_completion_tx: &mpsc::Sender<RouteBindCompletion>,
3684    metrics: &Arc<DispatchPathMetrics>,
3685    health_rollup_cache: &HealthRollupCache,
3686    push_senders: &PushSenders,
3687    dispatch: DispatchFn,
3688    user_config_path: Option<&Path>,
3689) -> Result<(), SubcError> {
3690    let request =
3691        serde_json::from_slice::<ModuleControlRequest>(&frame.body).map_err(SubcError::Json)?;
3692    match request {
3693        ModuleControlRequest::RouteBind {
3694            route_channel,
3695            epoch,
3696            target: _,
3697            identity,
3698            principal,
3699            consumer_capabilities,
3700            admission_facts: _,
3701        } => {
3702            let route_id = route_key(route_channel, epoch);
3703            if epoch == 0 {
3704                return send_route_bind_error(
3705                    tx,
3706                    frame,
3707                    "config_divergence",
3708                    "route bind uses an invalid channel generation",
3709                    metrics,
3710                )
3711                .await;
3712            }
3713            let mut bind_root_id = None;
3714            if let Some(installed_epoch) = installed_route_epochs.get(&route_channel).copied() {
3715                if installed_epoch >= epoch {
3716                    return send_route_bind_error(
3717                        tx,
3718                        frame,
3719                        "config_divergence",
3720                        "route bind generation is not newer than the installed generation",
3721                        metrics,
3722                    )
3723                    .await;
3724                }
3725
3726                let replacement_root = match ProjectRootId::from_path(&identity.project_root) {
3727                    Ok(root_id) => root_id,
3728                    Err(error) => {
3729                        return send_route_bind_error(
3730                            tx,
3731                            frame,
3732                            "config_divergence",
3733                            &format!("invalid route project root: {error}"),
3734                            metrics,
3735                        )
3736                        .await;
3737                    }
3738                };
3739                teardown_installed_route(
3740                    tx,
3741                    metrics,
3742                    executor,
3743                    route_key(route_channel, installed_epoch),
3744                    "higher-epoch RouteBind",
3745                    Some(&replacement_root),
3746                    installed_route_epochs,
3747                    routes,
3748                    root_channels,
3749                    bg_subs,
3750                    bg_sub_by_session,
3751                    bg_wake_pending,
3752                    pending_bash_asks,
3753                    live_roots,
3754                    route_bash_cancels,
3755                    active_tool_calls,
3756                    pending_binds,
3757                    retry_buffer,
3758                    push_buffer,
3759                    shutdown,
3760                )
3761                .await?;
3762                bind_root_id = Some(replacement_root);
3763            }
3764            if pending_binds.contains_key(&route_id) {
3765                return send_route_bind_error(
3766                    tx,
3767                    frame,
3768                    "config_divergence",
3769                    "route bind is already pending for channel",
3770                    metrics,
3771                )
3772                .await;
3773            }
3774            let bind_root_id = match bind_root_id {
3775                Some(root_id) => root_id,
3776                None => match ProjectRootId::from_path(&identity.project_root) {
3777                    Ok(root_id) => root_id,
3778                    Err(error) => {
3779                        return send_route_bind_error(
3780                            tx,
3781                            frame,
3782                            "config_divergence",
3783                            &format!("invalid route project root: {error}"),
3784                            metrics,
3785                        )
3786                        .await;
3787                    }
3788                },
3789            };
3790
3791            // Reconcile RootConfig: build a configure request from the bind
3792            // identity + forwarded config tiers and run it through the executor.
3793            let request_id = format!("subc-bind-{route_channel}");
3794            let bind_project_root = identity.project_root.clone();
3795            let bind_harness = identity.harness.clone();
3796            let bind_session = identity.session.clone();
3797            let bind_trust = trust_for_bind(&bind_harness, &principal);
3798            let bind_principal_id = principal_id(&principal);
3799            // Typed capability declaration from the consumer: the facade stamps it
3800            // from the MCP host's initialize-advertised capabilities. Absent
3801            // means no reverse-request capability — flat deny, fail-closed. A
3802            // consumer over-declaring only earns asks that TTL-deny.
3803            let consumer_elicitation_capable = consumer_capabilities
3804                .as_ref()
3805                .is_some_and(|capabilities| capabilities.iter().any(|c| c == "elicitation"));
3806            log::info!(
3807                "subc attach: route {} harness={} principal={} trust={} elicitation={}",
3808                route_channel,
3809                bind_harness,
3810                principal_label(&principal),
3811                bind_trust.label(),
3812                consumer_elicitation_capable
3813            );
3814
3815            // Config is single-per-project, read by AFT directly from the
3816            // CortexKit config files (user: ~/.config/cortexkit/aft.jsonc,
3817            // project: <root>/.cortexkit/aft.jsonc). Wire-relayed config tiers are
3818            // IGNORED entirely: a front (runner, mcp:*, or fed:*) cannot push config over
3819            // the wire. This is what makes config harness-INDEPENDENT — every
3820            // harness binding a project gets the identical on-disk config, so two
3821            // trust domains sharing the per-root actor can never diverge or
3822            // inherit each other's capabilities (the cross-bind escalation class).
3823            // Wire-relayed config tiers (if the protocol still carries them) are
3824            // ignored entirely; the per-tier trust boundary (user trusted, project
3825            // privileged-dropped) is applied to the FILE tiers in handle_configure.
3826            let local_tiers = crate::subc_config::read_local_cortexkit_config_tiers(
3827                user_config_path,
3828                Path::new(&bind_project_root),
3829            );
3830            let config_tiers: Vec<Value> = local_tiers
3831                .iter()
3832                .map(|t| json!({ "tier": t.tier, "source": t.source, "doc": t.doc }))
3833                .collect();
3834            let diagnostics_on_edit = diagnostics_on_edit_from_tiers(&local_tiers);
3835            let configure_json = json!({
3836                "id": request_id,
3837                "command": "configure",
3838                "project_root": bind_project_root,
3839                "harness": bind_harness,
3840                "session_id": bind_session.clone(),
3841                "config": config_tiers,
3842            });
3843            let configure_req = match serde_json::from_value::<RawRequest>(configure_json) {
3844                Ok(req) => req,
3845                Err(error) => {
3846                    return send_route_bind_error(
3847                        tx,
3848                        frame,
3849                        "config_divergence",
3850                        &format!("failed to build configure request: {error}"),
3851                        metrics,
3852                    )
3853                    .await;
3854                }
3855            };
3856
3857            let route_identity = RouteIdentity(Arc::new(RouteIdentityData {
3858                root: bind_root_id.clone(),
3859                project_root: PathBuf::from(&bind_project_root),
3860                harness: bind_harness.clone(),
3861                session: bind_session.clone(),
3862                trust: bind_trust,
3863                spawn_principal: AuthenticatedPrincipal::RouteBind {
3864                    trust: bind_trust.sandbox_trust(),
3865                    route_channel,
3866                    route_epoch: epoch,
3867                    project_root: PathBuf::from(&bind_project_root),
3868                    harness: bind_harness.clone(),
3869                    session_id: bind_session.clone(),
3870                    principal_id: bind_principal_id,
3871                },
3872                consumer_elicitation_capable,
3873            }));
3874            let configure_session = route_identity.session.clone();
3875            let root_was_live = live_roots.contains_key(&bind_root_id);
3876            let inserted_new_actor = register_actor_for_bind(
3877                shared_app,
3878                executor,
3879                push_senders,
3880                &bind_root_id,
3881                route_channel,
3882                root_was_live,
3883            );
3884
3885            let configure_request_id = configure_req.id.clone();
3886            installed_route_epochs.insert(route_channel, epoch);
3887            if let Some(meta) = live_roots.get_mut(&bind_root_id) {
3888                meta.maintenance_queued_kinds.clear();
3889                meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
3890            }
3891            let (configure_rx, configure_cancellation) = executor.submit_cancellable_async(
3892                bind_root_id.clone(),
3893                Lane::Mutating,
3894                configure_request_id.clone(),
3895                Box::new(move |ctx| {
3896                    log_ctx::with_session(Some(configure_session.clone()), || {
3897                        dispatch(configure_req, ctx)
3898                    })
3899                }),
3900            );
3901            pending_binds.insert(
3902                route_id,
3903                PendingBind {
3904                    bind_root_id: bind_root_id.clone(),
3905                    inserted_new_actor,
3906                    cancelled: false,
3907                    configure_request_id: configure_request_id.clone(),
3908                    started_at: Instant::now(),
3909                    warned_half_deadline: false,
3910                    deadline_reported: false,
3911                    corr: frame.header.corr,
3912                    ver: frame.header.ver,
3913                    flags: frame.header.flags,
3914                    cancellation: configure_cancellation,
3915                },
3916            );
3917
3918            let completion_tx = control_completion_tx.clone();
3919            let completion_identity = route_identity;
3920            let completion_root = bind_root_id.clone();
3921            let completion_route_channel = route_channel;
3922            let completion_ver = frame.header.ver;
3923            let completion_corr = frame.header.corr;
3924            let completion_flags = frame.header.flags;
3925            let completion_metrics = Arc::clone(metrics);
3926            tokio::spawn(async move {
3927                let _response_task = ResponseTaskGuard::new(&completion_metrics);
3928                let configure_response =
3929                    await_executor_response(configure_rx, configure_request_id.clone()).await;
3930                // Send the route-bind acknowledgment as soon as configure succeeds.
3931                // Installing completed search or callgraph builds only refreshes cached
3932                // read data, so a later maintenance pass can do it without delaying the
3933                // daemon's confirmation that the route is usable.
3934                let completion = RouteBindCompletion {
3935                    route: route_key(completion_route_channel, epoch),
3936                    identity: completion_identity,
3937                    bind_root_id: completion_root,
3938                    inserted_new_actor,
3939                    configure_response,
3940                    diagnostics_on_edit,
3941                    ver: completion_ver,
3942                    corr: completion_corr,
3943                    flags: completion_flags,
3944                };
3945                if send_counted_channel(
3946                    &completion_tx,
3947                    &completion_metrics.control_completion_queued,
3948                    completion,
3949                )
3950                .await
3951                .is_err()
3952                {
3953                    log::debug!(
3954                        "subc attach: dropped RouteBind completion for route {} after loop exit",
3955                        completion_route_channel
3956                    );
3957                }
3958            });
3959
3960            health_rollup_cache.refresh(executor, shared_app);
3961            Ok(())
3962        }
3963        ModuleControlRequest::HealthCheck {} => {
3964            metrics.record_bg_runtime(bg_subs.len(), bg_wake_pending.len());
3965            let report = build_health_report(
3966                health_rollup_cache,
3967                executor,
3968                pending_binds,
3969                metrics,
3970                shared_app,
3971            );
3972            let body = serde_json::to_vec(&ModuleControlResponse::from(report))
3973                .map_err(SubcError::Json)?;
3974            let response = Frame::build_with_version(
3975                frame.header.ver,
3976                FrameType::Response,
3977                frame.header.flags,
3978                0,
3979                0,
3980                frame.header.corr,
3981                body,
3982            )
3983            .map_err(SubcError::FrameBuild)?;
3984            send_frame(tx, metrics, response).await
3985        }
3986    }
3987}
3988
3989fn install_bash_compressor(ctx: &AppContext) {
3990    // Mirrors main.rs per-actor compressor installation for subc-created actors.
3991    let filter_registry_handle = ctx.shared_filter_registry();
3992    let compress_flag = ctx.bash_compress_flag();
3993    ctx.bash_background().set_compressor_with_exit_code(
3994        move |command: &str, output: String, exit_code: Option<i32>| {
3995            if !compress_flag.load(std::sync::atomic::Ordering::Relaxed) {
3996                return crate::compress::CompressionResult::new(output);
3997            }
3998            let registry_guard = match filter_registry_handle.read() {
3999                Ok(g) => g,
4000                Err(poisoned) => poisoned.into_inner(),
4001            };
4002            crate::compress::compress_with_registry_exit_code(
4003                command,
4004                &output,
4005                exit_code,
4006                &registry_guard,
4007            )
4008        },
4009    );
4010}
4011
4012fn diagnostics_on_edit_from_tiers(tiers: &[ConfigTier]) -> bool {
4013    let mut diagnostics_on_edit = false;
4014    for tier in tiers {
4015        if let Some(value) = diagnostics_on_edit_from_doc(&tier.doc) {
4016            diagnostics_on_edit = value;
4017        }
4018    }
4019    diagnostics_on_edit
4020}
4021
4022fn diagnostics_on_edit_from_doc(doc: &str) -> Option<bool> {
4023    let stripped = strip_jsonc(doc);
4024    let value = serde_json::from_str::<Value>(&stripped).ok()?;
4025    value
4026        .get("lsp")
4027        .and_then(Value::as_object)?
4028        .get("diagnostics_on_edit")
4029        .and_then(Value::as_bool)
4030}
4031
4032async fn send_route_bind_error(
4033    tx: &WriterSender,
4034    frame: &Frame,
4035    code: &str,
4036    message: &str,
4037    metrics: &DispatchPathMetrics,
4038) -> Result<(), SubcError> {
4039    send_route_bind_error_parts(
4040        tx,
4041        frame.header.ver,
4042        frame.header.corr,
4043        frame.header.flags,
4044        code,
4045        message,
4046        metrics,
4047    )
4048    .await
4049}
4050
4051async fn send_route_bind_error_parts(
4052    tx: &WriterSender,
4053    ver: u8,
4054    corr: u64,
4055    flags: Flags,
4056    code: &str,
4057    message: &str,
4058    metrics: &DispatchPathMetrics,
4059) -> Result<(), SubcError> {
4060    let response = build_error_frame(ver, 0, 0, corr, flags, code, message)?;
4061    send_reliable_writer_frame(tx, metrics, response, "RouteBind error").await?;
4062    log_route_bind_rejection(code, message);
4063    Ok(())
4064}
4065
4066/// Per-message rate limit for the bind-rejection warn line. A caller that
4067/// re-attaches a dead root forever turns this line into the entire readable
4068/// tail of the SHARED daemon log (measured 2026-08-09: 2.08M copies, ~40/sec,
4069/// 936MB log — other modules' incident lines pushed out of the tail).
4070/// The line itself stays byte-identical so external counters keep matching;
4071/// repeats inside the window are summarized with a suppressed count on the
4072/// next emission (volume stays diagnosable, per the log-diet convention).
4073fn log_route_bind_rejection(code: &str, message: &str) {
4074    const WINDOW: Duration = Duration::from_secs(60);
4075    static SUPPRESSED: OnceLock<StdMutex<HashMap<String, (Instant, u64)>>> = OnceLock::new();
4076    let map = SUPPRESSED.get_or_init(|| StdMutex::new(HashMap::new()));
4077    let mut map = match map.try_lock() {
4078        Ok(map) => map,
4079        // Contended: log unsuppressed rather than blocking or dropping.
4080        Err(_) => {
4081            log::warn!("subc attach: route bind rejected ({code}): {message}");
4082            return;
4083        }
4084    };
4085    let now = Instant::now();
4086    // Bound the map: dead roots churn, and an unbounded suppression map is
4087    // its own leak. Sweep expired entries once it grows past a fleet-sized
4088    // number of distinct rejection messages.
4089    if map.len() > 512 {
4090        map.retain(|_, (start, _)| now.duration_since(*start) < WINDOW);
4091    }
4092    match map.get_mut(message) {
4093        Some((window_start, suppressed)) if now.duration_since(*window_start) < WINDOW => {
4094            *suppressed += 1;
4095        }
4096        Some((window_start, suppressed)) => {
4097            if *suppressed > 0 {
4098                log::warn!(
4099                    "subc attach: route bind rejected ({code}): {message} (repeated {}x in last 60s)",
4100                    *suppressed
4101                );
4102            } else {
4103                log::warn!("subc attach: route bind rejected ({code}): {message}");
4104            }
4105            *window_start = now;
4106            *suppressed = 0;
4107        }
4108        None => {
4109            log::warn!("subc attach: route bind rejected ({code}): {message}");
4110            map.insert(message.to_string(), (now, 0));
4111        }
4112    }
4113}
4114
4115/// Route-channel tool call: `{name, arguments}` → executor lane → dispatch to
4116/// the sync command core → wrap the structured Response in a CallToolResult
4117/// `{content, isError}`. Tool-result mapping: the whole `{success, ...}` Response
4118/// serialized into ONE text block; `isError` carries `success == false`.
4119async fn handle_tool_call(
4120    tx: &WriterSender,
4121    frame: &Frame,
4122    mut phase_trace: PhaseTrace,
4123    routes: &HashMap<RouteChannel, RouteIdentity>,
4124    pending_binds: &HashMap<RouteChannel, PendingBind>,
4125    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4126    executor: &Arc<Executor>,
4127    active_tool_calls: &ActiveToolCalls,
4128    shutdown: &Arc<Notify>,
4129    connection_cancel: &PersistentCancelSignal,
4130    bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
4131    bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
4132    metrics: &Arc<DispatchPathMetrics>,
4133    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
4134    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
4135    next_bash_ask_corr: &mut u64,
4136    bg_subs: &mut HashMap<RouteChannel, BgSub>,
4137    bg_sub_by_session: &mut BgSubsBySession,
4138    bg_wake_pending: &mut HashSet<RouteChannel>,
4139    bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
4140    dispatch: DispatchFn,
4141    allow_native_passthrough: bool,
4142    tool_response_body_limit: usize,
4143) -> Result<(), SubcError> {
4144    let route_id = route_key(frame.header.channel, frame.header.epoch);
4145    if pending_binds.contains_key(&route_id) {
4146        let error = build_error_frame(
4147            frame.header.ver,
4148            frame.header.channel,
4149            frame.header.epoch,
4150            frame.header.corr,
4151            frame.header.flags,
4152            "route_not_bound",
4153            "route is not bound before tool call",
4154        )?;
4155        return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
4156    }
4157
4158    let Some(identity) = routes.get(&route_id).cloned() else {
4159        let error = build_error_frame(
4160            frame.header.ver,
4161            frame.header.channel,
4162            frame.header.epoch,
4163            frame.header.corr,
4164            frame.header.flags,
4165            "route_not_bound",
4166            "route is not bound before tool call",
4167        )?;
4168        return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
4169    };
4170    let restore_watcher = live_roots
4171        .get(&identity.root)
4172        .is_some_and(|meta| meta.idle_artifacts_evicted);
4173    if let Some(meta) = live_roots.get_mut(&identity.root) {
4174        meta.reactivate_bound();
4175    }
4176    if restore_watcher {
4177        if let Some(ctx) = executor.actor_context(&identity.root) {
4178            crate::commands::configure::ensure_project_watcher(&ctx);
4179        }
4180    }
4181
4182    let route_request =
4183        serde_json::from_slice::<RouteRequest>(&frame.body).map_err(SubcError::Json)?;
4184    if matches!(
4185        route_request,
4186        RouteRequest::BgEvents(BgEventsRequest {
4187            op: BgEventsOp::BgEvents
4188        })
4189    ) {
4190        if let Some(old_sub) = bg_subs.get(&route_id).cloned() {
4191            metrics.record_bg_subscription_ended(
4192                &old_sub.root,
4193                &old_sub.session,
4194                route_id,
4195                "resubscribe",
4196            );
4197            push::send_reliable_bg_stream_end(tx, metrics, route_id, &old_sub).await?;
4198        }
4199        if !identity.trust.allows_bash_observation() {
4200            bg_subs.remove(&route_id);
4201            bg_wake_pending.remove(&route_id);
4202            remove_bg_subscription_index(bg_sub_by_session, route_id, Some(&identity));
4203            let denied_sub = BgSub {
4204                corr: frame.header.corr,
4205                ver: frame.header.ver,
4206                flags: frame.header.flags,
4207                root: identity.root.clone(),
4208                session: identity.session.clone(),
4209            };
4210            metrics.record_bg_subscription_ended(
4211                &identity.root,
4212                &identity.session,
4213                route_id,
4214                "subscribe-denied",
4215            );
4216            push::send_reliable_bg_stream_end(tx, metrics, route_id, &denied_sub).await?;
4217            return Ok(());
4218        }
4219        bg_subs.insert(
4220            route_id,
4221            BgSub {
4222                corr: frame.header.corr,
4223                ver: frame.header.ver,
4224                flags: frame.header.flags,
4225                root: identity.root.clone(),
4226                session: identity.session.clone(),
4227            },
4228        );
4229        insert_bg_subscription_index(
4230            bg_sub_by_session,
4231            identity.root.clone(),
4232            identity.session.clone(),
4233            route_id,
4234        );
4235        metrics.record_bg_subscription_installed(&identity.root, &identity.session, route_id);
4236        push::arm_bg_wake(
4237            identity.root.clone(),
4238            identity.session.clone(),
4239            route_id,
4240            bg_wake_pending,
4241            bg_wake_epoch,
4242            metrics,
4243        );
4244        return Ok(());
4245    }
4246
4247    let RouteRequest::ToolCall(call) = route_request else {
4248        unreachable!("background event subscription returned above")
4249    };
4250    let bare_name = call.name;
4251    let arguments = strip_agent_preview_arg_owned(call.arguments);
4252    let format_context = crate::subc_format::FormatContext::from_tool_call(
4253        &bare_name,
4254        &arguments,
4255        identity.project_root.as_path(),
4256    );
4257
4258    let request_id = format!("subc-{}-{}", frame.header.channel, frame.header.corr);
4259    let bind_trust = identity.trust;
4260    let diagnostics_on_edit = live_roots
4261        .get(&identity.root)
4262        .map(|meta| meta.diagnostics_on_edit)
4263        .unwrap_or(false);
4264
4265    let requests_host = bare_name == "bash"
4266        && arguments
4267            .get("sandbox")
4268            .or_else(|| {
4269                arguments
4270                    .get("params")
4271                    .and_then(|params| params.get("sandbox"))
4272            })
4273            .and_then(Value::as_str)
4274            == Some("host");
4275    if matches!(bind_trust, BindTrust::Untrusted) && requests_host {
4276        let response = Response::error(
4277            request_id.clone(),
4278            "sandbox_escalation_denied",
4279            "sandbox host escalation is unavailable to untrusted principals",
4280        );
4281        let text = crate::subc_format::format_response_with_context(
4282            &bare_name,
4283            &response,
4284            &format_context,
4285        );
4286        let result = ToolCallResult { text, response };
4287        let response_frame = build_tool_response_frame_with_limit(
4288            frame.header.ver,
4289            route_id,
4290            frame.header.corr,
4291            frame.header.flags,
4292            &result,
4293            bind_trust,
4294            tool_response_body_limit,
4295        )?;
4296        return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4297    }
4298
4299    if matches!(bind_trust, BindTrust::Untrusted)
4300        && is_bash_family_tool(&bare_name)
4301        && (bare_name != "bash" || !identity.consumer_elicitation_capable)
4302    {
4303        let response = bash::bash_denied_untrusted_response(request_id.clone());
4304        let text = crate::subc_format::format_response_with_context(
4305            &bare_name,
4306            &response,
4307            &format_context,
4308        );
4309        let result = ToolCallResult { text, response };
4310        let response_frame = build_tool_response_frame_with_limit(
4311            frame.header.ver,
4312            route_id,
4313            frame.header.corr,
4314            frame.header.flags,
4315            &result,
4316            bind_trust,
4317            tool_response_body_limit,
4318        )?;
4319        return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4320    }
4321
4322    // A non-core name is NOT in the tool manifest. AFT fails closed and
4323    // does not trust subc to enforce the manifest: rejecting here is the
4324    // defense-in-depth backstop that prevents a forwarded native command
4325    // (e.g. `configure`, which would reach handle_configure and bypass
4326    // the RouteBind config-trust cap) from ever reaching dispatch. Only
4327    // the integration-test harness (run_subc_mode_for_test) opens this to
4328    // drive synthetic native commands through the executor.
4329    if !is_subc_agent_core_tool(&bare_name)
4330        && !is_subc_native_plumbing_tool(&bare_name)
4331        && !allow_native_passthrough
4332    {
4333        log::warn!(
4334            "subc tool call: rejecting non-manifest tool name {:?} on route {} (fail-closed)",
4335            bare_name,
4336            frame.header.channel
4337        );
4338        let response = Response::error(
4339            request_id.clone(),
4340            "unknown_tool",
4341            format!("tool {:?} is not in the AFT tool manifest", bare_name),
4342        );
4343        let text = crate::subc_format::format_response_with_context(
4344            &bare_name,
4345            &response,
4346            &format_context,
4347        );
4348        let result = ToolCallResult { text, response };
4349        let response_frame = build_tool_response_frame_with_limit(
4350            frame.header.ver,
4351            route_id,
4352            frame.header.corr,
4353            frame.header.flags,
4354            &result,
4355            bind_trust,
4356            tool_response_body_limit,
4357        )?;
4358        return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4359    }
4360
4361    if bare_name == "bash" {
4362        if matches!(bind_trust, BindTrust::Untrusted) {
4363            let plan = match bash::prepare_bash_elicitation_plan(
4364                &arguments,
4365                identity.project_root.as_path(),
4366            ) {
4367                Ok(plan) => plan,
4368                Err(error) => {
4369                    let response = Response::error(request_id.clone(), error.code, error.message);
4370                    let text = crate::subc_format::format_response_with_context(
4371                        &bare_name,
4372                        &response,
4373                        &format_context,
4374                    );
4375                    let result = ToolCallResult { text, response };
4376                    let response_frame = build_tool_response_frame_with_limit(
4377                        frame.header.ver,
4378                        route_id,
4379                        frame.header.corr,
4380                        frame.header.flags,
4381                        &result,
4382                        bind_trust,
4383                        tool_response_body_limit,
4384                    )?;
4385                    return send_reliable_writer_frame(
4386                        tx,
4387                        metrics,
4388                        response_frame,
4389                        "tool response",
4390                    )
4391                    .await;
4392                }
4393            };
4394
4395            let reverse_corr =
4396                allocate_reverse_corr(pending_bash_asks, route_id, next_bash_ask_corr);
4397            let ask_frame = build_bash_elicitation_request_frame(
4398                frame.header.ver,
4399                route_id,
4400                reverse_corr,
4401                frame.header.flags,
4402                &plan.command,
4403                &plan.asks,
4404            )?;
4405
4406            let meta = live_roots
4407                .entry(identity.root.clone())
4408                .or_insert_with(|| RootMeta::new(Instant::now()));
4409            meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
4410            meta.reactivate_bound();
4411
4412            let route_cancel =
4413                route_bash_cancels
4414                    .entry(route_id)
4415                    .or_insert_with(|| bash::RouteBashCancel {
4416                        token: PersistentCancelSignal::new(),
4417                        active_waits: 0,
4418                    });
4419            route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
4420            let cancel = bash::BashWaitCancel {
4421                connection: connection_cancel.clone(),
4422                route: route_cancel.token.clone(),
4423            };
4424            pending_bash_asks.insert(
4425                ReverseCorrKey {
4426                    route: route_id,
4427                    corr: reverse_corr,
4428                },
4429                PendingBashAsk {
4430                    route: route_id,
4431                    tool_corr: frame.header.corr,
4432                    tool_flags: frame.header.flags,
4433                    tool_ver: frame.header.ver,
4434                    root: identity.root.clone(),
4435                    project_root: identity.project_root.clone(),
4436                    session_id: identity.session.clone(),
4437                    spawn_principal: identity.spawn_principal.clone(),
4438                    edit_slot_survives: call.edit_slot_survives,
4439                    request_id,
4440                    arguments,
4441                    format_context,
4442                    cancel,
4443                    grants: plan.grants,
4444                    expires_at: Instant::now() + bash_elicitation_timeout(),
4445                },
4446            );
4447            return send_reliable_writer_frame(tx, metrics, ask_frame, "bash elicitation request")
4448                .await;
4449        }
4450
4451        let meta = live_roots
4452            .entry(identity.root.clone())
4453            .or_insert_with(|| RootMeta::new(Instant::now()));
4454        meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
4455        meta.reactivate_bound();
4456
4457        let route_cancel =
4458            route_bash_cancels
4459                .entry(route_id)
4460                .or_insert_with(|| bash::RouteBashCancel {
4461                    token: PersistentCancelSignal::new(),
4462                    active_waits: 0,
4463                });
4464        route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
4465        let cancel = bash::BashWaitCancel {
4466            connection: connection_cancel.clone(),
4467            route: route_cancel.token.clone(),
4468        };
4469
4470        bash::submit_deferred_bash(
4471            executor,
4472            bash_deferred_tx,
4473            bash_poll_touch_tx,
4474            metrics,
4475            dispatch,
4476            identity.root.clone(),
4477            identity.project_root.clone(),
4478            identity.session.clone(),
4479            request_id,
4480            route_id,
4481            frame.header.corr,
4482            frame.header.flags,
4483            frame.header.ver,
4484            arguments,
4485            format_context,
4486            cancel,
4487            bind_trust,
4488            identity.spawn_principal.clone(),
4489            call.edit_slot_survives,
4490            None,
4491        );
4492        return Ok(());
4493    }
4494
4495    let lane = command_lane(&bare_name);
4496    let tool_call_context = ToolCallContext {
4497        project_root: identity.project_root.clone(),
4498        session_id: Some(identity.session.clone()),
4499        request_id: request_id.clone(),
4500        diagnostics_on_edit,
4501        preview: call.preview,
4502        edit_slot_survives: call.edit_slot_survives,
4503        report_registration_downgrade: true,
4504    };
4505    let bare_name_for_frame = bare_name.clone();
4506    let identity_for_run = identity.clone();
4507    let completion_session = identity.session.clone();
4508    let completion_root = identity.project_root.clone();
4509    let request_id_for_force = request_id.clone();
4510    let format_context_for_frame = format_context.clone();
4511    let (tool_call_tx, tool_call_rx) = oneshot::channel::<ToolCallCompletion>();
4512    let cancellable_inspect = bare_name == "inspect";
4513    phase_trace.mark_executor_submitted();
4514    let job: crate::executor::ExecutorJob = Box::new(move |ctx| {
4515        phase_trace.mark_job_admitted();
4516        log_ctx::with_session(Some(identity_for_run.session.clone()), || {
4517            let run = || {
4518                let finalizer = |response: &mut Response| {
4519                    crate::response_finalize::finalize_response_with_bg_completions(
4520                        response,
4521                        ctx,
4522                        &identity_for_run.session,
4523                        &bare_name,
4524                        bind_trust.allows_bash_observation(),
4525                    );
4526                };
4527                match run_tool_call(
4528                    &bare_name,
4529                    arguments,
4530                    &format_context,
4531                    &tool_call_context,
4532                    ctx,
4533                    &dispatch,
4534                    Some(&finalizer),
4535                    Some(&mut phase_trace),
4536                ) {
4537                    ToolCallOutcome::Unary(result) => {
4538                        let response = result.response;
4539                        let _ = tool_call_tx.send(ToolCallCompletion {
4540                            text: result.text,
4541                            phase_trace,
4542                        });
4543                        response
4544                    }
4545                }
4546            };
4547            if matches!(bind_trust, BindTrust::Untrusted) {
4548                ctx.with_force_restrict(&request_id_for_force, run)
4549            } else {
4550                run()
4551            }
4552        })
4553    });
4554    let (rx, cancellation) = if cancellable_inspect {
4555        let (rx, cancellation) =
4556            executor.submit_cancellable_async(identity.root.clone(), lane, request_id.clone(), job);
4557        (rx, Some(cancellation))
4558    } else {
4559        (
4560            executor.submit_async(identity.root.clone(), lane, request_id.clone(), job),
4561            None,
4562        )
4563    };
4564    if let Some(cancellation) = cancellation {
4565        active_tool_calls
4566            .lock()
4567            .unwrap_or_else(std::sync::PoisonError::into_inner)
4568            .insert(
4569                (route_id, frame.header.corr),
4570                ActiveToolCall {
4571                    root_id: identity.root.clone(),
4572                    cancellation,
4573                },
4574            );
4575    }
4576    let completion_tx = tx.clone();
4577    let completion_shutdown = Arc::clone(shutdown);
4578    let route = route_id;
4579    let corr = frame.header.corr;
4580    let flags = frame.header.flags;
4581    let ver = frame.header.ver;
4582    let completion_metrics = Arc::clone(metrics);
4583    let active_tool_calls = Arc::clone(active_tool_calls);
4584    tokio::spawn(async move {
4585        let _response_task = ResponseTaskGuard::new(&completion_metrics);
4586        let response = await_executor_response(rx, request_id.clone()).await;
4587        if cancellable_inspect {
4588            finish_active_tool_call(&active_tool_calls, route, corr);
4589        }
4590        let (text, phase_trace) = match tool_call_rx.await {
4591            Ok(completion) => (completion.text, Some(completion.phase_trace)),
4592            Err(_) => (
4593                crate::subc_format::format_response_with_context(
4594                    &bare_name_for_frame,
4595                    &response,
4596                    &format_context_for_frame,
4597                ),
4598                None,
4599            ),
4600        };
4601        let result = ToolCallResult { text, response };
4602        let fatal = response_is_fatal_panic(&result.response);
4603        match build_tool_response_frame_with_limit(
4604            ver,
4605            route,
4606            corr,
4607            flags,
4608            &result,
4609            bind_trust,
4610            tool_response_body_limit,
4611        ) {
4612            Ok(response_frame) => {
4613                let send_result = if let Some(phase_trace) = phase_trace {
4614                    let trace = ToolResponseWriteTrace::new(
4615                        phase_trace,
4616                        bare_name_for_frame,
4617                        completion_root,
4618                        completion_session,
4619                        route.channel,
4620                        corr,
4621                    );
4622                    send_traced_tool_response_frame(
4623                        &completion_tx,
4624                        &completion_metrics,
4625                        response_frame,
4626                        trace,
4627                    )
4628                    .await
4629                } else {
4630                    send_reliable_writer_frame(
4631                        &completion_tx,
4632                        &completion_metrics,
4633                        response_frame,
4634                        "tool response",
4635                    )
4636                    .await
4637                };
4638                if let Err(error) = send_result {
4639                    log::warn!("subc attach: failed to queue tool response frame: {error}");
4640                }
4641            }
4642            Err(error) => {
4643                log::error!("subc attach: failed to build tool response frame: {error}");
4644            }
4645        }
4646        if fatal {
4647            signal_fatal_teardown(
4648                &completion_tx,
4649                Some(route),
4650                ver,
4651                corr,
4652                &completion_shutdown,
4653                &completion_metrics,
4654            )
4655            .await;
4656        }
4657    });
4658    Ok(())
4659}
4660
4661fn submit_maintenance_job(
4662    executor: &Arc<Executor>,
4663    root_id: ProjectRootId,
4664    kind: MaintenanceDrainKind,
4665    bg_sessions_to_check: Vec<(String, u64)>,
4666    completion_tx: &mpsc::Sender<MaintenanceCompletion>,
4667    metrics: &Arc<DispatchPathMetrics>,
4668) {
4669    let request_id = format!(
4670        "subc-maintenance-drain-{}-{}",
4671        kind.label(),
4672        root_id.as_path().to_string_lossy()
4673    );
4674    let response_id = request_id.clone();
4675    let completion_root_id = root_id.clone();
4676    let maintenance_generation = executor
4677        .actor_context(&root_id)
4678        .map(|ctx| ctx.configure_generation())
4679        .unwrap_or(0);
4680    let (outcome_tx, outcome_rx) = oneshot::channel::<MaintenanceJobOutcome>();
4681    // ConfigureTail runs deferred configure mutations and needs the actor
4682    // epoch write gate. The other drain kinds only mutate subsystem state
4683    // behind that subsystem's own lock, so they run on MaintenanceCommit and
4684    // overlap interactive reads instead of excluding them.
4685    let lane = match kind {
4686        MaintenanceDrainKind::ConfigureTail => Lane::Mutating,
4687        MaintenanceDrainKind::Watcher
4688        | MaintenanceDrainKind::Lsp
4689        | MaintenanceDrainKind::CompletionDrains => Lane::MaintenanceCommit,
4690    };
4691    let job: crate::executor::ExecutorJob = Box::new(move |ctx: &AppContext| {
4692        let outcome = match kind {
4693            MaintenanceDrainKind::Watcher => {
4694                let drained = runtime_drain::drain_watcher_events_bounded(
4695                    ctx,
4696                    runtime_drain::WATCHER_PATH_DRAIN_BATCH_CAP,
4697                );
4698                MaintenanceJobOutcome {
4699                    empty_bg_sessions: Vec::new(),
4700                    requeue_kind: drained.has_more.then_some(kind),
4701                }
4702            }
4703            MaintenanceDrainKind::Lsp => {
4704                let drained = runtime_drain::drain_lsp_events_bounded(
4705                    ctx,
4706                    runtime_drain::LSP_EVENT_DRAIN_BATCH_CAP,
4707                );
4708                MaintenanceJobOutcome {
4709                    empty_bg_sessions: Vec::new(),
4710                    requeue_kind: drained.has_more.then_some(kind),
4711                }
4712            }
4713            MaintenanceDrainKind::ConfigureTail => {
4714                runtime_drain::drain_deferred_configure_maintenance(ctx);
4715                runtime_drain::drain_configure_warning_events(ctx);
4716                MaintenanceJobOutcome::default()
4717            }
4718            MaintenanceDrainKind::CompletionDrains => {
4719                runtime_drain::drain_search_index_events(ctx);
4720                runtime_drain::drain_callgraph_store_events(ctx);
4721                runtime_drain::drain_semantic_index_events(ctx);
4722                runtime_drain::drain_semantic_refresh_events(ctx);
4723                runtime_drain::drain_inspect_events_for_generation(ctx, maintenance_generation);
4724                let empty_bg_sessions = bg_sessions_to_check
4725                    .into_iter()
4726                    .filter(|(session, _)| {
4727                        !ctx.bash_background()
4728                            .has_completions_for_session(Some(session.as_str()))
4729                    })
4730                    .collect();
4731                MaintenanceJobOutcome {
4732                    empty_bg_sessions,
4733                    requeue_kind: None,
4734                }
4735            }
4736        };
4737        let requeued = outcome.requeue_kind.is_some();
4738        let _ = outcome_tx.send(outcome);
4739        Response::success(
4740            response_id,
4741            json!({ "drained": true, "kind": kind.label(), "requeued": requeued }),
4742        )
4743    });
4744    let rx = match kind {
4745        MaintenanceDrainKind::Watcher => executor.submit_coalescable_maintenance_async(
4746            root_id,
4747            lane,
4748            request_id.clone(),
4749            crate::executor::MaintenanceCoalesceKey::WatcherDrain,
4750            job,
4751        ),
4752        MaintenanceDrainKind::Lsp => executor.submit_coalescable_maintenance_async(
4753            root_id,
4754            lane,
4755            request_id.clone(),
4756            crate::executor::MaintenanceCoalesceKey::LspDrain,
4757            job,
4758        ),
4759        MaintenanceDrainKind::ConfigureTail | MaintenanceDrainKind::CompletionDrains => {
4760            executor.submit_maintenance_async(root_id, lane, request_id.clone(), job)
4761        }
4762    };
4763    let completion_tx = completion_tx.clone();
4764    let completion_metrics = Arc::clone(metrics);
4765    tokio::spawn(async move {
4766        let _response_task = ResponseTaskGuard::new(&completion_metrics);
4767        let response = await_executor_response(rx, request_id).await;
4768        let outcome = outcome_rx.await.unwrap_or_default();
4769        let _ = send_counted_channel(
4770            &completion_tx,
4771            &completion_metrics.maintenance_queued,
4772            MaintenanceCompletion {
4773                root_id: completion_root_id,
4774                kind,
4775                response,
4776                empty_bg_sessions: outcome.empty_bg_sessions,
4777                requeue_kind: outcome.requeue_kind,
4778            },
4779        )
4780        .await;
4781    });
4782}
4783
4784async fn await_executor_response(rx: oneshot::Receiver<Response>, request_id: String) -> Response {
4785    rx.await
4786        .unwrap_or_else(|_| Response::error(request_id, "internal_error", "executor dropped"))
4787}
4788async fn signal_fatal_teardown(
4789    tx: &WriterSender,
4790    route: Option<RouteChannel>,
4791    ver: u8,
4792    corr: u64,
4793    shutdown: &Arc<Notify>,
4794    metrics: &DispatchPathMetrics,
4795) {
4796    if let Some(route) = route {
4797        if let Ok(frame) = build_goodbye_frame(ver, route.channel, route.epoch, corr) {
4798            if let Err(error) = send_frame(tx, metrics, frame).await {
4799                log::warn!(
4800                    "subc attach: failed to queue fatal route Goodbye for route {route}: {error}"
4801                );
4802            }
4803        }
4804    }
4805    if let Ok(frame) = build_goodbye_frame(ver, 0, 0, 0) {
4806        if let Err(error) = send_frame(tx, metrics, frame).await {
4807            log::warn!("subc attach: failed to queue fatal channel-0 Goodbye: {error}");
4808        }
4809    }
4810    shutdown.notify_one();
4811}
4812#[derive(Debug, Deserialize)]
4813#[serde(untagged)]
4814enum RouteRequest {
4815    BgEvents(BgEventsRequest),
4816    ToolCall(ToolCallRequest),
4817}
4818
4819#[derive(Debug, Deserialize)]
4820struct BgEventsRequest {
4821    op: BgEventsOp,
4822}
4823
4824#[derive(Debug, Deserialize)]
4825#[serde(rename_all = "snake_case")]
4826enum BgEventsOp {
4827    BgEvents,
4828}
4829
4830#[derive(Debug, Deserialize)]
4831struct ToolCallRequest {
4832    name: String,
4833    #[serde(default)]
4834    arguments: Value,
4835    /// Host-computed registration fact; kept outside agent-controlled arguments.
4836    #[serde(default)]
4837    edit_slot_survives: Option<bool>,
4838    /// Server-owned preview control (B1c-0): the plugin's mutation flow is
4839    /// preview -> permission ask -> apply. Dropping this field made "preview"
4840    /// calls mutate disk before the permission prompt and the subsequent
4841    /// apply fail with not-found.
4842    #[serde(default)]
4843    preview: bool,
4844}
4845
4846#[cfg(test)]
4847pub(crate) mod test_support {
4848    use super::*;
4849    use crate::bash_background::BgTaskStatus;
4850    use crate::protocol::{
4851        BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, ConfigureWarningsFrame,
4852        ProgressFrame, StatusChangedFrame,
4853    };
4854    use serde_json::json;
4855
4856    pub(super) fn test_root(name: &str) -> (tempfile::TempDir, ProjectRootId) {
4857        let dir = tempfile::Builder::new()
4858            .prefix(name)
4859            .tempdir()
4860            .expect("temp root");
4861        let root = ProjectRootId::from_path(dir.path()).expect("project root id");
4862        (dir, root)
4863    }
4864
4865    pub(super) fn test_ctx() -> Arc<AppContext> {
4866        Arc::new(AppContext::new(
4867            Box::new(crate::parser::TreeSitterProvider::new()),
4868            crate::config::Config::default(),
4869        ))
4870    }
4871
4872    #[test]
4873    fn cancelled_inspect_releases_lsp_lane_for_queued_bind() {
4874        let executor = Arc::new(Executor::new());
4875        let (_dir, root) = test_root("cancelled-inspect-lane");
4876        executor.register_actor(root.clone(), test_ctx());
4877        let active: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
4878        let route = RouteChannel {
4879            channel: 7,
4880            epoch: 1,
4881        };
4882        let (inspect_started_tx, inspect_started_rx) = std::sync::mpsc::sync_channel(1);
4883        let (_inspect_rx, cancellation) = executor.submit_cancellable_async(
4884            root.clone(),
4885            Lane::SerialLspStatus,
4886            "abandoned-inspect".to_string(),
4887            Box::new(move |_| {
4888                inspect_started_tx.send(()).expect("inspect starts");
4889                while !crate::executor::current_job_cancellation()
4890                    .is_some_and(|token| token.cancel_requested_before_commit())
4891                {
4892                    std::thread::sleep(Duration::from_millis(5));
4893                }
4894                Response::error(
4895                    "abandoned-inspect",
4896                    "inspect_interrupted",
4897                    "inspect request abandoned",
4898                )
4899            }),
4900        );
4901        active.lock().expect("active tool call map").insert(
4902            (route, 41),
4903            ActiveToolCall {
4904                root_id: root.clone(),
4905                cancellation,
4906            },
4907        );
4908        inspect_started_rx
4909            .recv_timeout(Duration::from_secs(1))
4910            .expect("inspect occupies the LSP lane");
4911
4912        let (bind_started_tx, bind_started_rx) = std::sync::mpsc::sync_channel(1);
4913        let bind = executor.submit(
4914            root,
4915            Lane::Mutating,
4916            "subc-bind-after-abandonment".to_string(),
4917            Box::new(move |_| {
4918                bind_started_tx.send(()).expect("bind starts");
4919                Response::success("bind", json!({}))
4920            }),
4921        );
4922        assert!(
4923            bind_started_rx
4924                .recv_timeout(Duration::from_millis(50))
4925                .is_err(),
4926            "bind must remain behind the live LSP reader"
4927        );
4928
4929        assert!(cancel_active_tool_call(
4930            &active,
4931            executor.as_ref(),
4932            route,
4933            41,
4934            "test abandonment"
4935        ));
4936        bind_started_rx
4937            .recv_timeout(Duration::from_secs(1))
4938            .expect("bind starts after inspect cancellation boundary");
4939        bind.recv_timeout(Duration::from_secs(1))
4940            .expect("bind completes after inspect cancellation");
4941        assert!(active.lock().expect("active tool call map").is_empty());
4942    }
4943
4944    pub(super) fn wait_for_watcher_count(ctx: &AppContext, expected: usize) {
4945        let deadline = Instant::now() + Duration::from_secs(30);
4946        loop {
4947            let observed = ctx.watcher_registry_count();
4948            if observed == expected {
4949                return;
4950            }
4951            assert!(
4952                Instant::now() < deadline,
4953                "watcher count did not settle before deadline: expected={expected}, observed={observed}"
4954            );
4955            std::thread::sleep(Duration::from_millis(50));
4956        }
4957    }
4958
4959    /// Sweep until `root` is forgotten, mirroring how production reaps.
4960    ///
4961    /// `reap_idle_roots` probes actor idleness with a try-lock and retains the
4962    /// root when the scheduler holds that lock — correct behavior, since the
4963    /// real caller sweeps on a timer and simply catches the root next tick. A
4964    /// test that asserts a single sweep succeeds is therefore asserting it wins
4965    /// a lock race that nothing in production depends on: `register_actor` wakes
4966    /// the scheduler, which grabs the same lock, and on a loaded runner that
4967    /// window is wide enough to lose. Sweep to the outcome instead.
4968    pub(super) fn reap_until_forgotten(
4969        root: &ProjectRootId,
4970        live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4971        pending_binds: &HashMap<RouteChannel, PendingBind>,
4972        root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
4973        executor: &Arc<Executor>,
4974        metrics: &DispatchPathMetrics,
4975    ) -> IdleReapOutcome {
4976        let deadline = Instant::now() + Duration::from_secs(30);
4977        loop {
4978            let outcome = reap_idle_roots(
4979                Instant::now(),
4980                live_roots,
4981                pending_binds,
4982                root_channels,
4983                executor,
4984                metrics,
4985            );
4986            if outcome.forgotten_deleted_roots.contains(root) {
4987                return outcome;
4988            }
4989            assert!(
4990                Instant::now() < deadline,
4991                "deleted root was never forgotten: {root:?}"
4992            );
4993            std::thread::sleep(Duration::from_millis(10));
4994        }
4995    }
4996
4997    pub(super) fn wait_for_actor_root_count(app: &App, expected: usize) {
4998        let deadline = Instant::now() + Duration::from_secs(30);
4999        loop {
5000            let observed = app.actor_root_count();
5001            if observed == expected {
5002                return;
5003            }
5004            assert!(
5005                Instant::now() < deadline,
5006                "actor root count did not settle before deadline: expected={expected}, observed={observed}"
5007            );
5008            std::thread::sleep(Duration::from_millis(50));
5009        }
5010    }
5011
5012    pub(super) fn status_frame(seq: u64) -> PushFrame {
5013        status_frame_with_session(seq, None)
5014    }
5015
5016    pub(super) fn status_frame_with_session(seq: u64, session_id: Option<&str>) -> PushFrame {
5017        PushFrame::StatusChanged(StatusChangedFrame {
5018            frame_type: "status_changed",
5019            session_id: session_id.map(str::to_string),
5020            snapshot: json!({ "seq": seq }),
5021        })
5022    }
5023
5024    pub(super) fn completion_frame(task_id: &str) -> PushFrame {
5025        completion_frame_with_session(task_id, "session-1")
5026    }
5027
5028    pub(super) fn completion_frame_with_session(task_id: &str, session_id: &str) -> PushFrame {
5029        PushFrame::BashCompleted(BashCompletedFrame {
5030            frame_type: "bash_completed",
5031            task_id: task_id.to_string(),
5032            session_id: session_id.to_string(),
5033            status: BgTaskStatus::Completed,
5034            exit_code: Some(0),
5035            command: format!("echo {task_id}"),
5036            output_preview: String::new(),
5037            output_truncated: false,
5038            original_tokens: None,
5039            compressed_tokens: None,
5040            tokens_skipped: false,
5041            status_reason: None,
5042        })
5043    }
5044
5045    pub(super) fn long_running_frame(task_id: &str, elapsed_ms: u64) -> PushFrame {
5046        long_running_frame_with_session(task_id, "session-1", elapsed_ms)
5047    }
5048
5049    pub(super) fn long_running_frame_with_session(
5050        task_id: &str,
5051        session_id: &str,
5052        elapsed_ms: u64,
5053    ) -> PushFrame {
5054        PushFrame::BashLongRunning(BashLongRunningFrame {
5055            frame_type: "bash_long_running",
5056            task_id: task_id.to_string(),
5057            session_id: session_id.to_string(),
5058            command: format!("sleep {elapsed_ms}"),
5059            elapsed_ms,
5060        })
5061    }
5062
5063    pub(super) fn pattern_match_frame(session_id: &str) -> PushFrame {
5064        PushFrame::BashPatternMatch(BashPatternMatchFrame {
5065            frame_type: "bash_pattern_match",
5066            task_id: "task-pattern".to_string(),
5067            session_id: session_id.to_string(),
5068            watch_id: "watch-1".to_string(),
5069            match_text: "needle".to_string(),
5070            match_offset: 7,
5071            context: "haystack needle".to_string(),
5072            once: true,
5073            reason: "pattern_match",
5074        })
5075    }
5076
5077    pub(super) fn configure_warnings_frame(session_id: Option<&str>) -> PushFrame {
5078        PushFrame::ConfigureWarnings(ConfigureWarningsFrame {
5079            frame_type: "configure_warnings",
5080            session_id: session_id.map(str::to_string),
5081            project_root: "/tmp/subc-test".to_string(),
5082            warnings: Vec::new(),
5083        })
5084    }
5085
5086    pub(super) fn route_identity(root: &ProjectRootId, session_id: &str) -> RouteIdentity {
5087        route_identity_with_trust(root, session_id, BindTrust::FirstParty)
5088    }
5089
5090    pub(super) fn route_identity_with_trust(
5091        root: &ProjectRootId,
5092        session_id: &str,
5093        trust: BindTrust,
5094    ) -> RouteIdentity {
5095        RouteIdentity(Arc::new(RouteIdentityData {
5096            root: root.clone(),
5097            project_root: root.as_path().to_path_buf(),
5098            harness: "opencode".to_string(),
5099            session: session_id.to_string(),
5100            trust,
5101            spawn_principal: AuthenticatedPrincipal::RouteBind {
5102                trust: trust.sandbox_trust(),
5103                route_channel: 0,
5104                route_epoch: 0,
5105                project_root: root.as_path().to_path_buf(),
5106                harness: "opencode".to_string(),
5107                session_id: session_id.to_string(),
5108                principal_id: Some(match trust {
5109                    BindTrust::FirstParty => "direct".to_string(),
5110                    BindTrust::Untrusted => "unverified".to_string(),
5111                }),
5112            },
5113            consumer_elicitation_capable: false,
5114        }))
5115    }
5116
5117    pub(super) fn progress_frame(request_id: &str, kind: ProgressKind, chunk: &str) -> PushFrame {
5118        PushFrame::Progress(ProgressFrame::new(request_id, kind, chunk))
5119    }
5120
5121    pub(super) fn status_seq(frame: &PushFrame) -> Option<u64> {
5122        match frame {
5123            PushFrame::StatusChanged(status) => status.snapshot.get("seq").and_then(|v| v.as_u64()),
5124            _ => None,
5125        }
5126    }
5127
5128    pub(super) fn completion_task(frame: &PushFrame) -> Option<&str> {
5129        match frame {
5130            PushFrame::BashCompleted(completion) => Some(completion.task_id.as_str()),
5131            _ => None,
5132        }
5133    }
5134
5135    pub(super) fn push_frame_task_id(frame: &Frame) -> Option<String> {
5136        let body: serde_json::Value = serde_json::from_slice(&frame.body).expect("push body");
5137        body.get("task_id")
5138            .and_then(serde_json::Value::as_str)
5139            .map(str::to_string)
5140    }
5141}
5142
5143#[cfg(test)]
5144mod tests {
5145    use super::test_support::{
5146        completion_frame, reap_until_forgotten, route_identity, test_ctx, test_root,
5147        wait_for_actor_root_count, wait_for_watcher_count,
5148    };
5149    use super::*;
5150    use crate::bash_background::BgTaskStatus;
5151
5152    fn attach_error(kind: io::ErrorKind) -> SubcError {
5153        SubcError::Connect {
5154            endpoint: "127.0.0.1:1".to_string(),
5155            source: io::Error::new(kind, "constructed attach failure"),
5156        }
5157    }
5158
5159    fn auth_io_error(kind: io::ErrorKind) -> SubcError {
5160        SubcError::Auth {
5161            endpoint: "127.0.0.1:1".to_string(),
5162            source: subc_transport::AuthError::Io {
5163                stage: subc_transport::AuthStage::ServerProof,
5164                source: io::Error::new(kind, "constructed auth failure"),
5165            },
5166        }
5167    }
5168
5169    #[test]
5170    fn initial_attach_error_classifier_distinguishes_transient_and_permanent_failures() {
5171        let transient_errors = vec![
5172            attach_error(io::ErrorKind::ConnectionRefused),
5173            attach_error(io::ErrorKind::TimedOut),
5174            attach_error(io::ErrorKind::ConnectionReset),
5175            auth_io_error(io::ErrorKind::ConnectionAborted),
5176            auth_io_error(io::ErrorKind::BrokenPipe),
5177            SubcError::Auth {
5178                endpoint: "127.0.0.1:1".to_string(),
5179                source: subc_transport::AuthError::UnexpectedEof {
5180                    stage: subc_transport::AuthStage::ServerProof,
5181                    expected: 4,
5182                    actual: 0,
5183                },
5184            },
5185            SubcError::Auth {
5186                endpoint: "127.0.0.1:1".to_string(),
5187                source: subc_transport::AuthError::Timeout {
5188                    stage: subc_transport::AuthStage::ServerProof,
5189                    deadline: AUTH_DEADLINE,
5190                },
5191            },
5192        ];
5193        for error in &transient_errors {
5194            assert_eq!(
5195                classify_attach_error(error),
5196                AttachErrorClass::Transient,
5197                "expected transient: {error}"
5198            );
5199        }
5200
5201        let permanent_errors = vec![
5202            attach_error(io::ErrorKind::PermissionDenied),
5203            auth_io_error(io::ErrorKind::InvalidData),
5204            SubcError::Auth {
5205                endpoint: "127.0.0.1:1".to_string(),
5206                source: subc_transport::AuthError::InvalidServerProof,
5207            },
5208            SubcError::Auth {
5209                endpoint: "127.0.0.1:1".to_string(),
5210                source: subc_transport::AuthError::DaemonIdMismatch,
5211            },
5212            SubcError::ConnectionFile {
5213                path: PathBuf::from("subc-connection.json"),
5214                source: subc_transport::ConnectionFileError::Invalid {
5215                    reason: "constructed invalid file".to_string(),
5216                },
5217            },
5218            SubcError::NoEndpoint {
5219                path: PathBuf::from("subc-connection.json"),
5220            },
5221            SubcError::InvalidEndpoint {
5222                path: PathBuf::from("subc-connection.json"),
5223                endpoint: "not-an-ip:1234".to_string(),
5224            },
5225        ];
5226        for error in &permanent_errors {
5227            assert_eq!(
5228                classify_attach_error(error),
5229                AttachErrorClass::Permanent,
5230                "expected permanent: {error}"
5231            );
5232        }
5233    }
5234
5235    #[test]
5236    fn incompatible_wire_version_is_rejected_before_tcp_connect() {
5237        let conn_dir = tempfile::tempdir().expect("connection tempdir");
5238        let conn_path = conn_dir.path().join("subc-connection.json");
5239        let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind listener");
5240        listener
5241            .set_nonblocking(true)
5242            .expect("set listener nonblocking");
5243        let port = listener.local_addr().expect("listener addr").port();
5244        connection_file::write_atomic(
5245            &conn_path,
5246            &connection_file::ConnectionInfo {
5247                schema: connection_file::SCHEMA_VERSION,
5248                wire_version: Some(PROTOCOL_VERSION.wrapping_add(1)),
5249                endpoints: vec![connection_file::Endpoint {
5250                    host: "127.0.0.1".to_string(),
5251                    port,
5252                }],
5253                key: vec![0x42; subc_transport::KEY_LEN],
5254                daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
5255                pid: std::process::id(),
5256                daemon_ver: "subc-test".to_string(),
5257            },
5258        )
5259        .expect("write connection file");
5260
5261        let runtime = tokio::runtime::Builder::new_current_thread()
5262            .enable_all()
5263            .build()
5264            .expect("test runtime");
5265        let result = runtime.block_on(connect_and_authenticate_with_policy(
5266            &conn_path,
5267            AttachRetryPolicy {
5268                budget: Duration::from_secs(1),
5269                initial_backoff: Duration::from_millis(5),
5270                max_backoff: Duration::from_millis(10),
5271                jitter_percent: 0,
5272            },
5273        ));
5274        assert!(matches!(
5275            result,
5276            Err(SubcError::ConnectionFile {
5277                source: connection_file::ConnectionFileError::WireVersionMismatch { .. },
5278                ..
5279            })
5280        ));
5281        assert!(matches!(
5282            listener.accept(),
5283            Err(error) if error.kind() == std::io::ErrorKind::WouldBlock
5284        ));
5285    }
5286
5287    #[test]
5288    fn initial_attach_unreachable_endpoint_retries_until_budget_then_fails_loud() {
5289        let conn_dir = tempfile::tempdir().expect("connection tempdir");
5290        let conn_path = conn_dir.path().join("subc-connection.json");
5291        let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve port");
5292        let port = listener.local_addr().expect("reserved addr").port();
5293        drop(listener);
5294        connection_file::write_atomic(
5295            &conn_path,
5296            &connection_file::ConnectionInfo {
5297                schema: connection_file::SCHEMA_VERSION,
5298                wire_version: Some(PROTOCOL_VERSION),
5299                endpoints: vec![connection_file::Endpoint {
5300                    host: "127.0.0.1".to_string(),
5301                    port,
5302                }],
5303                key: vec![0x42; subc_transport::KEY_LEN],
5304                daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
5305                pid: std::process::id(),
5306                daemon_ver: "subc-test".to_string(),
5307            },
5308        )
5309        .expect("write connection file");
5310
5311        let policy = AttachRetryPolicy {
5312            budget: Duration::from_millis(40),
5313            initial_backoff: Duration::from_millis(5),
5314            max_backoff: Duration::from_millis(10),
5315            jitter_percent: 0,
5316        };
5317        let runtime = tokio::runtime::Builder::new_current_thread()
5318            .enable_all()
5319            .build()
5320            .expect("test runtime");
5321        let started_at = Instant::now();
5322        let result = runtime.block_on(connect_and_authenticate_with_policy(&conn_path, policy));
5323        let elapsed = started_at.elapsed();
5324        let error = match result {
5325            Ok(_) => panic!("unreachable endpoint unexpectedly attached"),
5326            Err(error) => error,
5327        };
5328
5329        assert!(matches!(error, SubcError::Connect { .. }), "{error}");
5330        assert!(
5331            elapsed >= Duration::from_millis(35),
5332            "retry budget ended too early: {elapsed:?}"
5333        );
5334        assert!(
5335            elapsed < Duration::from_secs(1),
5336            "retry budget was not bounded: {elapsed:?}"
5337        );
5338    }
5339
5340    fn due_maintenance_jobs_without_actor_context(
5341        live_roots: &mut HashMap<ProjectRootId, RootMeta>,
5342        budget: usize,
5343        pending_bind_roots: &HashSet<ProjectRootId>,
5344    ) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
5345        due_maintenance_jobs(
5346            live_roots,
5347            None,
5348            &HashMap::new(),
5349            &HashSet::new(),
5350            budget,
5351            pending_bind_roots,
5352        )
5353    }
5354
5355    fn actor_ctx_with_dirty_search_index(
5356        root: &Path,
5357        storage: &Path,
5358        file_name: &str,
5359        old_contents: &str,
5360        new_contents: &str,
5361    ) -> (Arc<AppContext>, PathBuf, PathBuf) {
5362        let file = root.join(file_name);
5363        std::fs::write(&file, old_contents).expect("write source");
5364        let canonical_root = std::fs::canonicalize(root).expect("canonical root");
5365        let ctx = Arc::new(AppContext::new(
5366            Box::new(crate::parser::TreeSitterProvider::new()),
5367            Config {
5368                project_root: Some(root.to_path_buf()),
5369                storage_dir: Some(storage.to_path_buf()),
5370                ..Config::default()
5371            },
5372        ));
5373        ctx.set_canonical_cache_root(canonical_root.clone());
5374
5375        let cache_dir = crate::search_index::resolve_cache_dir(&canonical_root, Some(storage));
5376        let mut index = crate::search_index::SearchIndex::build(&canonical_root);
5377        let git_head = index.stored_git_head().map(str::to_owned);
5378        index.write_to_disk(&cache_dir, git_head.as_deref());
5379
5380        std::fs::write(&file, new_contents).expect("edit source");
5381        index.update_file(&file);
5382        *ctx.search_index()
5383            .write()
5384            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
5385        (ctx, canonical_root, cache_dir)
5386    }
5387
5388    #[test]
5389    fn graceful_shutdown_flushes_every_actor_search_index() {
5390        let storage = tempfile::tempdir().expect("storage tempdir");
5391        let (root1_dir, root1) = test_root("shutdown-flush-root-1");
5392        let (root2_dir, root2) = test_root("shutdown-flush-root-2");
5393        let (ctx1, canonical_root1, cache_dir1) = actor_ctx_with_dirty_search_index(
5394            root1_dir.path(),
5395            storage.path(),
5396            "alpha.txt",
5397            "old actor one token\n",
5398            "new actor one token\n",
5399        );
5400        let (ctx2, canonical_root2, cache_dir2) = actor_ctx_with_dirty_search_index(
5401            root2_dir.path(),
5402            storage.path(),
5403            "beta.txt",
5404            "old actor two token\n",
5405            "new actor two token\n",
5406        );
5407
5408        let executor = Executor::new();
5409        assert!(executor.register_actor(root1.clone(), Arc::clone(&ctx1)));
5410        assert!(executor.register_actor(root2.clone(), Arc::clone(&ctx2)));
5411
5412        flush_actor_indexes_on_graceful_shutdown(&executor.actor_contexts());
5413
5414        let mut restored1 =
5415            crate::search_index::SearchIndex::read_from_disk(&cache_dir1, &canonical_root1)
5416                .expect("load flushed root one index");
5417        restored1.ready = true;
5418        assert_eq!(
5419            restored1
5420                .grep("new actor one token", true, &[], &[], &canonical_root1, 10)
5421                .matches
5422                .len(),
5423            1,
5424            "graceful subc shutdown should flush the first root's trigram delta"
5425        );
5426
5427        let mut restored2 =
5428            crate::search_index::SearchIndex::read_from_disk(&cache_dir2, &canonical_root2)
5429                .expect("load flushed root two index");
5430        restored2.ready = true;
5431        assert_eq!(
5432            restored2
5433                .grep("new actor two token", true, &[], &[], &canonical_root2, 10)
5434                .matches
5435                .len(),
5436            1,
5437            "graceful subc shutdown should flush every registered root"
5438        );
5439    }
5440
5441    #[test]
5442    fn idle_root_reaper_closes_artifacts_and_stops_watcher() {
5443        let _ = env_logger::builder().is_test(true).try_init();
5444        let (root_dir, root) = test_root("idle-root-reaper");
5445        let storage = tempfile::tempdir().expect("storage tempdir");
5446        std::fs::write(
5447            root_dir.path().join("main.rs"),
5448            "fn entry() { leaf(); }\nfn leaf() {}\n",
5449        )
5450        .expect("source file");
5451        let canonical_root = std::fs::canonicalize(root_dir.path()).expect("canonical root");
5452        let app = App::default_shared();
5453        let ctx = Arc::new(AppContext::from_app(
5454            Arc::clone(&app),
5455            Config {
5456                project_root: Some(canonical_root.clone()),
5457                storage_dir: Some(storage.path().to_path_buf()),
5458                callgraph_store: true,
5459                search_index: true,
5460                ..Config::default()
5461            },
5462        ));
5463        ctx.set_canonical_cache_root(canonical_root.clone());
5464        let project_key = crate::search_index::artifact_cache_key(&canonical_root);
5465        crate::root_cache::configure_artifact_access(&canonical_root, &project_key, false);
5466        assert!(ctx
5467            .ensure_callgraph_store()
5468            .expect("build callgraph store")
5469            .is_some());
5470
5471        let cache_dir =
5472            crate::search_index::resolve_cache_dir(&canonical_root, Some(storage.path()));
5473        let mut index = crate::search_index::SearchIndex::build(&canonical_root);
5474        let git_head = index.stored_git_head().map(str::to_owned);
5475        index.write_to_disk(&cache_dir, git_head.as_deref());
5476        *ctx.search_index()
5477            .write()
5478            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
5479        // Seed a completed warm verification so the test can prove eviction
5480        // downgrades it to Strict rather than passing vacuously.
5481        let seeded_generation =
5482            crate::cache_freshness::artifact_generation(&cache_dir.join("cache.bin"))
5483                .expect("seeded artifact generation");
5484        crate::cache_freshness::record_verify_completed(
5485            &canonical_root,
5486            crate::cache_freshness::VerifyArtifact::Search,
5487            Some(seeded_generation),
5488        );
5489        assert!(
5490            matches!(
5491                crate::cache_freshness::warm_verify_plan(
5492                    canonical_root.as_path(),
5493                    crate::cache_freshness::VerifyArtifact::Search,
5494                    Some(seeded_generation),
5495                ),
5496                crate::cache_freshness::WarmVerifyPlan::Skip
5497            ),
5498            "memo must be warm before eviction for the downgrade assertion to bite"
5499        );
5500
5501        let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
5502        let _dispatch_tx = dispatch_tx;
5503        let shutdown = Arc::new(AtomicBool::new(false));
5504        let thread_shutdown = Arc::clone(&shutdown);
5505        let join = std::thread::spawn(move || {
5506            while !thread_shutdown.load(Ordering::SeqCst) {
5507                std::thread::yield_now();
5508            }
5509        });
5510        ctx.install_watcher_runtime(
5511            dispatch_rx,
5512            crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
5513        );
5514        wait_for_watcher_count(&ctx, 1);
5515
5516        let executor = Arc::new(Executor::new());
5517        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5518        ctx.mark_subc_unbound();
5519        let mut live_roots = HashMap::new();
5520        let mut meta = RootMeta::new(Instant::now());
5521        meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
5522        meta.unbound_quiesced = true;
5523        live_roots.insert(root.clone(), meta);
5524
5525        let message = idle_root_eviction_message(&root, &ctx.memory_root_snapshot(), None);
5526        assert!(message.contains("evicted idle root"));
5527        assert!(message.contains("freed ~"));
5528        assert!(message.contains("semantic"));
5529        assert!(!message.contains("semantic not estimated retained"));
5530        assert!(message.contains("trigram"));
5531        assert!(message.contains("retained: bash"));
5532        assert!(message.contains("parser_pool"));
5533
5534        assert_eq!(
5535            reap_idle_roots(
5536                Instant::now(),
5537                &mut live_roots,
5538                &HashMap::new(),
5539                &HashMap::new(),
5540                &executor,
5541                &DispatchPathMetrics::new(),
5542            )
5543            .evicted,
5544            1
5545        );
5546        assert!(ctx.search_index().read().unwrap().is_none());
5547        wait_for_watcher_count(&ctx, 0);
5548        // The watcher stopped with the eviction, so the idle interval is
5549        // unobserved: the pre-seeded warm-verify memo (Skip) must fall back to
5550        // strict content verification (stat-first would miss same-size,
5551        // preserved-mtime edits made while nobody was watching).
5552        assert!(
5553            matches!(
5554                crate::cache_freshness::warm_verify_plan(
5555                    canonical_root.as_path(),
5556                    crate::cache_freshness::VerifyArtifact::Search,
5557                    Some(seeded_generation),
5558                ),
5559                crate::cache_freshness::WarmVerifyPlan::Strict
5560            ),
5561            "idle eviction must force strict re-verification"
5562        );
5563        assert!(
5564            crate::search_index::SearchIndex::read_from_disk(&cache_dir, &canonical_root).is_some()
5565        );
5566        ctx.mark_subc_bound();
5567        assert!(ctx
5568            .ensure_callgraph_store()
5569            .expect("reopen callgraph store")
5570            .is_some());
5571        assert!(live_roots[&root].idle_artifacts_evicted);
5572    }
5573
5574    #[test]
5575    fn idle_root_reaper_applies_ttl_to_unbound_roots() {
5576        let (_root_dir, root) = test_root("idle-root-ttl-gate");
5577        let ctx = test_ctx();
5578        let executor = Arc::new(Executor::new());
5579        assert!(executor.register_actor(root.clone(), ctx));
5580        let ctx = executor.actor_context(&root).expect("actor context");
5581        ctx.mark_subc_unbound();
5582        let now = Instant::now();
5583        let mut meta = RootMeta::new(now);
5584        meta.unbound_quiesced = true;
5585        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5586
5587        // A recently-unbound root stays warm: a transient unbind (host
5588        // restart) must not pay the strict-verify + forced-rebuild teardown
5589        // on the next maintenance sweep.
5590        assert_eq!(
5591            reap_idle_roots(
5592                now,
5593                &mut live_roots,
5594                &HashMap::new(),
5595                &HashMap::new(),
5596                &executor,
5597                &DispatchPathMetrics::new(),
5598            )
5599            .evicted,
5600            0
5601        );
5602        assert!(!live_roots[&root].idle_artifacts_evicted);
5603
5604        // Past the TTL the same unbound root pays the full teardown. The
5605        // pending reconciliation paths retained across the transient-unbind
5606        // window would block eviction forever through
5607        // `artifact_eviction_blocked`; the reaper disposes them because the
5608        // strict gap invalidation subsumes their purpose.
5609        ctx.add_pending_search_index_paths([root.as_path().join("retained.rs")]);
5610        assert_eq!(
5611            reap_idle_roots(
5612                now + IDLE_ROOT_TTL,
5613                &mut live_roots,
5614                &HashMap::new(),
5615                &HashMap::new(),
5616                &executor,
5617                &DispatchPathMetrics::new(),
5618            )
5619            .evicted,
5620            1
5621        );
5622        assert!(live_roots[&root].idle_artifacts_evicted);
5623        assert!(
5624            ctx.take_pending_search_index_paths().is_empty(),
5625            "TTL eviction must dispose retained pending reconciliation paths"
5626        );
5627    }
5628
5629    #[test]
5630    fn blocked_ttl_eviction_restores_taken_pending_reconciliation_state() {
5631        let (_root_dir, root) = test_root("ttl-eviction-blocked-restore");
5632        let ctx = test_ctx();
5633        let executor = Arc::new(Executor::new());
5634        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5635        ctx.mark_subc_unbound();
5636
5637        // Retained pending path from the transient-unbind window plus a
5638        // SECONDARY eviction blocker (a non-ready resident search index, the
5639        // dirty-index blocker in artifact_eviction_blocked). Disposal must be
5640        // transactional: the blocked eviction may be followed by a rebind, and
5641        // the path is the only repair record for its consumed watcher event.
5642        let pending = root.as_path().join("edited-while-unbound.rs");
5643        ctx.add_pending_search_index_paths([pending.clone()]);
5644        let dirty_source = root.as_path().join("dirty.rs");
5645        std::fs::write(&dirty_source, "fn dirty() {}\n").expect("dirty source");
5646        let mut dirty = crate::search_index::SearchIndex::new();
5647        dirty.ready = true;
5648        dirty.update_file(&dirty_source);
5649        *ctx.search_index()
5650            .write()
5651            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(dirty);
5652        assert!(ctx.artifact_eviction_blocked());
5653
5654        let mut live_roots = HashMap::new();
5655        let mut meta = RootMeta::new(Instant::now());
5656        meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
5657        meta.unbound_quiesced = true;
5658        live_roots.insert(root.clone(), meta);
5659
5660        assert_eq!(
5661            reap_idle_roots(
5662                Instant::now(),
5663                &mut live_roots,
5664                &HashMap::new(),
5665                &HashMap::new(),
5666                &executor,
5667                &DispatchPathMetrics::new(),
5668            )
5669            .evicted,
5670            0,
5671            "the dirty index must still block this eviction"
5672        );
5673        assert_eq!(
5674            ctx.take_pending_search_index_paths(),
5675            vec![pending],
5676            "a blocked eviction must restore the taken pending paths"
5677        );
5678    }
5679
5680    #[test]
5681    fn idle_reap_with_bound_route_keeps_watcher_running() {
5682        let (_root_dir, root) = test_root("bound-root-reap-gate");
5683        let ctx = test_ctx();
5684        let executor = Arc::new(Executor::new());
5685        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5686
5687        let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
5688        let _dispatch_tx = dispatch_tx;
5689        let shutdown = Arc::new(AtomicBool::new(false));
5690        let thread_shutdown = Arc::clone(&shutdown);
5691        let join = std::thread::spawn(move || {
5692            while !thread_shutdown.load(Ordering::SeqCst) {
5693                std::thread::yield_now();
5694            }
5695        });
5696        ctx.install_watcher_runtime(
5697            dispatch_rx,
5698            crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
5699        );
5700
5701        let mut meta = RootMeta::new(Instant::now());
5702        meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
5703        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5704        let bound = HashMap::from([(root, HashSet::from([route_key(7, 1)]))]);
5705        assert_eq!(
5706            reap_idle_roots(
5707                Instant::now(),
5708                &mut live_roots,
5709                &HashMap::new(),
5710                &bound,
5711                &executor,
5712                &DispatchPathMetrics::new(),
5713            )
5714            .evicted,
5715            0
5716        );
5717        wait_for_watcher_count(&ctx, 1);
5718        ctx.stop_watcher_runtime_in_background();
5719        wait_for_watcher_count(&ctx, 0);
5720    }
5721
5722    #[test]
5723    fn deleted_root_with_bound_route_is_reclaimed_after_confirmation_and_routes_are_purged() {
5724        let (root_dir, root) = test_root("deleted-bound-root-reap");
5725        let executor = Arc::new(Executor::new());
5726        assert!(executor.register_actor(root.clone(), test_ctx()));
5727        root_dir.close().expect("delete project root");
5728
5729        let route = route_key(19, 3);
5730        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
5731        let cancel_signal = PersistentCancelSignal::new();
5732        let mut routes = HashMap::from([(route, route_identity(&root, "deleted-route"))]);
5733        let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
5734        let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
5735        let mut route_bash_cancels = HashMap::from([(
5736            route,
5737            bash::RouteBashCancel {
5738                token: cancel_signal.clone(),
5739                active_waits: 0,
5740            },
5741        )]);
5742        let metrics = DispatchPathMetrics::new();
5743
5744        let first = reap_idle_roots(
5745            Instant::now(),
5746            &mut live_roots,
5747            &HashMap::new(),
5748            &root_channels,
5749            &executor,
5750            &metrics,
5751        );
5752        assert!(first.forgotten_deleted_roots.is_empty());
5753        assert!(executor.actor_registered(&root));
5754
5755        let mut forgotten = Vec::new();
5756        for _ in 0..100 {
5757            let outcome = reap_idle_roots(
5758                Instant::now(),
5759                &mut live_roots,
5760                &HashMap::new(),
5761                &root_channels,
5762                &executor,
5763                &metrics,
5764            );
5765            if !outcome.forgotten_deleted_roots.is_empty() {
5766                forgotten = outcome.forgotten_deleted_roots;
5767                break;
5768            }
5769            std::thread::sleep(Duration::from_millis(10));
5770        }
5771        assert_eq!(forgotten, vec![root.clone()]);
5772        assert!(!executor.actor_registered(&root));
5773
5774        let mut retry_buffer = HashMap::new();
5775        let mut reclaimed_routes = ReclaimedRoutes::default();
5776        let mut session_identity = HashMap::new();
5777        let mut push_buffer = HashMap::new();
5778        let mut bg_subs = HashMap::from([(
5779            route,
5780            BgSub {
5781                corr: 77,
5782                ver: PROTOCOL_VERSION,
5783                flags: control_flags(),
5784                root: root.clone(),
5785                session: "deleted-route".to_string(),
5786            },
5787        )]);
5788        let mut bg_sub_by_session = HashMap::from([(
5789            (root.clone(), "deleted-route".to_string()),
5790            HashSet::from([route]),
5791        )]);
5792        let mut bg_wake_pending = HashSet::from([route]);
5793        let mut bg_wake_epoch = HashMap::new();
5794        let mut pending_bash_asks = HashMap::new();
5795        let active_tool_calls: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
5796        health::take_bg_observability_logs_for_test();
5797        purge_deleted_root_residents(
5798            &root,
5799            &mut routes,
5800            &mut root_channels,
5801            &mut installed_route_epochs,
5802            &mut route_bash_cancels,
5803            &active_tool_calls,
5804            executor.as_ref(),
5805            &mut retry_buffer,
5806            &mut reclaimed_routes,
5807            &mut session_identity,
5808            &mut push_buffer,
5809            &mut bg_subs,
5810            &mut bg_sub_by_session,
5811            &mut bg_wake_pending,
5812            &mut bg_wake_epoch,
5813            &mut pending_bash_asks,
5814            &metrics,
5815        );
5816
5817        assert!(routes.is_empty());
5818        assert!(root_channels.is_empty());
5819        assert!(installed_route_epochs.is_empty());
5820        assert!(route_bash_cancels.is_empty());
5821        assert!(reclaimed_routes.contains(route));
5822        assert!(cancel_signal.is_cancelled());
5823        assert_eq!(
5824            health::take_bg_observability_logs_for_test(),
5825            vec![format!(
5826                "subc bg subscription: ended root={} session=deleted-route channel=19@3 cause=root-reclaim suppressed=0",
5827                root.as_path().display()
5828            )]
5829        );
5830    }
5831
5832    /// The control for the deleted-root reclamation above: a root whose
5833    /// directory still EXISTS must stay retained while it holds a bound route,
5834    /// even with every other reap precondition satisfied. Without this, the
5835    /// suite cannot tell "reclaim roots that are provably gone" apart from
5836    /// "reap any root that looks idle" — the second would tear down live
5837    /// sessions, and both satisfy the deleted-root tests.
5838    #[test]
5839    fn live_root_with_bound_route_is_never_reclaimed() {
5840        let (_root_dir, root) = test_root("live-bound-root-retained");
5841        let ctx = test_ctx();
5842        ctx.mark_subc_unbound();
5843        let executor = Arc::new(Executor::new());
5844        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5845
5846        let route = RouteChannel {
5847            channel: 7,
5848            epoch: 1,
5849        };
5850        let mut meta = RootMeta::new(Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1));
5851        meta.unbound_quiesced = true;
5852        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5853        let root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
5854
5855        // Sweep three times because reclamation requires the root path to be
5856        // observed missing on two CONSECUTIVE sweeps. A single sweep would pass
5857        // here even if reclamation were wrongly unconditional, since the first
5858        // absence never reclaims on its own.
5859        for _ in 0..3 {
5860            let outcome = reap_idle_roots(
5861                Instant::now(),
5862                &mut live_roots,
5863                &HashMap::new(),
5864                &root_channels,
5865                &executor,
5866                &DispatchPathMetrics::new(),
5867            );
5868            assert!(
5869                outcome.forgotten_deleted_roots.is_empty(),
5870                "a root whose directory exists must never be forgotten"
5871            );
5872        }
5873
5874        assert!(live_roots.contains_key(&root), "live root must be retained");
5875        assert!(
5876            executor.actor_registered(&root),
5877            "live root's actor must survive"
5878        );
5879        assert!(
5880            root.as_path().exists(),
5881            "test vehicle must keep the directory alive; otherwise this control proves nothing"
5882        );
5883    }
5884
5885    #[test]
5886    fn deleted_root_is_not_reclaimed_on_first_absence_observation() {
5887        let (root_dir, root) = test_root("deleted-root-first-observation");
5888        let ctx = test_ctx();
5889        ctx.mark_subc_unbound();
5890        let executor = Arc::new(Executor::new());
5891        assert!(executor.register_actor(root.clone(), ctx));
5892        root_dir.close().expect("delete project root");
5893
5894        let mut meta = RootMeta::new(Instant::now());
5895        meta.unbound_quiesced = true;
5896        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5897        let outcome = reap_idle_roots(
5898            Instant::now(),
5899            &mut live_roots,
5900            &HashMap::new(),
5901            &HashMap::new(),
5902            &executor,
5903            &DispatchPathMetrics::new(),
5904        );
5905
5906        assert!(outcome.forgotten_deleted_roots.is_empty());
5907        assert!(live_roots.contains_key(&root));
5908        assert!(executor.actor_registered(&root));
5909    }
5910
5911    fn spawn_background_for_root(
5912        ctx: &AppContext,
5913        root: &ProjectRootId,
5914        storage: &tempfile::TempDir,
5915        session_id: &str,
5916    ) -> (String, u32) {
5917        // Windows refuses to delete a directory that is a running process's
5918        // cwd (ERROR_SHARING_VIOLATION), so the task must not live inside the
5919        // project root these tests delete. The kill path matches on the task's
5920        // registered project_root, not its cwd, so pointing the cwd at task
5921        // storage keeps the association under test intact.
5922        let command = if cfg!(windows) {
5923            // timeout.exe requires a console; ping is the standard sleep shim.
5924            "ping -n 31 127.0.0.1 > nul"
5925        } else {
5926            "sleep 30"
5927        };
5928        let task_id = ctx
5929            .bash_background()
5930            .spawn(
5931                crate::sandbox_spawn::SpawnPlan::Unsandboxed,
5932                command,
5933                session_id.to_string(),
5934                storage.path().to_path_buf(),
5935                HashMap::new(),
5936                Some(Duration::from_secs(60)),
5937                storage.path().to_path_buf(),
5938                8,
5939                true,
5940                false,
5941                Some(root.as_path().to_path_buf()),
5942            )
5943            .expect("spawn background task");
5944        let snapshot = ctx
5945            .bash_background()
5946            .status(
5947                &task_id,
5948                session_id,
5949                Some(root.as_path()),
5950                Some(storage.path()),
5951                0,
5952            )
5953            .expect("background task status");
5954        (task_id, snapshot.child_pid.expect("background child pid"))
5955    }
5956
5957    fn wait_for_background_exit(pid: u32) {
5958        let deadline = Instant::now() + Duration::from_secs(5);
5959        while crate::bash_background::process::is_process_alive(pid) {
5960            assert!(
5961                Instant::now() < deadline,
5962                "background task process survived kill"
5963            );
5964            std::thread::sleep(Duration::from_millis(20));
5965        }
5966    }
5967
5968    #[test]
5969    fn deleted_root_reclaims_background_task_after_two_absence_sweeps() {
5970        let (root_dir, root) = test_root("deleted-root-background-task");
5971        let storage = tempfile::tempdir().expect("task storage");
5972        let ctx = test_ctx();
5973        let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "reclaim-session");
5974        assert!(crate::bash_background::process::is_process_alive(pid));
5975
5976        let executor = Arc::new(Executor::new());
5977        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5978        assert!(executor.actor_is_idle(&root));
5979        root_dir.close().expect("delete project root");
5980        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
5981        let pending_binds = HashMap::new();
5982        let root_channels = HashMap::new();
5983        let metrics = DispatchPathMetrics::new();
5984
5985        let first = reap_idle_roots(
5986            Instant::now(),
5987            &mut live_roots,
5988            &pending_binds,
5989            &root_channels,
5990            &executor,
5991            &metrics,
5992        );
5993        assert!(first.forgotten_deleted_roots.is_empty());
5994        assert!(crate::bash_background::process::is_process_alive(pid));
5995
5996        let outcome = reap_until_forgotten(
5997            &root,
5998            &mut live_roots,
5999            &pending_binds,
6000            &root_channels,
6001            &executor,
6002            &metrics,
6003        );
6004        assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
6005        wait_for_background_exit(pid);
6006
6007        let snapshot = ctx
6008            .bash_background()
6009            .status(
6010                &task_id,
6011                "reclaim-session",
6012                Some(root.as_path()),
6013                Some(storage.path()),
6014                0,
6015            )
6016            .expect("reclaimed task status");
6017        assert_eq!(snapshot.info.status, BgTaskStatus::Killed);
6018        assert_eq!(
6019            snapshot.info.status_reason.as_deref(),
6020            Some(crate::bash_background::registry::ROOT_RECLAIMED_REASON)
6021        );
6022        assert_eq!(
6023            serde_json::to_value(&snapshot).expect("serialize bash status")["status_reason"],
6024            crate::bash_background::registry::ROOT_RECLAIMED_REASON
6025        );
6026        let completion = ctx
6027            .bash_background()
6028            .drain_completions_for_session(Some("reclaim-session"))
6029            .pop()
6030            .expect("reclaimed task completion");
6031        assert_eq!(
6032            completion.status_reason.as_deref(),
6033            Some(crate::bash_background::registry::ROOT_RECLAIMED_REASON)
6034        );
6035    }
6036
6037    #[test]
6038    fn existing_unbound_root_keeps_background_task_alive_across_sweeps() {
6039        let (root_dir, root) = test_root("existing-root-background-task");
6040        let storage = tempfile::tempdir().expect("task storage");
6041        let ctx = test_ctx();
6042        ctx.mark_subc_unbound();
6043        let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "existing-session");
6044
6045        let executor = Arc::new(Executor::new());
6046        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6047        let mut meta = RootMeta::new(
6048            Instant::now()
6049                .checked_sub(IDLE_ROOT_TTL + Duration::from_secs(1))
6050                .expect("old root timestamp"),
6051        );
6052        meta.unbound_quiesced = true;
6053        let mut live_roots = HashMap::from([(root.clone(), meta)]);
6054        let pending_binds = HashMap::new();
6055        let root_channels = HashMap::new();
6056        let metrics = DispatchPathMetrics::new();
6057
6058        for _ in 0..8 {
6059            reap_idle_roots(
6060                Instant::now(),
6061                &mut live_roots,
6062                &pending_binds,
6063                &root_channels,
6064                &executor,
6065                &metrics,
6066            );
6067            std::thread::sleep(Duration::from_millis(10));
6068        }
6069        assert!(root_dir.path().exists());
6070        assert!(crate::bash_background::process::is_process_alive(pid));
6071        let snapshot = ctx
6072            .bash_background()
6073            .status(
6074                &task_id,
6075                "existing-session",
6076                Some(root.as_path()),
6077                Some(storage.path()),
6078                0,
6079            )
6080            .expect("existing task status");
6081        assert_eq!(snapshot.info.status, BgTaskStatus::Running);
6082        let _ = ctx.bash_background().kill(&task_id, "existing-session");
6083        wait_for_background_exit(pid);
6084    }
6085
6086    #[test]
6087    fn restored_root_between_absence_sweeps_keeps_background_task_alive() {
6088        let (root_dir, root) = test_root("restored-root-background-task");
6089        let storage = tempfile::tempdir().expect("task storage");
6090        let ctx = test_ctx();
6091        let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "restored-session");
6092
6093        let executor = Arc::new(Executor::new());
6094        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6095        root_dir.close().expect("delete project root");
6096        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
6097        let pending_binds = HashMap::new();
6098        let root_channels = HashMap::new();
6099        let metrics = DispatchPathMetrics::new();
6100
6101        let first = reap_idle_roots(
6102            Instant::now(),
6103            &mut live_roots,
6104            &pending_binds,
6105            &root_channels,
6106            &executor,
6107            &metrics,
6108        );
6109        assert!(first.forgotten_deleted_roots.is_empty());
6110        std::fs::create_dir_all(root.as_path()).expect("restore project root");
6111        let second = reap_idle_roots(
6112            Instant::now(),
6113            &mut live_roots,
6114            &pending_binds,
6115            &root_channels,
6116            &executor,
6117            &metrics,
6118        );
6119        assert!(second.forgotten_deleted_roots.is_empty());
6120        assert!(crate::bash_background::process::is_process_alive(pid));
6121        let _ = ctx.bash_background().kill(&task_id, "restored-session");
6122        wait_for_background_exit(pid);
6123    }
6124
6125    #[test]
6126    fn observing_root_again_resets_deleted_sweep_confirmation() {
6127        let (root_dir, root) = test_root("deleted-root-observation-reset");
6128        let ctx = test_ctx();
6129        ctx.mark_subc_unbound();
6130        let executor = Arc::new(Executor::new());
6131        assert!(executor.register_actor(root.clone(), ctx));
6132        root_dir.close().expect("delete project root");
6133
6134        let mut meta = RootMeta::new(Instant::now());
6135        meta.unbound_quiesced = true;
6136        let mut live_roots = HashMap::from([(root.clone(), meta)]);
6137        let pending_binds = HashMap::new();
6138        let root_channels = HashMap::new();
6139        let metrics = DispatchPathMetrics::new();
6140
6141        let first = reap_idle_roots(
6142            Instant::now(),
6143            &mut live_roots,
6144            &pending_binds,
6145            &root_channels,
6146            &executor,
6147            &metrics,
6148        );
6149        assert!(first.forgotten_deleted_roots.is_empty());
6150
6151        std::fs::create_dir_all(root.as_path()).expect("restore project root");
6152        reap_idle_roots(
6153            Instant::now(),
6154            &mut live_roots,
6155            &pending_binds,
6156            &root_channels,
6157            &executor,
6158            &metrics,
6159        );
6160        std::fs::remove_dir_all(root.as_path()).expect("delete project root again");
6161
6162        let after_reset = reap_idle_roots(
6163            Instant::now(),
6164            &mut live_roots,
6165            &pending_binds,
6166            &root_channels,
6167            &executor,
6168            &metrics,
6169        );
6170        assert!(after_reset.forgotten_deleted_roots.is_empty());
6171        assert!(live_roots.contains_key(&root));
6172        assert!(executor.actor_registered(&root));
6173    }
6174
6175    #[test]
6176    fn deleted_idle_root_is_fully_forgotten_and_status_counts_drop() {
6177        let (root_dir, root) = test_root("deleted-root-reap");
6178        let app = App::default_shared();
6179        let ctx = Arc::new(AppContext::from_app(
6180            Arc::clone(&app),
6181            Config {
6182                project_root: Some(root.as_path().to_path_buf()),
6183                ..Config::default()
6184            },
6185        ));
6186        ctx.set_canonical_cache_root(root.as_path().to_path_buf());
6187        ctx.mark_subc_unbound();
6188        let executor = Arc::new(Executor::new());
6189        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6190        assert_eq!(app.actor_root_count(), 1);
6191        drop(ctx);
6192        root_dir.close().expect("delete project root");
6193
6194        let mut meta = RootMeta::new(Instant::now());
6195        meta.unbound_quiesced = true;
6196        let mut live_roots = HashMap::from([(root.clone(), meta)]);
6197        let outcome = reap_until_forgotten(
6198            &root,
6199            &mut live_roots,
6200            &HashMap::new(),
6201            &HashMap::new(),
6202            &executor,
6203            &DispatchPathMetrics::new(),
6204        );
6205        assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
6206        assert!(!executor.actor_registered(&root));
6207        assert!(!live_roots.contains_key(&root));
6208        wait_for_actor_root_count(&app, 0);
6209
6210        let status_ctx = AppContext::from_app(app, Config::default());
6211        let status = status_ctx.build_status_snapshot();
6212        assert_eq!(status["runtime"]["live_actor_roots"], 0);
6213        assert_eq!(status["runtime"]["open_routes"], 0);
6214    }
6215
6216    #[test]
6217    fn deleted_root_reap_blocker_census_is_exposed_in_health_metrics() {
6218        let (root_dir, root) = test_root("deleted-root-reap-census");
6219        let executor = Arc::new(Executor::new());
6220        assert!(executor.register_actor(root.clone(), test_ctx()));
6221        root_dir.close().expect("delete project root");
6222
6223        let mut live_roots = HashMap::from([(root, RootMeta::new(Instant::now()))]);
6224        let metrics = DispatchPathMetrics::new();
6225        let outcome = reap_idle_roots(
6226            Instant::now(),
6227            &mut live_roots,
6228            &HashMap::new(),
6229            &HashMap::new(),
6230            &executor,
6231            &metrics,
6232        );
6233        assert_eq!(outcome.evicted, 0);
6234
6235        let app = crate::context::App::default_shared();
6236        let health_rollup_cache = HealthRollupCache::new();
6237        health_rollup_cache.refresh(&executor, &app);
6238        let report = build_health_report(
6239            &health_rollup_cache,
6240            &executor,
6241            &HashMap::new(),
6242            &metrics,
6243            &app,
6244        );
6245        let reap = report
6246            .metrics
6247            .as_ref()
6248            .and_then(|metrics| metrics.get("reap"))
6249            .expect("reap health metrics");
6250        assert_eq!(reap["deleted_retained"].as_u64(), Some(1));
6251        assert_eq!(reap["blockers"]["absence_unconfirmed"].as_u64(), Some(1));
6252        assert_eq!(reap["blockers"]["unbound_quiesced"].as_u64(), Some(0));
6253        assert_eq!(reap["blockers"]["actor_busy"].as_u64(), Some(0));
6254    }
6255
6256    #[test]
6257    fn connection_exit_quiesces_queued_maintenance_and_deleted_root_is_purged() {
6258        let (root_dir, root) = test_root("connection-exit-deleted-root");
6259        let executor = Arc::new(Executor::new());
6260        assert!(executor.register_actor(root.clone(), test_ctx()));
6261
6262        let route = route_key(11, 1);
6263        let mut meta = RootMeta::new(Instant::now());
6264        meta.maintenance_pending = true;
6265        meta.maintenance_queued_kinds
6266            .push_back(MaintenanceDrainKind::CompletionDrains);
6267        let mut live_roots = HashMap::from([(root.clone(), meta)]);
6268        let mut pending_binds = HashMap::new();
6269        let mut routes = HashMap::from([(route, route_identity(&root, "abandoned"))]);
6270        let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
6271        let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
6272        let mut route_bash_cancels = HashMap::new();
6273        let active_tool_calls: ActiveToolCalls = Arc::new(StdMutex::new(HashMap::new()));
6274
6275        quiesce_connection_roots(
6276            &mut live_roots,
6277            &mut pending_binds,
6278            &mut routes,
6279            &mut root_channels,
6280            &mut installed_route_epochs,
6281            &mut route_bash_cancels,
6282            &active_tool_calls,
6283            &executor,
6284        );
6285        assert!(live_roots[&root].unbound_quiesced);
6286        assert!(!live_roots[&root].maintenance_pending);
6287        assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
6288        assert!(routes.is_empty());
6289        assert!(root_channels.is_empty());
6290
6291        root_dir.close().expect("delete project root");
6292        let metrics = DispatchPathMetrics::new();
6293        let outcome = reap_until_forgotten(
6294            &root,
6295            &mut live_roots,
6296            &pending_binds,
6297            &root_channels,
6298            &executor,
6299            &metrics,
6300        );
6301        let mut session_identity = HashMap::new();
6302        let mut push_buffer = HashMap::new();
6303        let mut bg_subs = HashMap::new();
6304        let mut bg_sub_by_session = HashMap::new();
6305        let mut bg_wake_pending = HashSet::new();
6306        let mut bg_wake_epoch = HashMap::new();
6307        let mut pending_bash_asks = HashMap::new();
6308        let mut retry_buffer = HashMap::new();
6309        let mut reclaimed_routes = ReclaimedRoutes::default();
6310        for forgotten in &outcome.forgotten_deleted_roots {
6311            purge_deleted_root_residents(
6312                forgotten,
6313                &mut routes,
6314                &mut root_channels,
6315                &mut installed_route_epochs,
6316                &mut route_bash_cancels,
6317                &active_tool_calls,
6318                executor.as_ref(),
6319                &mut retry_buffer,
6320                &mut reclaimed_routes,
6321                &mut session_identity,
6322                &mut push_buffer,
6323                &mut bg_subs,
6324                &mut bg_sub_by_session,
6325                &mut bg_wake_pending,
6326                &mut bg_wake_epoch,
6327                &mut pending_bash_asks,
6328                &metrics,
6329            );
6330        }
6331
6332        assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
6333        assert!(!executor.actor_registered(&root));
6334        assert!(!live_roots.contains_key(&root));
6335    }
6336
6337    #[test]
6338    fn unbound_root_quiesces_maintenance_without_removing_actor() {
6339        let (_root_dir, root) = test_root("unbound-root-quiesce");
6340        let ctx = test_ctx();
6341        let executor = Arc::new(Executor::new());
6342        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6343        let mut meta = RootMeta::new(Instant::now());
6344        meta.maintenance_pending = true;
6345        meta.maintenance_jobs_in_flight = 1;
6346        meta.maintenance_queued_kinds
6347            .push_back(MaintenanceDrainKind::ConfigureTail);
6348        let mut live_roots = HashMap::from([(root.clone(), meta)]);
6349        // Warm state planted before the unbind: quiesce must keep it.
6350        *ctx.search_index()
6351            .write()
6352            .unwrap_or_else(std::sync::PoisonError::into_inner) =
6353            Some(crate::search_index::SearchIndex::new());
6354        ctx.set_cache_writer_capabilities(true, true);
6355        let pending = root.as_path().join("pending.rs");
6356        ctx.add_pending_search_index_paths([pending.clone()]);
6357        // A warm verify memo must survive the transient unbind: the watcher
6358        // keeps running, so no unobserved window exists and the next warm
6359        // reload must not pay a strict full-corpus re-hash.
6360        let canonical_root = root.as_path().to_path_buf();
6361        let artifact = canonical_root.join("cache.bin");
6362        std::fs::write(&artifact, b"warm-artifact").expect("write artifact");
6363        let seeded_generation = crate::cache_freshness::artifact_generation(&artifact);
6364        crate::cache_freshness::record_verify_completed(
6365            &canonical_root,
6366            crate::cache_freshness::VerifyArtifact::Search,
6367            seeded_generation,
6368        );
6369        assert!(matches!(
6370            crate::cache_freshness::warm_verify_plan(
6371                &canonical_root,
6372                crate::cache_freshness::VerifyArtifact::Search,
6373                seeded_generation,
6374            ),
6375            crate::cache_freshness::WarmVerifyPlan::Skip
6376        ));
6377        // A live watcher runtime must survive quiesce (its events accumulate
6378        // for the rebind replay).
6379        let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
6380        let _dispatch_tx = dispatch_tx;
6381        let shutdown = Arc::new(AtomicBool::new(false));
6382        let thread_shutdown = Arc::clone(&shutdown);
6383        let join = std::thread::spawn(move || {
6384            while !thread_shutdown.load(Ordering::SeqCst) {
6385                std::thread::yield_now();
6386            }
6387        });
6388        ctx.install_watcher_runtime(
6389            dispatch_rx,
6390            crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
6391        );
6392        assert!(ctx.watcher_runtime_active());
6393
6394        quiesce_unbound_root(&root, &mut live_roots, &executor);
6395        let meta = &live_roots[&root];
6396        assert!(meta.unbound_quiesced);
6397        assert!(ctx.subc_unbound_quiesced());
6398        assert!(meta.maintenance_pending);
6399        assert!(meta.maintenance_queued_kinds.is_empty());
6400        assert!(executor.actor_registered(&root));
6401        // Transient unbind keeps the root warm: resident artifacts stay
6402        // resident, no forced callgraph rebuild is planted, and pending
6403        // reconciliation paths survive for the rebind replay.
6404        assert!(
6405            ctx.search_index()
6406                .read()
6407                .unwrap_or_else(std::sync::PoisonError::into_inner)
6408                .is_some(),
6409            "quiesce must not evict resident artifacts"
6410        );
6411        assert_eq!(
6412            ctx.pending_callgraph_store_force_token(),
6413            None,
6414            "quiesce must not force a callgraph rebuild"
6415        );
6416        assert_eq!(
6417            ctx.take_pending_search_index_paths(),
6418            vec![pending],
6419            "quiesce must retain pending watcher-derived paths"
6420        );
6421        assert!(
6422            matches!(
6423                crate::cache_freshness::warm_verify_plan(
6424                    &canonical_root,
6425                    crate::cache_freshness::VerifyArtifact::Search,
6426                    seeded_generation,
6427                ),
6428                crate::cache_freshness::WarmVerifyPlan::Skip
6429            ),
6430            "quiesce must not invalidate the warm verify memo"
6431        );
6432        assert!(
6433            ctx.watcher_runtime_active(),
6434            "quiesce must not stop a running watcher"
6435        );
6436        ctx.stop_watcher_runtime();
6437
6438        let meta = live_roots.get_mut(&root).expect("root metadata");
6439        note_maintenance_completion(
6440            meta,
6441            Some(MaintenanceDrainKind::ConfigureTail),
6442            false,
6443            meta.unbound_quiesced,
6444        );
6445        assert!(!meta.maintenance_pending);
6446        assert!(meta.maintenance_queued_kinds.is_empty());
6447    }
6448
6449    #[test]
6450    fn same_root_higher_epoch_replacement_does_not_quiesce_between_generations() {
6451        let (_dir, root) = test_root("same-root-replacement");
6452        let route = route_key(7, 1);
6453        let installed_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
6454        let root_channels = HashMap::new();
6455
6456        assert!(!route_removal_will_quiesce_root(
6457            &root,
6458            route,
6459            &installed_channels,
6460            false,
6461            Some(&root),
6462        ));
6463        assert!(route_removal_will_quiesce_root(
6464            &root,
6465            route,
6466            &installed_channels,
6467            false,
6468            None,
6469        ));
6470        assert!(!should_quiesce_removed_root(
6471            &root,
6472            &root_channels,
6473            false,
6474            Some(&root),
6475        ));
6476        assert!(should_quiesce_removed_root(
6477            &root,
6478            &root_channels,
6479            false,
6480            None,
6481        ));
6482        assert!(!should_quiesce_removed_root(
6483            &root,
6484            &root_channels,
6485            true,
6486            None,
6487        ));
6488    }
6489
6490    #[test]
6491    fn root_quiesces_only_after_its_last_route_is_removed_and_reactivates_on_bind() {
6492        let (_root_dir, root) = test_root("unbound-root-route-count");
6493        let executor = Arc::new(Executor::new());
6494        assert!(executor.register_actor(root.clone(), test_ctx()));
6495        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
6496        let mut root_channels = HashMap::from([(
6497            root.clone(),
6498            HashSet::from([route_key(7, 1), route_key(8, 1)]),
6499        )]);
6500
6501        remove_root_channel(&mut root_channels, &root, route_key(7, 1));
6502        if !root_channels.contains_key(&root) {
6503            quiesce_unbound_root(&root, &mut live_roots, &executor);
6504        }
6505        assert!(!live_roots[&root].unbound_quiesced);
6506
6507        remove_root_channel(&mut root_channels, &root, route_key(8, 1));
6508        if !root_channels.contains_key(&root) {
6509            quiesce_unbound_root(&root, &mut live_roots, &executor);
6510        }
6511        assert!(live_roots[&root].unbound_quiesced);
6512
6513        live_roots
6514            .get_mut(&root)
6515            .expect("root metadata")
6516            .note_activity();
6517        assert!(
6518            live_roots[&root].unbound_quiesced,
6519            "late asynchronous activity must not reactivate an unbound root"
6520        );
6521
6522        live_roots
6523            .get_mut(&root)
6524            .expect("root metadata")
6525            .reactivate_bound();
6526        assert!(!live_roots[&root].unbound_quiesced);
6527    }
6528
6529    #[test]
6530    fn allocator_pressure_relief_requires_every_root_to_be_idle() {
6531        let (_idle_dir, idle_root) = test_root("allocator-relief-idle");
6532        let (_active_dir, active_root) = test_root("allocator-relief-active");
6533        let now = Instant::now();
6534        let mut live_roots = HashMap::new();
6535        let mut idle = RootMeta::new(now);
6536        idle.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
6537        live_roots.insert(idle_root, idle);
6538        assert!(process_has_been_idle(now, &live_roots));
6539
6540        live_roots.insert(active_root.clone(), RootMeta::new(now));
6541        assert!(!process_has_been_idle(now, &live_roots));
6542
6543        let active = live_roots
6544            .get_mut(&active_root)
6545            .expect("active root metadata");
6546        active.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
6547        active.active_bash_waits = 1;
6548        assert!(!process_has_been_idle(now, &live_roots));
6549    }
6550
6551    #[test]
6552    fn pressure_relief_log_reports_before_and_after_measurements() {
6553        let allocator = crate::memory::AllocatorMemorySnapshot {
6554            status: "measured",
6555            bytes_in_use: Some(8 * 1024 * 1024),
6556            size_allocated: Some(12 * 1024 * 1024),
6557            retained_slack_bytes: Some(4 * 1024 * 1024),
6558            not_estimated: None,
6559        };
6560        let relief = crate::memory::AllocatorPressureRelief {
6561            bytes_released: 3 * 1024 * 1024,
6562            rss_before_bytes: Some(20 * 1024 * 1024),
6563            rss_after_bytes: Some(17 * 1024 * 1024),
6564            allocator_before: allocator.clone(),
6565            allocator_after: crate::memory::AllocatorMemorySnapshot {
6566                size_allocated: Some(9 * 1024 * 1024),
6567                retained_slack_bytes: Some(1024 * 1024),
6568                ..allocator
6569            },
6570        };
6571        let message = pressure_relief_label(&relief);
6572        assert!(message.contains("RSS 20.0 MB -> 17.0 MB"));
6573        assert!(message.contains("allocated 12.0 MB -> 9.0 MB"));
6574        assert!(message.contains("slack 4.0 MB -> 1.0 MB"));
6575        assert!(message.contains("reported 3.0 MB released"));
6576    }
6577
6578    #[test]
6579    fn due_maintenance_jobs_skip_poisoned_roots() {
6580        let (_healthy_dir, healthy_root) = test_root("maintenance-healthy");
6581        let (_poisoned_dir, poisoned_root) = test_root("maintenance-poisoned");
6582        let mut live_roots = HashMap::new();
6583        live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
6584        let mut poisoned_meta = RootMeta::new(Instant::now());
6585        poisoned_meta.maintenance_poisoned = true;
6586        live_roots.insert(poisoned_root.clone(), poisoned_meta);
6587
6588        let (due, deferred) = due_maintenance_jobs_without_actor_context(
6589            &mut live_roots,
6590            MAINTENANCE_SUBMIT_BUDGET,
6591            &HashSet::new(),
6592        );
6593
6594        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6595        assert!(due.iter().all(|(root, _)| root == &healthy_root));
6596        assert!(!deferred);
6597        assert!(live_roots[&healthy_root].maintenance_pending);
6598        assert_eq!(
6599            live_roots[&healthy_root].maintenance_jobs_in_flight,
6600            INITIAL_MAINTENANCE_JOB_COUNT
6601        );
6602        assert!(!live_roots[&poisoned_root].maintenance_pending);
6603    }
6604
6605    #[test]
6606    fn due_maintenance_jobs_do_not_restart_quiesced_root_work() {
6607        let (_dir, root) = test_root("maintenance-unbound");
6608        let mut meta = RootMeta::new(Instant::now());
6609        meta.unbound_quiesced = true;
6610        let mut live_roots = HashMap::from([(root.clone(), meta)]);
6611
6612        let (due, deferred) = due_maintenance_jobs_without_actor_context(
6613            &mut live_roots,
6614            MAINTENANCE_SUBMIT_BUDGET,
6615            &HashSet::new(),
6616        );
6617
6618        assert!(due.is_empty());
6619        assert!(!deferred);
6620        assert!(!live_roots[&root].maintenance_pending);
6621    }
6622
6623    #[test]
6624    fn idle_bg_subscription_queues_no_jobs_until_a_wake_arrives() {
6625        let (_dir, root) = test_root("maintenance-idle-bg-subscription");
6626        let ctx = test_ctx();
6627        assert!(!ctx.completion_drains_have_work());
6628
6629        let executor = Executor::new();
6630        assert!(executor.register_actor(root.clone(), ctx));
6631        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
6632        let session = "idle-session".to_string();
6633        let channel = route_key(17, 1);
6634        let metrics = DispatchPathMetrics::new();
6635        let bg_sub_by_session =
6636            HashMap::from([((root.clone(), session.clone()), HashSet::from([channel]))]);
6637        let mut bg_wake_pending = HashSet::new();
6638
6639        let (idle_tick_jobs, deferred) = due_maintenance_jobs(
6640            &mut live_roots,
6641            Some(&executor),
6642            &bg_sub_by_session,
6643            &bg_wake_pending,
6644            MAINTENANCE_SUBMIT_BUDGET,
6645            &HashSet::new(),
6646        );
6647        assert!(idle_tick_jobs.is_empty());
6648        assert!(!deferred);
6649        assert!(!live_roots[&root].maintenance_pending);
6650
6651        // A completion can arm its wake after the idle tick's probes. The wake
6652        // remains loop-owned state, so the following tick must observe it.
6653        let mut bg_wake_epoch = HashMap::new();
6654        push::arm_bg_wake(
6655            root.clone(),
6656            session,
6657            channel,
6658            &mut bg_wake_pending,
6659            &mut bg_wake_epoch,
6660            &metrics,
6661        );
6662        let (next_tick_jobs, deferred) = due_maintenance_jobs(
6663            &mut live_roots,
6664            Some(&executor),
6665            &bg_sub_by_session,
6666            &bg_wake_pending,
6667            MAINTENANCE_SUBMIT_BUDGET,
6668            &HashSet::new(),
6669        );
6670        assert_eq!(
6671            next_tick_jobs,
6672            vec![(root, MaintenanceDrainKind::CompletionDrains)]
6673        );
6674        assert!(!deferred);
6675    }
6676
6677    #[tokio::test]
6678    async fn subc_configure_tail_precedes_completed_search_install() {
6679        let root_dir = tempfile::tempdir().unwrap();
6680        let storage = tempfile::tempdir().unwrap();
6681        let root = ProjectRootId::from_path(root_dir.path()).unwrap();
6682        let (ctx, ignored_path) =
6683            runtime_drain::configure_search_order_context_for_test(root_dir.path(), storage.path());
6684        let ctx = Arc::new(ctx);
6685        assert!(!runtime_drain::watcher_path_is_ignored_by_current_matcher(
6686            &ctx,
6687            &ignored_path
6688        ));
6689
6690        let executor = Arc::new(Executor::new());
6691        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6692        let metrics = Arc::new(DispatchPathMetrics::new());
6693        let (completion_tx, mut completion_rx) = mpsc::channel(4);
6694        submit_maintenance_job(
6695            &executor,
6696            root.clone(),
6697            MaintenanceDrainKind::ConfigureTail,
6698            Vec::new(),
6699            &completion_tx,
6700            &metrics,
6701        );
6702        submit_maintenance_job(
6703            &executor,
6704            root,
6705            MaintenanceDrainKind::CompletionDrains,
6706            Vec::new(),
6707            &completion_tx,
6708            &metrics,
6709        );
6710
6711        let first = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
6712            .await
6713            .expect("configure-tail completion timed out")
6714            .expect("configure-tail completion channel closed");
6715        let second = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
6716            .await
6717            .expect("completion-drains completion timed out")
6718            .expect("completion-drains completion channel closed");
6719        assert!(first.response.id.contains("configure-tail"));
6720        assert!(second.response.id.contains("completion-drains"));
6721        assert!(runtime_drain::watcher_path_is_ignored_by_current_matcher(
6722            &ctx,
6723            &ignored_path
6724        ));
6725        assert_eq!(
6726            ctx.search_index()
6727                .read()
6728                .unwrap_or_else(std::sync::PoisonError::into_inner)
6729                .as_ref()
6730                .expect("completed search index installed")
6731                .file_count(),
6732            0,
6733            "configure must install the ignore matcher before pending paths replay"
6734        );
6735        ctx.stop_watcher_runtime();
6736    }
6737
6738    #[test]
6739    fn post_bind_configure_and_completion_jobs_are_queued_in_order() {
6740        let (_dir, root) = test_root("maintenance-post-bind");
6741        let mut live_roots = HashMap::new();
6742        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6743
6744        queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
6745        queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
6746
6747        let meta = live_roots.get(&root).expect("root metadata");
6748        assert!(meta.maintenance_pending);
6749        assert_eq!(meta.maintenance_jobs_in_flight, 0);
6750        assert_eq!(
6751            meta.maintenance_queued_kinds
6752                .iter()
6753                .copied()
6754                .collect::<Vec<_>>(),
6755            vec![
6756                MaintenanceDrainKind::ConfigureTail,
6757                MaintenanceDrainKind::CompletionDrains,
6758            ]
6759        );
6760
6761        let (due, deferred) = due_maintenance_jobs_without_actor_context(
6762            &mut live_roots,
6763            MAINTENANCE_SUBMIT_BUDGET,
6764            &HashSet::new(),
6765        );
6766
6767        assert_eq!(
6768            due,
6769            vec![
6770                (root.clone(), MaintenanceDrainKind::ConfigureTail),
6771                (root.clone(), MaintenanceDrainKind::CompletionDrains),
6772            ]
6773        );
6774        assert!(!deferred);
6775        assert_eq!(live_roots[&root].maintenance_jobs_in_flight, 2);
6776        assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
6777    }
6778
6779    #[test]
6780    fn due_maintenance_jobs_defers_unsubmitted_roots_without_marking_pending() {
6781        let mut live_roots = HashMap::new();
6782        let mut root_ids = Vec::new();
6783        let mut _dirs = Vec::new();
6784        for index in 0..4 {
6785            let (dir, root_id) = test_root(&format!("maintenance-budget-{index}"));
6786            live_roots.insert(root_id.clone(), RootMeta::new(Instant::now()));
6787            root_ids.push(root_id);
6788            _dirs.push(dir);
6789        }
6790
6791        let small_budget = INITIAL_MAINTENANCE_JOB_COUNT + 1;
6792        let (first_due, first_deferred) = due_maintenance_jobs_without_actor_context(
6793            &mut live_roots,
6794            small_budget,
6795            &HashSet::new(),
6796        );
6797
6798        assert_eq!(first_due.len(), small_budget);
6799        assert!(first_deferred);
6800        let first_due_set: HashSet<_> = first_due.into_iter().map(|(root, _)| root).collect();
6801        assert!(first_due_set
6802            .iter()
6803            .all(|root| live_roots[root].maintenance_pending));
6804        assert!(first_due_set
6805            .iter()
6806            .any(|root| !live_roots[root].maintenance_queued_kinds.is_empty()));
6807
6808        let all_roots: HashSet<_> = root_ids.into_iter().collect();
6809        let deferred_roots: HashSet<_> = all_roots.difference(&first_due_set).cloned().collect();
6810        assert!(deferred_roots
6811            .iter()
6812            .all(|root| !live_roots[root].maintenance_pending));
6813    }
6814
6815    #[test]
6816    fn due_maintenance_jobs_defers_pending_bind_roots() {
6817        let (_bind_dir, bind_root) = test_root("maintenance-pending-bind");
6818        let (_healthy_dir, healthy_root) = test_root("maintenance-no-bind");
6819        let mut live_roots = HashMap::new();
6820        live_roots.insert(bind_root.clone(), RootMeta::new(Instant::now()));
6821        live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
6822        let pending_bind_roots = HashSet::from([bind_root.clone()]);
6823
6824        let (due, deferred) = due_maintenance_jobs_without_actor_context(
6825            &mut live_roots,
6826            usize::MAX,
6827            &pending_bind_roots,
6828        );
6829
6830        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6831        assert!(due.iter().all(|(root, _)| root == &healthy_root));
6832        assert!(!deferred);
6833        assert!(!live_roots[&bind_root].maintenance_pending);
6834        assert!(live_roots[&bind_root].maintenance_queued_kinds.is_empty());
6835    }
6836
6837    #[test]
6838    fn maintenance_pending_survives_requeue_and_clears_after_final_batch() {
6839        let (_dir, root) = test_root("maintenance-requeue");
6840        let mut live_roots = HashMap::new();
6841        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6842        let (due, deferred) = due_maintenance_jobs_without_actor_context(
6843            &mut live_roots,
6844            usize::MAX,
6845            &HashSet::new(),
6846        );
6847        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6848        assert!(due.iter().all(|(due_root, _)| due_root == &root));
6849        assert!(!deferred);
6850
6851        let meta = live_roots.get_mut(&root).unwrap();
6852        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, false);
6853        assert!(meta.maintenance_pending);
6854        assert_eq!(
6855            meta.maintenance_jobs_in_flight,
6856            INITIAL_MAINTENANCE_JOB_COUNT - 1
6857        );
6858        assert_eq!(meta.maintenance_queued_kinds.len(), 1);
6859
6860        let (requeued, deferred) =
6861            due_maintenance_jobs_without_actor_context(&mut live_roots, 1, &HashSet::new());
6862        assert_eq!(
6863            requeued,
6864            vec![(root.clone(), MaintenanceDrainKind::Watcher)]
6865        );
6866        assert!(!deferred);
6867        let meta = live_roots.get_mut(&root).unwrap();
6868        assert_eq!(
6869            meta.maintenance_jobs_in_flight,
6870            INITIAL_MAINTENANCE_JOB_COUNT
6871        );
6872        assert!(meta.maintenance_queued_kinds.is_empty());
6873
6874        for _ in 0..INITIAL_MAINTENANCE_JOB_COUNT {
6875            note_maintenance_completion(meta, None, false, false);
6876        }
6877        assert!(!meta.maintenance_pending);
6878        assert_eq!(meta.maintenance_jobs_in_flight, 0);
6879    }
6880
6881    #[test]
6882    fn maintenance_requeue_drops_while_bind_is_pending() {
6883        let (_dir, root) = test_root("maintenance-bind-requeue");
6884        let mut live_roots = HashMap::new();
6885        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6886        let (due, _) = due_maintenance_jobs_without_actor_context(
6887            &mut live_roots,
6888            usize::MAX,
6889            &HashSet::new(),
6890        );
6891        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6892
6893        let meta = live_roots.get_mut(&root).unwrap();
6894        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, true);
6895
6896        assert_eq!(
6897            meta.maintenance_jobs_in_flight,
6898            INITIAL_MAINTENANCE_JOB_COUNT - 1
6899        );
6900        assert!(meta.maintenance_queued_kinds.is_empty());
6901        assert!(meta.maintenance_pending);
6902    }
6903
6904    #[test]
6905    fn parked_lsp_completion_never_requiesces_or_cancels_a_pending_bind() {
6906        let mut meta = RootMeta::new(Instant::now());
6907        meta.unbound_quiesced = true;
6908
6909        assert!(!should_requiesce_after_maintenance(
6910            &meta,
6911            MaintenanceDrainKind::Lsp,
6912            false,
6913        ));
6914        assert!(!should_requiesce_after_maintenance(
6915            &meta,
6916            MaintenanceDrainKind::ConfigureTail,
6917            true,
6918        ));
6919        assert!(should_requiesce_after_maintenance(
6920            &meta,
6921            MaintenanceDrainKind::ConfigureTail,
6922            false,
6923        ));
6924    }
6925
6926    #[test]
6927    fn maintenance_pending_clears_and_poison_stops_requeue_after_fatal() {
6928        let (_dir, root) = test_root("maintenance-fatal");
6929        let mut live_roots = HashMap::new();
6930        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6931        let (due, _) = due_maintenance_jobs_without_actor_context(
6932            &mut live_roots,
6933            usize::MAX,
6934            &HashSet::new(),
6935        );
6936        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6937
6938        let meta = live_roots.get_mut(&root).unwrap();
6939        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), true, false);
6940        assert!(meta.maintenance_poisoned);
6941        assert!(meta.maintenance_queued_kinds.is_empty());
6942
6943        for _ in 1..INITIAL_MAINTENANCE_JOB_COUNT {
6944            note_maintenance_completion(meta, None, false, false);
6945        }
6946        assert!(!meta.maintenance_pending);
6947        assert_eq!(meta.maintenance_jobs_in_flight, 0);
6948    }
6949
6950    #[test]
6951    fn trust_for_principal_matrix() {
6952        assert_eq!(
6953            trust_for_principal(&Some(Principal::Direct)),
6954            BindTrust::FirstParty
6955        );
6956        // Every first-party reserved id is asserted BY NAME, in one loop over
6957        // the full set, so two failure classes stay distinguishable: an empty
6958        // or broken allowlist reddens every name at once, while a dropped
6959        // single entry (the rename hazard) reddens exactly the missing name.
6960        // Both halves of each transitional rename pair stay listed until the
6961        // flip settles (see the allowlist comment).
6962        for module_id in [
6963            "llm-runner",
6964            "aft",
6965            "broca",
6966            "alfonso-core",
6967            "prefrontal",
6968            "prefrontal-core",
6969        ] {
6970            assert_eq!(
6971                trust_for_principal(&Some(Principal::Reserved {
6972                    module_id: module_id.to_string(),
6973                })),
6974                BindTrust::FirstParty,
6975                "reserved module id '{module_id}' must resolve to first-party trust"
6976            );
6977        }
6978        assert_eq!(
6979            trust_for_principal(&Some(Principal::Reserved {
6980                module_id: "subc-mcp".to_string(),
6981            })),
6982            BindTrust::Untrusted
6983        );
6984        assert_eq!(
6985            trust_for_principal(&Some(Principal::Reserved {
6986                module_id: "anything-unknown".to_string(),
6987            })),
6988            BindTrust::Untrusted
6989        );
6990        assert_eq!(
6991            trust_for_principal(&Some(Principal::Unverified)),
6992            BindTrust::Untrusted
6993        );
6994        assert_eq!(trust_for_principal(&None), BindTrust::Untrusted);
6995    }
6996
6997    #[test]
6998    fn fed_harness_class_maps_to_untrusted_regardless_of_fingerprint_value() {
6999        let principal = Some(Principal::Direct);
7000        let fingerprint_a = "fed:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
7001        let fingerprint_b = "fed:0123456789abcdef111111111111111111111111111111111111111111111111";
7002
7003        assert_eq!(
7004            trust_for_bind(fingerprint_a, &principal),
7005            BindTrust::Untrusted
7006        );
7007        assert_eq!(
7008            trust_for_bind(fingerprint_b, &principal),
7009            BindTrust::Untrusted
7010        );
7011    }
7012
7013    /// The table above proves `trust_for_principal` maps correctly, and the
7014    /// test above proves the `fed:` harness override wins — but neither
7015    /// exercises the ordinary path, so an implementation that ignored the
7016    /// principal entirely for non-fed harnesses would satisfy both. Pin the
7017    /// delegation itself: on a normal harness the verdict must still come from
7018    /// the principal, in both directions.
7019    #[test]
7020    fn trust_for_bind_delegates_to_the_principal_on_ordinary_harnesses() {
7021        for harness in ["opencode", "pi", "runner", "mcp:claude"] {
7022            assert_eq!(
7023                trust_for_bind(harness, &Some(Principal::Direct)),
7024                BindTrust::FirstParty,
7025                "a direct principal must stay first-party on {harness}"
7026            );
7027            assert_eq!(
7028                trust_for_bind(harness, &Some(Principal::Unverified)),
7029                BindTrust::Untrusted,
7030                "an unverified principal must stay untrusted on {harness}"
7031            );
7032            assert_eq!(
7033                trust_for_bind(harness, &None),
7034                BindTrust::Untrusted,
7035                "an absent principal must fail closed on {harness}"
7036            );
7037            assert_eq!(
7038                trust_for_bind(
7039                    harness,
7040                    &Some(Principal::Reserved {
7041                        module_id: "subc-mcp".to_string(),
7042                    })
7043                ),
7044                BindTrust::Untrusted,
7045                "a non-allowlisted reserved module must stay untrusted on {harness}"
7046            );
7047        }
7048    }
7049
7050    #[tokio::test]
7051    async fn persistent_cancel_resolves_when_fired_before_await() {
7052        // The lost-wakeup guard: cancel() fires exactly once via notify_waiters()
7053        // (no stored permit). A waiter that registers AFTER the cancel must still
7054        // observe it via the flag; a waiter racing the cancel must still be woken.
7055        let signal = PersistentCancelSignal::new();
7056        signal.cancel();
7057        // Fired before we ever call cancelled() — must return immediately, not park.
7058        tokio::time::timeout(Duration::from_secs(1), signal.cancelled())
7059            .await
7060            .expect("cancelled() must resolve when cancel fired beforehand");
7061
7062        // A fresh signal cancelled concurrently with an in-flight cancelled().
7063        let racing = PersistentCancelSignal::new();
7064        let racing_for_task = racing.clone();
7065        let waiter = tokio::spawn(async move { racing_for_task.cancelled().await });
7066        racing.cancel();
7067        tokio::time::timeout(Duration::from_secs(1), waiter)
7068            .await
7069            .expect("cancelled() must resolve when cancel races the await")
7070            .expect("waiter task panicked");
7071    }
7072
7073    #[test]
7074    fn ingress_epoch_validation_rejects_reclaimed_requests_and_drops_other_stale_epochs() {
7075        let installed = HashMap::from([(7, 9)]);
7076        let mut reclaimed = ReclaimedRoutes::default();
7077        reclaimed.insert(route_key(8, 1));
7078        for ty in [
7079            FrameType::Request,
7080            FrameType::Response,
7081            FrameType::Error,
7082            FrameType::Push,
7083            FrameType::Cancel,
7084            FrameType::Goodbye,
7085        ] {
7086            let body = if ty.is_pure_header() {
7087                Vec::new()
7088            } else {
7089                br#"{}"#.to_vec()
7090            };
7091            let stale = Frame::build(ty, control_flags(), 7, 8, 41, body).unwrap();
7092            assert!(
7093                !ingress_route_should_be_processed(&installed, &reclaimed, &stale),
7094                "{ty:?}"
7095            );
7096        }
7097
7098        let reclaimed_request = Frame::build(
7099            FrameType::Request,
7100            control_flags(),
7101            8,
7102            1,
7103            42,
7104            br#"{}"#.to_vec(),
7105        )
7106        .unwrap();
7107        assert!(ingress_route_should_be_processed(
7108            &installed,
7109            &reclaimed,
7110            &reclaimed_request
7111        ));
7112
7113        let never_installed = Frame::build(
7114            FrameType::Request,
7115            control_flags(),
7116            9,
7117            1,
7118            43,
7119            br#"{}"#.to_vec(),
7120        )
7121        .unwrap();
7122        assert!(!ingress_route_should_be_processed(
7123            &installed,
7124            &reclaimed,
7125            &never_installed
7126        ));
7127
7128        let current = Frame::build(
7129            FrameType::Request,
7130            control_flags(),
7131            7,
7132            9,
7133            43,
7134            br#"{}"#.to_vec(),
7135        )
7136        .unwrap();
7137        let control = Frame::build(FrameType::Ping, control_flags(), 0, 0, 44, Vec::new()).unwrap();
7138        assert!(ingress_route_should_be_processed(
7139            &installed, &reclaimed, &current
7140        ));
7141        assert!(ingress_route_should_be_processed(
7142            &installed, &reclaimed, &control
7143        ));
7144        assert_eq!(installed, HashMap::from([(7, 9)]));
7145    }
7146
7147    #[tokio::test]
7148    async fn route_bind_ack_precedes_route_egress_in_writer_queue() {
7149        let (_dir, root) = test_root("route-bind-b2-ordering");
7150        let route = route_key(7, 3);
7151        let identity = RouteIdentity(Arc::new(RouteIdentityData {
7152            root: root.clone(),
7153            project_root: root.as_path().to_path_buf(),
7154            harness: "opencode".to_string(),
7155            session: "b2-session".to_string(),
7156            trust: BindTrust::FirstParty,
7157            spawn_principal: AuthenticatedPrincipal::FirstParty,
7158            consumer_elicitation_capable: false,
7159        }));
7160        let replay_key = push::ReplayKey::from_identity(&identity);
7161        let completion = RouteBindCompletion {
7162            route,
7163            identity,
7164            bind_root_id: root.clone(),
7165            inserted_new_actor: false,
7166            configure_response: Response::success("subc-bind-7", json!({})),
7167            diagnostics_on_edit: false,
7168            ver: PROTOCOL_VERSION,
7169            corr: 91,
7170            flags: control_flags(),
7171        };
7172        let mut pending_binds = HashMap::from([(
7173            route,
7174            PendingBind {
7175                bind_root_id: root,
7176                inserted_new_actor: false,
7177                cancelled: false,
7178                configure_request_id: "subc-bind-7".to_string(),
7179                started_at: Instant::now(),
7180                warned_half_deadline: false,
7181                deadline_reported: false,
7182                corr: 91,
7183                ver: PROTOCOL_VERSION,
7184                flags: control_flags(),
7185                cancellation: crate::executor::JobCancellation::new(),
7186            },
7187        )]);
7188        let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
7189        let mut push_buffer =
7190            HashMap::from([(replay_key, VecDeque::from([completion_frame("b2-replay")]))]);
7191        let (writer_tx, mut writer_rx) = mpsc::channel(8);
7192        let metrics = Arc::new(DispatchPathMetrics::new());
7193
7194        handle_route_bind_completion(
7195            &writer_tx,
7196            completion,
7197            &mut HashMap::new(),
7198            &mut HashMap::new(),
7199            &mut HashMap::new(),
7200            &mut push_buffer,
7201            &mut HashMap::new(),
7202            &mut pending_binds,
7203            &mut installed_route_epochs,
7204            &Arc::new(Executor::new()),
7205            &Arc::new(Notify::new()),
7206            &metrics,
7207        )
7208        .await
7209        .unwrap();
7210
7211        let ack = writer_rx.try_recv().expect("RouteBindAck");
7212        assert_eq!(ack.header.ty, FrameType::Response);
7213        assert_eq!((ack.header.channel, ack.header.epoch), (0, 0));
7214        let route_frame = writer_rx.try_recv().expect("post-ack route frame");
7215        assert_eq!(route_frame.header.ty, FrameType::Push);
7216        assert_eq!(
7217            (route_frame.header.channel, route_frame.header.epoch),
7218            (route.channel, route.epoch)
7219        );
7220    }
7221}