Skip to main content

aft/subc/
mod.rs

1//! subc daemon attach — transport edge.
2//!
3//! When AFT is launched as `aft --subc <connection-file>`, it does NOT run the
4//! standalone NDJSON-over-stdin loop. Instead it connects to a running subc
5//! daemon over loopback TCP, authenticates with the pre-envelope HMAC handshake
6//! (`subc-transport`), then speaks the subc frame protocol (`subc-protocol`):
7//! ModuleHello → HelloAck (register as a tool provider), then a channel-0
8//! control loop (Ping/Pong, RouteBind) plus route-channel tool calls.
9//!
10//! Concurrency: subc routes tool calls through the executor. The tokio
11//! edge never dispatches against `AppContext` inline; per-actor executor lanes
12//! own the reader/mutator epoch, while a writer task serializes outbound frames.
13
14use std::collections::{HashMap, HashSet, VecDeque};
15use std::fmt;
16use std::io;
17use std::net::{IpAddr, SocketAddr};
18use std::ops::Deref;
19use std::path::{Path, PathBuf};
20use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
21use std::sync::{Arc, LazyLock, Mutex as StdMutex, OnceLock};
22use std::time::{Duration, Instant};
23
24use serde::Deserialize;
25use serde_json::{json, Value};
26
27use crate::config::Config;
28use crate::config_resolve::ConfigTier;
29use crate::context::{App, AppContext, ProgressSender, RootHealthSnapshot};
30use crate::executor::{Executor, Lane};
31use crate::fleet_status::{spawn_fleet_status_dial, FleetStatusClient};
32use crate::jsonc::strip_jsonc;
33use crate::log_ctx;
34use crate::path_identity::ProjectRootId;
35use crate::protocol::{ProgressKind, PushFrame, RawRequest, Response};
36use crate::run_tool_call::{
37    run_tool_call, strip_agent_preview_arg_owned, PhaseTrace, ToolCallContext, ToolCallOutcome,
38    ToolCallResult,
39};
40use crate::runtime_drain;
41use crate::sandbox_spawn::{AuthenticatedPrincipal, PrincipalTrust};
42
43use subc_protocol::manifest::{
44    Bindings, Concurrency, ExecutionMode, IdentityBinding, IdentityScope, ModuleManifest,
45    ProviderRole, StorageBinding, StorageKind, StorageScope, Tool, TrustTier,
46};
47use subc_protocol::session::{
48    HealthReport, HealthStatus, ModuleControlRequest, ModuleControlResponse,
49    MODULE_CONTROL_OP_HEALTH_CHECK,
50};
51use subc_protocol::{
52    ErrorBody, Flags, Frame, FrameType, ModuleHelloBody, Principal, Priority, MAX_FRAME_BODY_LEN,
53    PROTOCOL_VERSION,
54};
55use subc_transport::{authenticate_client, connection_file, read_frame, write_frame};
56use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
57use tokio::net::TcpStream;
58use tokio::sync::{mpsc, oneshot, Notify};
59use tokio::task::JoinHandle;
60
61/// Per-attempt handshake deadline. The initial attach loop has a separate total
62/// budget so a stalled peer cannot consume an unbounded supervisor launch window.
63const AUTH_DEADLINE: Duration = Duration::from_secs(5);
64const ATTACH_RETRY_BUDGET: Duration = Duration::from_secs(60);
65const ATTACH_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(250);
66const ATTACH_RETRY_MAX_BACKOFF: Duration = Duration::from_secs(5);
67const ATTACH_RETRY_JITTER_PERCENT: u64 = 20;
68
69/// Correlation id for the initial ModuleHello (channel 0).
70const HELLO_CORR: u64 = 1;
71
72/// Per-session in-memory replay cap for must-deliver Push frames. This covers
73/// detach/re-attach while AFT stays alive; cross-restart replay is phased later.
74const PUSH_BUFFER_MAX_PER_KEY: usize = 256;
75
76/// Bounded guard for control-frame sends. If the daemon stops reading and the
77/// writer queue stays full, tear the subc edge down instead of stalling the
78/// route loop indefinitely.
79const CONTROL_SEND_TIMEOUT: Duration = Duration::from_millis(250);
80
81/// Cadence for the loop's deadline-driven drain work (retry-buffer flush,
82/// bg-wake emission, maintenance submission). Checked at the top of every
83/// loop turn so busy select arms cannot starve it.
84const DRAIN_TICK_PERIOD: Duration = Duration::from_millis(250);
85
86/// Root-scoped stores and watcher runtimes are reopened lazily after this
87/// period without tool traffic. Keeping the value fixed avoids per-client
88/// eviction policies competing inside the module loop.
89const IDLE_ROOT_TTL: Duration = Duration::from_secs(30 * 60);
90
91const WRITER_QUEUE_CAPACITY: usize = 256;
92
93/// Keep reliable Push bursts from monopolizing the current-thread subc loop;
94/// any remaining must-deliver frames stay queued for the next loop turn.
95const RELIABLE_PUSH_DRAIN_BUDGET: usize = 32;
96
97/// Limit maintenance submissions per tick so background drains cannot delay
98/// control-plane work such as completed RouteBind acknowledgements.
99///
100/// The decomposed maintenance pass charges this budget by Mutating job, not by
101/// root. Size the default burst for one maintenance pass over eight live roots,
102/// while follow-up batches still re-enter the capped queue instead of bypassing
103/// the budget.
104const MAINTENANCE_SUBMIT_BUDGET: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len() * 8;
105const INITIAL_MAINTENANCE_DRAIN_KINDS: [MaintenanceDrainKind; 4] = [
106    MaintenanceDrainKind::Watcher,
107    MaintenanceDrainKind::Lsp,
108    MaintenanceDrainKind::ConfigureTail,
109    MaintenanceDrainKind::CompletionDrains,
110];
111#[cfg(test)]
112const INITIAL_MAINTENANCE_JOB_COUNT: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len();
113
114const RELIABLE_WRITER_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(10);
115const RELIABLE_WRITER_RETRY_MAX_BACKOFF: Duration = Duration::from_millis(250);
116
117const DISPATCH_PATH_BIND_WARN_AFTER: Duration = Duration::from_secs(6);
118const ROUTE_BIND_DEADLINE: Duration = Duration::from_secs(12);
119
120/// Small bounded memory of completed task ids used to suppress stale lossy
121/// long-running reminders that arrive after their reliable completion event.
122const COMPLETED_TASK_SUPPRESSION_MAX: usize = 4096;
123
124/// Bash foreground orchestration polls detached tasks with short read-lane jobs.
125/// The sleep between polls is outside the executor so no read or write worker is
126/// pinned while a foreground command is still running.
127const PENDING_POLL_INTERVAL: Duration = Duration::from_millis(100);
128
129/// Host elicitation asks fail closed if the MCP facade does not answer promptly.
130const BASH_ELICITATION_TIMEOUT: Duration = Duration::from_secs(60);
131const BASH_ELICITATION_CREATE_METHOD: &str = "elicitation/create";
132
133#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
134struct RouteChannel {
135    channel: u16,
136    epoch: u32,
137}
138
139impl fmt::Display for RouteChannel {
140    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
141        write!(f, "{}@{}", self.channel, self.epoch)
142    }
143}
144
145type PushEnvelope = (ProjectRootId, PushFrame);
146type LossyPushEnvelope = (u64, ProjectRootId, PushFrame);
147type RetryBuffer = HashMap<RouteChannel, VecDeque<(push::ReplayKey, PushFrame)>>;
148mod bash;
149mod health;
150mod manifest;
151mod push;
152mod wire;
153
154use self::health::{
155    build_health_report, warn_slow_pending_binds, DispatchPathMetrics, HealthRollupCache,
156    ReapBlockerCensus, ResponseTaskGuard, HEALTH_ROLLUP_TTL,
157};
158use self::manifest::{
159    build_manifest, command_lane, control_flags, control_ops, is_bash_family_tool,
160    is_subc_agent_core_tool, is_subc_native_plumbing_tool,
161};
162pub use self::wire::SubcError;
163
164/// Test-only view of the fail-closed tool-call gate: would `name` be admitted
165/// on a bound route (as an agent tool or native plumbing)? Used by the
166/// plugin-send drift guard in `subc_plumbing_drift_test.rs`.
167pub fn is_tool_call_admitted_for_test(name: &str) -> bool {
168    manifest::is_subc_agent_core_tool(name) || manifest::is_subc_native_plumbing_tool(name)
169}
170use self::wire::{
171    build_error_frame, build_goodbye_frame, build_tool_response_frame,
172    build_tool_response_frame_with_limit, decrement_counted_channel, response_is_fatal_panic,
173    response_message, send_counted_channel, send_frame, send_reliable_writer_frame,
174    send_traced_tool_response_frame, ToolResponseWriteTrace, WriterFrame, WriterSender,
175};
176
177struct DecodedFrame {
178    frame: Frame,
179    phase_trace: PhaseTrace,
180}
181
182struct ToolCallCompletion {
183    text: String,
184    phase_trace: PhaseTrace,
185}
186
187#[derive(Clone)]
188struct PushSenders {
189    lossy_tx: mpsc::Sender<LossyPushEnvelope>,
190    reliable_tx: mpsc::UnboundedSender<PushEnvelope>,
191    lossy_overflow: Arc<push::LossyOverflow>,
192    lossy_seq: Arc<AtomicU64>,
193    fleet_status_client: FleetStatusClient,
194}
195
196#[derive(Clone)]
197struct PersistentCancelSignal {
198    inner: Arc<PersistentCancelInner>,
199}
200
201struct PersistentCancelInner {
202    cancelled: AtomicBool,
203    notify: Notify,
204}
205
206impl PersistentCancelSignal {
207    fn new() -> Self {
208        Self {
209            inner: Arc::new(PersistentCancelInner {
210                cancelled: AtomicBool::new(false),
211                notify: Notify::new(),
212            }),
213        }
214    }
215
216    fn cancel(&self) {
217        if !self.inner.cancelled.swap(true, Ordering::SeqCst) {
218            self.inner.notify.notify_waiters();
219        }
220    }
221
222    fn is_cancelled(&self) -> bool {
223        self.inner.cancelled.load(Ordering::SeqCst)
224    }
225
226    async fn cancelled(&self) {
227        // `enable()` REGISTERS this waiter before we read the flag, closing the
228        // lost-wakeup window: `notify_waiters()` only wakes already-registered
229        // waiters and stores no permit, so without enable() a `cancel()` firing
230        // between the flag read and `.await` would be missed and the future
231        // would park forever (cancel() fires only once). With enable(), a cancel
232        // racing the flag read still wakes the registered waiter. The loop is a
233        // belt-and-suspenders re-check on spurious wakeups.
234        loop {
235            let notified = self.inner.notify.notified();
236            tokio::pin!(notified);
237            notified.as_mut().enable();
238            if self.is_cancelled() {
239                return;
240            }
241            notified.await;
242        }
243    }
244}
245
246#[derive(Debug, Clone, Copy, PartialEq, Eq)]
247pub(crate) enum BindTrust {
248    FirstParty,
249    Untrusted,
250}
251
252impl BindTrust {
253    fn allows_bash_observation(self) -> bool {
254        matches!(self, Self::FirstParty)
255    }
256
257    fn label(self) -> &'static str {
258        match self {
259            Self::FirstParty => "first_party",
260            Self::Untrusted => "untrusted",
261        }
262    }
263
264    fn sandbox_trust(self) -> PrincipalTrust {
265        match self {
266            Self::FirstParty => PrincipalTrust::FirstParty,
267            Self::Untrusted => PrincipalTrust::Untrusted,
268        }
269    }
270}
271
272pub(super) fn trust_for_principal(principal: &Option<Principal>) -> BindTrust {
273    match principal {
274        Some(Principal::Direct) => BindTrust::FirstParty,
275        // Module renames are flag-days: the daemon registry refuses duplicate
276        // active ids, so a renaming module cannot advertise both names during
277        // its transition. This allowlist is DIALLED, not dialling — it must
278        // accept a module's NEW name in a released binary before the module
279        // starts using it, and the old name stays until the flip has settled.
280        // That is why transitional pairs appear here: llm-runner/broca was the
281        // previous rename, alfonso-core/prefrontal is the current one. When
282        // retiring an old name, confirm the fleet no longer spawns it — a
283        // stale entry here is inert, but a missing one silently downgrades a
284        // first-party module to Untrusted and revokes its bash access.
285        Some(Principal::Reserved { module_id })
286            if module_id == "llm-runner"
287                || module_id == "aft"
288                || module_id == "broca"
289                || module_id == "alfonso-core"
290                || module_id == "prefrontal"
291                || module_id == "prefrontal-core" =>
292        {
293            BindTrust::FirstParty
294        }
295        Some(Principal::Reserved { .. }) | Some(Principal::Unverified) | None => {
296            BindTrust::Untrusted
297        }
298    }
299}
300
301fn harness_forces_untrusted(harness: &str) -> bool {
302    harness.starts_with("fed:")
303}
304
305pub(super) fn trust_for_bind(harness: &str, principal: &Option<Principal>) -> BindTrust {
306    if harness_forces_untrusted(harness) {
307        BindTrust::Untrusted
308    } else {
309        trust_for_principal(principal)
310    }
311}
312
313fn principal_id(principal: &Option<Principal>) -> Option<String> {
314    match principal {
315        Some(Principal::Direct) => Some("direct".to_string()),
316        Some(Principal::Reserved { module_id }) => Some(format!("reserved:{module_id}")),
317        Some(Principal::Unverified) => Some("unverified".to_string()),
318        None => None,
319    }
320}
321
322fn principal_label(principal: &Option<Principal>) -> String {
323    principal_id(principal).unwrap_or_else(|| "absent".to_string())
324}
325
326#[derive(Debug)]
327/// Per-root route metadata owned by the subc loop. The `active_bash_waits` field
328/// counts detached bash processes that are still being observed for this root.
329/// Any future logic that evicts roots based on idle time must not evict a root
330/// while this count is greater than zero, because a foreground bash response may
331/// still arrive later.
332struct RootMeta {
333    maintenance_pending: bool,
334    maintenance_jobs_in_flight: usize,
335    maintenance_queued_kinds: VecDeque<MaintenanceDrainKind>,
336    maintenance_last_submitted: Option<Instant>,
337    maintenance_poisoned: bool,
338    last_touched: Instant,
339    diagnostics_on_edit: bool,
340    active_bash_waits: usize,
341    idle_artifacts_evicted: bool,
342    unbound_quiesced: bool,
343    consecutive_missing_sweeps: u8,
344}
345
346#[derive(Debug)]
347struct PendingBind {
348    bind_root_id: ProjectRootId,
349    inserted_new_actor: bool,
350    cancelled: bool,
351    configure_request_id: String,
352    started_at: Instant,
353    warned_half_deadline: bool,
354    deadline_reported: bool,
355    corr: u64,
356    ver: u8,
357    flags: Flags,
358    /// Exact-job cancellation for the submitted configure: Goodbye and
359    /// deadline expiry cancel the executor job operationally (queued jobs are
360    /// removed, running configures return at their next checkpoint) instead of
361    /// only marking bookkeeping.
362    cancellation: crate::executor::JobCancellation,
363}
364
365struct RouteBindCompletion {
366    route: RouteChannel,
367    identity: RouteIdentity,
368    bind_root_id: ProjectRootId,
369    inserted_new_actor: bool,
370    configure_response: Response,
371    diagnostics_on_edit: bool,
372    ver: u8,
373    corr: u64,
374    flags: Flags,
375}
376
377#[derive(Debug, Clone)]
378struct RouteIdentity(Arc<RouteIdentityData>);
379
380#[derive(Debug)]
381struct RouteIdentityData {
382    root: ProjectRootId,
383    project_root: PathBuf,
384    harness: String,
385    session: String,
386    trust: BindTrust,
387    spawn_principal: AuthenticatedPrincipal,
388    consumer_elicitation_capable: bool,
389}
390
391impl Deref for RouteIdentity {
392    type Target = RouteIdentityData;
393
394    fn deref(&self) -> &Self::Target {
395        &self.0
396    }
397}
398
399#[derive(Debug, Clone)]
400struct RetainedSessionIdentity {
401    harness: String,
402    trust: BindTrust,
403}
404
405#[derive(Clone)]
406struct BgSub {
407    corr: u64,
408    ver: u8,
409    flags: Flags,
410    root: ProjectRootId,
411    session: String,
412}
413
414// A session can be observed by multiple long-lived consumer records. Retain
415// every route so each wake uses the correlation captured by that route's BgSub.
416type BgSubsBySession = HashMap<(ProjectRootId, String), HashSet<RouteChannel>>;
417
418struct MaintenanceCompletion {
419    root_id: ProjectRootId,
420    kind: MaintenanceDrainKind,
421    response: Response,
422    empty_bg_sessions: Vec<(String, u64)>,
423    requeue_kind: Option<MaintenanceDrainKind>,
424}
425
426#[derive(Clone, Copy, Debug, PartialEq, Eq)]
427enum MaintenanceDrainKind {
428    Watcher,
429    Lsp,
430    ConfigureTail,
431    CompletionDrains,
432}
433
434impl MaintenanceDrainKind {
435    fn label(self) -> &'static str {
436        match self {
437            Self::Watcher => "watcher",
438            Self::Lsp => "lsp",
439            Self::ConfigureTail => "configure-tail",
440            Self::CompletionDrains => "completion-drains",
441        }
442    }
443}
444
445#[derive(Debug, Default)]
446struct MaintenanceJobOutcome {
447    empty_bg_sessions: Vec<(String, u64)>,
448    requeue_kind: Option<MaintenanceDrainKind>,
449}
450
451#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
452struct ReverseCorrKey {
453    route: RouteChannel,
454    corr: u64,
455}
456
457struct PendingBashAsk {
458    route: RouteChannel,
459    tool_corr: u64,
460    tool_flags: Flags,
461    tool_ver: u8,
462    root: ProjectRootId,
463    project_root: PathBuf,
464    session_id: String,
465    spawn_principal: AuthenticatedPrincipal,
466    edit_slot_survives: Option<bool>,
467    request_id: String,
468    arguments: Value,
469    format_context: crate::subc_format::FormatContext,
470    cancel: bash::BashWaitCancel,
471    grants: Vec<String>,
472    expires_at: Instant,
473}
474
475impl RootMeta {
476    fn new(now: Instant) -> Self {
477        Self {
478            maintenance_pending: false,
479            maintenance_jobs_in_flight: 0,
480            maintenance_queued_kinds: VecDeque::new(),
481            maintenance_last_submitted: None,
482            maintenance_poisoned: false,
483            last_touched: now,
484            diagnostics_on_edit: false,
485            active_bash_waits: 0,
486            idle_artifacts_evicted: false,
487            unbound_quiesced: false,
488            consecutive_missing_sweeps: 0,
489        }
490    }
491
492    fn note_activity(&mut self) {
493        self.last_touched = Instant::now();
494    }
495
496    fn reactivate_bound(&mut self) {
497        self.note_activity();
498        self.idle_artifacts_evicted = false;
499        self.unbound_quiesced = false;
500    }
501}
502
503fn due_maintenance_jobs(
504    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
505    executor: Option<&Executor>,
506    bg_sub_by_session: &BgSubsBySession,
507    bg_wake_pending: &HashSet<RouteChannel>,
508    budget: usize,
509    pending_bind_roots: &HashSet<ProjectRootId>,
510) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
511    let mut jobs = Vec::new();
512    let mut deferred = false;
513    let mut roots = live_roots.keys().cloned().collect::<Vec<_>>();
514    roots.sort_by(|left, right| {
515        let left_last = live_roots
516            .get(left)
517            .and_then(|meta| meta.maintenance_last_submitted);
518        let right_last = live_roots
519            .get(right)
520            .and_then(|meta| meta.maintenance_last_submitted);
521        left_last
522            .cmp(&right_last)
523            .then_with(|| left.as_path().cmp(right.as_path()))
524    });
525
526    for root_id in roots {
527        let Some(meta) = live_roots.get_mut(&root_id) else {
528            continue;
529        };
530        if meta.maintenance_poisoned {
531            continue;
532        }
533
534        if pending_bind_roots.contains(&root_id) {
535            if meta.maintenance_pending || !meta.maintenance_queued_kinds.is_empty() {
536                deferred = true;
537            }
538            continue;
539        }
540
541        if !meta.maintenance_pending {
542            if jobs.len() >= budget {
543                deferred = true;
544                continue;
545            }
546            // Only enqueue kinds with pending work. Probes are cheap and
547            // fail-open (contended sources count as pending), so an idle root
548            // costs four probes per tick instead of four dispatched jobs.
549            let executor_actor_context =
550                executor.and_then(|executor| executor.actor_context(&root_id));
551            let root_has_pending_bg_wake =
552                bg_sub_by_session.iter().any(|((sub_root, _), channels)| {
553                    sub_root == &root_id
554                        && channels
555                            .iter()
556                            .any(|channel| bg_wake_pending.contains(channel))
557                });
558            let kinds_with_work: Vec<MaintenanceDrainKind> = match executor_actor_context {
559                Some(ctx) => INITIAL_MAINTENANCE_DRAIN_KINDS
560                    .into_iter()
561                    .filter(|kind| {
562                        if meta.unbound_quiesced && !matches!(kind, MaintenanceDrainKind::Lsp) {
563                            return false;
564                        }
565                        match kind {
566                            MaintenanceDrainKind::Watcher => ctx.watcher_drain_has_work(),
567                            MaintenanceDrainKind::Lsp => ctx.lsp_drain_has_work(),
568                            MaintenanceDrainKind::ConfigureTail => ctx.configure_tail_has_work(),
569                            // Every CompletionDrains source is visible at this enqueue site:
570                            // AppContext probes completion queues, this loop owns bg wakes,
571                            // and queued continuations bypass probing via maintenance_pending.
572                            // New drain sources must expose a probe here rather than making
573                            // every subscribed root fail open again.
574                            MaintenanceDrainKind::CompletionDrains => {
575                                root_has_pending_bg_wake || ctx.completion_drains_have_work()
576                            }
577                        }
578                    })
579                    .collect(),
580                None if meta.unbound_quiesced => Vec::new(),
581                // No context handle (actor gone mid-tick): enqueue everything.
582                None => INITIAL_MAINTENANCE_DRAIN_KINDS.to_vec(),
583            };
584            if kinds_with_work.is_empty() {
585                continue;
586            }
587            meta.maintenance_pending = true;
588            meta.maintenance_queued_kinds.extend(kinds_with_work);
589        }
590
591        while let Some(kind) = meta.maintenance_queued_kinds.pop_front() {
592            if jobs.len() >= budget {
593                meta.maintenance_queued_kinds.push_front(kind);
594                deferred = true;
595                break;
596            }
597            meta.maintenance_jobs_in_flight += 1;
598            meta.maintenance_last_submitted = Some(Instant::now());
599            jobs.push((root_id.clone(), kind));
600        }
601
602        meta.maintenance_pending =
603            meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
604    }
605
606    (jobs, deferred)
607}
608
609fn eviction_estimate_label(estimate: &crate::memory::MemoryEstimate) -> String {
610    match estimate.estimated_bytes {
611        Some(bytes) => format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
612        None if estimate.status == "busy" => "busy".to_string(),
613        None => "not estimated".to_string(),
614    }
615}
616
617fn optional_memory_label(bytes: Option<u64>) -> String {
618    bytes.map_or_else(
619        || "not estimated".to_string(),
620        |bytes| format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
621    )
622}
623
624fn pressure_relief_label(relief: &crate::memory::AllocatorPressureRelief) -> String {
625    format!(
626        "; allocator pressure relief: RSS {} -> {}, in-use {} -> {}, allocated {} -> {}, slack {} -> {}, allocator reported {:.1} MB released",
627        optional_memory_label(relief.rss_before_bytes),
628        optional_memory_label(relief.rss_after_bytes),
629        optional_memory_label(relief.allocator_before.bytes_in_use),
630        optional_memory_label(relief.allocator_after.bytes_in_use),
631        optional_memory_label(relief.allocator_before.size_allocated),
632        optional_memory_label(relief.allocator_after.size_allocated),
633        optional_memory_label(relief.allocator_before.retained_slack_bytes),
634        optional_memory_label(relief.allocator_after.retained_slack_bytes),
635        relief.bytes_released as f64 / (1024.0 * 1024.0),
636    )
637}
638
639fn idle_root_eviction_message(
640    root_id: &ProjectRootId,
641    memory: &crate::memory::RootMemorySnapshot,
642    pressure_relief: Option<&crate::memory::AllocatorPressureRelief>,
643) -> String {
644    // Bash, LSP, and parser state remain resident. The freed total is deliberately
645    // only the known-byte portion of handles eviction actually drops.
646    let freed_bytes = [
647        &memory.semantic,
648        &memory.trigram,
649        &memory.symbols,
650        &memory.callgraph,
651        &memory.inspect,
652    ]
653    .iter()
654    .filter_map(|estimate| estimate.estimated_bytes)
655    .fold(0u64, u64::saturating_add);
656    let mut message = format!(
657        "evicted idle root {}: freed ~{:.1} MB (semantic {}, trigram {}, symbols {}, callgraph {}, inspect {}; retained: bash {}, lsp {}, parser_pool {})",
658        root_id.as_path().display(),
659        freed_bytes as f64 / (1024.0 * 1024.0),
660        eviction_estimate_label(&memory.semantic),
661        eviction_estimate_label(&memory.trigram),
662        eviction_estimate_label(&memory.symbols),
663        eviction_estimate_label(&memory.callgraph),
664        eviction_estimate_label(&memory.inspect),
665        eviction_estimate_label(&memory.bash),
666        eviction_estimate_label(&memory.lsp),
667        eviction_estimate_label(&memory.parser_pool),
668    );
669    if let Some(pressure_relief) = pressure_relief {
670        message.push_str(&pressure_relief_label(pressure_relief));
671    }
672    message
673}
674
675fn process_has_been_idle(now: Instant, live_roots: &HashMap<ProjectRootId, RootMeta>) -> bool {
676    !live_roots.is_empty()
677        && live_roots.values().all(|meta| {
678            now.saturating_duration_since(meta.last_touched) >= IDLE_ROOT_TTL
679                && meta.active_bash_waits == 0
680                && !meta.maintenance_pending
681                && meta.maintenance_queued_kinds.is_empty()
682        })
683}
684
685fn allocator_pressure_relief_after_idle_sweep(
686    now: Instant,
687    live_roots: &HashMap<ProjectRootId, RootMeta>,
688    executor: &Executor,
689) -> Option<crate::memory::AllocatorPressureRelief> {
690    if !process_has_been_idle(now, live_roots)
691        || live_roots.keys().any(|root_id| {
692            executor
693                .actor_context(root_id)
694                .is_some_and(|ctx| ctx.artifact_eviction_blocked())
695        })
696    {
697        return None;
698    }
699
700    #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu")))]
701    {
702        Some(crate::memory::relieve_allocator_pressure())
703    }
704    #[cfg(not(any(target_os = "macos", all(target_os = "linux", target_env = "gnu"))))]
705    {
706        None
707    }
708}
709
710fn quiesce_unbound_root(
711    root_id: &ProjectRootId,
712    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
713    executor: &Arc<Executor>,
714) {
715    let Some(meta) = live_roots.get_mut(root_id) else {
716        return;
717    };
718
719    let ctx = executor.actor_context(root_id);
720    if let Some(ctx) = ctx.as_ref() {
721        // Close lifecycle admission before touching scheduler queues. A running
722        // ConfigureTail cannot release gates, install a watcher, or reserve a
723        // callgraph build after this transition becomes visible.
724        ctx.mark_subc_unbound();
725    }
726    let cancelled = executor.cancel_queued_maintenance(root_id);
727    // Transient unbind keeps the root WARM: the watcher stays running (its
728    // events accumulate and replay on rebind, so no unobserved gap exists) and
729    // resident artifacts stay resident. Host restarts unbind every root and
730    // rebind seconds later; stopping the watcher here would force strict
731    // re-verification plus a full callgraph rebuild on every restart. The
732    // expensive teardown (watcher stop + gap invalidation) belongs to the
733    // idle-TTL reaper and the root-deleted path.
734    let discarded = ctx
735        .map(|ctx| crate::commands::configure::cancel_deferred_configure_maintenance(&ctx))
736        .unwrap_or(0);
737    meta.unbound_quiesced = true;
738    meta.maintenance_queued_kinds.clear();
739    meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
740    log::info!(
741        "subc attach: quiesced unbound root {} (cancelled {} queued maintenance job(s), cancelled {} configure maintenance job(s)); cause=goodbye_unbound",
742        root_id.as_path().display(),
743        cancelled,
744        discarded
745    );
746}
747
748#[allow(clippy::too_many_arguments)]
749fn quiesce_connection_roots(
750    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
751    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
752    routes: &mut HashMap<RouteChannel, RouteIdentity>,
753    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
754    installed_route_epochs: &mut HashMap<u16, u32>,
755    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
756    executor: &Arc<Executor>,
757) {
758    for cancel in route_bash_cancels.values() {
759        cancel.token.cancel();
760    }
761    route_bash_cancels.clear();
762
763    let mut roots = live_roots.keys().cloned().collect::<HashSet<_>>();
764    for pending in pending_binds.values_mut() {
765        pending.cancelled = true;
766        roots.insert(pending.bind_root_id.clone());
767        let _ = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
768    }
769
770    // A connection exit abandons every installed route at once. Close lifecycle
771    // admission before cancelling maintenance so no deferred worker can restore
772    // root activity after the loop-owned route tables disappear.
773    for root_id in roots {
774        if live_roots.contains_key(&root_id) {
775            quiesce_unbound_root(&root_id, live_roots, executor);
776        } else if let Some(ctx) = executor.actor_context(&root_id) {
777            ctx.mark_subc_unbound();
778            executor.cancel_queued_maintenance(&root_id);
779            crate::commands::configure::cancel_deferred_configure_maintenance(&ctx);
780        }
781    }
782
783    routes.clear();
784    root_channels.clear();
785    installed_route_epochs.clear();
786}
787
788/// Per-channel epoch watermarks for roots reclaimed without a client Goodbye.
789/// The 16-bit channel space bounds this map, and no root identity or resource
790/// handle is retained. It exists only so late requests receive a typed error.
791#[derive(Debug, Default)]
792struct ReclaimedRoutes {
793    highest_epoch_by_channel: HashMap<u16, u32>,
794}
795
796impl ReclaimedRoutes {
797    fn insert(&mut self, route: RouteChannel) {
798        self.highest_epoch_by_channel
799            .entry(route.channel)
800            .and_modify(|epoch| *epoch = (*epoch).max(route.epoch))
801            .or_insert(route.epoch);
802    }
803
804    fn contains(&self, route: RouteChannel) -> bool {
805        self.highest_epoch_by_channel
806            .get(&route.channel)
807            .is_some_and(|epoch| route.epoch <= *epoch)
808    }
809}
810
811#[derive(Debug, Default)]
812struct IdleReapOutcome {
813    evicted: usize,
814    forgotten_deleted_roots: Vec<ProjectRootId>,
815}
816
817fn reap_idle_roots(
818    now: Instant,
819    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
820    pending_binds: &HashMap<RouteChannel, PendingBind>,
821    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
822    executor: &Arc<Executor>,
823    metrics: &DispatchPathMetrics,
824) -> IdleReapOutcome {
825    let pending_bind_roots = pending_binds
826        .values()
827        .map(|pending| pending.bind_root_id.clone())
828        .collect::<HashSet<_>>();
829    let mut census = ReapBlockerCensus::default();
830    let mut candidates = Vec::new();
831
832    for (root_id, meta) in live_roots.iter_mut() {
833        let deleted = !root_id.as_path().exists();
834        if deleted {
835            // A missing directory makes a bound route obsolete, but one failed
836            // lookup is not enough evidence to tear down a client-visible actor.
837            // Requiring two maintenance sweeps protects atomic replacement and
838            // transient filesystem failures; observing the path resets the proof.
839            // Absence also covers renames: a task's cwd handle can follow the
840            // moved directory while the registered path disappears. The old
841            // path is deliberately treated as a retired root identity, so the
842            // reaper accepts killing such tasks; rename a project only with no
843            // live tasks rather than relying on cwd-resolution heuristics.
844            meta.consecutive_missing_sweeps = meta.consecutive_missing_sweeps.saturating_add(1);
845        } else {
846            meta.consecutive_missing_sweeps = 0;
847        }
848        let deletion_confirmed = meta.consecutive_missing_sweeps >= 2;
849        let has_bound_route = root_channels
850            .get(root_id)
851            .is_some_and(|channels| !channels.is_empty());
852        let has_pending_bind = pending_bind_roots.contains(root_id);
853
854        if deleted {
855            let mut retained = false;
856            if !deletion_confirmed {
857                census.absence_unconfirmed += 1;
858                retained = true;
859            }
860            // Once absence is confirmed, the directory cannot serve this route
861            // again. Neither a stale route nor the lack of normal unbind cleanup
862            // justifies retaining the root; purge removes the route after retirement.
863            if meta.active_bash_waits > 0 {
864                census.bash_waits += 1;
865                retained = true;
866            }
867            if meta.maintenance_pending {
868                census.maintenance_pending += 1;
869                retained = true;
870            }
871            if !meta.maintenance_queued_kinds.is_empty() {
872                census.maintenance_queued += 1;
873                retained = true;
874            }
875            if has_pending_bind {
876                census.pending_binds += 1;
877                retained = true;
878            }
879            match executor.try_actor_is_idle(root_id) {
880                Some(true) => {}
881                Some(false) => {
882                    census.actor_busy += 1;
883                    retained = true;
884                }
885                None => {
886                    census.actor_state_busy += 1;
887                    retained = true;
888                }
889            }
890            if retained {
891                census.deleted_retained += 1;
892                continue;
893            }
894        } else {
895            // Route teardown marks the lifecycle admission gate before the last
896            // channel disappears. Requiring zero bound channels and a quiesced
897            // lifecycle prevents a still-bound root from losing its watcher.
898            if has_bound_route
899                || !meta.unbound_quiesced
900                || meta.idle_artifacts_evicted
901                || now.saturating_duration_since(meta.last_touched) < IDLE_ROOT_TTL
902                || meta.active_bash_waits > 0
903                || meta.maintenance_pending
904                || !meta.maintenance_queued_kinds.is_empty()
905                || has_pending_bind
906                || !executor.actor_is_idle(root_id)
907            {
908                continue;
909            }
910        }
911        candidates.push((root_id.clone(), deleted));
912    }
913
914    let mut reaped = Vec::new();
915    let mut forgotten_deleted_roots = Vec::new();
916    for (root_id, deleted) in candidates {
917        let Some(ctx) = executor.actor_context(&root_id) else {
918            if deleted {
919                census.deleted_retained += 1;
920                census.actor_busy += 1;
921            }
922            continue;
923        };
924        // A TTL-aged unbound root retained its watcher-derived pending paths
925        // across the transient-unbind window. Strict gap invalidation subsumes
926        // them, but every abort path must restore them because a rebind can
927        // still happen until eviction commits.
928        //
929        // After two consecutive directory-absence scans confirm that the
930        // root is gone, terminate its background task before checking the
931        // artifact-eviction gate. The task can otherwise keep the root's
932        // artifacts in use; cleanup first lets confirmed reclamation finish
933        // without weakening the gate for unrelated active work.
934        if deleted {
935            ctx.bash_background()
936                .kill_running_tasks_for_root(root_id.as_path());
937        }
938        let taken_pending = Some(ctx.take_pending_reconciliation_state());
939        if ctx.artifact_eviction_blocked() {
940            if let Some(pending) = taken_pending {
941                ctx.restore_pending_reconciliation_state(pending);
942            }
943            if deleted {
944                census.deleted_retained += 1;
945                census.artifact_eviction_blocked += 1;
946            }
947            continue;
948        }
949        let memory_before = ctx.memory_root_snapshot();
950        if !ctx.evict_idle_artifacts() {
951            if let Some(pending) = taken_pending {
952                ctx.restore_pending_reconciliation_state(pending);
953            }
954            if deleted {
955                census.deleted_retained += 1;
956                census.artifact_eviction_failed += 1;
957            }
958            continue;
959        }
960        drop(taken_pending);
961        ctx.stop_watcher_runtime_in_background();
962        // Edits during watcher downtime are unobserved. Advance publication
963        // epochs and force strict verification before any later warm reload.
964        ctx.invalidate_artifacts_after_watcher_gap();
965
966        if deleted {
967            if executor.retire_idle_actor_in_background(&root_id) {
968                live_roots.remove(&root_id);
969                forgotten_deleted_roots.push(root_id.clone());
970            } else {
971                census.deleted_retained += 1;
972                census.actor_busy += 1;
973            }
974        } else {
975            if let Some(meta) = live_roots.get_mut(&root_id) {
976                meta.idle_artifacts_evicted = true;
977            }
978            ctx.release_idle_reopenable_resources_in_background();
979        }
980        reaped.push((root_id, memory_before));
981    }
982
983    metrics.record_reap(census);
984    if census.deleted_retained > 0 {
985        log::info!(
986            "subc attach: retained {} deleted root(s) during idle reap; blockers={}",
987            census.deleted_retained,
988            census.blocker_histogram()
989        );
990    }
991
992    let pressure_relief = (!reaped.is_empty())
993        .then(|| allocator_pressure_relief_after_idle_sweep(now, live_roots, executor))
994        .flatten();
995    for (root_id, memory_before) in &reaped {
996        log::info!(
997            "{}",
998            idle_root_eviction_message(root_id, memory_before, pressure_relief.as_ref())
999        );
1000    }
1001    IdleReapOutcome {
1002        evicted: reaped.len(),
1003        forgotten_deleted_roots,
1004    }
1005}
1006
1007#[allow(clippy::too_many_arguments)]
1008fn purge_deleted_root_residents(
1009    root_id: &ProjectRootId,
1010    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1011    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1012    installed_route_epochs: &mut HashMap<u16, u32>,
1013    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1014    retry_buffer: &mut RetryBuffer,
1015    reclaimed_routes: &mut ReclaimedRoutes,
1016    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1017    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1018    bg_subs: &mut HashMap<RouteChannel, BgSub>,
1019    bg_sub_by_session: &mut BgSubsBySession,
1020    bg_wake_pending: &mut HashSet<RouteChannel>,
1021    bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
1022    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1023    metrics: &DispatchPathMetrics,
1024) {
1025    let mut stale_routes = root_channels.get(root_id).cloned().unwrap_or_default();
1026    stale_routes.extend(
1027        routes
1028            .iter()
1029            .filter_map(|(route, identity)| (&identity.root == root_id).then_some(*route)),
1030    );
1031    stale_routes.extend(
1032        bg_sub_by_session
1033            .iter()
1034            .filter(|((root, _), _)| root == root_id)
1035            .flat_map(|(_, routes)| routes.iter().copied()),
1036    );
1037    stale_routes.extend(
1038        pending_bash_asks
1039            .values()
1040            .filter_map(|ask| (&ask.root == root_id).then_some(ask.route)),
1041    );
1042
1043    for route in stale_routes {
1044        reclaimed_routes.insert(route);
1045        remove_installed_route(installed_route_epochs, route);
1046        remove_route_channel(routes, root_channels, route);
1047        if let Some(cancel) = route_bash_cancels.remove(&route) {
1048            cancel.token.cancel();
1049        }
1050        retry_buffer.remove(&route);
1051        if let Some(sub) = bg_subs.remove(&route) {
1052            metrics.record_bg_subscription_ended(&sub.root, &sub.session, route, "root-reclaim");
1053        }
1054        bg_wake_pending.remove(&route);
1055    }
1056    root_channels.remove(root_id);
1057    session_identity.retain(|(root, _), _| root != root_id);
1058    push_buffer.retain(|key, _| &key.root != root_id);
1059    bg_wake_epoch.retain(|(root, _), _| root != root_id);
1060    pending_bash_asks.retain(|_, ask| &ask.root != root_id);
1061    bg_sub_by_session.retain(|(root, _), _| root != root_id);
1062
1063    log::info!(
1064        "subc attach: fully forgot deleted root {}; cause=absence_reclaim",
1065        root_id.as_path().display()
1066    );
1067}
1068
1069#[allow(clippy::too_many_arguments)]
1070fn submit_due_maintenance_jobs(
1071    executor: &Arc<Executor>,
1072    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1073    pending_binds: &HashMap<RouteChannel, PendingBind>,
1074    bg_sub_by_session: &BgSubsBySession,
1075    bg_wake_pending: &HashSet<RouteChannel>,
1076    bg_wake_epoch: &HashMap<(ProjectRootId, String), u64>,
1077    maintenance_tx: &mpsc::Sender<MaintenanceCompletion>,
1078    metrics: &Arc<DispatchPathMetrics>,
1079) {
1080    let pending_bind_roots = pending_binds
1081        .values()
1082        .map(|pending| pending.bind_root_id.clone())
1083        .collect::<HashSet<_>>();
1084    let (due_jobs, deferred_jobs) = due_maintenance_jobs(
1085        live_roots,
1086        Some(executor),
1087        bg_sub_by_session,
1088        bg_wake_pending,
1089        MAINTENANCE_SUBMIT_BUDGET,
1090        &pending_bind_roots,
1091    );
1092    if deferred_jobs {
1093        metrics
1094            .maintenance_budget_deferrals
1095            .fetch_add(1, Ordering::Relaxed);
1096    }
1097    for (root_id, kind) in due_jobs {
1098        let bg_sessions_to_check = if kind == MaintenanceDrainKind::CompletionDrains {
1099            bg_sub_by_session
1100                .iter()
1101                .filter_map(|((root, session), _)| {
1102                    if root == &root_id {
1103                        Some((
1104                            session.clone(),
1105                            bg_wake_epoch
1106                                .get(&(root_id.clone(), session.clone()))
1107                                .copied()
1108                                .unwrap_or(0),
1109                        ))
1110                    } else {
1111                        None
1112                    }
1113                })
1114                .collect()
1115        } else {
1116            Vec::new()
1117        };
1118        submit_maintenance_job(
1119            executor,
1120            root_id,
1121            kind,
1122            bg_sessions_to_check,
1123            maintenance_tx,
1124            metrics,
1125        );
1126    }
1127}
1128
1129fn should_requiesce_after_maintenance(
1130    meta: &RootMeta,
1131    completed_kind: MaintenanceDrainKind,
1132    bind_pending: bool,
1133) -> bool {
1134    meta.unbound_quiesced && completed_kind != MaintenanceDrainKind::Lsp && !bind_pending
1135}
1136
1137fn note_maintenance_completion(
1138    meta: &mut RootMeta,
1139    requeue_kind: Option<MaintenanceDrainKind>,
1140    fatal: bool,
1141    defer_requeue: bool,
1142) {
1143    if fatal {
1144        meta.maintenance_poisoned = true;
1145    }
1146
1147    if let Some(kind) = requeue_kind.filter(|_| !meta.maintenance_poisoned && !defer_requeue) {
1148        meta.maintenance_queued_kinds.push_back(kind);
1149    }
1150
1151    meta.maintenance_jobs_in_flight = meta.maintenance_jobs_in_flight.saturating_sub(1);
1152    meta.maintenance_pending =
1153        meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
1154}
1155
1156fn route_key(channel: u16, epoch: u32) -> RouteChannel {
1157    RouteChannel { channel, epoch }
1158}
1159
1160fn remove_installed_route(installed_epochs: &mut HashMap<u16, u32>, route: RouteChannel) {
1161    if installed_epochs.get(&route.channel).copied() == Some(route.epoch) {
1162        installed_epochs.remove(&route.channel);
1163    }
1164}
1165
1166fn ingress_route_should_be_processed(
1167    installed_epochs: &HashMap<u16, u32>,
1168    reclaimed_routes: &ReclaimedRoutes,
1169    frame: &Frame,
1170) -> bool {
1171    if frame.header.channel == 0
1172        || installed_epochs.get(&frame.header.channel).copied() == Some(frame.header.epoch)
1173    {
1174        return true;
1175    }
1176
1177    // A late request for a reclaimed root reaches the normal unknown-route
1178    // handler, which returns the typed `route_not_bound` error. Other stale or
1179    // never-installed generations remain silent so they cannot affect a newer
1180    // route or change the protocol's rejected-bind behavior.
1181    frame.header.ty == FrameType::Request
1182        && reclaimed_routes.contains(route_key(frame.header.channel, frame.header.epoch))
1183}
1184
1185fn bash_elicitation_timeout() -> Duration {
1186    if cfg!(debug_assertions) {
1187        if let Ok(raw) = std::env::var("AFT_TEST_SUBC_BASH_ELICITATION_TTL_MS") {
1188            if let Ok(ms) = raw.parse::<u64>() {
1189                if ms > 0 {
1190                    return Duration::from_millis(ms);
1191                }
1192            }
1193        }
1194    }
1195    BASH_ELICITATION_TIMEOUT
1196}
1197
1198fn allocate_reverse_corr(
1199    pending_bash_asks: &HashMap<ReverseCorrKey, PendingBashAsk>,
1200    route: RouteChannel,
1201    next_corr: &mut u64,
1202) -> u64 {
1203    loop {
1204        let corr = *next_corr;
1205        *next_corr = (*next_corr).wrapping_add(1).max(1);
1206        if !pending_bash_asks.contains_key(&ReverseCorrKey { route, corr }) {
1207            return corr;
1208        }
1209    }
1210}
1211
1212fn bash_permission_kind_label(kind: &crate::bash_permissions::PermissionKind) -> &'static str {
1213    match kind {
1214        crate::bash_permissions::PermissionKind::ExternalDirectory => "external directory",
1215        crate::bash_permissions::PermissionKind::Bash => "bash",
1216    }
1217}
1218
1219fn bash_elicitation_patterns(asks: &[crate::bash_permissions::PermissionAsk]) -> Vec<String> {
1220    let mut patterns = Vec::new();
1221    let mut seen = HashSet::new();
1222    for ask in asks {
1223        for pattern in ask.patterns.iter().chain(ask.always.iter()) {
1224            if seen.insert(pattern.clone()) {
1225                patterns.push(pattern.clone());
1226            }
1227        }
1228    }
1229    patterns
1230}
1231
1232fn bash_elicitation_message(
1233    command: &str,
1234    asks: &[crate::bash_permissions::PermissionAsk],
1235) -> String {
1236    let command = command.split_whitespace().collect::<Vec<_>>().join(" ");
1237    let patterns = bash_elicitation_patterns(asks);
1238    let pattern_text = if patterns.is_empty() {
1239        "no matched permission patterns".to_string()
1240    } else {
1241        patterns.join(", ")
1242    };
1243    let ask_kinds = asks
1244        .iter()
1245        .map(|ask| bash_permission_kind_label(&ask.kind))
1246        .collect::<HashSet<_>>()
1247        .into_iter()
1248        .collect::<Vec<_>>()
1249        .join(", ");
1250    if ask_kinds.is_empty() {
1251        format!("Allow bash command `{command}`? Matched patterns: {pattern_text}")
1252    } else {
1253        format!("Allow bash command `{command}`? Matched {ask_kinds} patterns: {pattern_text}")
1254    }
1255}
1256
1257fn bash_elicitation_request_body(
1258    command: &str,
1259    asks: &[crate::bash_permissions::PermissionAsk],
1260) -> Value {
1261    json!({
1262        "method": BASH_ELICITATION_CREATE_METHOD,
1263        "params": {
1264            "mode": "form",
1265            "message": bash_elicitation_message(command, asks),
1266            "requestedSchema": {
1267                "type": "object",
1268                "properties": {
1269                    "decision": {
1270                        "type": "string",
1271                        "enum": ["allow", "deny"],
1272                        "description": "Choose allow to run this bash command once, or deny to block it."
1273                    }
1274                },
1275                "required": ["decision"],
1276                "additionalProperties": false
1277            },
1278            "_meta": {
1279                "aft": {
1280                    "tool": "bash",
1281                    "command": command,
1282                    "asks": asks
1283                }
1284            }
1285        }
1286    })
1287}
1288
1289fn build_bash_elicitation_request_frame(
1290    ver: u8,
1291    route: RouteChannel,
1292    corr: u64,
1293    flags: Flags,
1294    command: &str,
1295    asks: &[crate::bash_permissions::PermissionAsk],
1296) -> Result<Frame, SubcError> {
1297    let body = bash_elicitation_request_body(command, asks);
1298    Frame::build_with_version(
1299        ver,
1300        FrameType::Request,
1301        flags,
1302        route.channel,
1303        route.epoch,
1304        corr,
1305        serde_json::to_vec(&body).map_err(SubcError::Json)?,
1306    )
1307    .map_err(SubcError::FrameBuild)
1308}
1309
1310fn bash_elicitation_reply_is_allow(body: &[u8]) -> bool {
1311    let Ok(value) = serde_json::from_slice::<Value>(body) else {
1312        return false;
1313    };
1314    flat_bash_elicitation_reply_is_allow(&value) || mcp_bash_elicitation_reply_is_allow(&value)
1315}
1316
1317fn flat_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1318    let Some(object) = value.as_object() else {
1319        return false;
1320    };
1321    object.len() == 1 && object.get("decision").and_then(Value::as_str) == Some("allow")
1322}
1323
1324fn mcp_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1325    let Some(object) = value.as_object() else {
1326        return false;
1327    };
1328    if object.len() != 2 || object.get("action").and_then(Value::as_str) != Some("accept") {
1329        return false;
1330    }
1331    let Some(content) = object.get("content").and_then(Value::as_object) else {
1332        return false;
1333    };
1334    content.len() == 1 && content.get("decision").and_then(Value::as_str) == Some("allow")
1335}
1336
1337#[allow(clippy::too_many_arguments)]
1338async fn settle_pending_bash_ask_denied(
1339    tx: &WriterSender,
1340    pending: PendingBashAsk,
1341    routes: &HashMap<RouteChannel, RouteIdentity>,
1342    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1343    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1344    shutdown: &Arc<Notify>,
1345    metrics: &DispatchPathMetrics,
1346) -> Result<(), SubcError> {
1347    let completion = bash::bash_denied_untrusted_completion(
1348        pending.route,
1349        pending.tool_corr,
1350        pending.tool_flags,
1351        pending.tool_ver,
1352        pending.root,
1353        pending.request_id,
1354        pending.format_context,
1355    );
1356    bash::handle_bash_deferred_completion(
1357        tx,
1358        completion,
1359        routes,
1360        live_roots,
1361        route_bash_cancels,
1362        shutdown,
1363        metrics,
1364    )
1365    .await
1366}
1367
1368fn take_pending_bash_asks_for_route(
1369    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1370    route: RouteChannel,
1371) -> Vec<PendingBashAsk> {
1372    let keys = pending_bash_asks
1373        .keys()
1374        .copied()
1375        .filter(|key| key.route == route)
1376        .collect::<Vec<_>>();
1377    keys.into_iter()
1378        .filter_map(|key| pending_bash_asks.remove(&key))
1379        .collect()
1380}
1381
1382#[allow(clippy::too_many_arguments)]
1383async fn settle_pending_bash_asks_for_route(
1384    tx: &WriterSender,
1385    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1386    route: RouteChannel,
1387    routes: &HashMap<RouteChannel, RouteIdentity>,
1388    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1389    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1390    shutdown: &Arc<Notify>,
1391    metrics: &DispatchPathMetrics,
1392) -> Result<(), SubcError> {
1393    for pending in take_pending_bash_asks_for_route(pending_bash_asks, route) {
1394        settle_pending_bash_ask_denied(
1395            tx,
1396            pending,
1397            routes,
1398            live_roots,
1399            route_bash_cancels,
1400            shutdown,
1401            metrics,
1402        )
1403        .await?;
1404    }
1405    Ok(())
1406}
1407
1408#[allow(clippy::too_many_arguments)]
1409async fn settle_all_pending_bash_asks(
1410    tx: &WriterSender,
1411    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1412    routes: &HashMap<RouteChannel, RouteIdentity>,
1413    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1414    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1415    shutdown: &Arc<Notify>,
1416    metrics: &DispatchPathMetrics,
1417) -> Result<(), SubcError> {
1418    let pending = pending_bash_asks
1419        .drain()
1420        .map(|(_, pending)| pending)
1421        .collect::<Vec<_>>();
1422    for pending in pending {
1423        settle_pending_bash_ask_denied(
1424            tx,
1425            pending,
1426            routes,
1427            live_roots,
1428            route_bash_cancels,
1429            shutdown,
1430            metrics,
1431        )
1432        .await?;
1433    }
1434    Ok(())
1435}
1436
1437#[allow(clippy::too_many_arguments)]
1438async fn expire_pending_bash_asks(
1439    tx: &WriterSender,
1440    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1441    routes: &HashMap<RouteChannel, RouteIdentity>,
1442    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1443    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1444    shutdown: &Arc<Notify>,
1445    metrics: &DispatchPathMetrics,
1446) -> Result<(), SubcError> {
1447    let now = Instant::now();
1448    let expired = pending_bash_asks
1449        .iter()
1450        .filter_map(|(key, pending)| (pending.expires_at <= now).then_some(*key))
1451        .collect::<Vec<_>>();
1452    for key in expired {
1453        if let Some(pending) = pending_bash_asks.remove(&key) {
1454            log::debug!(
1455                "subc attach: bash elicitation request {} on route {} expired fail-closed",
1456                key.corr,
1457                pending.route
1458            );
1459            settle_pending_bash_ask_denied(
1460                tx,
1461                pending,
1462                routes,
1463                live_roots,
1464                route_bash_cancels,
1465                shutdown,
1466                metrics,
1467            )
1468            .await?;
1469        }
1470    }
1471    Ok(())
1472}
1473
1474#[allow(clippy::too_many_arguments)]
1475async fn handle_bash_elicitation_reply(
1476    tx: &WriterSender,
1477    frame: &Frame,
1478    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1479    routes: &HashMap<RouteChannel, RouteIdentity>,
1480    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1481    executor: &Arc<Executor>,
1482    shutdown: &Arc<Notify>,
1483    bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
1484    bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
1485    metrics: &Arc<DispatchPathMetrics>,
1486    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1487    dispatch: DispatchFn,
1488) -> Result<(), SubcError> {
1489    let key = ReverseCorrKey {
1490        route: route_key(frame.header.channel, frame.header.epoch),
1491        corr: frame.header.corr,
1492    };
1493    let Some(pending) = pending_bash_asks.remove(&key) else {
1494        return Ok(());
1495    };
1496
1497    if frame.header.ty == FrameType::Response && bash_elicitation_reply_is_allow(&frame.body) {
1498        if routes.contains_key(&key.route) {
1499            bash::submit_deferred_bash(
1500                executor,
1501                bash_deferred_tx,
1502                bash_poll_touch_tx,
1503                metrics,
1504                dispatch,
1505                pending.root,
1506                pending.project_root,
1507                pending.session_id,
1508                pending.request_id,
1509                pending.route,
1510                pending.tool_corr,
1511                pending.tool_flags,
1512                pending.tool_ver,
1513                pending.arguments,
1514                pending.format_context,
1515                pending.cancel,
1516                BindTrust::Untrusted,
1517                pending.spawn_principal,
1518                pending.edit_slot_survives,
1519                Some(pending.grants),
1520            );
1521            return Ok(());
1522        }
1523        log::debug!(
1524            "subc attach: dropping allowed bash elicitation reply {} for unbound route {}",
1525            key.corr,
1526            pending.route
1527        );
1528    }
1529
1530    settle_pending_bash_ask_denied(
1531        tx,
1532        pending,
1533        routes,
1534        live_roots,
1535        route_bash_cancels,
1536        shutdown,
1537        metrics,
1538    )
1539    .await
1540}
1541
1542#[allow(clippy::too_many_arguments)]
1543async fn cancel_pending_bash_ask_for_tool_call(
1544    tx: &WriterSender,
1545    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1546    route: RouteChannel,
1547    tool_corr: u64,
1548    routes: &HashMap<RouteChannel, RouteIdentity>,
1549    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1550    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1551    shutdown: &Arc<Notify>,
1552    metrics: &DispatchPathMetrics,
1553) -> Result<(), SubcError> {
1554    let keys = pending_bash_asks
1555        .iter()
1556        .filter_map(|(key, pending)| {
1557            (key.route == route && pending.tool_corr == tool_corr).then_some(*key)
1558        })
1559        .collect::<Vec<_>>();
1560    for key in keys {
1561        if let Some(pending) = pending_bash_asks.remove(&key) {
1562            settle_pending_bash_ask_denied(
1563                tx,
1564                pending,
1565                routes,
1566                live_roots,
1567                route_bash_cancels,
1568                shutdown,
1569                metrics,
1570            )
1571            .await?;
1572        }
1573    }
1574    Ok(())
1575}
1576
1577fn remove_root_channel(
1578    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1579    root: &ProjectRootId,
1580    channel: RouteChannel,
1581) {
1582    let remove_root = if let Some(channels) = root_channels.get_mut(root) {
1583        channels.remove(&channel);
1584        channels.is_empty()
1585    } else {
1586        false
1587    };
1588    if remove_root {
1589        root_channels.remove(root);
1590    }
1591}
1592
1593fn remove_route_channel(
1594    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1595    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1596    channel: RouteChannel,
1597) -> Option<RouteIdentity> {
1598    let removed = routes.remove(&channel);
1599    if let Some(identity) = &removed {
1600        remove_root_channel(root_channels, &identity.root, channel);
1601    }
1602    removed
1603}
1604
1605fn insert_route_channel(
1606    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1607    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1608    channel: RouteChannel,
1609    identity: RouteIdentity,
1610) {
1611    if let Some(previous) = routes.insert(channel, identity.clone()) {
1612        remove_root_channel(root_channels, &previous.root, channel);
1613    }
1614    root_channels
1615        .entry(identity.root.clone())
1616        .or_default()
1617        .insert(channel);
1618}
1619
1620fn insert_bg_subscription_index(
1621    bg_sub_by_session: &mut BgSubsBySession,
1622    root: ProjectRootId,
1623    session: String,
1624    channel: RouteChannel,
1625) {
1626    bg_sub_by_session
1627        .entry((root, session))
1628        .or_default()
1629        .insert(channel);
1630}
1631
1632fn remove_bg_subscription_index(
1633    bg_sub_by_session: &mut BgSubsBySession,
1634    channel: RouteChannel,
1635    identity: Option<&RouteIdentity>,
1636) {
1637    if let Some(identity) = identity {
1638        let key = (identity.root.clone(), identity.session.clone());
1639        let remove_key = bg_sub_by_session.get_mut(&key).is_some_and(|channels| {
1640            channels.remove(&channel);
1641            channels.is_empty()
1642        });
1643        if remove_key {
1644            bg_sub_by_session.remove(&key);
1645        }
1646    } else {
1647        bg_sub_by_session.retain(|_, channels| {
1648            channels.remove(&channel);
1649            !channels.is_empty()
1650        });
1651    }
1652}
1653
1654fn route_removal_will_quiesce_root(
1655    root: &ProjectRootId,
1656    route: RouteChannel,
1657    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1658    has_pending_bind: bool,
1659    replacement_root: Option<&ProjectRootId>,
1660) -> bool {
1661    let removes_last_route = root_channels
1662        .get(root)
1663        .is_some_and(|channels| channels.len() == 1 && channels.contains(&route));
1664    removes_last_route && !has_pending_bind && replacement_root != Some(root)
1665}
1666
1667fn should_quiesce_removed_root(
1668    root: &ProjectRootId,
1669    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1670    has_pending_bind: bool,
1671    replacement_root: Option<&ProjectRootId>,
1672) -> bool {
1673    !root_channels.contains_key(root) && !has_pending_bind && replacement_root != Some(root)
1674}
1675
1676async fn end_bg_subscription(
1677    writer_tx: &WriterSender,
1678    metrics: &DispatchPathMetrics,
1679    bg_subs: &mut HashMap<RouteChannel, BgSub>,
1680    bg_sub_by_session: &mut BgSubsBySession,
1681    bg_wake_pending: &mut HashSet<RouteChannel>,
1682    channel: RouteChannel,
1683    identity: Option<&RouteIdentity>,
1684    cause: &str,
1685) -> Result<(), SubcError> {
1686    if let Some(sub) = bg_subs.remove(&channel) {
1687        bg_wake_pending.remove(&channel);
1688        remove_bg_subscription_index(bg_sub_by_session, channel, identity);
1689        metrics.record_bg_subscription_ended(&sub.root, &sub.session, channel, cause);
1690        push::send_reliable_bg_stream_end(writer_tx, metrics, channel, &sub).await?;
1691    }
1692    Ok(())
1693}
1694
1695#[allow(clippy::too_many_arguments)]
1696async fn teardown_installed_route(
1697    tx: &WriterSender,
1698    metrics: &DispatchPathMetrics,
1699    executor: &Arc<Executor>,
1700    channel: RouteChannel,
1701    cancellation_reason: &str,
1702    replacement_root: Option<&ProjectRootId>,
1703    installed_route_epochs: &mut HashMap<u16, u32>,
1704    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1705    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1706    bg_subs: &mut HashMap<RouteChannel, BgSub>,
1707    bg_sub_by_session: &mut BgSubsBySession,
1708    bg_wake_pending: &mut HashSet<RouteChannel>,
1709    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1710    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1711    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1712    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
1713    retry_buffer: &mut RetryBuffer,
1714    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1715    shutdown: &Arc<Notify>,
1716) -> Result<(), SubcError> {
1717    remove_installed_route(installed_route_epochs, channel);
1718    let bg_end_cause = match cancellation_reason {
1719        "Goodbye" => "goodbye",
1720        "higher-epoch RouteBind" => "higher-epoch",
1721        other => other,
1722    };
1723    end_bg_subscription(
1724        tx,
1725        metrics,
1726        bg_subs,
1727        bg_sub_by_session,
1728        bg_wake_pending,
1729        channel,
1730        routes.get(&channel),
1731        bg_end_cause,
1732    )
1733    .await?;
1734    settle_pending_bash_asks_for_route(
1735        tx,
1736        pending_bash_asks,
1737        channel,
1738        routes,
1739        live_roots,
1740        route_bash_cancels,
1741        shutdown,
1742        metrics,
1743    )
1744    .await?;
1745    if let Some(cancel) = route_bash_cancels.remove(&channel) {
1746        cancel.token.cancel();
1747    }
1748    if let Some(pending) = pending_binds.get_mut(&channel) {
1749        pending.cancelled = true;
1750        let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
1751        log::debug!(
1752            "subc attach: cancelled pending RouteBind for route {} on {cancellation_reason} (configure job: {outcome:?})",
1753            channel.channel
1754        );
1755    }
1756    let migrated = push::migrate_retry_buffer_to_push_buffer(retry_buffer, channel, push_buffer);
1757    if let Some(identity) = routes.get(&channel) {
1758        let has_pending_bind = pending_binds
1759            .values()
1760            .any(|pending| pending.bind_root_id == identity.root);
1761        if route_removal_will_quiesce_root(
1762            &identity.root,
1763            channel,
1764            root_channels,
1765            has_pending_bind,
1766            replacement_root,
1767        ) {
1768            if let Some(ctx) = executor.actor_context(&identity.root) {
1769                // Fence deferred admissions before the final route disappears
1770                // from the loop-owned routing tables.
1771                ctx.mark_subc_unbound();
1772            }
1773        }
1774    }
1775    if let Some(identity) = remove_route_channel(routes, root_channels, channel) {
1776        let session_still_routed = routes
1777            .values()
1778            .any(|route| route.root == identity.root && route.session == identity.session);
1779        if !session_still_routed {
1780            if let Some(ctx) = executor.actor_context(&identity.root) {
1781                ctx.hashline_bindings()
1782                    .teardown(identity.root.as_path(), &identity.session);
1783            }
1784        }
1785        if migrated > 0 {
1786            log::debug!(
1787                "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from route {} into detach replay",
1788                channel.channel
1789            );
1790        }
1791        if let Some(meta) = live_roots.get_mut(&identity.root) {
1792            let idle_for = meta.last_touched.elapsed();
1793            meta.note_activity();
1794            log::debug!(
1795                "subc attach: route {} torn down for root {} harness {} session {} (last touched {:?} ago)",
1796                channel.channel,
1797                identity.root.as_path().display(),
1798                identity.harness,
1799                identity.session,
1800                idle_for
1801            );
1802        } else {
1803            log::debug!(
1804                "subc attach: route {} torn down for root {} harness {} session {}",
1805                channel.channel,
1806                identity.root.as_path().display(),
1807                identity.harness,
1808                identity.session
1809            );
1810        }
1811        let has_pending_bind = pending_binds
1812            .values()
1813            .any(|pending| pending.bind_root_id == identity.root);
1814        if should_quiesce_removed_root(
1815            &identity.root,
1816            root_channels,
1817            has_pending_bind,
1818            replacement_root,
1819        ) {
1820            quiesce_unbound_root(&identity.root, live_roots, executor);
1821        }
1822    } else {
1823        if migrated > 0 {
1824            log::debug!(
1825                "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from unbound route {} into detach replay",
1826                channel.channel
1827            );
1828        }
1829        log::debug!("subc attach: unbound route {} torn down", channel.channel);
1830    }
1831    Ok(())
1832}
1833
1834fn remember_session_identity(
1835    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1836    identity: &RouteIdentity,
1837) {
1838    let key = (identity.root.clone(), identity.session.clone());
1839    if matches!(identity.trust, BindTrust::Untrusted)
1840        && session_identity
1841            .get(&key)
1842            .is_some_and(|retained| matches!(retained.trust, BindTrust::FirstParty))
1843    {
1844        return;
1845    }
1846
1847    // Retained after route Goodbye so reliable session-scoped frames emitted while
1848    // the session is detached can still be keyed by the full (root,harness,session)
1849    // replay triple. Untrusted binds never overwrite a retained first-party
1850    // session identity, because bash completion replay is an observation channel.
1851    session_identity.insert(
1852        key,
1853        RetainedSessionIdentity {
1854            harness: identity.harness.clone(),
1855            trust: identity.trust,
1856        },
1857    );
1858}
1859
1860fn replay_key_for_session(
1861    session_identity: &HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1862    root: &ProjectRootId,
1863    session: &str,
1864) -> Option<(push::ReplayKey, BindTrust)> {
1865    let retained = session_identity.get(&(root.clone(), session.to_string()))?;
1866    Some((
1867        push::ReplayKey {
1868            root: root.clone(),
1869            harness: retained.harness.clone(),
1870            session: session.to_string(),
1871        },
1872        retained.trust,
1873    ))
1874}
1875/// Sync command dispatch, passed in from `main` (the binary owns the command
1876/// table). Invoked only inside executor jobs in subc mode.
1877pub type DispatchFn = fn(RawRequest, &AppContext) -> Response;
1878
1879#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1880enum ModuleLoopExit {
1881    Graceful,
1882    SkipSearchFlush,
1883}
1884
1885/// Entry point for `aft --subc <connection-file>`. Synchronous on the outside;
1886/// owns an isolated current-thread tokio runtime for the async transport.
1887/// Returns `Err` (fail-loud) on any connect/auth/protocol failure — we never
1888/// fall back to the standalone loop, to avoid split-brain index state.
1889pub fn run_subc_mode(
1890    connection_file_path: &Path,
1891    ctx: Arc<AppContext>,
1892    executor: Arc<Executor>,
1893    dispatch: DispatchFn,
1894    user_config_path: Option<PathBuf>,
1895) -> Result<(), SubcError> {
1896    // Production NEVER allows non-manifest tool names on route channels: AFT
1897    // fails closed and does not trust subc to enforce the manifest. The
1898    // test-only harness sets this through `run_subc_mode_for_test`.
1899    run_subc_mode_inner(
1900        connection_file_path,
1901        ctx,
1902        executor,
1903        dispatch,
1904        user_config_path,
1905        false,
1906        MAX_FRAME_BODY_LEN as usize,
1907    )
1908}
1909
1910fn run_subc_mode_inner(
1911    connection_file_path: &Path,
1912    ctx: Arc<AppContext>,
1913    executor: Arc<Executor>,
1914    dispatch: DispatchFn,
1915    user_config_path: Option<PathBuf>,
1916    allow_native_passthrough: bool,
1917    tool_response_body_limit: usize,
1918) -> Result<(), SubcError> {
1919    let runtime = tokio::runtime::Builder::new_current_thread()
1920        .enable_all()
1921        .build()
1922        .map_err(SubcError::Runtime)?;
1923
1924    let executor_for_loop = Arc::clone(&executor);
1925    let loop_result = runtime.block_on(async move {
1926        let shared_app = ctx.app();
1927        drop(ctx);
1928        let stream = connect_and_authenticate(connection_file_path).await?;
1929        log::info!(
1930            "subc attach: authenticated to daemon via {}",
1931            connection_file_path.display()
1932        );
1933        let (read_half, write_half) = tokio::io::split(stream);
1934        run_module_loop(
1935            read_half,
1936            write_half,
1937            connection_file_path,
1938            shared_app,
1939            executor_for_loop,
1940            dispatch,
1941            user_config_path,
1942            allow_native_passthrough,
1943            tool_response_body_limit,
1944        )
1945        .await
1946    });
1947
1948    let actor_contexts = executor.actor_contexts();
1949    if matches!(loop_result, Ok(ModuleLoopExit::Graceful)) {
1950        // EOF/Goodbye teardown flushes each root's index deltas and queued
1951        // callgraph refreshes. Fatal/panic teardown skips this best-effort work.
1952        flush_actor_indexes_on_graceful_shutdown(&actor_contexts);
1953    }
1954    for actor_ctx in &actor_contexts {
1955        actor_ctx.lsp().shutdown_all();
1956        actor_ctx.bash_background().detach();
1957    }
1958
1959    loop_result.map(|_| ())
1960}
1961
1962fn flush_actor_indexes_on_graceful_shutdown(actor_contexts: &[Arc<AppContext>]) {
1963    for actor_ctx in actor_contexts {
1964        let _ = actor_ctx.flush_search_index_on_graceful_shutdown();
1965    }
1966    let _ = crate::callgraph_store::flush_callgraph_store_refreshes_on_graceful_shutdown();
1967}
1968
1969/// Test-only entry that enables the non-manifest native-command passthrough on
1970/// route channels. Integration tests drive synthetic native commands (`glob`,
1971/// `callers`, `subc_test_echo_session`, …) through the executor to exercise
1972/// mechanics; production callers use [`run_subc_mode`], which fails closed.
1973#[doc(hidden)]
1974pub fn run_subc_mode_for_test(
1975    connection_file_path: &Path,
1976    ctx: Arc<AppContext>,
1977    executor: Arc<Executor>,
1978    dispatch: DispatchFn,
1979    user_config_path: Option<PathBuf>,
1980) -> Result<(), SubcError> {
1981    run_subc_mode_inner(
1982        connection_file_path,
1983        ctx,
1984        executor,
1985        dispatch,
1986        user_config_path,
1987        true,
1988        MAX_FRAME_BODY_LEN as usize,
1989    )
1990}
1991
1992/// Test-only entry that lowers the effective tool-response body limit without
1993/// allocating a 64 MiB fixture. The fixed fallback envelope needs 4 KiB of room.
1994#[doc(hidden)]
1995pub fn run_subc_mode_for_test_with_response_body_limit(
1996    connection_file_path: &Path,
1997    ctx: Arc<AppContext>,
1998    executor: Arc<Executor>,
1999    dispatch: DispatchFn,
2000    user_config_path: Option<PathBuf>,
2001    tool_response_body_limit: usize,
2002) -> Result<(), SubcError> {
2003    assert!((4 * 1_024..=MAX_FRAME_BODY_LEN as usize).contains(&tool_response_body_limit));
2004    run_subc_mode_inner(
2005        connection_file_path,
2006        ctx,
2007        executor,
2008        dispatch,
2009        user_config_path,
2010        true,
2011        tool_response_body_limit,
2012    )
2013}
2014
2015#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2016enum AttachErrorClass {
2017    Transient,
2018    Permanent,
2019}
2020
2021impl fmt::Display for AttachErrorClass {
2022    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2023        match self {
2024            Self::Transient => f.write_str("transient"),
2025            Self::Permanent => f.write_str("permanent"),
2026        }
2027    }
2028}
2029
2030#[derive(Clone, Copy)]
2031struct AttachRetryPolicy {
2032    budget: Duration,
2033    initial_backoff: Duration,
2034    max_backoff: Duration,
2035    jitter_percent: u64,
2036}
2037
2038const ATTACH_RETRY_POLICY: AttachRetryPolicy = AttachRetryPolicy {
2039    budget: ATTACH_RETRY_BUDGET,
2040    initial_backoff: ATTACH_RETRY_INITIAL_BACKOFF,
2041    max_backoff: ATTACH_RETRY_MAX_BACKOFF,
2042    jitter_percent: ATTACH_RETRY_JITTER_PERCENT,
2043};
2044
2045/// Retry only failures that can be caused by a daemon bounce or an interrupted
2046/// handshake. Protocol and credential failures are permanent for this process.
2047fn classify_attach_error(error: &SubcError) -> AttachErrorClass {
2048    let transient = match error {
2049        SubcError::Connect { source, .. } => is_transient_attach_io(source.kind()),
2050        SubcError::Auth { source, .. } => match source {
2051            subc_transport::AuthError::Timeout { .. }
2052            | subc_transport::AuthError::UnexpectedEof { .. } => true,
2053            subc_transport::AuthError::Io { source, .. } => is_transient_attach_io(source.kind()),
2054            _ => false,
2055        },
2056        _ => false,
2057    };
2058    if transient {
2059        AttachErrorClass::Transient
2060    } else {
2061        AttachErrorClass::Permanent
2062    }
2063}
2064
2065fn is_transient_attach_io(kind: io::ErrorKind) -> bool {
2066    matches!(
2067        kind,
2068        io::ErrorKind::ConnectionRefused
2069            | io::ErrorKind::TimedOut
2070            | io::ErrorKind::ConnectionReset
2071            | io::ErrorKind::ConnectionAborted
2072            | io::ErrorKind::BrokenPipe
2073            | io::ErrorKind::UnexpectedEof
2074    )
2075}
2076
2077/// Read the connection file → resolve the first endpoint → TCP connect → HMAC
2078/// handshake. Transient initial-attach failures retry on fresh sockets and reread
2079/// the file so a daemon bounce can publish a new endpoint or authentication key.
2080async fn connect_and_authenticate(connection_file_path: &Path) -> Result<TcpStream, SubcError> {
2081    connect_and_authenticate_with_policy(connection_file_path, ATTACH_RETRY_POLICY).await
2082}
2083
2084async fn connect_and_authenticate_with_policy(
2085    connection_file_path: &Path,
2086    policy: AttachRetryPolicy,
2087) -> Result<TcpStream, SubcError> {
2088    let started_at = Instant::now();
2089    let deadline = started_at + policy.budget;
2090    let mut attempt = 0_u32;
2091    let mut backoff = policy.initial_backoff;
2092    let mut history = Vec::new();
2093
2094    loop {
2095        attempt = attempt.saturating_add(1);
2096        let error = match connect_and_authenticate_once(connection_file_path, deadline).await {
2097            Ok(stream) => return Ok(stream),
2098            Err(error) => error,
2099        };
2100        let class = classify_attach_error(&error);
2101        let error_text = error.to_string().lines().collect::<Vec<_>>().join(" ");
2102        history.push(format!("attempt {attempt} [{class}]: {error_text}"));
2103
2104        if class == AttachErrorClass::Permanent {
2105            log_attach_final_failure(started_at.elapsed(), &history);
2106            return Err(error);
2107        }
2108
2109        let remaining = deadline.saturating_duration_since(Instant::now());
2110        if remaining.is_zero() {
2111            log_attach_final_failure(started_at.elapsed(), &history);
2112            return Err(error);
2113        }
2114
2115        let delay = jittered_attach_delay(backoff, policy.jitter_percent, attempt).min(remaining);
2116        log::info!(
2117            "subc attach retry: attempt {attempt} failed; error_class={class}; error={error_text}; next_delay={delay:?}"
2118        );
2119        tokio::time::sleep(delay).await;
2120
2121        if Instant::now() >= deadline {
2122            log_attach_final_failure(started_at.elapsed(), &history);
2123            return Err(error);
2124        }
2125        backoff = backoff.saturating_mul(2).min(policy.max_backoff);
2126    }
2127}
2128
2129fn jittered_attach_delay(base: Duration, jitter_percent: u64, attempt: u32) -> Duration {
2130    let jitter_percent = jitter_percent.min(100);
2131    if jitter_percent == 0 {
2132        return base;
2133    }
2134
2135    let mut random_bytes = [0_u8; 8];
2136    let random = if getrandom::fill(&mut random_bytes).is_ok() {
2137        u64::from_le_bytes(random_bytes)
2138    } else {
2139        let timestamp = std::time::SystemTime::now()
2140            .duration_since(std::time::UNIX_EPOCH)
2141            .unwrap_or_default()
2142            .subsec_nanos();
2143        u64::from(timestamp) ^ u64::from(attempt)
2144    };
2145    let span = jitter_percent.saturating_mul(2).saturating_add(1);
2146    let multiplier_percent = 100 - jitter_percent + random % span;
2147    let base_millis = u64::try_from(base.as_millis()).unwrap_or(u64::MAX);
2148    Duration::from_millis(base_millis.saturating_mul(multiplier_percent) / 100)
2149}
2150
2151fn log_attach_final_failure(elapsed: Duration, history: &[String]) {
2152    log::error!(
2153        "subc initial attach failed after {} attempt(s) in {elapsed:?}; attempt history: {}",
2154        history.len(),
2155        history.join(" | ")
2156    );
2157}
2158
2159async fn connect_and_authenticate_once(
2160    connection_file_path: &Path,
2161    deadline: Instant,
2162) -> Result<TcpStream, SubcError> {
2163    // This read intentionally lives inside the per-attempt function. The daemon
2164    // publishes connection files atomically and may change both port and key.
2165    let conn = connection_file::read_for_client(connection_file_path).map_err(|source| {
2166        SubcError::ConnectionFile {
2167            path: connection_file_path.to_path_buf(),
2168            source,
2169        }
2170    })?;
2171
2172    let endpoint = conn
2173        .endpoints
2174        .first()
2175        .ok_or_else(|| SubcError::NoEndpoint {
2176            path: connection_file_path.to_path_buf(),
2177        })?;
2178    let endpoint_label = format!("{}:{}", endpoint.host, endpoint.port);
2179    let ip = endpoint
2180        .host
2181        .parse::<IpAddr>()
2182        .map_err(|_| SubcError::InvalidEndpoint {
2183            path: connection_file_path.to_path_buf(),
2184            endpoint: endpoint_label.clone(),
2185        })?;
2186    let addr = SocketAddr::new(ip, endpoint.port);
2187
2188    let connect_budget = deadline.saturating_duration_since(Instant::now());
2189    let mut stream = tokio::time::timeout(connect_budget, TcpStream::connect(addr))
2190        .await
2191        .map_err(|_| SubcError::Connect {
2192            endpoint: endpoint_label.clone(),
2193            source: io::Error::new(
2194                io::ErrorKind::TimedOut,
2195                "initial subc attach retry budget elapsed during TCP connect",
2196            ),
2197        })?
2198        .map_err(|source| SubcError::Connect {
2199            endpoint: endpoint_label.clone(),
2200            source,
2201        })?;
2202    stream
2203        .set_nodelay(true)
2204        .map_err(|source| SubcError::Connect {
2205            endpoint: endpoint_label.clone(),
2206            source,
2207        })?;
2208
2209    let auth_budget = AUTH_DEADLINE.min(deadline.saturating_duration_since(Instant::now()));
2210    authenticate_client(&mut stream, &conn, auth_budget)
2211        .await
2212        .map_err(|source| SubcError::Auth {
2213            endpoint: endpoint_label,
2214            source,
2215        })?;
2216
2217    Ok(stream)
2218}
2219
2220#[allow(clippy::too_many_arguments)]
2221async fn process_route_bind_completion(
2222    writer_tx: &WriterSender,
2223    completion: RouteBindCompletion,
2224    routes: &mut HashMap<RouteChannel, RouteIdentity>,
2225    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2226    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2227    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2228    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2229    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2230    installed_route_epochs: &mut HashMap<u16, u32>,
2231    executor: &Arc<Executor>,
2232    shutdown: &Arc<Notify>,
2233    metrics: &Arc<DispatchPathMetrics>,
2234) -> Result<(), SubcError> {
2235    decrement_counted_channel(&metrics.control_completion_queued);
2236    handle_route_bind_completion(
2237        writer_tx,
2238        completion,
2239        routes,
2240        root_channels,
2241        session_identity,
2242        push_buffer,
2243        live_roots,
2244        pending_binds,
2245        installed_route_epochs,
2246        executor,
2247        shutdown,
2248        metrics,
2249    )
2250    .await
2251}
2252
2253#[allow(clippy::too_many_arguments)]
2254async fn drain_pending_route_bind_completions(
2255    control_completion_rx: &mut mpsc::Receiver<RouteBindCompletion>,
2256    writer_tx: &WriterSender,
2257    routes: &mut HashMap<RouteChannel, RouteIdentity>,
2258    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2259    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2260    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2261    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2262    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2263    installed_route_epochs: &mut HashMap<u16, u32>,
2264    executor: &Arc<Executor>,
2265    shutdown: &Arc<Notify>,
2266    metrics: &Arc<DispatchPathMetrics>,
2267) -> Result<usize, SubcError> {
2268    let mut drained = 0;
2269    while let Ok(completion) = control_completion_rx.try_recv() {
2270        process_route_bind_completion(
2271            writer_tx,
2272            completion,
2273            routes,
2274            root_channels,
2275            session_identity,
2276            push_buffer,
2277            live_roots,
2278            pending_binds,
2279            installed_route_epochs,
2280            executor,
2281            shutdown,
2282            metrics,
2283        )
2284        .await?;
2285        drained += 1;
2286    }
2287    Ok(drained)
2288}
2289
2290/// ModuleHello → HelloAck → control/route loop. Runs until the daemon closes
2291/// the connection (EOF), sends channel-0 Goodbye, or a fatal mutating executor
2292/// response requests whole-connection teardown.
2293async fn run_module_loop<R, W>(
2294    mut read: R,
2295    mut write: W,
2296    connection_file_path: &Path,
2297    shared_app: Arc<App>,
2298    executor: Arc<Executor>,
2299    dispatch: DispatchFn,
2300    user_config_path: Option<PathBuf>,
2301    allow_native_passthrough: bool,
2302    tool_response_body_limit: usize,
2303) -> Result<ModuleLoopExit, SubcError>
2304where
2305    R: AsyncRead + Unpin + Send + 'static,
2306    W: AsyncWrite + Unpin + Send + 'static,
2307{
2308    // ModuleHello: register as a tool provider and advertise the supported control-plane operations.
2309    // Echo the one-time launch nonce the daemon injected via SUBC_LAUNCH_NONCE so a
2310    // reserved module_id's HELLO is accepted; absent for non-reserved/self-connect.
2311    let hello = ModuleHelloBody {
2312        manifest: build_manifest(),
2313        protocol_ver: PROTOCOL_VERSION,
2314        control_ops: control_ops(),
2315        launch_nonce: std::env::var("SUBC_LAUNCH_NONCE").ok(),
2316    };
2317    let hello_frame = Frame::build(
2318        FrameType::Hello,
2319        control_flags(),
2320        0,
2321        0,
2322        HELLO_CORR,
2323        serde_json::to_vec(&hello).map_err(SubcError::Json)?,
2324    )
2325    .map_err(SubcError::FrameBuild)?;
2326    write_frame(&mut write, &hello_frame)
2327        .await
2328        .map_err(SubcError::FrameIo)?;
2329
2330    // Expect HelloAck (registered) or a channel-0 Error (manifest/version reject).
2331    match read_frame(&mut read).await.map_err(SubcError::FrameIo)? {
2332        None => return Err(SubcError::ClosedBeforeHelloAck),
2333        Some(frame) => match frame.header.ty {
2334            FrameType::HelloAck => {
2335                log::info!("subc attach: registered (HelloAck received)");
2336            }
2337            FrameType::Error => {
2338                let body = serde_json::from_slice::<ErrorBody>(&frame.body).ok();
2339                return Err(SubcError::HelloRejected { body });
2340            }
2341            other => return Err(SubcError::UnexpectedFrame { ty: other }),
2342        },
2343    }
2344
2345    let dispatch_path_metrics = Arc::new(DispatchPathMetrics::new());
2346    let (writer_tx, writer_rx) = mpsc::channel::<WriterFrame>(WRITER_QUEUE_CAPACITY);
2347    let writer_task = spawn_writer_task(write, writer_rx, Arc::clone(&dispatch_path_metrics));
2348    // `read_frame` is NOT cancellation-safe, so it must never sit directly inside
2349    // the `select!` below: a drain-interval tick (or shutdown) firing while a
2350    // frame is mid-transit would drop the partially-consumed bytes and desync the
2351    // stream (the next read would parse a body byte as a frame header). A
2352    // dedicated reader task owns the socket, reads whole frames sequentially, and
2353    // forwards them over a channel; the loop selects on the cancel-safe `recv()`.
2354    let (reader_tx, mut reader_rx) = mpsc::channel::<Result<DecodedFrame, SubcError>>(256);
2355    let reader_task = spawn_reader_task(read, reader_tx);
2356    let shutdown = Arc::new(Notify::new());
2357    // Drain-tick deadline is tracked manually and checked at the TOP of every
2358    // loop turn rather than as an Interval select arm: the select below is
2359    // `biased` (bind completions first), and biased polling means a saturated
2360    // higher arm (sustained lossy push traffic keeps lossy_rx always-ready)
2361    // would starve every arm below it, including a timer arm — leaving
2362    // backpressured reliable frames parked in the retry buffer past their
2363    // delivery deadline. The pre-turn check cannot be starved by arm order;
2364    // the sleep_until arm below only exists to wake an otherwise-idle loop.
2365    let mut next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2366    let mut next_maintenance_at = next_drain_at;
2367    // Rate-limit stamp for opportunistic allocator slack relief (checked on the
2368    // maintenance tick; policy shared with standalone via memory.rs).
2369    #[cfg(any(target_os = "macos", target_os = "linux"))]
2370    let mut last_slack_relief: Option<std::time::Instant> = None;
2371    let (maintenance_tx, mut maintenance_rx) = mpsc::channel::<MaintenanceCompletion>(256);
2372    let (bash_deferred_tx, mut bash_deferred_rx) =
2373        mpsc::channel::<bash::BashDeferredCompletion>(256);
2374    let (bash_poll_touch_tx, mut bash_poll_touch_rx) = mpsc::channel::<ProjectRootId>(256);
2375    let (control_completion_tx, mut control_completion_rx) =
2376        mpsc::channel::<RouteBindCompletion>(256);
2377    let (lossy_tx, mut lossy_rx) = mpsc::channel::<LossyPushEnvelope>(1024);
2378    let lossy_overflow = Arc::new(push::LossyOverflow::default());
2379    let lossy_seq = Arc::new(AtomicU64::new(0));
2380    let (reliable_tx, mut reliable_rx) = mpsc::unbounded_channel::<PushEnvelope>();
2381    let (fleet_status_client, fleet_status_task) =
2382        spawn_fleet_status_dial(connection_file_path, 64);
2383    let push_senders = PushSenders {
2384        lossy_tx,
2385        reliable_tx,
2386        lossy_overflow: Arc::clone(&lossy_overflow),
2387        lossy_seq,
2388        fleet_status_client: fleet_status_client.clone(),
2389    };
2390    let connection_cancel = PersistentCancelSignal::new();
2391    let mut installed_route_epochs: HashMap<u16, u32> = HashMap::new();
2392    let mut routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
2393    let mut bg_subs: HashMap<RouteChannel, BgSub> = HashMap::new();
2394    let mut bg_sub_by_session: BgSubsBySession = HashMap::new();
2395    let mut bg_wake_pending: HashSet<RouteChannel> = HashSet::new();
2396    let mut bg_wake_epoch: HashMap<(ProjectRootId, String), u64> = HashMap::new();
2397    let mut root_channels: HashMap<ProjectRootId, HashSet<RouteChannel>> = HashMap::new();
2398    let mut session_identity: HashMap<(ProjectRootId, String), RetainedSessionIdentity> =
2399        HashMap::new();
2400    let mut push_buffer: HashMap<push::ReplayKey, VecDeque<PushFrame>> = HashMap::new();
2401    let mut retry_buffer: RetryBuffer = HashMap::new();
2402    let mut reclaimed_routes = ReclaimedRoutes::default();
2403    let mut completed_tasks = push::CompletedTaskIds::default();
2404    let mut live_roots: HashMap<ProjectRootId, RootMeta> = HashMap::new();
2405    let mut pending_binds: HashMap<RouteChannel, PendingBind> = HashMap::new();
2406    let mut pending_bash_asks: HashMap<ReverseCorrKey, PendingBashAsk> = HashMap::new();
2407    let mut next_bash_ask_corr: u64 = 1;
2408    let mut route_bash_cancels: HashMap<RouteChannel, bash::RouteBashCancel> = HashMap::new();
2409    let health_rollup_cache = HealthRollupCache::new();
2410    health_rollup_cache.refresh(&executor, &shared_app);
2411    let mut next_health_rollup_at = tokio::time::Instant::now() + HEALTH_ROLLUP_TTL;
2412
2413    let loop_result: Result<ModuleLoopExit, SubcError> = loop {
2414        shared_app.set_open_route_count(routes.len());
2415        if tokio::time::Instant::now() >= next_health_rollup_at {
2416            health_rollup_cache.refresh(&executor, &shared_app);
2417            next_health_rollup_at = tokio::time::Instant::now() + HEALTH_ROLLUP_TTL;
2418        }
2419        crate::logging::perf_tick(Some(&executor));
2420        dispatch_path_metrics.mark_frame_loop_tick();
2421        if let Err(error) = expire_pending_bash_asks(
2422            &writer_tx,
2423            &mut pending_bash_asks,
2424            &routes,
2425            &mut live_roots,
2426            &mut route_bash_cancels,
2427            &shutdown,
2428            &dispatch_path_metrics,
2429        )
2430        .await
2431        {
2432            break Err(error);
2433        }
2434
2435        // RouteBind completions are control-plane unblockers. Drain any completed
2436        // binds before entering other branch work so Push and maintenance bursts
2437        // can only add one loop-turn of latency.
2438        match drain_pending_route_bind_completions(
2439            &mut control_completion_rx,
2440            &writer_tx,
2441            &mut routes,
2442            &mut root_channels,
2443            &mut session_identity,
2444            &mut push_buffer,
2445            &mut live_roots,
2446            &mut pending_binds,
2447            &mut installed_route_epochs,
2448            &executor,
2449            &shutdown,
2450            &dispatch_path_metrics,
2451        )
2452        .await
2453        {
2454            Ok(drained) => {
2455                if drained > 0 {
2456                    next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2457                    health_rollup_cache.refresh(&executor, &shared_app);
2458                    next_health_rollup_at = tokio::time::Instant::now() + HEALTH_ROLLUP_TTL;
2459                }
2460            }
2461            Err(error) => break Err(error),
2462        }
2463
2464        if tokio::time::Instant::now() >= next_drain_at {
2465            push::emit_bg_event_wakes(
2466                &writer_tx,
2467                &dispatch_path_metrics,
2468                &bg_subs,
2469                &mut bg_wake_pending,
2470            );
2471            warn_slow_pending_binds(&mut pending_binds, &executor);
2472            if let Err(error) = expire_overdue_route_binds(
2473                &writer_tx,
2474                &executor,
2475                &mut pending_binds,
2476                &mut installed_route_epochs,
2477                &dispatch_path_metrics,
2478            )
2479            .await
2480            {
2481                break Err(error);
2482            }
2483
2484            let retried = push::drain_retry_buffers_for_bound_routes(
2485                &writer_tx,
2486                &dispatch_path_metrics,
2487                &routes,
2488                &mut retry_buffer,
2489            );
2490            if retried > 0 {
2491                log::debug!(
2492                    "subc attach: retried {retried} reliable Push frame(s) after writer backpressure"
2493                );
2494            }
2495
2496            next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2497        }
2498
2499        // A lossy emitter may place its newest update in the overflow buffer
2500        // when the bounded channel is full, while this receive loop is draining
2501        // the channel. Drain overflow before selecting again so that raced
2502        // update is delivered on the next timer tick instead of waiting for
2503        // another lossy enqueue.
2504        let overflow_batch = lossy_overflow.drain();
2505        if !overflow_batch.is_empty() {
2506            let (_, deferred) = push::drain_reliable_push_turn(
2507                &writer_tx,
2508                &dispatch_path_metrics,
2509                &routes,
2510                &root_channels,
2511                &session_identity,
2512                &mut retry_buffer,
2513                &mut push_buffer,
2514                &mut completed_tasks,
2515                &bg_sub_by_session,
2516                &mut bg_wake_pending,
2517                &mut bg_wake_epoch,
2518                &mut reliable_rx,
2519                None,
2520            );
2521            if deferred {
2522                tokio::task::yield_now().await;
2523            }
2524
2525            let mut batch = Vec::new();
2526            while let Ok(item) = lossy_rx.try_recv() {
2527                batch.push(item);
2528            }
2529            batch.extend(overflow_batch);
2530            push::process_lossy_push_envelope_batch(
2531                &writer_tx,
2532                &dispatch_path_metrics,
2533                &routes,
2534                &root_channels,
2535                &completed_tasks,
2536                batch,
2537            );
2538        }
2539
2540        tokio::select! {
2541            biased;
2542            Some(completion) = control_completion_rx.recv() => {
2543                if let Err(error) = process_route_bind_completion(
2544                    &writer_tx,
2545                    completion,
2546                    &mut routes,
2547                    &mut root_channels,
2548                    &mut session_identity,
2549                    &mut push_buffer,
2550                    &mut live_roots,
2551                    &mut pending_binds,
2552                    &mut installed_route_epochs,
2553                    &executor,
2554                    &shutdown,
2555                    &dispatch_path_metrics,
2556                )
2557                .await
2558                {
2559                    break Err(error);
2560                }
2561                next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2562                next_health_rollup_at = tokio::time::Instant::now();
2563            }
2564            _ = shutdown.notified() => {
2565                log::warn!("subc attach: fatal executor response requested teardown");
2566                break Ok(ModuleLoopExit::SkipSearchFlush);
2567            }
2568            maybe_frame = reader_rx.recv() => {
2569                let frame = match maybe_frame {
2570                    None => {
2571                        log::info!("subc attach: daemon closed connection");
2572                        break Ok(ModuleLoopExit::Graceful);
2573                    }
2574                    Some(Err(error)) => break Err(error),
2575                    Some(Ok(frame)) => frame,
2576                };
2577                let phase_trace = frame.phase_trace;
2578                let frame = frame.frame;
2579
2580                if !ingress_route_should_be_processed(
2581                    &installed_route_epochs,
2582                    &reclaimed_routes,
2583                    &frame,
2584                ) {
2585                    log::debug!(
2586                        "subc attach: silently dropping {:?} for uninstalled route {}@{}",
2587                        frame.header.ty,
2588                        frame.header.channel,
2589                        frame.header.epoch
2590                    );
2591                    continue;
2592                }
2593
2594                match frame.header.ty {
2595                    FrameType::Ping if frame.header.channel == 0 => {
2596                        let pong = match Frame::build_with_version(
2597                            frame.header.ver,
2598                            FrameType::Pong,
2599                            frame.header.flags,
2600                            0,
2601                            0,
2602                            frame.header.corr,
2603                            Vec::new(),
2604                        ) {
2605                            Ok(pong) => pong,
2606                            Err(error) => break Err(SubcError::FrameBuild(error)),
2607                        };
2608                        if let Err(error) = send_frame(&writer_tx, &dispatch_path_metrics, pong).await {
2609                            break Err(error);
2610                        }
2611                    }
2612                    FrameType::Goodbye if frame.header.channel == 0 => {
2613                        log::info!("subc attach: received channel-0 Goodbye");
2614                        break Ok(ModuleLoopExit::Graceful);
2615                    }
2616                    FrameType::Goodbye => {
2617                        let channel = route_key(frame.header.channel, frame.header.epoch);
2618                        if let Err(error) = teardown_installed_route(
2619                            &writer_tx,
2620                            &dispatch_path_metrics,
2621                            &executor,
2622                            channel,
2623                            "Goodbye",
2624                            None,
2625                            &mut installed_route_epochs,
2626                            &mut routes,
2627                            &mut root_channels,
2628                            &mut bg_subs,
2629                            &mut bg_sub_by_session,
2630                            &mut bg_wake_pending,
2631                            &mut pending_bash_asks,
2632                            &mut live_roots,
2633                            &mut route_bash_cancels,
2634                            &mut pending_binds,
2635                            &mut retry_buffer,
2636                            &mut push_buffer,
2637                            &shutdown,
2638                        )
2639                        .await
2640                        {
2641                            break Err(error);
2642                        }
2643                    }
2644                    FrameType::Response | FrameType::Error if frame.header.channel != 0 => {
2645                        if let Err(error) = handle_bash_elicitation_reply(
2646                            &writer_tx,
2647                            &frame,
2648                            &mut pending_bash_asks,
2649                            &routes,
2650                            &mut live_roots,
2651                            &executor,
2652                            &shutdown,
2653                            &bash_deferred_tx,
2654                            &bash_poll_touch_tx,
2655                            &dispatch_path_metrics,
2656                            &mut route_bash_cancels,
2657                            dispatch,
2658                        )
2659                        .await
2660                        {
2661                            break Err(error);
2662                        }
2663                    }
2664                    FrameType::Request if frame.header.channel == 0 => {
2665                        if let Err(error) = handle_control_request(
2666                            &writer_tx,
2667                            &frame,
2668                            &shared_app,
2669                            &executor,
2670                            &mut live_roots,
2671                            &mut pending_binds,
2672                            &mut installed_route_epochs,
2673                            &mut routes,
2674                            &mut root_channels,
2675                            &mut bg_subs,
2676                            &mut bg_sub_by_session,
2677                            &mut bg_wake_pending,
2678                            &mut pending_bash_asks,
2679                            &mut route_bash_cancels,
2680                            &mut retry_buffer,
2681                            &mut push_buffer,
2682                            &shutdown,
2683                            &control_completion_tx,
2684                            &dispatch_path_metrics,
2685                            &health_rollup_cache,
2686                            &push_senders,
2687                            dispatch,
2688                            user_config_path.as_deref(),
2689                        )
2690                        .await
2691                        {
2692                            break Err(error);
2693                        }
2694                    }
2695                    FrameType::Request => {
2696                        if let Err(error) = handle_tool_call(
2697                            &writer_tx,
2698                            &frame,
2699                            phase_trace,
2700                            &routes,
2701                            &pending_binds,
2702                            &mut live_roots,
2703                            &executor,
2704                            &shutdown,
2705                            &connection_cancel,
2706                            &bash_deferred_tx,
2707                            &bash_poll_touch_tx,
2708                            &dispatch_path_metrics,
2709                            &mut route_bash_cancels,
2710                            &mut pending_bash_asks,
2711                            &mut next_bash_ask_corr,
2712                            &mut bg_subs,
2713                            &mut bg_sub_by_session,
2714                            &mut bg_wake_pending,
2715                            &mut bg_wake_epoch,
2716                            dispatch,
2717                            allow_native_passthrough,
2718                            tool_response_body_limit,
2719                        )
2720                        .await
2721                        {
2722                            break Err(error);
2723                        }
2724                    }
2725                    FrameType::Cancel => {
2726                        let channel = route_key(frame.header.channel, frame.header.epoch);
2727                        if bg_subs.contains_key(&channel) {
2728                            if let Err(error) = end_bg_subscription(
2729                                &writer_tx,
2730                                &dispatch_path_metrics,
2731                                &mut bg_subs,
2732                                &mut bg_sub_by_session,
2733                                &mut bg_wake_pending,
2734                                channel,
2735                                routes.get(&channel),
2736                                "cancel",
2737                            )
2738                            .await
2739                            {
2740                                break Err(error);
2741                            }
2742                        }
2743                        if let Err(error) = cancel_pending_bash_ask_for_tool_call(
2744                            &writer_tx,
2745                            &mut pending_bash_asks,
2746                            channel,
2747                            frame.header.corr,
2748                            &routes,
2749                            &mut live_roots,
2750                            &mut route_bash_cancels,
2751                            &shutdown,
2752                            &dispatch_path_metrics,
2753                        )
2754                        .await
2755                        {
2756                            break Err(error);
2757                        }
2758                    }
2759                    // Incoming push messages are ignored here. Cancel frames only
2760                    // stop pending bash elicitation requests; executor-level
2761                    // cancellation for tool calls that are already running is not
2762                    // implemented.
2763                    _ => {}
2764                }
2765            }
2766            Some((root_id, frame)) = reliable_rx.recv() => {
2767                // Reliable Push frames are FIFO and must-deliver, but draining an
2768                // unbounded burst in one current-thread turn can starve RouteBind
2769                // completions. The budget defers excess frames, never drops them.
2770                let (_, deferred) = push::drain_reliable_push_turn(
2771                    &writer_tx,
2772                    &dispatch_path_metrics,
2773                    &routes,
2774                    &root_channels,
2775                    &session_identity,
2776                    &mut retry_buffer,
2777                    &mut push_buffer,
2778                    &mut completed_tasks,
2779                    &bg_sub_by_session,
2780                    &mut bg_wake_pending,
2781                    &mut bg_wake_epoch,
2782                    &mut reliable_rx,
2783                    Some((root_id, frame)),
2784                );
2785                if deferred {
2786                    tokio::task::yield_now().await;
2787                }
2788            }
2789            Some((order, root_id, frame)) = lossy_rx.recv() => {
2790                // When both push lanes have work, handle a small reliable slice before lossy work.
2791                // That ordering lets completed task ids suppress stale BashLongRunning frames.
2792                // The slice stays bounded so reliable bursts cannot monopolize this loop turn.
2793                let (_, deferred) = push::drain_reliable_push_turn(
2794                    &writer_tx,
2795                    &dispatch_path_metrics,
2796                    &routes,
2797                    &root_channels,
2798                    &session_identity,
2799                    &mut retry_buffer,
2800                    &mut push_buffer,
2801                    &mut completed_tasks,
2802                    &bg_sub_by_session,
2803                    &mut bg_wake_pending,
2804                    &mut bg_wake_epoch,
2805                    &mut reliable_rx,
2806                    None,
2807                );
2808                if deferred {
2809                    tokio::task::yield_now().await;
2810                }
2811
2812                // Drain the currently queued burst in one loop turn so lossy
2813                // status/progress updates can be merged before reaching subc's
2814                // shared egress queue. Each lossy frame gets a sequence number
2815                // before it goes to the channel or overflow buffer, so the
2816                // combined batch is sorted back into producer order before
2817                // coalescing drops stale updates for the same key.
2818                let mut batch = vec![(order, root_id, frame)];
2819                while let Ok(item) = lossy_rx.try_recv() {
2820                    batch.push(item);
2821                }
2822                batch.extend(lossy_overflow.drain());
2823                push::process_lossy_push_envelope_batch(
2824                    &writer_tx,
2825                    &dispatch_path_metrics,
2826                    &routes,
2827                    &root_channels,
2828                    &completed_tasks,
2829                    batch,
2830                );
2831            }
2832            Some(done) = bash_deferred_rx.recv() => {
2833                decrement_counted_channel(&dispatch_path_metrics.bash_deferred_queued);
2834                if let Err(error) = bash::handle_bash_deferred_completion(
2835                    &writer_tx,
2836                    done,
2837                    &routes,
2838                    &mut live_roots,
2839                    &mut route_bash_cancels,
2840                    &shutdown,
2841                    &dispatch_path_metrics,
2842                )
2843                .await
2844                {
2845                    break Err(error);
2846                }
2847            }
2848            Some(root_id) = bash_poll_touch_rx.recv() => {
2849                decrement_counted_channel(&dispatch_path_metrics.bash_poll_touch_queued);
2850                if let Some(meta) = live_roots.get_mut(&root_id) {
2851                    meta.note_activity();
2852                }
2853            }
2854            Some(completion) = maintenance_rx.recv() => {
2855                decrement_counted_channel(&dispatch_path_metrics.maintenance_queued);
2856                let root_id = completion.root_id.clone();
2857                let response = completion.response;
2858                let response_is_fatal = response_is_fatal_panic(&response);
2859                let bind_pending = pending_binds
2860                    .values()
2861                    .any(|pending| pending.bind_root_id == root_id);
2862                let requiesce = if let Some(meta) = live_roots.get_mut(&root_id) {
2863                    let defer_requeue = meta.unbound_quiesced || bind_pending;
2864                    note_maintenance_completion(
2865                        meta,
2866                        completion.requeue_kind,
2867                        response_is_fatal,
2868                        defer_requeue,
2869                    );
2870                    should_requiesce_after_maintenance(meta, completion.kind, bind_pending)
2871                } else {
2872                    false
2873                };
2874                if requiesce {
2875                    quiesce_unbound_root(&root_id, &mut live_roots, &executor);
2876                }
2877                push::clear_stale_bg_wakes_for_empty_sessions(
2878                    &root_id,
2879                    &completion.empty_bg_sessions,
2880                    &bg_sub_by_session,
2881                    &mut bg_wake_pending,
2882                    &bg_wake_epoch,
2883                );
2884                if response_is_fatal {
2885                    if let Some(meta) = live_roots.get_mut(&root_id) {
2886                        meta.maintenance_poisoned = true;
2887                    }
2888                    log::warn!(
2889                        "subc attach: maintenance drain observed a fatal actor; deferring teardown until a route request can receive actor_fatal"
2890                    );
2891                }
2892            }
2893            _ = tokio::time::sleep_until(next_drain_at) => {
2894                // Wakes an otherwise-idle loop so the pre-turn drain check
2895                // above runs on schedule; the drain work itself lives there.
2896            }
2897            _ = tokio::time::sleep_until(next_maintenance_at) => {
2898                // Delay cache-draining maintenance until any already-ready
2899                // inbound route/control messages and push completions have run,
2900                // so maintenance does not block the actor from handling the
2901                // first request that arrives after a route bind is acknowledged.
2902                crate::logging::maybe_sweep_logs();
2903                let reaped_lsp_children = shared_app
2904                    .lsp_child_registry()
2905                    .reap_children_with_gone_cwd_or_reclaimed_root();
2906                if reaped_lsp_children > 0 {
2907                    log::warn!(
2908                        "subc attach: reaped {reaped_lsp_children} LSP child process group(s) with a deleted cwd or reclaimed root"
2909                    );
2910                }
2911                let reap = reap_idle_roots(
2912                    Instant::now(),
2913                    &mut live_roots,
2914                    &pending_binds,
2915                    &root_channels,
2916                    &executor,
2917                    &dispatch_path_metrics,
2918                );
2919                for root_id in &reap.forgotten_deleted_roots {
2920                    purge_deleted_root_residents(
2921                        root_id,
2922                        &mut routes,
2923                        &mut root_channels,
2924                        &mut installed_route_epochs,
2925                        &mut route_bash_cancels,
2926                        &mut retry_buffer,
2927                        &mut reclaimed_routes,
2928                        &mut session_identity,
2929                        &mut push_buffer,
2930                        &mut bg_subs,
2931                        &mut bg_sub_by_session,
2932                        &mut bg_wake_pending,
2933                        &mut bg_wake_epoch,
2934                        &mut pending_bash_asks,
2935                        &dispatch_path_metrics,
2936                    );
2937                }
2938                if reap.evicted > 0 {
2939                    log::debug!("subc attach: reaped {} idle root(s)", reap.evicted);
2940                }
2941                submit_due_maintenance_jobs(
2942                    &executor,
2943                    &mut live_roots,
2944                    &pending_binds,
2945                    &bg_sub_by_session,
2946                    &bg_wake_pending,
2947                    &bg_wake_epoch,
2948                    &maintenance_tx,
2949                    &dispatch_path_metrics,
2950                );
2951                // Opportunistic allocator relief, independent of the idle
2952                // sweep: the sweep's whole-process idle gate never opens while
2953                // any session stays active, which let freed warm-up arenas sit
2954                // resident for the process lifetime (5.1 GB RSS over ~600 MB
2955                // live). Slack threshold + spacing live in memory.rs; the pass
2956                // itself runs on a detached thread.
2957                #[cfg(any(target_os = "macos", target_os = "linux"))]
2958                {
2959                    let now_std = std::time::Instant::now();
2960                    if crate::memory::spawn_allocator_slack_relief_if_due(
2961                        last_slack_relief,
2962                        now_std,
2963                    ) {
2964                        last_slack_relief = Some(now_std);
2965                    }
2966                }
2967                next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2968            }
2969        }
2970    };
2971
2972    shared_app.set_open_route_count(0);
2973
2974    connection_cancel.cancel();
2975    // Channel-0 Goodbye, EOF, and fatal exits bypass per-route Goodbye. Settle
2976    // their root lifecycle state before loop-owned routing metadata is dropped.
2977    quiesce_connection_roots(
2978        &mut live_roots,
2979        &mut pending_binds,
2980        &mut routes,
2981        &mut root_channels,
2982        &mut installed_route_epochs,
2983        &mut route_bash_cancels,
2984        &executor,
2985    );
2986
2987    fleet_status_client.set_route_live(false);
2988    fleet_status_task.abort();
2989    let _ = fleet_status_task.await;
2990
2991    let mut loop_result = loop_result;
2992    if !pending_bash_asks.is_empty() {
2993        let no_routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
2994        if let Err(error) = settle_all_pending_bash_asks(
2995            &writer_tx,
2996            &mut pending_bash_asks,
2997            &no_routes,
2998            &mut live_roots,
2999            &mut route_bash_cancels,
3000            &shutdown,
3001            &dispatch_path_metrics,
3002        )
3003        .await
3004        {
3005            loop_result = loop_result.and(Err(error));
3006        }
3007    }
3008
3009    // The reader task may be parked on `read_frame`; abort it (we are done with
3010    // the connection) and flush the writer.
3011    reader_task.abort();
3012    drop(writer_tx);
3013    let writer_result = finish_writer_task(writer_task).await;
3014    loop_result.and_then(|exit| writer_result.map(|_| exit))
3015}
3016
3017fn spawn_writer_task<W>(
3018    mut write: W,
3019    mut rx: mpsc::Receiver<WriterFrame>,
3020    metrics: Arc<DispatchPathMetrics>,
3021) -> JoinHandle<Result<(), subc_transport::FrameIoError>>
3022where
3023    W: AsyncWrite + Unpin + Send + 'static,
3024{
3025    tokio::spawn(async move {
3026        let mut write_buffer = Vec::new();
3027        while let Some(mut queued) = rx.recv().await {
3028            let measure = queued.tool_response_trace.is_some();
3029            let dequeued = measure.then(Instant::now);
3030            metrics.writer_active.store(true, Ordering::Relaxed);
3031            decrement_counted_channel(&metrics.writer_queued);
3032            let write_timing = write_frame_contiguous(
3033                &mut write,
3034                queued.frame(),
3035                queued.body(),
3036                &mut write_buffer,
3037                measure,
3038            )
3039            .await;
3040            metrics.writer_active.store(false, Ordering::Relaxed);
3041            let write_timing = write_timing?;
3042
3043            if let (Some(trace), Some(dequeued), Some(write_timing)) =
3044                (queued.tool_response_trace.take(), dequeued, write_timing)
3045            {
3046                if let Some(completed) = trace.finish(
3047                    dequeued,
3048                    write_timing.write_started,
3049                    write_timing.write_finished,
3050                    write_timing.frame_bytes,
3051                ) {
3052                    log_ctx::with_session(Some(completed.session), || {
3053                        crate::logging::note_tool_call_trace(
3054                            &completed.name,
3055                            &completed.root,
3056                            completed.channel,
3057                            completed.corr,
3058                            completed.phases,
3059                        );
3060                    });
3061                }
3062            }
3063        }
3064        Ok(())
3065    })
3066}
3067
3068struct FrameWriteTiming {
3069    write_started: Instant,
3070    write_finished: Instant,
3071    frame_bytes: usize,
3072}
3073
3074/// Encode one complete frame into the existing reusable buffer and write it
3075/// without interleaving bytes from another channel. Timing is collected only
3076/// for tool responses, so Push and control frames add no clock reads.
3077async fn write_frame_contiguous<W>(
3078    writer: &mut W,
3079    frame: &Frame,
3080    body: &[u8],
3081    buffer: &mut Vec<u8>,
3082    measure: bool,
3083) -> Result<Option<FrameWriteTiming>, subc_transport::FrameIoError>
3084where
3085    W: AsyncWrite + Unpin,
3086{
3087    if frame.header.len as usize != body.len() {
3088        return Err(subc_transport::FrameIoError::BodyLengthMismatch {
3089            header_len: frame.header.len,
3090            body_len: body.len(),
3091        });
3092    }
3093
3094    let header = frame.header.encode();
3095    buffer.clear();
3096    buffer.reserve(header.len() + body.len());
3097    buffer.extend_from_slice(&header);
3098    buffer.extend_from_slice(body);
3099    let write_started = measure.then(Instant::now);
3100    writer
3101        .write_all(buffer)
3102        .await
3103        .map_err(subc_transport::FrameIoError::Io)?;
3104    Ok(write_started.map(|write_started| FrameWriteTiming {
3105        write_started,
3106        write_finished: Instant::now(),
3107        frame_bytes: buffer.len(),
3108    }))
3109}
3110
3111fn spawn_reader_task<R>(
3112    mut read: R,
3113    tx: mpsc::Sender<Result<DecodedFrame, SubcError>>,
3114) -> JoinHandle<()>
3115where
3116    R: AsyncRead + Unpin + Send + 'static,
3117{
3118    tokio::spawn(async move {
3119        loop {
3120            match read_frame(&mut read).await {
3121                Ok(Some(frame)) => {
3122                    let decoded = DecodedFrame {
3123                        frame,
3124                        phase_trace: PhaseTrace::new(Instant::now()),
3125                    };
3126                    if tx.send(Ok(decoded)).await.is_err() {
3127                        return;
3128                    }
3129                }
3130                Ok(None) => {
3131                    // EOF: let the loop observe channel close as "daemon closed".
3132                    return;
3133                }
3134                Err(error) => {
3135                    // A killed daemon surfaces as ConnectionReset (RST) on
3136                    // Windows where Unix delivers a clean EOF (FIN); a
3137                    // mid-teardown daemon can also abort the socket. Both mean
3138                    // "daemon went away", not a wire fault — normalize them to
3139                    // the clean-close path so module exit behavior matches
3140                    // across platforms (same class subc-core fixed in d33d9a71).
3141                    if let subc_transport::FrameIoError::Io(io_error) = &error {
3142                        if matches!(
3143                            io_error.kind(),
3144                            std::io::ErrorKind::ConnectionReset
3145                                | std::io::ErrorKind::ConnectionAborted
3146                        ) {
3147                            log::info!(
3148                                "subc attach: connection reset by daemon; treating as close"
3149                            );
3150                            return;
3151                        }
3152                    }
3153                    let _ = tx.send(Err(SubcError::FrameIo(error))).await;
3154                    return;
3155                }
3156            }
3157        }
3158    })
3159}
3160
3161async fn finish_writer_task(
3162    mut writer_task: JoinHandle<Result<(), subc_transport::FrameIoError>>,
3163) -> Result<(), SubcError> {
3164    match tokio::time::timeout(Duration::from_millis(100), &mut writer_task).await {
3165        Ok(Ok(Ok(()))) => Ok(()),
3166        Ok(Ok(Err(error))) => Err(SubcError::FrameIo(error)),
3167        Ok(Err(error)) => Err(SubcError::WriterJoin(error)),
3168        Err(_) => {
3169            writer_task.abort();
3170            Ok(())
3171        }
3172    }
3173}
3174
3175fn register_actor_for_bind(
3176    shared_app: &Arc<App>,
3177    executor: &Arc<Executor>,
3178    push_senders: &PushSenders,
3179    bind_root_id: &ProjectRootId,
3180    route_channel: u16,
3181    root_was_live: bool,
3182) -> bool {
3183    if executor.actor_registered(bind_root_id) {
3184        log::debug!(
3185            "subc attach: reusing actor for route {} root {}",
3186            route_channel,
3187            bind_root_id.as_path().display()
3188        );
3189        return false;
3190    }
3191
3192    if root_was_live {
3193        log::warn!(
3194            "subc attach: recreating missing actor for live root {} on route {}",
3195            bind_root_id.as_path().display(),
3196            route_channel
3197        );
3198    }
3199
3200    let actor_ctx = Arc::new(AppContext::from_app(
3201        Arc::clone(shared_app),
3202        Config::default(),
3203    ));
3204    install_bash_compressor(&actor_ctx);
3205    actor_ctx.install_fleet_status_client(Some(push_senders.fleet_status_client.clone()));
3206    actor_ctx.set_progress_sender(Some(push::progress_sender_for_root(
3207        push_senders.clone(),
3208        bind_root_id.clone(),
3209    )));
3210    let inserted = executor.register_actor(bind_root_id.clone(), Arc::clone(&actor_ctx));
3211    drop(actor_ctx);
3212    if inserted {
3213        // Do not insert into live_roots until configure succeeds: live_roots
3214        // drives maintenance, and a half-configured new actor must not be
3215        // maintenance-eligible before its route/session identity exists.
3216        log::debug!(
3217            "subc attach: registered actor for route {} root {}",
3218            route_channel,
3219            bind_root_id.as_path().display()
3220        );
3221    } else {
3222        log::debug!(
3223            "subc attach: actor appeared while binding route {} root {}; reusing it",
3224            route_channel,
3225            bind_root_id.as_path().display()
3226        );
3227    }
3228    inserted
3229}
3230
3231fn rollback_pending_bind_actor(
3232    executor: &Arc<Executor>,
3233    live_roots: &HashMap<ProjectRootId, RootMeta>,
3234    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3235    root_id: &ProjectRootId,
3236    inserted_new_actor: bool,
3237) {
3238    if !inserted_new_actor || live_roots.contains_key(root_id) {
3239        return;
3240    }
3241
3242    if let Some((route, pending)) = pending_binds
3243        .iter_mut()
3244        .find(|(_, pending)| &pending.bind_root_id == root_id)
3245    {
3246        pending.inserted_new_actor = true;
3247        log::debug!(
3248            "subc attach: transferred rollback ownership for root {} to pending route {}",
3249            root_id.as_path().display(),
3250            route
3251        );
3252        return;
3253    }
3254
3255    executor.remove_actor(root_id);
3256}
3257
3258fn route_bind_error_code_for_configure_response(response: &Response) -> &'static str {
3259    match response.data.get("code").and_then(|code| code.as_str()) {
3260        // Preserve typed configure rejections across the bind boundary: a
3261        // malformed fed fingerprint means a federation-module bug or
3262        // fingerprint-format drift, and the fed side matches on the code rather
3263        // than parsing prose.
3264        Some("bad_harness_fingerprint") => "bad_harness_fingerprint",
3265        // Cache-key probe failures are transient (fd pressure, git spawn
3266        // contention); the client retries the bind rather than treating the
3267        // root as permanently divergent.
3268        Some("cache_key_probe_failed") => "cache_key_probe_failed",
3269        // Actor lifecycle gaps are transient from the daemon/client viewpoint:
3270        // a fresh bind can create or join a healthy actor, so do not classify
3271        // them as permanent config divergence.
3272        Some("actor_not_registered" | "actor_fatal") => "actor_not_ready",
3273        _ => "config_divergence",
3274    }
3275}
3276
3277fn queue_post_bind_configure_and_completion_maintenance(
3278    root_id: &ProjectRootId,
3279    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3280) {
3281    let Some(meta) = live_roots.get_mut(root_id) else {
3282        return;
3283    };
3284    if meta.maintenance_poisoned || meta.maintenance_pending {
3285        return;
3286    }
3287
3288    meta.maintenance_pending = true;
3289    meta.maintenance_queued_kinds
3290        .push_back(MaintenanceDrainKind::ConfigureTail);
3291    meta.maintenance_queued_kinds
3292        .push_back(MaintenanceDrainKind::CompletionDrains);
3293}
3294
3295#[allow(clippy::too_many_arguments)]
3296async fn handle_route_bind_completion(
3297    tx: &WriterSender,
3298    completion: RouteBindCompletion,
3299    routes: &mut HashMap<RouteChannel, RouteIdentity>,
3300    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
3301    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
3302    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
3303    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3304    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3305    installed_route_epochs: &mut HashMap<u16, u32>,
3306    executor: &Arc<Executor>,
3307    shutdown: &Arc<Notify>,
3308    metrics: &Arc<DispatchPathMetrics>,
3309) -> Result<(), SubcError> {
3310    let route_id = completion.route;
3311    let Some(pending) = pending_binds.remove(&route_id) else {
3312        log::warn!(
3313            "subc attach: dropping RouteBind completion for non-pending route {}",
3314            completion.route
3315        );
3316        rollback_pending_bind_actor(
3317            executor,
3318            live_roots,
3319            pending_binds,
3320            &completion.bind_root_id,
3321            completion.inserted_new_actor,
3322        );
3323        let has_pending_bind = pending_binds
3324            .values()
3325            .any(|pending| pending.bind_root_id == completion.bind_root_id);
3326        if !root_channels
3327            .get(&completion.bind_root_id)
3328            .is_some_and(|channels| !channels.is_empty())
3329            && !has_pending_bind
3330        {
3331            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3332        }
3333        remove_installed_route(installed_route_epochs, route_id);
3334        return Ok(());
3335    };
3336
3337    if pending.bind_root_id != completion.bind_root_id {
3338        log::warn!(
3339            "subc attach: pending RouteBind root mismatch for route {} (pending {} completion {})",
3340            completion.route,
3341            pending.bind_root_id.as_path().display(),
3342            completion.bind_root_id.as_path().display()
3343        );
3344    }
3345
3346    let inserted_new_actor = pending.inserted_new_actor || completion.inserted_new_actor;
3347    if pending.cancelled {
3348        rollback_pending_bind_actor(
3349            executor,
3350            live_roots,
3351            pending_binds,
3352            &completion.bind_root_id,
3353            inserted_new_actor,
3354        );
3355        let has_pending_bind = pending_binds
3356            .values()
3357            .any(|pending| pending.bind_root_id == completion.bind_root_id);
3358        if !root_channels
3359            .get(&completion.bind_root_id)
3360            .is_some_and(|channels| !channels.is_empty())
3361            && !has_pending_bind
3362        {
3363            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3364        }
3365        log::debug!(
3366            "subc attach: discarded completed RouteBind for cancelled route {} root {}",
3367            completion.route,
3368            completion.bind_root_id.as_path().display()
3369        );
3370        remove_installed_route(installed_route_epochs, route_id);
3371        return Ok(());
3372    }
3373
3374    let failure = if !completion.configure_response.success {
3375        Some((
3376            &completion.configure_response,
3377            "configure failed during route bind",
3378        ))
3379    } else {
3380        None
3381    };
3382
3383    if let Some((response, fallback)) = failure {
3384        rollback_pending_bind_actor(
3385            executor,
3386            live_roots,
3387            pending_binds,
3388            &completion.bind_root_id,
3389            inserted_new_actor,
3390        );
3391        let has_pending_bind = pending_binds
3392            .values()
3393            .any(|pending| pending.bind_root_id == completion.bind_root_id);
3394        if !root_channels
3395            .get(&completion.bind_root_id)
3396            .is_some_and(|channels| !channels.is_empty())
3397            && !has_pending_bind
3398        {
3399            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3400        }
3401        let message = response_message(response, fallback);
3402        let fatal = response_is_fatal_panic(response);
3403        let error_code = route_bind_error_code_for_configure_response(response);
3404        send_route_bind_error_parts(
3405            tx,
3406            completion.ver,
3407            completion.corr,
3408            completion.flags,
3409            error_code,
3410            &message,
3411            metrics,
3412        )
3413        .await?;
3414        remove_installed_route(installed_route_epochs, route_id);
3415        if fatal {
3416            signal_fatal_teardown(
3417                tx,
3418                Some(completion.route),
3419                completion.ver,
3420                completion.corr,
3421                shutdown,
3422                metrics,
3423            )
3424            .await;
3425        }
3426        return Ok(());
3427    }
3428
3429    remember_session_identity(session_identity, &completion.identity);
3430    let replay_key = push::ReplayKey::from_identity(&completion.identity);
3431    let bind_trust = completion.identity.trust;
3432    insert_route_channel(routes, root_channels, route_id, completion.identity);
3433    let restore_watcher = live_roots
3434        .get(&completion.bind_root_id)
3435        .is_some_and(|meta| meta.idle_artifacts_evicted || meta.unbound_quiesced);
3436    live_roots
3437        .entry(completion.bind_root_id.clone())
3438        .and_modify(|meta| {
3439            meta.reactivate_bound();
3440            meta.diagnostics_on_edit = completion.diagnostics_on_edit;
3441            meta.maintenance_poisoned = false;
3442        })
3443        .or_insert_with(|| RootMeta::new(Instant::now()));
3444    if let Some(meta) = live_roots.get_mut(&completion.bind_root_id) {
3445        meta.diagnostics_on_edit = completion.diagnostics_on_edit;
3446        meta.maintenance_poisoned = false;
3447    }
3448    if let Some(ctx) = executor.actor_context(&completion.bind_root_id) {
3449        ctx.mark_subc_bound();
3450        if restore_watcher {
3451            crate::commands::configure::ensure_project_watcher(&ctx);
3452        }
3453    }
3454
3455    let ack =
3456        serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).map_err(SubcError::Json)?;
3457    let response = Frame::build_with_version(
3458        completion.ver,
3459        FrameType::Response,
3460        control_flags(),
3461        0,
3462        0,
3463        completion.corr,
3464        ack,
3465    )
3466    .map_err(SubcError::FrameBuild)?;
3467    send_reliable_writer_frame(tx, metrics, response, "RouteBindAck").await?;
3468    queue_post_bind_configure_and_completion_maintenance(&completion.bind_root_id, live_roots);
3469    let replayed = push::replay_buffered_push_frames(
3470        tx,
3471        metrics,
3472        route_id,
3473        push_buffer,
3474        &replay_key,
3475        bind_trust,
3476    );
3477    if replayed > 0 {
3478        log::debug!(
3479            "subc attach: replayed {} buffered Push frame(s) to route {} root {} harness {} session {}",
3480            replayed,
3481            completion.route,
3482            replay_key.root.as_path().display(),
3483            replay_key.harness,
3484            replay_key.session
3485        );
3486    }
3487    log::info!(
3488        "subc attach: route {} bound to root {}",
3489        completion.route,
3490        completion.bind_root_id.as_path().display()
3491    );
3492    Ok(())
3493}
3494
3495async fn expire_overdue_route_binds(
3496    tx: &WriterSender,
3497    executor: &Arc<Executor>,
3498    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3499    installed_route_epochs: &mut HashMap<u16, u32>,
3500    metrics: &DispatchPathMetrics,
3501) -> Result<(), SubcError> {
3502    let now = Instant::now();
3503    let expired: Vec<_> = pending_binds
3504        .iter()
3505        .filter_map(|(route, pending)| {
3506            let age = now.saturating_duration_since(pending.started_at);
3507            (!pending.deadline_reported && age >= ROUTE_BIND_DEADLINE).then(|| {
3508                (
3509                    *route,
3510                    pending.corr,
3511                    pending.ver,
3512                    pending.flags,
3513                    pending.bind_root_id.clone(),
3514                    pending.configure_request_id.clone(),
3515                    age,
3516                )
3517            })
3518        })
3519        .collect();
3520
3521    for (route, corr, ver, flags, root_id, configure_request_id, age) in expired {
3522        if let Some(pending) = pending_binds.get_mut(&route) {
3523            pending.cancelled = true;
3524            pending.deadline_reported = true;
3525            let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
3526            log::debug!(
3527                "subc attach: cancelled overdue RouteBind configure for route {route} ({outcome:?})"
3528            );
3529        }
3530        remove_installed_route(installed_route_epochs, route);
3531        let age_ms = age.as_millis().min(u128::from(u64::MAX)) as u64;
3532        let deadline_ms = ROUTE_BIND_DEADLINE.as_millis();
3533        send_route_bind_error_parts(
3534            tx,
3535            ver,
3536            corr,
3537            flags,
3538            "actor_not_ready",
3539            &format!("route bind deadline exceeded after {age_ms}ms (deadline {deadline_ms}ms)"),
3540            metrics,
3541        )
3542        .await?;
3543        log::warn!(
3544            "subc attach: route {} bind for root {} exceeded {}ms deadline (configure_request_id={})",
3545            route,
3546            root_id.as_path().display(),
3547            deadline_ms,
3548            configure_request_id
3549        );
3550    }
3551
3552    Ok(())
3553}
3554
3555/// channel-0 control requests: RouteBind plus the cached health probe. RouteBind
3556/// still reconciles the route's RootConfig through the executor's Mutating lane
3557/// and resolves completion on a loop-owned control-completion channel so slow
3558/// configure jobs do not block the transport loop.
3559#[allow(clippy::too_many_arguments)]
3560async fn handle_control_request(
3561    tx: &WriterSender,
3562    frame: &Frame,
3563    shared_app: &Arc<App>,
3564    executor: &Arc<Executor>,
3565    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3566    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3567    installed_route_epochs: &mut HashMap<u16, u32>,
3568    routes: &mut HashMap<RouteChannel, RouteIdentity>,
3569    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
3570    bg_subs: &mut HashMap<RouteChannel, BgSub>,
3571    bg_sub_by_session: &mut BgSubsBySession,
3572    bg_wake_pending: &mut HashSet<RouteChannel>,
3573    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
3574    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
3575    retry_buffer: &mut RetryBuffer,
3576    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
3577    shutdown: &Arc<Notify>,
3578    control_completion_tx: &mpsc::Sender<RouteBindCompletion>,
3579    metrics: &Arc<DispatchPathMetrics>,
3580    health_rollup_cache: &HealthRollupCache,
3581    push_senders: &PushSenders,
3582    dispatch: DispatchFn,
3583    user_config_path: Option<&Path>,
3584) -> Result<(), SubcError> {
3585    let request =
3586        serde_json::from_slice::<ModuleControlRequest>(&frame.body).map_err(SubcError::Json)?;
3587    match request {
3588        ModuleControlRequest::RouteBind {
3589            route_channel,
3590            epoch,
3591            target: _,
3592            identity,
3593            principal,
3594            consumer_capabilities,
3595            admission_facts: _,
3596        } => {
3597            let route_id = route_key(route_channel, epoch);
3598            if epoch == 0 {
3599                return send_route_bind_error(
3600                    tx,
3601                    frame,
3602                    "config_divergence",
3603                    "route bind uses an invalid channel generation",
3604                    metrics,
3605                )
3606                .await;
3607            }
3608            let mut bind_root_id = None;
3609            if let Some(installed_epoch) = installed_route_epochs.get(&route_channel).copied() {
3610                if installed_epoch >= epoch {
3611                    return send_route_bind_error(
3612                        tx,
3613                        frame,
3614                        "config_divergence",
3615                        "route bind generation is not newer than the installed generation",
3616                        metrics,
3617                    )
3618                    .await;
3619                }
3620
3621                let replacement_root = match ProjectRootId::from_path(&identity.project_root) {
3622                    Ok(root_id) => root_id,
3623                    Err(error) => {
3624                        return send_route_bind_error(
3625                            tx,
3626                            frame,
3627                            "config_divergence",
3628                            &format!("invalid route project root: {error}"),
3629                            metrics,
3630                        )
3631                        .await;
3632                    }
3633                };
3634                teardown_installed_route(
3635                    tx,
3636                    metrics,
3637                    executor,
3638                    route_key(route_channel, installed_epoch),
3639                    "higher-epoch RouteBind",
3640                    Some(&replacement_root),
3641                    installed_route_epochs,
3642                    routes,
3643                    root_channels,
3644                    bg_subs,
3645                    bg_sub_by_session,
3646                    bg_wake_pending,
3647                    pending_bash_asks,
3648                    live_roots,
3649                    route_bash_cancels,
3650                    pending_binds,
3651                    retry_buffer,
3652                    push_buffer,
3653                    shutdown,
3654                )
3655                .await?;
3656                bind_root_id = Some(replacement_root);
3657            }
3658            if pending_binds.contains_key(&route_id) {
3659                return send_route_bind_error(
3660                    tx,
3661                    frame,
3662                    "config_divergence",
3663                    "route bind is already pending for channel",
3664                    metrics,
3665                )
3666                .await;
3667            }
3668            let bind_root_id = match bind_root_id {
3669                Some(root_id) => root_id,
3670                None => match ProjectRootId::from_path(&identity.project_root) {
3671                    Ok(root_id) => root_id,
3672                    Err(error) => {
3673                        return send_route_bind_error(
3674                            tx,
3675                            frame,
3676                            "config_divergence",
3677                            &format!("invalid route project root: {error}"),
3678                            metrics,
3679                        )
3680                        .await;
3681                    }
3682                },
3683            };
3684
3685            // Reconcile RootConfig: build a configure request from the bind
3686            // identity + forwarded config tiers and run it through the executor.
3687            let request_id = format!("subc-bind-{route_channel}");
3688            let bind_project_root = identity.project_root.clone();
3689            let bind_harness = identity.harness.clone();
3690            let bind_session = identity.session.clone();
3691            let bind_trust = trust_for_bind(&bind_harness, &principal);
3692            let bind_principal_id = principal_id(&principal);
3693            // Typed capability declaration from the consumer: the facade stamps it
3694            // from the MCP host's initialize-advertised capabilities. Absent
3695            // means no reverse-request capability — flat deny, fail-closed. A
3696            // consumer over-declaring only earns asks that TTL-deny.
3697            let consumer_elicitation_capable = consumer_capabilities
3698                .as_ref()
3699                .is_some_and(|capabilities| capabilities.iter().any(|c| c == "elicitation"));
3700            log::info!(
3701                "subc attach: route {} harness={} principal={} trust={} elicitation={}",
3702                route_channel,
3703                bind_harness,
3704                principal_label(&principal),
3705                bind_trust.label(),
3706                consumer_elicitation_capable
3707            );
3708
3709            // Config is single-per-project, read by AFT directly from the
3710            // CortexKit config files (user: ~/.config/cortexkit/aft.jsonc,
3711            // project: <root>/.cortexkit/aft.jsonc). Wire-relayed config tiers are
3712            // IGNORED entirely: a front (runner, mcp:*, or fed:*) cannot push config over
3713            // the wire. This is what makes config harness-INDEPENDENT — every
3714            // harness binding a project gets the identical on-disk config, so two
3715            // trust domains sharing the per-root actor can never diverge or
3716            // inherit each other's capabilities (the cross-bind escalation class).
3717            // Wire-relayed config tiers (if the protocol still carries them) are
3718            // ignored entirely; the per-tier trust boundary (user trusted, project
3719            // privileged-dropped) is applied to the FILE tiers in handle_configure.
3720            let local_tiers = crate::subc_config::read_local_cortexkit_config_tiers(
3721                user_config_path,
3722                Path::new(&bind_project_root),
3723            );
3724            let config_tiers: Vec<Value> = local_tiers
3725                .iter()
3726                .map(|t| json!({ "tier": t.tier, "source": t.source, "doc": t.doc }))
3727                .collect();
3728            let diagnostics_on_edit = diagnostics_on_edit_from_tiers(&local_tiers);
3729            let configure_json = json!({
3730                "id": request_id,
3731                "command": "configure",
3732                "project_root": bind_project_root,
3733                "harness": bind_harness,
3734                "session_id": bind_session.clone(),
3735                "config": config_tiers,
3736            });
3737            let configure_req = match serde_json::from_value::<RawRequest>(configure_json) {
3738                Ok(req) => req,
3739                Err(error) => {
3740                    return send_route_bind_error(
3741                        tx,
3742                        frame,
3743                        "config_divergence",
3744                        &format!("failed to build configure request: {error}"),
3745                        metrics,
3746                    )
3747                    .await;
3748                }
3749            };
3750
3751            let route_identity = RouteIdentity(Arc::new(RouteIdentityData {
3752                root: bind_root_id.clone(),
3753                project_root: PathBuf::from(&bind_project_root),
3754                harness: bind_harness.clone(),
3755                session: bind_session.clone(),
3756                trust: bind_trust,
3757                spawn_principal: AuthenticatedPrincipal::RouteBind {
3758                    trust: bind_trust.sandbox_trust(),
3759                    route_channel,
3760                    route_epoch: epoch,
3761                    project_root: PathBuf::from(&bind_project_root),
3762                    harness: bind_harness.clone(),
3763                    session_id: bind_session.clone(),
3764                    principal_id: bind_principal_id,
3765                },
3766                consumer_elicitation_capable,
3767            }));
3768            let configure_session = route_identity.session.clone();
3769            let root_was_live = live_roots.contains_key(&bind_root_id);
3770            let inserted_new_actor = register_actor_for_bind(
3771                shared_app,
3772                executor,
3773                push_senders,
3774                &bind_root_id,
3775                route_channel,
3776                root_was_live,
3777            );
3778
3779            let configure_request_id = configure_req.id.clone();
3780            installed_route_epochs.insert(route_channel, epoch);
3781            if let Some(meta) = live_roots.get_mut(&bind_root_id) {
3782                meta.maintenance_queued_kinds.clear();
3783                meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
3784            }
3785            let (configure_rx, configure_cancellation) = executor.submit_cancellable_async(
3786                bind_root_id.clone(),
3787                Lane::Mutating,
3788                configure_request_id.clone(),
3789                Box::new(move |ctx| {
3790                    log_ctx::with_session(Some(configure_session.clone()), || {
3791                        dispatch(configure_req, ctx)
3792                    })
3793                }),
3794            );
3795            pending_binds.insert(
3796                route_id,
3797                PendingBind {
3798                    bind_root_id: bind_root_id.clone(),
3799                    inserted_new_actor,
3800                    cancelled: false,
3801                    configure_request_id: configure_request_id.clone(),
3802                    started_at: Instant::now(),
3803                    warned_half_deadline: false,
3804                    deadline_reported: false,
3805                    corr: frame.header.corr,
3806                    ver: frame.header.ver,
3807                    flags: frame.header.flags,
3808                    cancellation: configure_cancellation,
3809                },
3810            );
3811
3812            let completion_tx = control_completion_tx.clone();
3813            let completion_identity = route_identity;
3814            let completion_root = bind_root_id.clone();
3815            let completion_route_channel = route_channel;
3816            let completion_ver = frame.header.ver;
3817            let completion_corr = frame.header.corr;
3818            let completion_flags = frame.header.flags;
3819            let completion_metrics = Arc::clone(metrics);
3820            tokio::spawn(async move {
3821                let _response_task = ResponseTaskGuard::new(&completion_metrics);
3822                let configure_response =
3823                    await_executor_response(configure_rx, configure_request_id.clone()).await;
3824                // Send the route-bind acknowledgment as soon as configure succeeds.
3825                // Installing completed search or callgraph builds only refreshes cached
3826                // read data, so a later maintenance pass can do it without delaying the
3827                // daemon's confirmation that the route is usable.
3828                let completion = RouteBindCompletion {
3829                    route: route_key(completion_route_channel, epoch),
3830                    identity: completion_identity,
3831                    bind_root_id: completion_root,
3832                    inserted_new_actor,
3833                    configure_response,
3834                    diagnostics_on_edit,
3835                    ver: completion_ver,
3836                    corr: completion_corr,
3837                    flags: completion_flags,
3838                };
3839                if send_counted_channel(
3840                    &completion_tx,
3841                    &completion_metrics.control_completion_queued,
3842                    completion,
3843                )
3844                .await
3845                .is_err()
3846                {
3847                    log::debug!(
3848                        "subc attach: dropped RouteBind completion for route {} after loop exit",
3849                        completion_route_channel
3850                    );
3851                }
3852            });
3853
3854            health_rollup_cache.refresh(executor, shared_app);
3855            Ok(())
3856        }
3857        ModuleControlRequest::HealthCheck {} => {
3858            metrics.record_bg_runtime(bg_subs.len(), bg_wake_pending.len());
3859            let report = build_health_report(
3860                health_rollup_cache,
3861                executor,
3862                pending_binds,
3863                metrics,
3864                shared_app,
3865            );
3866            let body = serde_json::to_vec(&ModuleControlResponse::from(report))
3867                .map_err(SubcError::Json)?;
3868            let response = Frame::build_with_version(
3869                frame.header.ver,
3870                FrameType::Response,
3871                frame.header.flags,
3872                0,
3873                0,
3874                frame.header.corr,
3875                body,
3876            )
3877            .map_err(SubcError::FrameBuild)?;
3878            send_frame(tx, metrics, response).await
3879        }
3880    }
3881}
3882
3883fn install_bash_compressor(ctx: &AppContext) {
3884    // Mirrors main.rs per-actor compressor installation for subc-created actors.
3885    let filter_registry_handle = ctx.shared_filter_registry();
3886    let compress_flag = ctx.bash_compress_flag();
3887    ctx.bash_background().set_compressor_with_exit_code(
3888        move |command: &str, output: String, exit_code: Option<i32>| {
3889            if !compress_flag.load(std::sync::atomic::Ordering::Relaxed) {
3890                return crate::compress::CompressionResult::new(output);
3891            }
3892            let registry_guard = match filter_registry_handle.read() {
3893                Ok(g) => g,
3894                Err(poisoned) => poisoned.into_inner(),
3895            };
3896            crate::compress::compress_with_registry_exit_code(
3897                command,
3898                &output,
3899                exit_code,
3900                &registry_guard,
3901            )
3902        },
3903    );
3904}
3905
3906fn diagnostics_on_edit_from_tiers(tiers: &[ConfigTier]) -> bool {
3907    let mut diagnostics_on_edit = false;
3908    for tier in tiers {
3909        if let Some(value) = diagnostics_on_edit_from_doc(&tier.doc) {
3910            diagnostics_on_edit = value;
3911        }
3912    }
3913    diagnostics_on_edit
3914}
3915
3916fn diagnostics_on_edit_from_doc(doc: &str) -> Option<bool> {
3917    let stripped = strip_jsonc(doc);
3918    let value = serde_json::from_str::<Value>(&stripped).ok()?;
3919    value
3920        .get("lsp")
3921        .and_then(Value::as_object)?
3922        .get("diagnostics_on_edit")
3923        .and_then(Value::as_bool)
3924}
3925
3926async fn send_route_bind_error(
3927    tx: &WriterSender,
3928    frame: &Frame,
3929    code: &str,
3930    message: &str,
3931    metrics: &DispatchPathMetrics,
3932) -> Result<(), SubcError> {
3933    send_route_bind_error_parts(
3934        tx,
3935        frame.header.ver,
3936        frame.header.corr,
3937        frame.header.flags,
3938        code,
3939        message,
3940        metrics,
3941    )
3942    .await
3943}
3944
3945async fn send_route_bind_error_parts(
3946    tx: &WriterSender,
3947    ver: u8,
3948    corr: u64,
3949    flags: Flags,
3950    code: &str,
3951    message: &str,
3952    metrics: &DispatchPathMetrics,
3953) -> Result<(), SubcError> {
3954    let response = build_error_frame(ver, 0, 0, corr, flags, code, message)?;
3955    send_reliable_writer_frame(tx, metrics, response, "RouteBind error").await?;
3956    log_route_bind_rejection(code, message);
3957    Ok(())
3958}
3959
3960/// Per-message rate limit for the bind-rejection warn line. A caller that
3961/// re-attaches a dead root forever turns this line into the entire readable
3962/// tail of the SHARED daemon log (measured 2026-08-09: 2.08M copies, ~40/sec,
3963/// 936MB log — other modules' incident lines pushed out of the tail).
3964/// The line itself stays byte-identical so external counters keep matching;
3965/// repeats inside the window are summarized with a suppressed count on the
3966/// next emission (volume stays diagnosable, per the log-diet convention).
3967fn log_route_bind_rejection(code: &str, message: &str) {
3968    const WINDOW: Duration = Duration::from_secs(60);
3969    static SUPPRESSED: OnceLock<StdMutex<HashMap<String, (Instant, u64)>>> = OnceLock::new();
3970    let map = SUPPRESSED.get_or_init(|| StdMutex::new(HashMap::new()));
3971    let mut map = match map.try_lock() {
3972        Ok(map) => map,
3973        // Contended: log unsuppressed rather than blocking or dropping.
3974        Err(_) => {
3975            log::warn!("subc attach: route bind rejected ({code}): {message}");
3976            return;
3977        }
3978    };
3979    let now = Instant::now();
3980    // Bound the map: dead roots churn, and an unbounded suppression map is
3981    // its own leak. Sweep expired entries once it grows past a fleet-sized
3982    // number of distinct rejection messages.
3983    if map.len() > 512 {
3984        map.retain(|_, (start, _)| now.duration_since(*start) < WINDOW);
3985    }
3986    match map.get_mut(message) {
3987        Some((window_start, suppressed)) if now.duration_since(*window_start) < WINDOW => {
3988            *suppressed += 1;
3989        }
3990        Some((window_start, suppressed)) => {
3991            if *suppressed > 0 {
3992                log::warn!(
3993                    "subc attach: route bind rejected ({code}): {message} (repeated {}x in last 60s)",
3994                    *suppressed
3995                );
3996            } else {
3997                log::warn!("subc attach: route bind rejected ({code}): {message}");
3998            }
3999            *window_start = now;
4000            *suppressed = 0;
4001        }
4002        None => {
4003            log::warn!("subc attach: route bind rejected ({code}): {message}");
4004            map.insert(message.to_string(), (now, 0));
4005        }
4006    }
4007}
4008
4009/// Route-channel tool call: `{name, arguments}` → executor lane → dispatch to
4010/// the sync command core → wrap the structured Response in a CallToolResult
4011/// `{content, isError}`. Tool-result mapping: the whole `{success, ...}` Response
4012/// serialized into ONE text block; `isError` carries `success == false`.
4013async fn handle_tool_call(
4014    tx: &WriterSender,
4015    frame: &Frame,
4016    mut phase_trace: PhaseTrace,
4017    routes: &HashMap<RouteChannel, RouteIdentity>,
4018    pending_binds: &HashMap<RouteChannel, PendingBind>,
4019    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4020    executor: &Arc<Executor>,
4021    shutdown: &Arc<Notify>,
4022    connection_cancel: &PersistentCancelSignal,
4023    bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
4024    bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
4025    metrics: &Arc<DispatchPathMetrics>,
4026    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
4027    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
4028    next_bash_ask_corr: &mut u64,
4029    bg_subs: &mut HashMap<RouteChannel, BgSub>,
4030    bg_sub_by_session: &mut BgSubsBySession,
4031    bg_wake_pending: &mut HashSet<RouteChannel>,
4032    bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
4033    dispatch: DispatchFn,
4034    allow_native_passthrough: bool,
4035    tool_response_body_limit: usize,
4036) -> Result<(), SubcError> {
4037    let route_id = route_key(frame.header.channel, frame.header.epoch);
4038    if pending_binds.contains_key(&route_id) {
4039        let error = build_error_frame(
4040            frame.header.ver,
4041            frame.header.channel,
4042            frame.header.epoch,
4043            frame.header.corr,
4044            frame.header.flags,
4045            "route_not_bound",
4046            "route is not bound before tool call",
4047        )?;
4048        return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
4049    }
4050
4051    let Some(identity) = routes.get(&route_id).cloned() else {
4052        let error = build_error_frame(
4053            frame.header.ver,
4054            frame.header.channel,
4055            frame.header.epoch,
4056            frame.header.corr,
4057            frame.header.flags,
4058            "route_not_bound",
4059            "route is not bound before tool call",
4060        )?;
4061        return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
4062    };
4063    let restore_watcher = live_roots
4064        .get(&identity.root)
4065        .is_some_and(|meta| meta.idle_artifacts_evicted);
4066    if let Some(meta) = live_roots.get_mut(&identity.root) {
4067        meta.reactivate_bound();
4068    }
4069    if restore_watcher {
4070        if let Some(ctx) = executor.actor_context(&identity.root) {
4071            crate::commands::configure::ensure_project_watcher(&ctx);
4072        }
4073    }
4074
4075    let route_request =
4076        serde_json::from_slice::<RouteRequest>(&frame.body).map_err(SubcError::Json)?;
4077    if matches!(
4078        route_request,
4079        RouteRequest::BgEvents(BgEventsRequest {
4080            op: BgEventsOp::BgEvents
4081        })
4082    ) {
4083        if let Some(old_sub) = bg_subs.get(&route_id).cloned() {
4084            metrics.record_bg_subscription_ended(
4085                &old_sub.root,
4086                &old_sub.session,
4087                route_id,
4088                "resubscribe",
4089            );
4090            push::send_reliable_bg_stream_end(tx, metrics, route_id, &old_sub).await?;
4091        }
4092        if !identity.trust.allows_bash_observation() {
4093            bg_subs.remove(&route_id);
4094            bg_wake_pending.remove(&route_id);
4095            remove_bg_subscription_index(bg_sub_by_session, route_id, Some(&identity));
4096            let denied_sub = BgSub {
4097                corr: frame.header.corr,
4098                ver: frame.header.ver,
4099                flags: frame.header.flags,
4100                root: identity.root.clone(),
4101                session: identity.session.clone(),
4102            };
4103            metrics.record_bg_subscription_ended(
4104                &identity.root,
4105                &identity.session,
4106                route_id,
4107                "subscribe-denied",
4108            );
4109            push::send_reliable_bg_stream_end(tx, metrics, route_id, &denied_sub).await?;
4110            return Ok(());
4111        }
4112        bg_subs.insert(
4113            route_id,
4114            BgSub {
4115                corr: frame.header.corr,
4116                ver: frame.header.ver,
4117                flags: frame.header.flags,
4118                root: identity.root.clone(),
4119                session: identity.session.clone(),
4120            },
4121        );
4122        insert_bg_subscription_index(
4123            bg_sub_by_session,
4124            identity.root.clone(),
4125            identity.session.clone(),
4126            route_id,
4127        );
4128        metrics.record_bg_subscription_installed(&identity.root, &identity.session, route_id);
4129        push::arm_bg_wake(
4130            identity.root.clone(),
4131            identity.session.clone(),
4132            route_id,
4133            bg_wake_pending,
4134            bg_wake_epoch,
4135            metrics,
4136        );
4137        return Ok(());
4138    }
4139
4140    let RouteRequest::ToolCall(call) = route_request else {
4141        unreachable!("background event subscription returned above")
4142    };
4143    let bare_name = call.name;
4144    let arguments = strip_agent_preview_arg_owned(call.arguments);
4145    let format_context = crate::subc_format::FormatContext::from_tool_call(
4146        &bare_name,
4147        &arguments,
4148        identity.project_root.as_path(),
4149    );
4150
4151    let request_id = format!("subc-{}-{}", frame.header.channel, frame.header.corr);
4152    let bind_trust = identity.trust;
4153    let diagnostics_on_edit = live_roots
4154        .get(&identity.root)
4155        .map(|meta| meta.diagnostics_on_edit)
4156        .unwrap_or(false);
4157
4158    let requests_host = bare_name == "bash"
4159        && arguments
4160            .get("sandbox")
4161            .or_else(|| {
4162                arguments
4163                    .get("params")
4164                    .and_then(|params| params.get("sandbox"))
4165            })
4166            .and_then(Value::as_str)
4167            == Some("host");
4168    if matches!(bind_trust, BindTrust::Untrusted) && requests_host {
4169        let response = Response::error(
4170            request_id.clone(),
4171            "sandbox_escalation_denied",
4172            "sandbox host escalation is unavailable to untrusted principals",
4173        );
4174        let text = crate::subc_format::format_response_with_context(
4175            &bare_name,
4176            &response,
4177            &format_context,
4178        );
4179        let result = ToolCallResult { text, response };
4180        let response_frame = build_tool_response_frame_with_limit(
4181            frame.header.ver,
4182            route_id,
4183            frame.header.corr,
4184            frame.header.flags,
4185            &result,
4186            bind_trust,
4187            tool_response_body_limit,
4188        )?;
4189        return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4190    }
4191
4192    if matches!(bind_trust, BindTrust::Untrusted)
4193        && is_bash_family_tool(&bare_name)
4194        && (bare_name != "bash" || !identity.consumer_elicitation_capable)
4195    {
4196        let response = bash::bash_denied_untrusted_response(request_id.clone());
4197        let text = crate::subc_format::format_response_with_context(
4198            &bare_name,
4199            &response,
4200            &format_context,
4201        );
4202        let result = ToolCallResult { text, response };
4203        let response_frame = build_tool_response_frame_with_limit(
4204            frame.header.ver,
4205            route_id,
4206            frame.header.corr,
4207            frame.header.flags,
4208            &result,
4209            bind_trust,
4210            tool_response_body_limit,
4211        )?;
4212        return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4213    }
4214
4215    // A non-core name is NOT in the tool manifest. AFT fails closed and
4216    // does not trust subc to enforce the manifest: rejecting here is the
4217    // defense-in-depth backstop that prevents a forwarded native command
4218    // (e.g. `configure`, which would reach handle_configure and bypass
4219    // the RouteBind config-trust cap) from ever reaching dispatch. Only
4220    // the integration-test harness (run_subc_mode_for_test) opens this to
4221    // drive synthetic native commands through the executor.
4222    if !is_subc_agent_core_tool(&bare_name)
4223        && !is_subc_native_plumbing_tool(&bare_name)
4224        && !allow_native_passthrough
4225    {
4226        log::warn!(
4227            "subc tool call: rejecting non-manifest tool name {:?} on route {} (fail-closed)",
4228            bare_name,
4229            frame.header.channel
4230        );
4231        let response = Response::error(
4232            request_id.clone(),
4233            "unknown_tool",
4234            format!("tool {:?} is not in the AFT tool manifest", bare_name),
4235        );
4236        let text = crate::subc_format::format_response_with_context(
4237            &bare_name,
4238            &response,
4239            &format_context,
4240        );
4241        let result = ToolCallResult { text, response };
4242        let response_frame = build_tool_response_frame_with_limit(
4243            frame.header.ver,
4244            route_id,
4245            frame.header.corr,
4246            frame.header.flags,
4247            &result,
4248            bind_trust,
4249            tool_response_body_limit,
4250        )?;
4251        return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4252    }
4253
4254    if bare_name == "bash" {
4255        if matches!(bind_trust, BindTrust::Untrusted) {
4256            let plan = match bash::prepare_bash_elicitation_plan(
4257                &arguments,
4258                identity.project_root.as_path(),
4259            ) {
4260                Ok(plan) => plan,
4261                Err(error) => {
4262                    let response = Response::error(request_id.clone(), error.code, error.message);
4263                    let text = crate::subc_format::format_response_with_context(
4264                        &bare_name,
4265                        &response,
4266                        &format_context,
4267                    );
4268                    let result = ToolCallResult { text, response };
4269                    let response_frame = build_tool_response_frame_with_limit(
4270                        frame.header.ver,
4271                        route_id,
4272                        frame.header.corr,
4273                        frame.header.flags,
4274                        &result,
4275                        bind_trust,
4276                        tool_response_body_limit,
4277                    )?;
4278                    return send_reliable_writer_frame(
4279                        tx,
4280                        metrics,
4281                        response_frame,
4282                        "tool response",
4283                    )
4284                    .await;
4285                }
4286            };
4287
4288            let reverse_corr =
4289                allocate_reverse_corr(pending_bash_asks, route_id, next_bash_ask_corr);
4290            let ask_frame = build_bash_elicitation_request_frame(
4291                frame.header.ver,
4292                route_id,
4293                reverse_corr,
4294                frame.header.flags,
4295                &plan.command,
4296                &plan.asks,
4297            )?;
4298
4299            let meta = live_roots
4300                .entry(identity.root.clone())
4301                .or_insert_with(|| RootMeta::new(Instant::now()));
4302            meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
4303            meta.reactivate_bound();
4304
4305            let route_cancel =
4306                route_bash_cancels
4307                    .entry(route_id)
4308                    .or_insert_with(|| bash::RouteBashCancel {
4309                        token: PersistentCancelSignal::new(),
4310                        active_waits: 0,
4311                    });
4312            route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
4313            let cancel = bash::BashWaitCancel {
4314                connection: connection_cancel.clone(),
4315                route: route_cancel.token.clone(),
4316            };
4317            pending_bash_asks.insert(
4318                ReverseCorrKey {
4319                    route: route_id,
4320                    corr: reverse_corr,
4321                },
4322                PendingBashAsk {
4323                    route: route_id,
4324                    tool_corr: frame.header.corr,
4325                    tool_flags: frame.header.flags,
4326                    tool_ver: frame.header.ver,
4327                    root: identity.root.clone(),
4328                    project_root: identity.project_root.clone(),
4329                    session_id: identity.session.clone(),
4330                    spawn_principal: identity.spawn_principal.clone(),
4331                    edit_slot_survives: call.edit_slot_survives,
4332                    request_id,
4333                    arguments,
4334                    format_context,
4335                    cancel,
4336                    grants: plan.grants,
4337                    expires_at: Instant::now() + bash_elicitation_timeout(),
4338                },
4339            );
4340            return send_reliable_writer_frame(tx, metrics, ask_frame, "bash elicitation request")
4341                .await;
4342        }
4343
4344        let meta = live_roots
4345            .entry(identity.root.clone())
4346            .or_insert_with(|| RootMeta::new(Instant::now()));
4347        meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
4348        meta.reactivate_bound();
4349
4350        let route_cancel =
4351            route_bash_cancels
4352                .entry(route_id)
4353                .or_insert_with(|| bash::RouteBashCancel {
4354                    token: PersistentCancelSignal::new(),
4355                    active_waits: 0,
4356                });
4357        route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
4358        let cancel = bash::BashWaitCancel {
4359            connection: connection_cancel.clone(),
4360            route: route_cancel.token.clone(),
4361        };
4362
4363        bash::submit_deferred_bash(
4364            executor,
4365            bash_deferred_tx,
4366            bash_poll_touch_tx,
4367            metrics,
4368            dispatch,
4369            identity.root.clone(),
4370            identity.project_root.clone(),
4371            identity.session.clone(),
4372            request_id,
4373            route_id,
4374            frame.header.corr,
4375            frame.header.flags,
4376            frame.header.ver,
4377            arguments,
4378            format_context,
4379            cancel,
4380            bind_trust,
4381            identity.spawn_principal.clone(),
4382            call.edit_slot_survives,
4383            None,
4384        );
4385        return Ok(());
4386    }
4387
4388    let lane = command_lane(&bare_name);
4389    let tool_call_context = ToolCallContext {
4390        project_root: identity.project_root.clone(),
4391        session_id: Some(identity.session.clone()),
4392        request_id: request_id.clone(),
4393        diagnostics_on_edit,
4394        preview: call.preview,
4395        edit_slot_survives: call.edit_slot_survives,
4396        report_registration_downgrade: true,
4397    };
4398    let bare_name_for_frame = bare_name.clone();
4399    let identity_for_run = identity.clone();
4400    let completion_session = identity.session.clone();
4401    let completion_root = identity.project_root.clone();
4402    let request_id_for_force = request_id.clone();
4403    let format_context_for_frame = format_context.clone();
4404    let (tool_call_tx, tool_call_rx) = oneshot::channel::<ToolCallCompletion>();
4405    phase_trace.mark_executor_submitted();
4406    let rx = executor.submit_async(
4407        identity.root.clone(),
4408        lane,
4409        request_id.clone(),
4410        Box::new(move |ctx| {
4411            phase_trace.mark_job_admitted();
4412            log_ctx::with_session(Some(identity_for_run.session.clone()), || {
4413                let run = || {
4414                    let finalizer = |response: &mut Response| {
4415                        crate::response_finalize::finalize_response_with_bg_completions(
4416                            response,
4417                            ctx,
4418                            &identity_for_run.session,
4419                            &bare_name,
4420                            bind_trust.allows_bash_observation(),
4421                        );
4422                    };
4423                    match run_tool_call(
4424                        &bare_name,
4425                        arguments,
4426                        &format_context,
4427                        &tool_call_context,
4428                        ctx,
4429                        &dispatch,
4430                        Some(&finalizer),
4431                        Some(&mut phase_trace),
4432                    ) {
4433                        ToolCallOutcome::Unary(result) => {
4434                            let response = result.response;
4435                            let _ = tool_call_tx.send(ToolCallCompletion {
4436                                text: result.text,
4437                                phase_trace,
4438                            });
4439                            response
4440                        }
4441                    }
4442                };
4443                if matches!(bind_trust, BindTrust::Untrusted) {
4444                    ctx.with_force_restrict(&request_id_for_force, run)
4445                } else {
4446                    run()
4447                }
4448            })
4449        }),
4450    );
4451    let completion_tx = tx.clone();
4452    let completion_shutdown = Arc::clone(shutdown);
4453    let route = route_id;
4454    let corr = frame.header.corr;
4455    let flags = frame.header.flags;
4456    let ver = frame.header.ver;
4457    let completion_metrics = Arc::clone(metrics);
4458    tokio::spawn(async move {
4459        let _response_task = ResponseTaskGuard::new(&completion_metrics);
4460        let response = await_executor_response(rx, request_id.clone()).await;
4461        let (text, phase_trace) = match tool_call_rx.await {
4462            Ok(completion) => (completion.text, Some(completion.phase_trace)),
4463            Err(_) => (
4464                crate::subc_format::format_response_with_context(
4465                    &bare_name_for_frame,
4466                    &response,
4467                    &format_context_for_frame,
4468                ),
4469                None,
4470            ),
4471        };
4472        let result = ToolCallResult { text, response };
4473        let fatal = response_is_fatal_panic(&result.response);
4474        match build_tool_response_frame_with_limit(
4475            ver,
4476            route,
4477            corr,
4478            flags,
4479            &result,
4480            bind_trust,
4481            tool_response_body_limit,
4482        ) {
4483            Ok(response_frame) => {
4484                let send_result = if let Some(phase_trace) = phase_trace {
4485                    let trace = ToolResponseWriteTrace::new(
4486                        phase_trace,
4487                        bare_name_for_frame,
4488                        completion_root,
4489                        completion_session,
4490                        route.channel,
4491                        corr,
4492                    );
4493                    send_traced_tool_response_frame(
4494                        &completion_tx,
4495                        &completion_metrics,
4496                        response_frame,
4497                        trace,
4498                    )
4499                    .await
4500                } else {
4501                    send_reliable_writer_frame(
4502                        &completion_tx,
4503                        &completion_metrics,
4504                        response_frame,
4505                        "tool response",
4506                    )
4507                    .await
4508                };
4509                if let Err(error) = send_result {
4510                    log::warn!("subc attach: failed to queue tool response frame: {error}");
4511                }
4512            }
4513            Err(error) => {
4514                log::error!("subc attach: failed to build tool response frame: {error}");
4515            }
4516        }
4517        if fatal {
4518            signal_fatal_teardown(
4519                &completion_tx,
4520                Some(route),
4521                ver,
4522                corr,
4523                &completion_shutdown,
4524                &completion_metrics,
4525            )
4526            .await;
4527        }
4528    });
4529    Ok(())
4530}
4531
4532fn submit_maintenance_job(
4533    executor: &Arc<Executor>,
4534    root_id: ProjectRootId,
4535    kind: MaintenanceDrainKind,
4536    bg_sessions_to_check: Vec<(String, u64)>,
4537    completion_tx: &mpsc::Sender<MaintenanceCompletion>,
4538    metrics: &Arc<DispatchPathMetrics>,
4539) {
4540    let request_id = format!(
4541        "subc-maintenance-drain-{}-{}",
4542        kind.label(),
4543        root_id.as_path().to_string_lossy()
4544    );
4545    let response_id = request_id.clone();
4546    let completion_root_id = root_id.clone();
4547    let maintenance_generation = executor
4548        .actor_context(&root_id)
4549        .map(|ctx| ctx.configure_generation())
4550        .unwrap_or(0);
4551    let (outcome_tx, outcome_rx) = oneshot::channel::<MaintenanceJobOutcome>();
4552    // ConfigureTail runs deferred configure mutations and needs the actor
4553    // epoch write gate. The other drain kinds only mutate subsystem state
4554    // behind that subsystem's own lock, so they run on MaintenanceCommit and
4555    // overlap interactive reads instead of excluding them.
4556    let lane = match kind {
4557        MaintenanceDrainKind::ConfigureTail => Lane::Mutating,
4558        MaintenanceDrainKind::Watcher
4559        | MaintenanceDrainKind::Lsp
4560        | MaintenanceDrainKind::CompletionDrains => Lane::MaintenanceCommit,
4561    };
4562    let job: crate::executor::ExecutorJob = Box::new(move |ctx: &AppContext| {
4563        let outcome = match kind {
4564            MaintenanceDrainKind::Watcher => {
4565                let drained = runtime_drain::drain_watcher_events_bounded(
4566                    ctx,
4567                    runtime_drain::WATCHER_PATH_DRAIN_BATCH_CAP,
4568                );
4569                MaintenanceJobOutcome {
4570                    empty_bg_sessions: Vec::new(),
4571                    requeue_kind: drained.has_more.then_some(kind),
4572                }
4573            }
4574            MaintenanceDrainKind::Lsp => {
4575                let drained = runtime_drain::drain_lsp_events_bounded(
4576                    ctx,
4577                    runtime_drain::LSP_EVENT_DRAIN_BATCH_CAP,
4578                );
4579                MaintenanceJobOutcome {
4580                    empty_bg_sessions: Vec::new(),
4581                    requeue_kind: drained.has_more.then_some(kind),
4582                }
4583            }
4584            MaintenanceDrainKind::ConfigureTail => {
4585                runtime_drain::drain_deferred_configure_maintenance(ctx);
4586                runtime_drain::drain_configure_warning_events(ctx);
4587                MaintenanceJobOutcome::default()
4588            }
4589            MaintenanceDrainKind::CompletionDrains => {
4590                runtime_drain::drain_search_index_events(ctx);
4591                runtime_drain::drain_callgraph_store_events(ctx);
4592                runtime_drain::drain_semantic_index_events(ctx);
4593                runtime_drain::drain_semantic_refresh_events(ctx);
4594                runtime_drain::drain_inspect_events_for_generation(ctx, maintenance_generation);
4595                let empty_bg_sessions = bg_sessions_to_check
4596                    .into_iter()
4597                    .filter(|(session, _)| {
4598                        !ctx.bash_background()
4599                            .has_completions_for_session(Some(session.as_str()))
4600                    })
4601                    .collect();
4602                MaintenanceJobOutcome {
4603                    empty_bg_sessions,
4604                    requeue_kind: None,
4605                }
4606            }
4607        };
4608        let requeued = outcome.requeue_kind.is_some();
4609        let _ = outcome_tx.send(outcome);
4610        Response::success(
4611            response_id,
4612            json!({ "drained": true, "kind": kind.label(), "requeued": requeued }),
4613        )
4614    });
4615    let rx = match kind {
4616        MaintenanceDrainKind::Watcher => executor.submit_coalescable_maintenance_async(
4617            root_id,
4618            lane,
4619            request_id.clone(),
4620            crate::executor::MaintenanceCoalesceKey::WatcherDrain,
4621            job,
4622        ),
4623        MaintenanceDrainKind::Lsp => executor.submit_coalescable_maintenance_async(
4624            root_id,
4625            lane,
4626            request_id.clone(),
4627            crate::executor::MaintenanceCoalesceKey::LspDrain,
4628            job,
4629        ),
4630        MaintenanceDrainKind::ConfigureTail | MaintenanceDrainKind::CompletionDrains => {
4631            executor.submit_maintenance_async(root_id, lane, request_id.clone(), job)
4632        }
4633    };
4634    let completion_tx = completion_tx.clone();
4635    let completion_metrics = Arc::clone(metrics);
4636    tokio::spawn(async move {
4637        let _response_task = ResponseTaskGuard::new(&completion_metrics);
4638        let response = await_executor_response(rx, request_id).await;
4639        let outcome = outcome_rx.await.unwrap_or_default();
4640        let _ = send_counted_channel(
4641            &completion_tx,
4642            &completion_metrics.maintenance_queued,
4643            MaintenanceCompletion {
4644                root_id: completion_root_id,
4645                kind,
4646                response,
4647                empty_bg_sessions: outcome.empty_bg_sessions,
4648                requeue_kind: outcome.requeue_kind,
4649            },
4650        )
4651        .await;
4652    });
4653}
4654
4655async fn await_executor_response(rx: oneshot::Receiver<Response>, request_id: String) -> Response {
4656    rx.await
4657        .unwrap_or_else(|_| Response::error(request_id, "internal_error", "executor dropped"))
4658}
4659async fn signal_fatal_teardown(
4660    tx: &WriterSender,
4661    route: Option<RouteChannel>,
4662    ver: u8,
4663    corr: u64,
4664    shutdown: &Arc<Notify>,
4665    metrics: &DispatchPathMetrics,
4666) {
4667    if let Some(route) = route {
4668        if let Ok(frame) = build_goodbye_frame(ver, route.channel, route.epoch, corr) {
4669            if let Err(error) = send_frame(tx, metrics, frame).await {
4670                log::warn!(
4671                    "subc attach: failed to queue fatal route Goodbye for route {route}: {error}"
4672                );
4673            }
4674        }
4675    }
4676    if let Ok(frame) = build_goodbye_frame(ver, 0, 0, 0) {
4677        if let Err(error) = send_frame(tx, metrics, frame).await {
4678            log::warn!("subc attach: failed to queue fatal channel-0 Goodbye: {error}");
4679        }
4680    }
4681    shutdown.notify_one();
4682}
4683#[derive(Debug, Deserialize)]
4684#[serde(untagged)]
4685enum RouteRequest {
4686    BgEvents(BgEventsRequest),
4687    ToolCall(ToolCallRequest),
4688}
4689
4690#[derive(Debug, Deserialize)]
4691struct BgEventsRequest {
4692    op: BgEventsOp,
4693}
4694
4695#[derive(Debug, Deserialize)]
4696#[serde(rename_all = "snake_case")]
4697enum BgEventsOp {
4698    BgEvents,
4699}
4700
4701#[derive(Debug, Deserialize)]
4702struct ToolCallRequest {
4703    name: String,
4704    #[serde(default)]
4705    arguments: Value,
4706    /// Host-computed registration fact; kept outside agent-controlled arguments.
4707    #[serde(default)]
4708    edit_slot_survives: Option<bool>,
4709    /// Server-owned preview control (B1c-0): the plugin's mutation flow is
4710    /// preview -> permission ask -> apply. Dropping this field made "preview"
4711    /// calls mutate disk before the permission prompt and the subsequent
4712    /// apply fail with not-found.
4713    #[serde(default)]
4714    preview: bool,
4715}
4716
4717#[cfg(test)]
4718pub(crate) mod test_support {
4719    use super::*;
4720    use crate::bash_background::BgTaskStatus;
4721    use crate::protocol::{
4722        BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, ConfigureWarningsFrame,
4723        ProgressFrame, StatusChangedFrame,
4724    };
4725    use serde_json::json;
4726
4727    pub(super) fn test_root(name: &str) -> (tempfile::TempDir, ProjectRootId) {
4728        let dir = tempfile::Builder::new()
4729            .prefix(name)
4730            .tempdir()
4731            .expect("temp root");
4732        let root = ProjectRootId::from_path(dir.path()).expect("project root id");
4733        (dir, root)
4734    }
4735
4736    pub(super) fn test_ctx() -> Arc<AppContext> {
4737        Arc::new(AppContext::new(
4738            Box::new(crate::parser::TreeSitterProvider::new()),
4739            crate::config::Config::default(),
4740        ))
4741    }
4742
4743    pub(super) fn wait_for_watcher_count(ctx: &AppContext, expected: usize) {
4744        let deadline = Instant::now() + Duration::from_secs(30);
4745        loop {
4746            let observed = ctx.watcher_registry_count();
4747            if observed == expected {
4748                return;
4749            }
4750            assert!(
4751                Instant::now() < deadline,
4752                "watcher count did not settle before deadline: expected={expected}, observed={observed}"
4753            );
4754            std::thread::sleep(Duration::from_millis(50));
4755        }
4756    }
4757
4758    /// Sweep until `root` is forgotten, mirroring how production reaps.
4759    ///
4760    /// `reap_idle_roots` probes actor idleness with a try-lock and retains the
4761    /// root when the scheduler holds that lock — correct behavior, since the
4762    /// real caller sweeps on a timer and simply catches the root next tick. A
4763    /// test that asserts a single sweep succeeds is therefore asserting it wins
4764    /// a lock race that nothing in production depends on: `register_actor` wakes
4765    /// the scheduler, which grabs the same lock, and on a loaded runner that
4766    /// window is wide enough to lose. Sweep to the outcome instead.
4767    pub(super) fn reap_until_forgotten(
4768        root: &ProjectRootId,
4769        live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4770        pending_binds: &HashMap<RouteChannel, PendingBind>,
4771        root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
4772        executor: &Arc<Executor>,
4773        metrics: &DispatchPathMetrics,
4774    ) -> IdleReapOutcome {
4775        let deadline = Instant::now() + Duration::from_secs(30);
4776        loop {
4777            let outcome = reap_idle_roots(
4778                Instant::now(),
4779                live_roots,
4780                pending_binds,
4781                root_channels,
4782                executor,
4783                metrics,
4784            );
4785            if outcome.forgotten_deleted_roots.contains(root) {
4786                return outcome;
4787            }
4788            assert!(
4789                Instant::now() < deadline,
4790                "deleted root was never forgotten: {root:?}"
4791            );
4792            std::thread::sleep(Duration::from_millis(10));
4793        }
4794    }
4795
4796    pub(super) fn wait_for_actor_root_count(app: &App, expected: usize) {
4797        let deadline = Instant::now() + Duration::from_secs(30);
4798        loop {
4799            let observed = app.actor_root_count();
4800            if observed == expected {
4801                return;
4802            }
4803            assert!(
4804                Instant::now() < deadline,
4805                "actor root count did not settle before deadline: expected={expected}, observed={observed}"
4806            );
4807            std::thread::sleep(Duration::from_millis(50));
4808        }
4809    }
4810
4811    pub(super) fn status_frame(seq: u64) -> PushFrame {
4812        status_frame_with_session(seq, None)
4813    }
4814
4815    pub(super) fn status_frame_with_session(seq: u64, session_id: Option<&str>) -> PushFrame {
4816        PushFrame::StatusChanged(StatusChangedFrame {
4817            frame_type: "status_changed",
4818            session_id: session_id.map(str::to_string),
4819            snapshot: json!({ "seq": seq }),
4820        })
4821    }
4822
4823    pub(super) fn completion_frame(task_id: &str) -> PushFrame {
4824        completion_frame_with_session(task_id, "session-1")
4825    }
4826
4827    pub(super) fn completion_frame_with_session(task_id: &str, session_id: &str) -> PushFrame {
4828        PushFrame::BashCompleted(BashCompletedFrame {
4829            frame_type: "bash_completed",
4830            task_id: task_id.to_string(),
4831            session_id: session_id.to_string(),
4832            status: BgTaskStatus::Completed,
4833            exit_code: Some(0),
4834            command: format!("echo {task_id}"),
4835            output_preview: String::new(),
4836            output_truncated: false,
4837            original_tokens: None,
4838            compressed_tokens: None,
4839            tokens_skipped: false,
4840            status_reason: None,
4841        })
4842    }
4843
4844    pub(super) fn long_running_frame(task_id: &str, elapsed_ms: u64) -> PushFrame {
4845        long_running_frame_with_session(task_id, "session-1", elapsed_ms)
4846    }
4847
4848    pub(super) fn long_running_frame_with_session(
4849        task_id: &str,
4850        session_id: &str,
4851        elapsed_ms: u64,
4852    ) -> PushFrame {
4853        PushFrame::BashLongRunning(BashLongRunningFrame {
4854            frame_type: "bash_long_running",
4855            task_id: task_id.to_string(),
4856            session_id: session_id.to_string(),
4857            command: format!("sleep {elapsed_ms}"),
4858            elapsed_ms,
4859        })
4860    }
4861
4862    pub(super) fn pattern_match_frame(session_id: &str) -> PushFrame {
4863        PushFrame::BashPatternMatch(BashPatternMatchFrame {
4864            frame_type: "bash_pattern_match",
4865            task_id: "task-pattern".to_string(),
4866            session_id: session_id.to_string(),
4867            watch_id: "watch-1".to_string(),
4868            match_text: "needle".to_string(),
4869            match_offset: 7,
4870            context: "haystack needle".to_string(),
4871            once: true,
4872            reason: "pattern_match",
4873        })
4874    }
4875
4876    pub(super) fn configure_warnings_frame(session_id: Option<&str>) -> PushFrame {
4877        PushFrame::ConfigureWarnings(ConfigureWarningsFrame {
4878            frame_type: "configure_warnings",
4879            session_id: session_id.map(str::to_string),
4880            project_root: "/tmp/subc-test".to_string(),
4881            warnings: Vec::new(),
4882        })
4883    }
4884
4885    pub(super) fn route_identity(root: &ProjectRootId, session_id: &str) -> RouteIdentity {
4886        route_identity_with_trust(root, session_id, BindTrust::FirstParty)
4887    }
4888
4889    pub(super) fn route_identity_with_trust(
4890        root: &ProjectRootId,
4891        session_id: &str,
4892        trust: BindTrust,
4893    ) -> RouteIdentity {
4894        RouteIdentity(Arc::new(RouteIdentityData {
4895            root: root.clone(),
4896            project_root: root.as_path().to_path_buf(),
4897            harness: "opencode".to_string(),
4898            session: session_id.to_string(),
4899            trust,
4900            spawn_principal: AuthenticatedPrincipal::RouteBind {
4901                trust: trust.sandbox_trust(),
4902                route_channel: 0,
4903                route_epoch: 0,
4904                project_root: root.as_path().to_path_buf(),
4905                harness: "opencode".to_string(),
4906                session_id: session_id.to_string(),
4907                principal_id: Some(match trust {
4908                    BindTrust::FirstParty => "direct".to_string(),
4909                    BindTrust::Untrusted => "unverified".to_string(),
4910                }),
4911            },
4912            consumer_elicitation_capable: false,
4913        }))
4914    }
4915
4916    pub(super) fn progress_frame(request_id: &str, kind: ProgressKind, chunk: &str) -> PushFrame {
4917        PushFrame::Progress(ProgressFrame::new(request_id, kind, chunk))
4918    }
4919
4920    pub(super) fn status_seq(frame: &PushFrame) -> Option<u64> {
4921        match frame {
4922            PushFrame::StatusChanged(status) => status.snapshot.get("seq").and_then(|v| v.as_u64()),
4923            _ => None,
4924        }
4925    }
4926
4927    pub(super) fn completion_task(frame: &PushFrame) -> Option<&str> {
4928        match frame {
4929            PushFrame::BashCompleted(completion) => Some(completion.task_id.as_str()),
4930            _ => None,
4931        }
4932    }
4933
4934    pub(super) fn push_frame_task_id(frame: &Frame) -> Option<String> {
4935        let body: serde_json::Value = serde_json::from_slice(&frame.body).expect("push body");
4936        body.get("task_id")
4937            .and_then(serde_json::Value::as_str)
4938            .map(str::to_string)
4939    }
4940}
4941
4942#[cfg(test)]
4943mod tests {
4944    use super::test_support::{
4945        completion_frame, reap_until_forgotten, route_identity, test_ctx, test_root,
4946        wait_for_actor_root_count, wait_for_watcher_count,
4947    };
4948    use super::*;
4949    use crate::bash_background::BgTaskStatus;
4950
4951    fn attach_error(kind: io::ErrorKind) -> SubcError {
4952        SubcError::Connect {
4953            endpoint: "127.0.0.1:1".to_string(),
4954            source: io::Error::new(kind, "constructed attach failure"),
4955        }
4956    }
4957
4958    fn auth_io_error(kind: io::ErrorKind) -> SubcError {
4959        SubcError::Auth {
4960            endpoint: "127.0.0.1:1".to_string(),
4961            source: subc_transport::AuthError::Io {
4962                stage: subc_transport::AuthStage::ServerProof,
4963                source: io::Error::new(kind, "constructed auth failure"),
4964            },
4965        }
4966    }
4967
4968    #[test]
4969    fn initial_attach_error_classifier_distinguishes_transient_and_permanent_failures() {
4970        let transient_errors = vec![
4971            attach_error(io::ErrorKind::ConnectionRefused),
4972            attach_error(io::ErrorKind::TimedOut),
4973            attach_error(io::ErrorKind::ConnectionReset),
4974            auth_io_error(io::ErrorKind::ConnectionAborted),
4975            auth_io_error(io::ErrorKind::BrokenPipe),
4976            SubcError::Auth {
4977                endpoint: "127.0.0.1:1".to_string(),
4978                source: subc_transport::AuthError::UnexpectedEof {
4979                    stage: subc_transport::AuthStage::ServerProof,
4980                    expected: 4,
4981                    actual: 0,
4982                },
4983            },
4984            SubcError::Auth {
4985                endpoint: "127.0.0.1:1".to_string(),
4986                source: subc_transport::AuthError::Timeout {
4987                    stage: subc_transport::AuthStage::ServerProof,
4988                    deadline: AUTH_DEADLINE,
4989                },
4990            },
4991        ];
4992        for error in &transient_errors {
4993            assert_eq!(
4994                classify_attach_error(error),
4995                AttachErrorClass::Transient,
4996                "expected transient: {error}"
4997            );
4998        }
4999
5000        let permanent_errors = vec![
5001            attach_error(io::ErrorKind::PermissionDenied),
5002            auth_io_error(io::ErrorKind::InvalidData),
5003            SubcError::Auth {
5004                endpoint: "127.0.0.1:1".to_string(),
5005                source: subc_transport::AuthError::InvalidServerProof,
5006            },
5007            SubcError::Auth {
5008                endpoint: "127.0.0.1:1".to_string(),
5009                source: subc_transport::AuthError::DaemonIdMismatch,
5010            },
5011            SubcError::ConnectionFile {
5012                path: PathBuf::from("subc-connection.json"),
5013                source: subc_transport::ConnectionFileError::Invalid {
5014                    reason: "constructed invalid file".to_string(),
5015                },
5016            },
5017            SubcError::NoEndpoint {
5018                path: PathBuf::from("subc-connection.json"),
5019            },
5020            SubcError::InvalidEndpoint {
5021                path: PathBuf::from("subc-connection.json"),
5022                endpoint: "not-an-ip:1234".to_string(),
5023            },
5024        ];
5025        for error in &permanent_errors {
5026            assert_eq!(
5027                classify_attach_error(error),
5028                AttachErrorClass::Permanent,
5029                "expected permanent: {error}"
5030            );
5031        }
5032    }
5033
5034    #[test]
5035    fn incompatible_wire_version_is_rejected_before_tcp_connect() {
5036        let conn_dir = tempfile::tempdir().expect("connection tempdir");
5037        let conn_path = conn_dir.path().join("subc-connection.json");
5038        let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind listener");
5039        listener
5040            .set_nonblocking(true)
5041            .expect("set listener nonblocking");
5042        let port = listener.local_addr().expect("listener addr").port();
5043        connection_file::write_atomic(
5044            &conn_path,
5045            &connection_file::ConnectionInfo {
5046                schema: connection_file::SCHEMA_VERSION,
5047                wire_version: Some(PROTOCOL_VERSION.wrapping_add(1)),
5048                endpoints: vec![connection_file::Endpoint {
5049                    host: "127.0.0.1".to_string(),
5050                    port,
5051                }],
5052                key: vec![0x42; subc_transport::KEY_LEN],
5053                daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
5054                pid: std::process::id(),
5055                daemon_ver: "subc-test".to_string(),
5056            },
5057        )
5058        .expect("write connection file");
5059
5060        let runtime = tokio::runtime::Builder::new_current_thread()
5061            .enable_all()
5062            .build()
5063            .expect("test runtime");
5064        let result = runtime.block_on(connect_and_authenticate_with_policy(
5065            &conn_path,
5066            AttachRetryPolicy {
5067                budget: Duration::from_secs(1),
5068                initial_backoff: Duration::from_millis(5),
5069                max_backoff: Duration::from_millis(10),
5070                jitter_percent: 0,
5071            },
5072        ));
5073        assert!(matches!(
5074            result,
5075            Err(SubcError::ConnectionFile {
5076                source: connection_file::ConnectionFileError::WireVersionMismatch { .. },
5077                ..
5078            })
5079        ));
5080        assert!(matches!(
5081            listener.accept(),
5082            Err(error) if error.kind() == std::io::ErrorKind::WouldBlock
5083        ));
5084    }
5085
5086    #[test]
5087    fn initial_attach_unreachable_endpoint_retries_until_budget_then_fails_loud() {
5088        let conn_dir = tempfile::tempdir().expect("connection tempdir");
5089        let conn_path = conn_dir.path().join("subc-connection.json");
5090        let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve port");
5091        let port = listener.local_addr().expect("reserved addr").port();
5092        drop(listener);
5093        connection_file::write_atomic(
5094            &conn_path,
5095            &connection_file::ConnectionInfo {
5096                schema: connection_file::SCHEMA_VERSION,
5097                wire_version: Some(PROTOCOL_VERSION),
5098                endpoints: vec![connection_file::Endpoint {
5099                    host: "127.0.0.1".to_string(),
5100                    port,
5101                }],
5102                key: vec![0x42; subc_transport::KEY_LEN],
5103                daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
5104                pid: std::process::id(),
5105                daemon_ver: "subc-test".to_string(),
5106            },
5107        )
5108        .expect("write connection file");
5109
5110        let policy = AttachRetryPolicy {
5111            budget: Duration::from_millis(40),
5112            initial_backoff: Duration::from_millis(5),
5113            max_backoff: Duration::from_millis(10),
5114            jitter_percent: 0,
5115        };
5116        let runtime = tokio::runtime::Builder::new_current_thread()
5117            .enable_all()
5118            .build()
5119            .expect("test runtime");
5120        let started_at = Instant::now();
5121        let result = runtime.block_on(connect_and_authenticate_with_policy(&conn_path, policy));
5122        let elapsed = started_at.elapsed();
5123        let error = match result {
5124            Ok(_) => panic!("unreachable endpoint unexpectedly attached"),
5125            Err(error) => error,
5126        };
5127
5128        assert!(matches!(error, SubcError::Connect { .. }), "{error}");
5129        assert!(
5130            elapsed >= Duration::from_millis(35),
5131            "retry budget ended too early: {elapsed:?}"
5132        );
5133        assert!(
5134            elapsed < Duration::from_secs(1),
5135            "retry budget was not bounded: {elapsed:?}"
5136        );
5137    }
5138
5139    fn due_maintenance_jobs_without_actor_context(
5140        live_roots: &mut HashMap<ProjectRootId, RootMeta>,
5141        budget: usize,
5142        pending_bind_roots: &HashSet<ProjectRootId>,
5143    ) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
5144        due_maintenance_jobs(
5145            live_roots,
5146            None,
5147            &HashMap::new(),
5148            &HashSet::new(),
5149            budget,
5150            pending_bind_roots,
5151        )
5152    }
5153
5154    fn actor_ctx_with_dirty_search_index(
5155        root: &Path,
5156        storage: &Path,
5157        file_name: &str,
5158        old_contents: &str,
5159        new_contents: &str,
5160    ) -> (Arc<AppContext>, PathBuf, PathBuf) {
5161        let file = root.join(file_name);
5162        std::fs::write(&file, old_contents).expect("write source");
5163        let canonical_root = std::fs::canonicalize(root).expect("canonical root");
5164        let ctx = Arc::new(AppContext::new(
5165            Box::new(crate::parser::TreeSitterProvider::new()),
5166            Config {
5167                project_root: Some(root.to_path_buf()),
5168                storage_dir: Some(storage.to_path_buf()),
5169                ..Config::default()
5170            },
5171        ));
5172        ctx.set_canonical_cache_root(canonical_root.clone());
5173
5174        let cache_dir = crate::search_index::resolve_cache_dir(&canonical_root, Some(storage));
5175        let mut index = crate::search_index::SearchIndex::build(&canonical_root);
5176        let git_head = index.stored_git_head().map(str::to_owned);
5177        index.write_to_disk(&cache_dir, git_head.as_deref());
5178
5179        std::fs::write(&file, new_contents).expect("edit source");
5180        index.update_file(&file);
5181        *ctx.search_index()
5182            .write()
5183            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
5184        (ctx, canonical_root, cache_dir)
5185    }
5186
5187    #[test]
5188    fn graceful_shutdown_flushes_every_actor_search_index() {
5189        let storage = tempfile::tempdir().expect("storage tempdir");
5190        let (root1_dir, root1) = test_root("shutdown-flush-root-1");
5191        let (root2_dir, root2) = test_root("shutdown-flush-root-2");
5192        let (ctx1, canonical_root1, cache_dir1) = actor_ctx_with_dirty_search_index(
5193            root1_dir.path(),
5194            storage.path(),
5195            "alpha.txt",
5196            "old actor one token\n",
5197            "new actor one token\n",
5198        );
5199        let (ctx2, canonical_root2, cache_dir2) = actor_ctx_with_dirty_search_index(
5200            root2_dir.path(),
5201            storage.path(),
5202            "beta.txt",
5203            "old actor two token\n",
5204            "new actor two token\n",
5205        );
5206
5207        let executor = Executor::new();
5208        assert!(executor.register_actor(root1.clone(), Arc::clone(&ctx1)));
5209        assert!(executor.register_actor(root2.clone(), Arc::clone(&ctx2)));
5210
5211        flush_actor_indexes_on_graceful_shutdown(&executor.actor_contexts());
5212
5213        let mut restored1 =
5214            crate::search_index::SearchIndex::read_from_disk(&cache_dir1, &canonical_root1)
5215                .expect("load flushed root one index");
5216        restored1.ready = true;
5217        assert_eq!(
5218            restored1
5219                .grep("new actor one token", true, &[], &[], &canonical_root1, 10)
5220                .matches
5221                .len(),
5222            1,
5223            "graceful subc shutdown should flush the first root's trigram delta"
5224        );
5225
5226        let mut restored2 =
5227            crate::search_index::SearchIndex::read_from_disk(&cache_dir2, &canonical_root2)
5228                .expect("load flushed root two index");
5229        restored2.ready = true;
5230        assert_eq!(
5231            restored2
5232                .grep("new actor two token", true, &[], &[], &canonical_root2, 10)
5233                .matches
5234                .len(),
5235            1,
5236            "graceful subc shutdown should flush every registered root"
5237        );
5238    }
5239
5240    #[test]
5241    fn idle_root_reaper_closes_artifacts_and_stops_watcher() {
5242        let _ = env_logger::builder().is_test(true).try_init();
5243        let (root_dir, root) = test_root("idle-root-reaper");
5244        let storage = tempfile::tempdir().expect("storage tempdir");
5245        std::fs::write(
5246            root_dir.path().join("main.rs"),
5247            "fn entry() { leaf(); }\nfn leaf() {}\n",
5248        )
5249        .expect("source file");
5250        let canonical_root = std::fs::canonicalize(root_dir.path()).expect("canonical root");
5251        let app = App::default_shared();
5252        let ctx = Arc::new(AppContext::from_app(
5253            Arc::clone(&app),
5254            Config {
5255                project_root: Some(canonical_root.clone()),
5256                storage_dir: Some(storage.path().to_path_buf()),
5257                callgraph_store: true,
5258                search_index: true,
5259                ..Config::default()
5260            },
5261        ));
5262        ctx.set_canonical_cache_root(canonical_root.clone());
5263        let project_key = crate::search_index::artifact_cache_key(&canonical_root);
5264        crate::root_cache::configure_artifact_access(&canonical_root, &project_key, false);
5265        assert!(ctx
5266            .ensure_callgraph_store()
5267            .expect("build callgraph store")
5268            .is_some());
5269
5270        let cache_dir =
5271            crate::search_index::resolve_cache_dir(&canonical_root, Some(storage.path()));
5272        let mut index = crate::search_index::SearchIndex::build(&canonical_root);
5273        let git_head = index.stored_git_head().map(str::to_owned);
5274        index.write_to_disk(&cache_dir, git_head.as_deref());
5275        *ctx.search_index()
5276            .write()
5277            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
5278        // Seed a completed warm verification so the test can prove eviction
5279        // downgrades it to Strict rather than passing vacuously.
5280        let seeded_generation =
5281            crate::cache_freshness::artifact_generation(&cache_dir.join("cache.bin"))
5282                .expect("seeded artifact generation");
5283        crate::cache_freshness::record_verify_completed(
5284            &canonical_root,
5285            crate::cache_freshness::VerifyArtifact::Search,
5286            Some(seeded_generation),
5287        );
5288        assert!(
5289            matches!(
5290                crate::cache_freshness::warm_verify_plan(
5291                    canonical_root.as_path(),
5292                    crate::cache_freshness::VerifyArtifact::Search,
5293                    Some(seeded_generation),
5294                ),
5295                crate::cache_freshness::WarmVerifyPlan::Skip
5296            ),
5297            "memo must be warm before eviction for the downgrade assertion to bite"
5298        );
5299
5300        let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
5301        let _dispatch_tx = dispatch_tx;
5302        let shutdown = Arc::new(AtomicBool::new(false));
5303        let thread_shutdown = Arc::clone(&shutdown);
5304        let join = std::thread::spawn(move || {
5305            while !thread_shutdown.load(Ordering::SeqCst) {
5306                std::thread::yield_now();
5307            }
5308        });
5309        ctx.install_watcher_runtime(
5310            dispatch_rx,
5311            crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
5312        );
5313        wait_for_watcher_count(&ctx, 1);
5314
5315        let executor = Arc::new(Executor::new());
5316        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5317        ctx.mark_subc_unbound();
5318        let mut live_roots = HashMap::new();
5319        let mut meta = RootMeta::new(Instant::now());
5320        meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
5321        meta.unbound_quiesced = true;
5322        live_roots.insert(root.clone(), meta);
5323
5324        let message = idle_root_eviction_message(&root, &ctx.memory_root_snapshot(), None);
5325        assert!(message.contains("evicted idle root"));
5326        assert!(message.contains("freed ~"));
5327        assert!(message.contains("semantic"));
5328        assert!(!message.contains("semantic not estimated retained"));
5329        assert!(message.contains("trigram"));
5330        assert!(message.contains("retained: bash"));
5331        assert!(message.contains("parser_pool"));
5332
5333        assert_eq!(
5334            reap_idle_roots(
5335                Instant::now(),
5336                &mut live_roots,
5337                &HashMap::new(),
5338                &HashMap::new(),
5339                &executor,
5340                &DispatchPathMetrics::new(),
5341            )
5342            .evicted,
5343            1
5344        );
5345        assert!(ctx.search_index().read().unwrap().is_none());
5346        wait_for_watcher_count(&ctx, 0);
5347        // The watcher stopped with the eviction, so the idle interval is
5348        // unobserved: the pre-seeded warm-verify memo (Skip) must fall back to
5349        // strict content verification (stat-first would miss same-size,
5350        // preserved-mtime edits made while nobody was watching).
5351        assert!(
5352            matches!(
5353                crate::cache_freshness::warm_verify_plan(
5354                    canonical_root.as_path(),
5355                    crate::cache_freshness::VerifyArtifact::Search,
5356                    Some(seeded_generation),
5357                ),
5358                crate::cache_freshness::WarmVerifyPlan::Strict
5359            ),
5360            "idle eviction must force strict re-verification"
5361        );
5362        assert!(
5363            crate::search_index::SearchIndex::read_from_disk(&cache_dir, &canonical_root).is_some()
5364        );
5365        ctx.mark_subc_bound();
5366        assert!(ctx
5367            .ensure_callgraph_store()
5368            .expect("reopen callgraph store")
5369            .is_some());
5370        assert!(live_roots[&root].idle_artifacts_evicted);
5371    }
5372
5373    #[test]
5374    fn idle_root_reaper_applies_ttl_to_unbound_roots() {
5375        let (_root_dir, root) = test_root("idle-root-ttl-gate");
5376        let ctx = test_ctx();
5377        let executor = Arc::new(Executor::new());
5378        assert!(executor.register_actor(root.clone(), ctx));
5379        let ctx = executor.actor_context(&root).expect("actor context");
5380        ctx.mark_subc_unbound();
5381        let now = Instant::now();
5382        let mut meta = RootMeta::new(now);
5383        meta.unbound_quiesced = true;
5384        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5385
5386        // A recently-unbound root stays warm: a transient unbind (host
5387        // restart) must not pay the strict-verify + forced-rebuild teardown
5388        // on the next maintenance sweep.
5389        assert_eq!(
5390            reap_idle_roots(
5391                now,
5392                &mut live_roots,
5393                &HashMap::new(),
5394                &HashMap::new(),
5395                &executor,
5396                &DispatchPathMetrics::new(),
5397            )
5398            .evicted,
5399            0
5400        );
5401        assert!(!live_roots[&root].idle_artifacts_evicted);
5402
5403        // Past the TTL the same unbound root pays the full teardown. The
5404        // pending reconciliation paths retained across the transient-unbind
5405        // window would block eviction forever through
5406        // `artifact_eviction_blocked`; the reaper disposes them because the
5407        // strict gap invalidation subsumes their purpose.
5408        ctx.add_pending_search_index_paths([root.as_path().join("retained.rs")]);
5409        assert_eq!(
5410            reap_idle_roots(
5411                now + IDLE_ROOT_TTL,
5412                &mut live_roots,
5413                &HashMap::new(),
5414                &HashMap::new(),
5415                &executor,
5416                &DispatchPathMetrics::new(),
5417            )
5418            .evicted,
5419            1
5420        );
5421        assert!(live_roots[&root].idle_artifacts_evicted);
5422        assert!(
5423            ctx.take_pending_search_index_paths().is_empty(),
5424            "TTL eviction must dispose retained pending reconciliation paths"
5425        );
5426    }
5427
5428    #[test]
5429    fn blocked_ttl_eviction_restores_taken_pending_reconciliation_state() {
5430        let (_root_dir, root) = test_root("ttl-eviction-blocked-restore");
5431        let ctx = test_ctx();
5432        let executor = Arc::new(Executor::new());
5433        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5434        ctx.mark_subc_unbound();
5435
5436        // Retained pending path from the transient-unbind window plus a
5437        // SECONDARY eviction blocker (a non-ready resident search index, the
5438        // dirty-index blocker in artifact_eviction_blocked). Disposal must be
5439        // transactional: the blocked eviction may be followed by a rebind, and
5440        // the path is the only repair record for its consumed watcher event.
5441        let pending = root.as_path().join("edited-while-unbound.rs");
5442        ctx.add_pending_search_index_paths([pending.clone()]);
5443        let dirty_source = root.as_path().join("dirty.rs");
5444        std::fs::write(&dirty_source, "fn dirty() {}\n").expect("dirty source");
5445        let mut dirty = crate::search_index::SearchIndex::new();
5446        dirty.ready = true;
5447        dirty.update_file(&dirty_source);
5448        *ctx.search_index()
5449            .write()
5450            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(dirty);
5451        assert!(ctx.artifact_eviction_blocked());
5452
5453        let mut live_roots = HashMap::new();
5454        let mut meta = RootMeta::new(Instant::now());
5455        meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
5456        meta.unbound_quiesced = true;
5457        live_roots.insert(root.clone(), meta);
5458
5459        assert_eq!(
5460            reap_idle_roots(
5461                Instant::now(),
5462                &mut live_roots,
5463                &HashMap::new(),
5464                &HashMap::new(),
5465                &executor,
5466                &DispatchPathMetrics::new(),
5467            )
5468            .evicted,
5469            0,
5470            "the dirty index must still block this eviction"
5471        );
5472        assert_eq!(
5473            ctx.take_pending_search_index_paths(),
5474            vec![pending],
5475            "a blocked eviction must restore the taken pending paths"
5476        );
5477    }
5478
5479    #[test]
5480    fn idle_reap_with_bound_route_keeps_watcher_running() {
5481        let (_root_dir, root) = test_root("bound-root-reap-gate");
5482        let ctx = test_ctx();
5483        let executor = Arc::new(Executor::new());
5484        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5485
5486        let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
5487        let _dispatch_tx = dispatch_tx;
5488        let shutdown = Arc::new(AtomicBool::new(false));
5489        let thread_shutdown = Arc::clone(&shutdown);
5490        let join = std::thread::spawn(move || {
5491            while !thread_shutdown.load(Ordering::SeqCst) {
5492                std::thread::yield_now();
5493            }
5494        });
5495        ctx.install_watcher_runtime(
5496            dispatch_rx,
5497            crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
5498        );
5499
5500        let mut meta = RootMeta::new(Instant::now());
5501        meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
5502        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5503        let bound = HashMap::from([(root, HashSet::from([route_key(7, 1)]))]);
5504        assert_eq!(
5505            reap_idle_roots(
5506                Instant::now(),
5507                &mut live_roots,
5508                &HashMap::new(),
5509                &bound,
5510                &executor,
5511                &DispatchPathMetrics::new(),
5512            )
5513            .evicted,
5514            0
5515        );
5516        wait_for_watcher_count(&ctx, 1);
5517        ctx.stop_watcher_runtime_in_background();
5518        wait_for_watcher_count(&ctx, 0);
5519    }
5520
5521    #[test]
5522    fn deleted_root_with_bound_route_is_reclaimed_after_confirmation_and_routes_are_purged() {
5523        let (root_dir, root) = test_root("deleted-bound-root-reap");
5524        let executor = Arc::new(Executor::new());
5525        assert!(executor.register_actor(root.clone(), test_ctx()));
5526        root_dir.close().expect("delete project root");
5527
5528        let route = route_key(19, 3);
5529        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
5530        let cancel_signal = PersistentCancelSignal::new();
5531        let mut routes = HashMap::from([(route, route_identity(&root, "deleted-route"))]);
5532        let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
5533        let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
5534        let mut route_bash_cancels = HashMap::from([(
5535            route,
5536            bash::RouteBashCancel {
5537                token: cancel_signal.clone(),
5538                active_waits: 0,
5539            },
5540        )]);
5541        let metrics = DispatchPathMetrics::new();
5542
5543        let first = reap_idle_roots(
5544            Instant::now(),
5545            &mut live_roots,
5546            &HashMap::new(),
5547            &root_channels,
5548            &executor,
5549            &metrics,
5550        );
5551        assert!(first.forgotten_deleted_roots.is_empty());
5552        assert!(executor.actor_registered(&root));
5553
5554        let mut forgotten = Vec::new();
5555        for _ in 0..100 {
5556            let outcome = reap_idle_roots(
5557                Instant::now(),
5558                &mut live_roots,
5559                &HashMap::new(),
5560                &root_channels,
5561                &executor,
5562                &metrics,
5563            );
5564            if !outcome.forgotten_deleted_roots.is_empty() {
5565                forgotten = outcome.forgotten_deleted_roots;
5566                break;
5567            }
5568            std::thread::sleep(Duration::from_millis(10));
5569        }
5570        assert_eq!(forgotten, vec![root.clone()]);
5571        assert!(!executor.actor_registered(&root));
5572
5573        let mut retry_buffer = HashMap::new();
5574        let mut reclaimed_routes = ReclaimedRoutes::default();
5575        let mut session_identity = HashMap::new();
5576        let mut push_buffer = HashMap::new();
5577        let mut bg_subs = HashMap::from([(
5578            route,
5579            BgSub {
5580                corr: 77,
5581                ver: PROTOCOL_VERSION,
5582                flags: control_flags(),
5583                root: root.clone(),
5584                session: "deleted-route".to_string(),
5585            },
5586        )]);
5587        let mut bg_sub_by_session = HashMap::from([(
5588            (root.clone(), "deleted-route".to_string()),
5589            HashSet::from([route]),
5590        )]);
5591        let mut bg_wake_pending = HashSet::from([route]);
5592        let mut bg_wake_epoch = HashMap::new();
5593        let mut pending_bash_asks = HashMap::new();
5594        health::take_bg_observability_logs_for_test();
5595        purge_deleted_root_residents(
5596            &root,
5597            &mut routes,
5598            &mut root_channels,
5599            &mut installed_route_epochs,
5600            &mut route_bash_cancels,
5601            &mut retry_buffer,
5602            &mut reclaimed_routes,
5603            &mut session_identity,
5604            &mut push_buffer,
5605            &mut bg_subs,
5606            &mut bg_sub_by_session,
5607            &mut bg_wake_pending,
5608            &mut bg_wake_epoch,
5609            &mut pending_bash_asks,
5610            &metrics,
5611        );
5612
5613        assert!(routes.is_empty());
5614        assert!(root_channels.is_empty());
5615        assert!(installed_route_epochs.is_empty());
5616        assert!(route_bash_cancels.is_empty());
5617        assert!(reclaimed_routes.contains(route));
5618        assert!(cancel_signal.is_cancelled());
5619        assert_eq!(
5620            health::take_bg_observability_logs_for_test(),
5621            vec![format!(
5622                "subc bg subscription: ended root={} session=deleted-route channel=19@3 cause=root-reclaim suppressed=0",
5623                root.as_path().display()
5624            )]
5625        );
5626    }
5627
5628    /// The control for the deleted-root reclamation above: a root whose
5629    /// directory still EXISTS must stay retained while it holds a bound route,
5630    /// even with every other reap precondition satisfied. Without this, the
5631    /// suite cannot tell "reclaim roots that are provably gone" apart from
5632    /// "reap any root that looks idle" — the second would tear down live
5633    /// sessions, and both satisfy the deleted-root tests.
5634    #[test]
5635    fn live_root_with_bound_route_is_never_reclaimed() {
5636        let (_root_dir, root) = test_root("live-bound-root-retained");
5637        let ctx = test_ctx();
5638        ctx.mark_subc_unbound();
5639        let executor = Arc::new(Executor::new());
5640        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5641
5642        let route = RouteChannel {
5643            channel: 7,
5644            epoch: 1,
5645        };
5646        let mut meta = RootMeta::new(Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1));
5647        meta.unbound_quiesced = true;
5648        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5649        let root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
5650
5651        // Sweep three times because reclamation requires the root path to be
5652        // observed missing on two CONSECUTIVE sweeps. A single sweep would pass
5653        // here even if reclamation were wrongly unconditional, since the first
5654        // absence never reclaims on its own.
5655        for _ in 0..3 {
5656            let outcome = reap_idle_roots(
5657                Instant::now(),
5658                &mut live_roots,
5659                &HashMap::new(),
5660                &root_channels,
5661                &executor,
5662                &DispatchPathMetrics::new(),
5663            );
5664            assert!(
5665                outcome.forgotten_deleted_roots.is_empty(),
5666                "a root whose directory exists must never be forgotten"
5667            );
5668        }
5669
5670        assert!(live_roots.contains_key(&root), "live root must be retained");
5671        assert!(
5672            executor.actor_registered(&root),
5673            "live root's actor must survive"
5674        );
5675        assert!(
5676            root.as_path().exists(),
5677            "test vehicle must keep the directory alive; otherwise this control proves nothing"
5678        );
5679    }
5680
5681    #[test]
5682    fn deleted_root_is_not_reclaimed_on_first_absence_observation() {
5683        let (root_dir, root) = test_root("deleted-root-first-observation");
5684        let ctx = test_ctx();
5685        ctx.mark_subc_unbound();
5686        let executor = Arc::new(Executor::new());
5687        assert!(executor.register_actor(root.clone(), ctx));
5688        root_dir.close().expect("delete project root");
5689
5690        let mut meta = RootMeta::new(Instant::now());
5691        meta.unbound_quiesced = true;
5692        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5693        let outcome = reap_idle_roots(
5694            Instant::now(),
5695            &mut live_roots,
5696            &HashMap::new(),
5697            &HashMap::new(),
5698            &executor,
5699            &DispatchPathMetrics::new(),
5700        );
5701
5702        assert!(outcome.forgotten_deleted_roots.is_empty());
5703        assert!(live_roots.contains_key(&root));
5704        assert!(executor.actor_registered(&root));
5705    }
5706
5707    fn spawn_background_for_root(
5708        ctx: &AppContext,
5709        root: &ProjectRootId,
5710        storage: &tempfile::TempDir,
5711        session_id: &str,
5712    ) -> (String, u32) {
5713        // Windows refuses to delete a directory that is a running process's
5714        // cwd (ERROR_SHARING_VIOLATION), so the task must not live inside the
5715        // project root these tests delete. The kill path matches on the task's
5716        // registered project_root, not its cwd, so pointing the cwd at task
5717        // storage keeps the association under test intact.
5718        let command = if cfg!(windows) {
5719            // timeout.exe requires a console; ping is the standard sleep shim.
5720            "ping -n 31 127.0.0.1 > nul"
5721        } else {
5722            "sleep 30"
5723        };
5724        let task_id = ctx
5725            .bash_background()
5726            .spawn(
5727                crate::sandbox_spawn::SpawnPlan::Unsandboxed,
5728                command,
5729                session_id.to_string(),
5730                storage.path().to_path_buf(),
5731                HashMap::new(),
5732                Some(Duration::from_secs(60)),
5733                storage.path().to_path_buf(),
5734                8,
5735                true,
5736                false,
5737                Some(root.as_path().to_path_buf()),
5738            )
5739            .expect("spawn background task");
5740        let snapshot = ctx
5741            .bash_background()
5742            .status(
5743                &task_id,
5744                session_id,
5745                Some(root.as_path()),
5746                Some(storage.path()),
5747                0,
5748            )
5749            .expect("background task status");
5750        (task_id, snapshot.child_pid.expect("background child pid"))
5751    }
5752
5753    fn wait_for_background_exit(pid: u32) {
5754        let deadline = Instant::now() + Duration::from_secs(5);
5755        while crate::bash_background::process::is_process_alive(pid) {
5756            assert!(
5757                Instant::now() < deadline,
5758                "background task process survived kill"
5759            );
5760            std::thread::sleep(Duration::from_millis(20));
5761        }
5762    }
5763
5764    #[test]
5765    fn deleted_root_reclaims_background_task_after_two_absence_sweeps() {
5766        let (root_dir, root) = test_root("deleted-root-background-task");
5767        let storage = tempfile::tempdir().expect("task storage");
5768        let ctx = test_ctx();
5769        let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "reclaim-session");
5770        assert!(crate::bash_background::process::is_process_alive(pid));
5771
5772        let executor = Arc::new(Executor::new());
5773        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5774        assert!(executor.actor_is_idle(&root));
5775        root_dir.close().expect("delete project root");
5776        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
5777        let pending_binds = HashMap::new();
5778        let root_channels = HashMap::new();
5779        let metrics = DispatchPathMetrics::new();
5780
5781        let first = reap_idle_roots(
5782            Instant::now(),
5783            &mut live_roots,
5784            &pending_binds,
5785            &root_channels,
5786            &executor,
5787            &metrics,
5788        );
5789        assert!(first.forgotten_deleted_roots.is_empty());
5790        assert!(crate::bash_background::process::is_process_alive(pid));
5791
5792        let outcome = reap_until_forgotten(
5793            &root,
5794            &mut live_roots,
5795            &pending_binds,
5796            &root_channels,
5797            &executor,
5798            &metrics,
5799        );
5800        assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
5801        wait_for_background_exit(pid);
5802
5803        let snapshot = ctx
5804            .bash_background()
5805            .status(
5806                &task_id,
5807                "reclaim-session",
5808                Some(root.as_path()),
5809                Some(storage.path()),
5810                0,
5811            )
5812            .expect("reclaimed task status");
5813        assert_eq!(snapshot.info.status, BgTaskStatus::Killed);
5814        assert_eq!(
5815            snapshot.info.status_reason.as_deref(),
5816            Some(crate::bash_background::registry::ROOT_RECLAIMED_REASON)
5817        );
5818        assert_eq!(
5819            serde_json::to_value(&snapshot).expect("serialize bash status")["status_reason"],
5820            crate::bash_background::registry::ROOT_RECLAIMED_REASON
5821        );
5822        let completion = ctx
5823            .bash_background()
5824            .drain_completions_for_session(Some("reclaim-session"))
5825            .pop()
5826            .expect("reclaimed task completion");
5827        assert_eq!(
5828            completion.status_reason.as_deref(),
5829            Some(crate::bash_background::registry::ROOT_RECLAIMED_REASON)
5830        );
5831    }
5832
5833    #[test]
5834    fn existing_unbound_root_keeps_background_task_alive_across_sweeps() {
5835        let (root_dir, root) = test_root("existing-root-background-task");
5836        let storage = tempfile::tempdir().expect("task storage");
5837        let ctx = test_ctx();
5838        ctx.mark_subc_unbound();
5839        let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "existing-session");
5840
5841        let executor = Arc::new(Executor::new());
5842        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5843        let mut meta = RootMeta::new(
5844            Instant::now()
5845                .checked_sub(IDLE_ROOT_TTL + Duration::from_secs(1))
5846                .expect("old root timestamp"),
5847        );
5848        meta.unbound_quiesced = true;
5849        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5850        let pending_binds = HashMap::new();
5851        let root_channels = HashMap::new();
5852        let metrics = DispatchPathMetrics::new();
5853
5854        for _ in 0..8 {
5855            reap_idle_roots(
5856                Instant::now(),
5857                &mut live_roots,
5858                &pending_binds,
5859                &root_channels,
5860                &executor,
5861                &metrics,
5862            );
5863            std::thread::sleep(Duration::from_millis(10));
5864        }
5865        assert!(root_dir.path().exists());
5866        assert!(crate::bash_background::process::is_process_alive(pid));
5867        let snapshot = ctx
5868            .bash_background()
5869            .status(
5870                &task_id,
5871                "existing-session",
5872                Some(root.as_path()),
5873                Some(storage.path()),
5874                0,
5875            )
5876            .expect("existing task status");
5877        assert_eq!(snapshot.info.status, BgTaskStatus::Running);
5878        let _ = ctx.bash_background().kill(&task_id, "existing-session");
5879        wait_for_background_exit(pid);
5880    }
5881
5882    #[test]
5883    fn restored_root_between_absence_sweeps_keeps_background_task_alive() {
5884        let (root_dir, root) = test_root("restored-root-background-task");
5885        let storage = tempfile::tempdir().expect("task storage");
5886        let ctx = test_ctx();
5887        let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "restored-session");
5888
5889        let executor = Arc::new(Executor::new());
5890        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5891        root_dir.close().expect("delete project root");
5892        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
5893        let pending_binds = HashMap::new();
5894        let root_channels = HashMap::new();
5895        let metrics = DispatchPathMetrics::new();
5896
5897        let first = reap_idle_roots(
5898            Instant::now(),
5899            &mut live_roots,
5900            &pending_binds,
5901            &root_channels,
5902            &executor,
5903            &metrics,
5904        );
5905        assert!(first.forgotten_deleted_roots.is_empty());
5906        std::fs::create_dir_all(root.as_path()).expect("restore project root");
5907        let second = reap_idle_roots(
5908            Instant::now(),
5909            &mut live_roots,
5910            &pending_binds,
5911            &root_channels,
5912            &executor,
5913            &metrics,
5914        );
5915        assert!(second.forgotten_deleted_roots.is_empty());
5916        assert!(crate::bash_background::process::is_process_alive(pid));
5917        let _ = ctx.bash_background().kill(&task_id, "restored-session");
5918        wait_for_background_exit(pid);
5919    }
5920
5921    #[test]
5922    fn observing_root_again_resets_deleted_sweep_confirmation() {
5923        let (root_dir, root) = test_root("deleted-root-observation-reset");
5924        let ctx = test_ctx();
5925        ctx.mark_subc_unbound();
5926        let executor = Arc::new(Executor::new());
5927        assert!(executor.register_actor(root.clone(), ctx));
5928        root_dir.close().expect("delete project root");
5929
5930        let mut meta = RootMeta::new(Instant::now());
5931        meta.unbound_quiesced = true;
5932        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5933        let pending_binds = HashMap::new();
5934        let root_channels = HashMap::new();
5935        let metrics = DispatchPathMetrics::new();
5936
5937        let first = reap_idle_roots(
5938            Instant::now(),
5939            &mut live_roots,
5940            &pending_binds,
5941            &root_channels,
5942            &executor,
5943            &metrics,
5944        );
5945        assert!(first.forgotten_deleted_roots.is_empty());
5946
5947        std::fs::create_dir_all(root.as_path()).expect("restore project root");
5948        reap_idle_roots(
5949            Instant::now(),
5950            &mut live_roots,
5951            &pending_binds,
5952            &root_channels,
5953            &executor,
5954            &metrics,
5955        );
5956        std::fs::remove_dir_all(root.as_path()).expect("delete project root again");
5957
5958        let after_reset = reap_idle_roots(
5959            Instant::now(),
5960            &mut live_roots,
5961            &pending_binds,
5962            &root_channels,
5963            &executor,
5964            &metrics,
5965        );
5966        assert!(after_reset.forgotten_deleted_roots.is_empty());
5967        assert!(live_roots.contains_key(&root));
5968        assert!(executor.actor_registered(&root));
5969    }
5970
5971    #[test]
5972    fn deleted_idle_root_is_fully_forgotten_and_status_counts_drop() {
5973        let (root_dir, root) = test_root("deleted-root-reap");
5974        let app = App::default_shared();
5975        let ctx = Arc::new(AppContext::from_app(
5976            Arc::clone(&app),
5977            Config {
5978                project_root: Some(root.as_path().to_path_buf()),
5979                ..Config::default()
5980            },
5981        ));
5982        ctx.set_canonical_cache_root(root.as_path().to_path_buf());
5983        ctx.mark_subc_unbound();
5984        let executor = Arc::new(Executor::new());
5985        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5986        assert_eq!(app.actor_root_count(), 1);
5987        drop(ctx);
5988        root_dir.close().expect("delete project root");
5989
5990        let mut meta = RootMeta::new(Instant::now());
5991        meta.unbound_quiesced = true;
5992        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5993        let outcome = reap_until_forgotten(
5994            &root,
5995            &mut live_roots,
5996            &HashMap::new(),
5997            &HashMap::new(),
5998            &executor,
5999            &DispatchPathMetrics::new(),
6000        );
6001        assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
6002        assert!(!executor.actor_registered(&root));
6003        assert!(!live_roots.contains_key(&root));
6004        wait_for_actor_root_count(&app, 0);
6005
6006        let status_ctx = AppContext::from_app(app, Config::default());
6007        let status = status_ctx.build_status_snapshot();
6008        assert_eq!(status["runtime"]["live_actor_roots"], 0);
6009        assert_eq!(status["runtime"]["open_routes"], 0);
6010    }
6011
6012    #[test]
6013    fn deleted_root_reap_blocker_census_is_exposed_in_health_metrics() {
6014        let (root_dir, root) = test_root("deleted-root-reap-census");
6015        let executor = Arc::new(Executor::new());
6016        assert!(executor.register_actor(root.clone(), test_ctx()));
6017        root_dir.close().expect("delete project root");
6018
6019        let mut live_roots = HashMap::from([(root, RootMeta::new(Instant::now()))]);
6020        let metrics = DispatchPathMetrics::new();
6021        let outcome = reap_idle_roots(
6022            Instant::now(),
6023            &mut live_roots,
6024            &HashMap::new(),
6025            &HashMap::new(),
6026            &executor,
6027            &metrics,
6028        );
6029        assert_eq!(outcome.evicted, 0);
6030
6031        let app = crate::context::App::default_shared();
6032        let health_rollup_cache = HealthRollupCache::new();
6033        health_rollup_cache.refresh(&executor, &app);
6034        let report = build_health_report(
6035            &health_rollup_cache,
6036            &executor,
6037            &HashMap::new(),
6038            &metrics,
6039            &app,
6040        );
6041        let reap = report
6042            .metrics
6043            .as_ref()
6044            .and_then(|metrics| metrics.get("reap"))
6045            .expect("reap health metrics");
6046        assert_eq!(reap["deleted_retained"].as_u64(), Some(1));
6047        assert_eq!(reap["blockers"]["absence_unconfirmed"].as_u64(), Some(1));
6048        assert_eq!(reap["blockers"]["unbound_quiesced"].as_u64(), Some(0));
6049        assert_eq!(reap["blockers"]["actor_busy"].as_u64(), Some(0));
6050    }
6051
6052    #[test]
6053    fn connection_exit_quiesces_queued_maintenance_and_deleted_root_is_purged() {
6054        let (root_dir, root) = test_root("connection-exit-deleted-root");
6055        let executor = Arc::new(Executor::new());
6056        assert!(executor.register_actor(root.clone(), test_ctx()));
6057
6058        let route = route_key(11, 1);
6059        let mut meta = RootMeta::new(Instant::now());
6060        meta.maintenance_pending = true;
6061        meta.maintenance_queued_kinds
6062            .push_back(MaintenanceDrainKind::CompletionDrains);
6063        let mut live_roots = HashMap::from([(root.clone(), meta)]);
6064        let mut pending_binds = HashMap::new();
6065        let mut routes = HashMap::from([(route, route_identity(&root, "abandoned"))]);
6066        let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
6067        let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
6068        let mut route_bash_cancels = HashMap::new();
6069
6070        quiesce_connection_roots(
6071            &mut live_roots,
6072            &mut pending_binds,
6073            &mut routes,
6074            &mut root_channels,
6075            &mut installed_route_epochs,
6076            &mut route_bash_cancels,
6077            &executor,
6078        );
6079        assert!(live_roots[&root].unbound_quiesced);
6080        assert!(!live_roots[&root].maintenance_pending);
6081        assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
6082        assert!(routes.is_empty());
6083        assert!(root_channels.is_empty());
6084
6085        root_dir.close().expect("delete project root");
6086        let metrics = DispatchPathMetrics::new();
6087        let outcome = reap_until_forgotten(
6088            &root,
6089            &mut live_roots,
6090            &pending_binds,
6091            &root_channels,
6092            &executor,
6093            &metrics,
6094        );
6095        let mut session_identity = HashMap::new();
6096        let mut push_buffer = HashMap::new();
6097        let mut bg_subs = HashMap::new();
6098        let mut bg_sub_by_session = HashMap::new();
6099        let mut bg_wake_pending = HashSet::new();
6100        let mut bg_wake_epoch = HashMap::new();
6101        let mut pending_bash_asks = HashMap::new();
6102        let mut retry_buffer = HashMap::new();
6103        let mut reclaimed_routes = ReclaimedRoutes::default();
6104        for forgotten in &outcome.forgotten_deleted_roots {
6105            purge_deleted_root_residents(
6106                forgotten,
6107                &mut routes,
6108                &mut root_channels,
6109                &mut installed_route_epochs,
6110                &mut route_bash_cancels,
6111                &mut retry_buffer,
6112                &mut reclaimed_routes,
6113                &mut session_identity,
6114                &mut push_buffer,
6115                &mut bg_subs,
6116                &mut bg_sub_by_session,
6117                &mut bg_wake_pending,
6118                &mut bg_wake_epoch,
6119                &mut pending_bash_asks,
6120                &metrics,
6121            );
6122        }
6123
6124        assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
6125        assert!(!executor.actor_registered(&root));
6126        assert!(!live_roots.contains_key(&root));
6127    }
6128
6129    #[test]
6130    fn unbound_root_quiesces_maintenance_without_removing_actor() {
6131        let (_root_dir, root) = test_root("unbound-root-quiesce");
6132        let ctx = test_ctx();
6133        let executor = Arc::new(Executor::new());
6134        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6135        let mut meta = RootMeta::new(Instant::now());
6136        meta.maintenance_pending = true;
6137        meta.maintenance_jobs_in_flight = 1;
6138        meta.maintenance_queued_kinds
6139            .push_back(MaintenanceDrainKind::ConfigureTail);
6140        let mut live_roots = HashMap::from([(root.clone(), meta)]);
6141        // Warm state planted before the unbind: quiesce must keep it.
6142        *ctx.search_index()
6143            .write()
6144            .unwrap_or_else(std::sync::PoisonError::into_inner) =
6145            Some(crate::search_index::SearchIndex::new());
6146        ctx.set_cache_writer_capabilities(true, true);
6147        let pending = root.as_path().join("pending.rs");
6148        ctx.add_pending_search_index_paths([pending.clone()]);
6149        // A warm verify memo must survive the transient unbind: the watcher
6150        // keeps running, so no unobserved window exists and the next warm
6151        // reload must not pay a strict full-corpus re-hash.
6152        let canonical_root = root.as_path().to_path_buf();
6153        let artifact = canonical_root.join("cache.bin");
6154        std::fs::write(&artifact, b"warm-artifact").expect("write artifact");
6155        let seeded_generation = crate::cache_freshness::artifact_generation(&artifact);
6156        crate::cache_freshness::record_verify_completed(
6157            &canonical_root,
6158            crate::cache_freshness::VerifyArtifact::Search,
6159            seeded_generation,
6160        );
6161        assert!(matches!(
6162            crate::cache_freshness::warm_verify_plan(
6163                &canonical_root,
6164                crate::cache_freshness::VerifyArtifact::Search,
6165                seeded_generation,
6166            ),
6167            crate::cache_freshness::WarmVerifyPlan::Skip
6168        ));
6169        // A live watcher runtime must survive quiesce (its events accumulate
6170        // for the rebind replay).
6171        let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
6172        let _dispatch_tx = dispatch_tx;
6173        let shutdown = Arc::new(AtomicBool::new(false));
6174        let thread_shutdown = Arc::clone(&shutdown);
6175        let join = std::thread::spawn(move || {
6176            while !thread_shutdown.load(Ordering::SeqCst) {
6177                std::thread::yield_now();
6178            }
6179        });
6180        ctx.install_watcher_runtime(
6181            dispatch_rx,
6182            crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
6183        );
6184        assert!(ctx.watcher_runtime_active());
6185
6186        quiesce_unbound_root(&root, &mut live_roots, &executor);
6187        let meta = &live_roots[&root];
6188        assert!(meta.unbound_quiesced);
6189        assert!(ctx.subc_unbound_quiesced());
6190        assert!(meta.maintenance_pending);
6191        assert!(meta.maintenance_queued_kinds.is_empty());
6192        assert!(executor.actor_registered(&root));
6193        // Transient unbind keeps the root warm: resident artifacts stay
6194        // resident, no forced callgraph rebuild is planted, and pending
6195        // reconciliation paths survive for the rebind replay.
6196        assert!(
6197            ctx.search_index()
6198                .read()
6199                .unwrap_or_else(std::sync::PoisonError::into_inner)
6200                .is_some(),
6201            "quiesce must not evict resident artifacts"
6202        );
6203        assert_eq!(
6204            ctx.pending_callgraph_store_force_token(),
6205            None,
6206            "quiesce must not force a callgraph rebuild"
6207        );
6208        assert_eq!(
6209            ctx.take_pending_search_index_paths(),
6210            vec![pending],
6211            "quiesce must retain pending watcher-derived paths"
6212        );
6213        assert!(
6214            matches!(
6215                crate::cache_freshness::warm_verify_plan(
6216                    &canonical_root,
6217                    crate::cache_freshness::VerifyArtifact::Search,
6218                    seeded_generation,
6219                ),
6220                crate::cache_freshness::WarmVerifyPlan::Skip
6221            ),
6222            "quiesce must not invalidate the warm verify memo"
6223        );
6224        assert!(
6225            ctx.watcher_runtime_active(),
6226            "quiesce must not stop a running watcher"
6227        );
6228        ctx.stop_watcher_runtime();
6229
6230        let meta = live_roots.get_mut(&root).expect("root metadata");
6231        note_maintenance_completion(
6232            meta,
6233            Some(MaintenanceDrainKind::ConfigureTail),
6234            false,
6235            meta.unbound_quiesced,
6236        );
6237        assert!(!meta.maintenance_pending);
6238        assert!(meta.maintenance_queued_kinds.is_empty());
6239    }
6240
6241    #[test]
6242    fn same_root_higher_epoch_replacement_does_not_quiesce_between_generations() {
6243        let (_dir, root) = test_root("same-root-replacement");
6244        let route = route_key(7, 1);
6245        let installed_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
6246        let root_channels = HashMap::new();
6247
6248        assert!(!route_removal_will_quiesce_root(
6249            &root,
6250            route,
6251            &installed_channels,
6252            false,
6253            Some(&root),
6254        ));
6255        assert!(route_removal_will_quiesce_root(
6256            &root,
6257            route,
6258            &installed_channels,
6259            false,
6260            None,
6261        ));
6262        assert!(!should_quiesce_removed_root(
6263            &root,
6264            &root_channels,
6265            false,
6266            Some(&root),
6267        ));
6268        assert!(should_quiesce_removed_root(
6269            &root,
6270            &root_channels,
6271            false,
6272            None,
6273        ));
6274        assert!(!should_quiesce_removed_root(
6275            &root,
6276            &root_channels,
6277            true,
6278            None,
6279        ));
6280    }
6281
6282    #[test]
6283    fn root_quiesces_only_after_its_last_route_is_removed_and_reactivates_on_bind() {
6284        let (_root_dir, root) = test_root("unbound-root-route-count");
6285        let executor = Arc::new(Executor::new());
6286        assert!(executor.register_actor(root.clone(), test_ctx()));
6287        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
6288        let mut root_channels = HashMap::from([(
6289            root.clone(),
6290            HashSet::from([route_key(7, 1), route_key(8, 1)]),
6291        )]);
6292
6293        remove_root_channel(&mut root_channels, &root, route_key(7, 1));
6294        if !root_channels.contains_key(&root) {
6295            quiesce_unbound_root(&root, &mut live_roots, &executor);
6296        }
6297        assert!(!live_roots[&root].unbound_quiesced);
6298
6299        remove_root_channel(&mut root_channels, &root, route_key(8, 1));
6300        if !root_channels.contains_key(&root) {
6301            quiesce_unbound_root(&root, &mut live_roots, &executor);
6302        }
6303        assert!(live_roots[&root].unbound_quiesced);
6304
6305        live_roots
6306            .get_mut(&root)
6307            .expect("root metadata")
6308            .note_activity();
6309        assert!(
6310            live_roots[&root].unbound_quiesced,
6311            "late asynchronous activity must not reactivate an unbound root"
6312        );
6313
6314        live_roots
6315            .get_mut(&root)
6316            .expect("root metadata")
6317            .reactivate_bound();
6318        assert!(!live_roots[&root].unbound_quiesced);
6319    }
6320
6321    #[test]
6322    fn allocator_pressure_relief_requires_every_root_to_be_idle() {
6323        let (_idle_dir, idle_root) = test_root("allocator-relief-idle");
6324        let (_active_dir, active_root) = test_root("allocator-relief-active");
6325        let now = Instant::now();
6326        let mut live_roots = HashMap::new();
6327        let mut idle = RootMeta::new(now);
6328        idle.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
6329        live_roots.insert(idle_root, idle);
6330        assert!(process_has_been_idle(now, &live_roots));
6331
6332        live_roots.insert(active_root.clone(), RootMeta::new(now));
6333        assert!(!process_has_been_idle(now, &live_roots));
6334
6335        let active = live_roots
6336            .get_mut(&active_root)
6337            .expect("active root metadata");
6338        active.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
6339        active.active_bash_waits = 1;
6340        assert!(!process_has_been_idle(now, &live_roots));
6341    }
6342
6343    #[test]
6344    fn pressure_relief_log_reports_before_and_after_measurements() {
6345        let allocator = crate::memory::AllocatorMemorySnapshot {
6346            status: "measured",
6347            bytes_in_use: Some(8 * 1024 * 1024),
6348            size_allocated: Some(12 * 1024 * 1024),
6349            retained_slack_bytes: Some(4 * 1024 * 1024),
6350            not_estimated: None,
6351        };
6352        let relief = crate::memory::AllocatorPressureRelief {
6353            bytes_released: 3 * 1024 * 1024,
6354            rss_before_bytes: Some(20 * 1024 * 1024),
6355            rss_after_bytes: Some(17 * 1024 * 1024),
6356            allocator_before: allocator.clone(),
6357            allocator_after: crate::memory::AllocatorMemorySnapshot {
6358                size_allocated: Some(9 * 1024 * 1024),
6359                retained_slack_bytes: Some(1024 * 1024),
6360                ..allocator
6361            },
6362        };
6363        let message = pressure_relief_label(&relief);
6364        assert!(message.contains("RSS 20.0 MB -> 17.0 MB"));
6365        assert!(message.contains("allocated 12.0 MB -> 9.0 MB"));
6366        assert!(message.contains("slack 4.0 MB -> 1.0 MB"));
6367        assert!(message.contains("reported 3.0 MB released"));
6368    }
6369
6370    #[test]
6371    fn due_maintenance_jobs_skip_poisoned_roots() {
6372        let (_healthy_dir, healthy_root) = test_root("maintenance-healthy");
6373        let (_poisoned_dir, poisoned_root) = test_root("maintenance-poisoned");
6374        let mut live_roots = HashMap::new();
6375        live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
6376        let mut poisoned_meta = RootMeta::new(Instant::now());
6377        poisoned_meta.maintenance_poisoned = true;
6378        live_roots.insert(poisoned_root.clone(), poisoned_meta);
6379
6380        let (due, deferred) = due_maintenance_jobs_without_actor_context(
6381            &mut live_roots,
6382            MAINTENANCE_SUBMIT_BUDGET,
6383            &HashSet::new(),
6384        );
6385
6386        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6387        assert!(due.iter().all(|(root, _)| root == &healthy_root));
6388        assert!(!deferred);
6389        assert!(live_roots[&healthy_root].maintenance_pending);
6390        assert_eq!(
6391            live_roots[&healthy_root].maintenance_jobs_in_flight,
6392            INITIAL_MAINTENANCE_JOB_COUNT
6393        );
6394        assert!(!live_roots[&poisoned_root].maintenance_pending);
6395    }
6396
6397    #[test]
6398    fn due_maintenance_jobs_do_not_restart_quiesced_root_work() {
6399        let (_dir, root) = test_root("maintenance-unbound");
6400        let mut meta = RootMeta::new(Instant::now());
6401        meta.unbound_quiesced = true;
6402        let mut live_roots = HashMap::from([(root.clone(), meta)]);
6403
6404        let (due, deferred) = due_maintenance_jobs_without_actor_context(
6405            &mut live_roots,
6406            MAINTENANCE_SUBMIT_BUDGET,
6407            &HashSet::new(),
6408        );
6409
6410        assert!(due.is_empty());
6411        assert!(!deferred);
6412        assert!(!live_roots[&root].maintenance_pending);
6413    }
6414
6415    #[test]
6416    fn idle_bg_subscription_queues_no_jobs_until_a_wake_arrives() {
6417        let (_dir, root) = test_root("maintenance-idle-bg-subscription");
6418        let ctx = test_ctx();
6419        assert!(!ctx.completion_drains_have_work());
6420
6421        let executor = Executor::new();
6422        assert!(executor.register_actor(root.clone(), ctx));
6423        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
6424        let session = "idle-session".to_string();
6425        let channel = route_key(17, 1);
6426        let metrics = DispatchPathMetrics::new();
6427        let bg_sub_by_session =
6428            HashMap::from([((root.clone(), session.clone()), HashSet::from([channel]))]);
6429        let mut bg_wake_pending = HashSet::new();
6430
6431        let (idle_tick_jobs, deferred) = due_maintenance_jobs(
6432            &mut live_roots,
6433            Some(&executor),
6434            &bg_sub_by_session,
6435            &bg_wake_pending,
6436            MAINTENANCE_SUBMIT_BUDGET,
6437            &HashSet::new(),
6438        );
6439        assert!(idle_tick_jobs.is_empty());
6440        assert!(!deferred);
6441        assert!(!live_roots[&root].maintenance_pending);
6442
6443        // A completion can arm its wake after the idle tick's probes. The wake
6444        // remains loop-owned state, so the following tick must observe it.
6445        let mut bg_wake_epoch = HashMap::new();
6446        push::arm_bg_wake(
6447            root.clone(),
6448            session,
6449            channel,
6450            &mut bg_wake_pending,
6451            &mut bg_wake_epoch,
6452            &metrics,
6453        );
6454        let (next_tick_jobs, deferred) = due_maintenance_jobs(
6455            &mut live_roots,
6456            Some(&executor),
6457            &bg_sub_by_session,
6458            &bg_wake_pending,
6459            MAINTENANCE_SUBMIT_BUDGET,
6460            &HashSet::new(),
6461        );
6462        assert_eq!(
6463            next_tick_jobs,
6464            vec![(root, MaintenanceDrainKind::CompletionDrains)]
6465        );
6466        assert!(!deferred);
6467    }
6468
6469    #[tokio::test]
6470    async fn subc_configure_tail_precedes_completed_search_install() {
6471        let root_dir = tempfile::tempdir().unwrap();
6472        let storage = tempfile::tempdir().unwrap();
6473        let root = ProjectRootId::from_path(root_dir.path()).unwrap();
6474        let (ctx, ignored_path) =
6475            runtime_drain::configure_search_order_context_for_test(root_dir.path(), storage.path());
6476        let ctx = Arc::new(ctx);
6477        assert!(!runtime_drain::watcher_path_is_ignored_by_current_matcher(
6478            &ctx,
6479            &ignored_path
6480        ));
6481
6482        let executor = Arc::new(Executor::new());
6483        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6484        let metrics = Arc::new(DispatchPathMetrics::new());
6485        let (completion_tx, mut completion_rx) = mpsc::channel(4);
6486        submit_maintenance_job(
6487            &executor,
6488            root.clone(),
6489            MaintenanceDrainKind::ConfigureTail,
6490            Vec::new(),
6491            &completion_tx,
6492            &metrics,
6493        );
6494        submit_maintenance_job(
6495            &executor,
6496            root,
6497            MaintenanceDrainKind::CompletionDrains,
6498            Vec::new(),
6499            &completion_tx,
6500            &metrics,
6501        );
6502
6503        let first = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
6504            .await
6505            .expect("configure-tail completion timed out")
6506            .expect("configure-tail completion channel closed");
6507        let second = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
6508            .await
6509            .expect("completion-drains completion timed out")
6510            .expect("completion-drains completion channel closed");
6511        assert!(first.response.id.contains("configure-tail"));
6512        assert!(second.response.id.contains("completion-drains"));
6513        assert!(runtime_drain::watcher_path_is_ignored_by_current_matcher(
6514            &ctx,
6515            &ignored_path
6516        ));
6517        assert_eq!(
6518            ctx.search_index()
6519                .read()
6520                .unwrap_or_else(std::sync::PoisonError::into_inner)
6521                .as_ref()
6522                .expect("completed search index installed")
6523                .file_count(),
6524            0,
6525            "configure must install the ignore matcher before pending paths replay"
6526        );
6527        ctx.stop_watcher_runtime();
6528    }
6529
6530    #[test]
6531    fn post_bind_configure_and_completion_jobs_are_queued_in_order() {
6532        let (_dir, root) = test_root("maintenance-post-bind");
6533        let mut live_roots = HashMap::new();
6534        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6535
6536        queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
6537        queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
6538
6539        let meta = live_roots.get(&root).expect("root metadata");
6540        assert!(meta.maintenance_pending);
6541        assert_eq!(meta.maintenance_jobs_in_flight, 0);
6542        assert_eq!(
6543            meta.maintenance_queued_kinds
6544                .iter()
6545                .copied()
6546                .collect::<Vec<_>>(),
6547            vec![
6548                MaintenanceDrainKind::ConfigureTail,
6549                MaintenanceDrainKind::CompletionDrains,
6550            ]
6551        );
6552
6553        let (due, deferred) = due_maintenance_jobs_without_actor_context(
6554            &mut live_roots,
6555            MAINTENANCE_SUBMIT_BUDGET,
6556            &HashSet::new(),
6557        );
6558
6559        assert_eq!(
6560            due,
6561            vec![
6562                (root.clone(), MaintenanceDrainKind::ConfigureTail),
6563                (root.clone(), MaintenanceDrainKind::CompletionDrains),
6564            ]
6565        );
6566        assert!(!deferred);
6567        assert_eq!(live_roots[&root].maintenance_jobs_in_flight, 2);
6568        assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
6569    }
6570
6571    #[test]
6572    fn due_maintenance_jobs_defers_unsubmitted_roots_without_marking_pending() {
6573        let mut live_roots = HashMap::new();
6574        let mut root_ids = Vec::new();
6575        let mut _dirs = Vec::new();
6576        for index in 0..4 {
6577            let (dir, root_id) = test_root(&format!("maintenance-budget-{index}"));
6578            live_roots.insert(root_id.clone(), RootMeta::new(Instant::now()));
6579            root_ids.push(root_id);
6580            _dirs.push(dir);
6581        }
6582
6583        let small_budget = INITIAL_MAINTENANCE_JOB_COUNT + 1;
6584        let (first_due, first_deferred) = due_maintenance_jobs_without_actor_context(
6585            &mut live_roots,
6586            small_budget,
6587            &HashSet::new(),
6588        );
6589
6590        assert_eq!(first_due.len(), small_budget);
6591        assert!(first_deferred);
6592        let first_due_set: HashSet<_> = first_due.into_iter().map(|(root, _)| root).collect();
6593        assert!(first_due_set
6594            .iter()
6595            .all(|root| live_roots[root].maintenance_pending));
6596        assert!(first_due_set
6597            .iter()
6598            .any(|root| !live_roots[root].maintenance_queued_kinds.is_empty()));
6599
6600        let all_roots: HashSet<_> = root_ids.into_iter().collect();
6601        let deferred_roots: HashSet<_> = all_roots.difference(&first_due_set).cloned().collect();
6602        assert!(deferred_roots
6603            .iter()
6604            .all(|root| !live_roots[root].maintenance_pending));
6605    }
6606
6607    #[test]
6608    fn due_maintenance_jobs_defers_pending_bind_roots() {
6609        let (_bind_dir, bind_root) = test_root("maintenance-pending-bind");
6610        let (_healthy_dir, healthy_root) = test_root("maintenance-no-bind");
6611        let mut live_roots = HashMap::new();
6612        live_roots.insert(bind_root.clone(), RootMeta::new(Instant::now()));
6613        live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
6614        let pending_bind_roots = HashSet::from([bind_root.clone()]);
6615
6616        let (due, deferred) = due_maintenance_jobs_without_actor_context(
6617            &mut live_roots,
6618            usize::MAX,
6619            &pending_bind_roots,
6620        );
6621
6622        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6623        assert!(due.iter().all(|(root, _)| root == &healthy_root));
6624        assert!(!deferred);
6625        assert!(!live_roots[&bind_root].maintenance_pending);
6626        assert!(live_roots[&bind_root].maintenance_queued_kinds.is_empty());
6627    }
6628
6629    #[test]
6630    fn maintenance_pending_survives_requeue_and_clears_after_final_batch() {
6631        let (_dir, root) = test_root("maintenance-requeue");
6632        let mut live_roots = HashMap::new();
6633        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6634        let (due, deferred) = due_maintenance_jobs_without_actor_context(
6635            &mut live_roots,
6636            usize::MAX,
6637            &HashSet::new(),
6638        );
6639        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6640        assert!(due.iter().all(|(due_root, _)| due_root == &root));
6641        assert!(!deferred);
6642
6643        let meta = live_roots.get_mut(&root).unwrap();
6644        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, false);
6645        assert!(meta.maintenance_pending);
6646        assert_eq!(
6647            meta.maintenance_jobs_in_flight,
6648            INITIAL_MAINTENANCE_JOB_COUNT - 1
6649        );
6650        assert_eq!(meta.maintenance_queued_kinds.len(), 1);
6651
6652        let (requeued, deferred) =
6653            due_maintenance_jobs_without_actor_context(&mut live_roots, 1, &HashSet::new());
6654        assert_eq!(
6655            requeued,
6656            vec![(root.clone(), MaintenanceDrainKind::Watcher)]
6657        );
6658        assert!(!deferred);
6659        let meta = live_roots.get_mut(&root).unwrap();
6660        assert_eq!(
6661            meta.maintenance_jobs_in_flight,
6662            INITIAL_MAINTENANCE_JOB_COUNT
6663        );
6664        assert!(meta.maintenance_queued_kinds.is_empty());
6665
6666        for _ in 0..INITIAL_MAINTENANCE_JOB_COUNT {
6667            note_maintenance_completion(meta, None, false, false);
6668        }
6669        assert!(!meta.maintenance_pending);
6670        assert_eq!(meta.maintenance_jobs_in_flight, 0);
6671    }
6672
6673    #[test]
6674    fn maintenance_requeue_drops_while_bind_is_pending() {
6675        let (_dir, root) = test_root("maintenance-bind-requeue");
6676        let mut live_roots = HashMap::new();
6677        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6678        let (due, _) = due_maintenance_jobs_without_actor_context(
6679            &mut live_roots,
6680            usize::MAX,
6681            &HashSet::new(),
6682        );
6683        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6684
6685        let meta = live_roots.get_mut(&root).unwrap();
6686        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, true);
6687
6688        assert_eq!(
6689            meta.maintenance_jobs_in_flight,
6690            INITIAL_MAINTENANCE_JOB_COUNT - 1
6691        );
6692        assert!(meta.maintenance_queued_kinds.is_empty());
6693        assert!(meta.maintenance_pending);
6694    }
6695
6696    #[test]
6697    fn parked_lsp_completion_never_requiesces_or_cancels_a_pending_bind() {
6698        let mut meta = RootMeta::new(Instant::now());
6699        meta.unbound_quiesced = true;
6700
6701        assert!(!should_requiesce_after_maintenance(
6702            &meta,
6703            MaintenanceDrainKind::Lsp,
6704            false,
6705        ));
6706        assert!(!should_requiesce_after_maintenance(
6707            &meta,
6708            MaintenanceDrainKind::ConfigureTail,
6709            true,
6710        ));
6711        assert!(should_requiesce_after_maintenance(
6712            &meta,
6713            MaintenanceDrainKind::ConfigureTail,
6714            false,
6715        ));
6716    }
6717
6718    #[test]
6719    fn maintenance_pending_clears_and_poison_stops_requeue_after_fatal() {
6720        let (_dir, root) = test_root("maintenance-fatal");
6721        let mut live_roots = HashMap::new();
6722        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6723        let (due, _) = due_maintenance_jobs_without_actor_context(
6724            &mut live_roots,
6725            usize::MAX,
6726            &HashSet::new(),
6727        );
6728        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6729
6730        let meta = live_roots.get_mut(&root).unwrap();
6731        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), true, false);
6732        assert!(meta.maintenance_poisoned);
6733        assert!(meta.maintenance_queued_kinds.is_empty());
6734
6735        for _ in 1..INITIAL_MAINTENANCE_JOB_COUNT {
6736            note_maintenance_completion(meta, None, false, false);
6737        }
6738        assert!(!meta.maintenance_pending);
6739        assert_eq!(meta.maintenance_jobs_in_flight, 0);
6740    }
6741
6742    #[test]
6743    fn trust_for_principal_matrix() {
6744        assert_eq!(
6745            trust_for_principal(&Some(Principal::Direct)),
6746            BindTrust::FirstParty
6747        );
6748        // Every first-party reserved id is asserted BY NAME, in one loop over
6749        // the full set, so two failure classes stay distinguishable: an empty
6750        // or broken allowlist reddens every name at once, while a dropped
6751        // single entry (the rename hazard) reddens exactly the missing name.
6752        // Both halves of each transitional rename pair stay listed until the
6753        // flip settles (see the allowlist comment).
6754        for module_id in [
6755            "llm-runner",
6756            "aft",
6757            "broca",
6758            "alfonso-core",
6759            "prefrontal",
6760            "prefrontal-core",
6761        ] {
6762            assert_eq!(
6763                trust_for_principal(&Some(Principal::Reserved {
6764                    module_id: module_id.to_string(),
6765                })),
6766                BindTrust::FirstParty,
6767                "reserved module id '{module_id}' must resolve to first-party trust"
6768            );
6769        }
6770        assert_eq!(
6771            trust_for_principal(&Some(Principal::Reserved {
6772                module_id: "subc-mcp".to_string(),
6773            })),
6774            BindTrust::Untrusted
6775        );
6776        assert_eq!(
6777            trust_for_principal(&Some(Principal::Reserved {
6778                module_id: "anything-unknown".to_string(),
6779            })),
6780            BindTrust::Untrusted
6781        );
6782        assert_eq!(
6783            trust_for_principal(&Some(Principal::Unverified)),
6784            BindTrust::Untrusted
6785        );
6786        assert_eq!(trust_for_principal(&None), BindTrust::Untrusted);
6787    }
6788
6789    #[test]
6790    fn fed_harness_class_maps_to_untrusted_regardless_of_fingerprint_value() {
6791        let principal = Some(Principal::Direct);
6792        let fingerprint_a = "fed:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
6793        let fingerprint_b = "fed:0123456789abcdef111111111111111111111111111111111111111111111111";
6794
6795        assert_eq!(
6796            trust_for_bind(fingerprint_a, &principal),
6797            BindTrust::Untrusted
6798        );
6799        assert_eq!(
6800            trust_for_bind(fingerprint_b, &principal),
6801            BindTrust::Untrusted
6802        );
6803    }
6804
6805    /// The table above proves `trust_for_principal` maps correctly, and the
6806    /// test above proves the `fed:` harness override wins — but neither
6807    /// exercises the ordinary path, so an implementation that ignored the
6808    /// principal entirely for non-fed harnesses would satisfy both. Pin the
6809    /// delegation itself: on a normal harness the verdict must still come from
6810    /// the principal, in both directions.
6811    #[test]
6812    fn trust_for_bind_delegates_to_the_principal_on_ordinary_harnesses() {
6813        for harness in ["opencode", "pi", "runner", "mcp:claude"] {
6814            assert_eq!(
6815                trust_for_bind(harness, &Some(Principal::Direct)),
6816                BindTrust::FirstParty,
6817                "a direct principal must stay first-party on {harness}"
6818            );
6819            assert_eq!(
6820                trust_for_bind(harness, &Some(Principal::Unverified)),
6821                BindTrust::Untrusted,
6822                "an unverified principal must stay untrusted on {harness}"
6823            );
6824            assert_eq!(
6825                trust_for_bind(harness, &None),
6826                BindTrust::Untrusted,
6827                "an absent principal must fail closed on {harness}"
6828            );
6829            assert_eq!(
6830                trust_for_bind(
6831                    harness,
6832                    &Some(Principal::Reserved {
6833                        module_id: "subc-mcp".to_string(),
6834                    })
6835                ),
6836                BindTrust::Untrusted,
6837                "a non-allowlisted reserved module must stay untrusted on {harness}"
6838            );
6839        }
6840    }
6841
6842    #[tokio::test]
6843    async fn persistent_cancel_resolves_when_fired_before_await() {
6844        // The lost-wakeup guard: cancel() fires exactly once via notify_waiters()
6845        // (no stored permit). A waiter that registers AFTER the cancel must still
6846        // observe it via the flag; a waiter racing the cancel must still be woken.
6847        let signal = PersistentCancelSignal::new();
6848        signal.cancel();
6849        // Fired before we ever call cancelled() — must return immediately, not park.
6850        tokio::time::timeout(Duration::from_secs(1), signal.cancelled())
6851            .await
6852            .expect("cancelled() must resolve when cancel fired beforehand");
6853
6854        // A fresh signal cancelled concurrently with an in-flight cancelled().
6855        let racing = PersistentCancelSignal::new();
6856        let racing_for_task = racing.clone();
6857        let waiter = tokio::spawn(async move { racing_for_task.cancelled().await });
6858        racing.cancel();
6859        tokio::time::timeout(Duration::from_secs(1), waiter)
6860            .await
6861            .expect("cancelled() must resolve when cancel races the await")
6862            .expect("waiter task panicked");
6863    }
6864
6865    #[test]
6866    fn ingress_epoch_validation_rejects_reclaimed_requests_and_drops_other_stale_epochs() {
6867        let installed = HashMap::from([(7, 9)]);
6868        let mut reclaimed = ReclaimedRoutes::default();
6869        reclaimed.insert(route_key(8, 1));
6870        for ty in [
6871            FrameType::Request,
6872            FrameType::Response,
6873            FrameType::Error,
6874            FrameType::Push,
6875            FrameType::Cancel,
6876            FrameType::Goodbye,
6877        ] {
6878            let body = if ty.is_pure_header() {
6879                Vec::new()
6880            } else {
6881                br#"{}"#.to_vec()
6882            };
6883            let stale = Frame::build(ty, control_flags(), 7, 8, 41, body).unwrap();
6884            assert!(
6885                !ingress_route_should_be_processed(&installed, &reclaimed, &stale),
6886                "{ty:?}"
6887            );
6888        }
6889
6890        let reclaimed_request = Frame::build(
6891            FrameType::Request,
6892            control_flags(),
6893            8,
6894            1,
6895            42,
6896            br#"{}"#.to_vec(),
6897        )
6898        .unwrap();
6899        assert!(ingress_route_should_be_processed(
6900            &installed,
6901            &reclaimed,
6902            &reclaimed_request
6903        ));
6904
6905        let never_installed = Frame::build(
6906            FrameType::Request,
6907            control_flags(),
6908            9,
6909            1,
6910            43,
6911            br#"{}"#.to_vec(),
6912        )
6913        .unwrap();
6914        assert!(!ingress_route_should_be_processed(
6915            &installed,
6916            &reclaimed,
6917            &never_installed
6918        ));
6919
6920        let current = Frame::build(
6921            FrameType::Request,
6922            control_flags(),
6923            7,
6924            9,
6925            43,
6926            br#"{}"#.to_vec(),
6927        )
6928        .unwrap();
6929        let control = Frame::build(FrameType::Ping, control_flags(), 0, 0, 44, Vec::new()).unwrap();
6930        assert!(ingress_route_should_be_processed(
6931            &installed, &reclaimed, &current
6932        ));
6933        assert!(ingress_route_should_be_processed(
6934            &installed, &reclaimed, &control
6935        ));
6936        assert_eq!(installed, HashMap::from([(7, 9)]));
6937    }
6938
6939    #[tokio::test]
6940    async fn route_bind_ack_precedes_route_egress_in_writer_queue() {
6941        let (_dir, root) = test_root("route-bind-b2-ordering");
6942        let route = route_key(7, 3);
6943        let identity = RouteIdentity(Arc::new(RouteIdentityData {
6944            root: root.clone(),
6945            project_root: root.as_path().to_path_buf(),
6946            harness: "opencode".to_string(),
6947            session: "b2-session".to_string(),
6948            trust: BindTrust::FirstParty,
6949            spawn_principal: AuthenticatedPrincipal::FirstParty,
6950            consumer_elicitation_capable: false,
6951        }));
6952        let replay_key = push::ReplayKey::from_identity(&identity);
6953        let completion = RouteBindCompletion {
6954            route,
6955            identity,
6956            bind_root_id: root.clone(),
6957            inserted_new_actor: false,
6958            configure_response: Response::success("subc-bind-7", json!({})),
6959            diagnostics_on_edit: false,
6960            ver: PROTOCOL_VERSION,
6961            corr: 91,
6962            flags: control_flags(),
6963        };
6964        let mut pending_binds = HashMap::from([(
6965            route,
6966            PendingBind {
6967                bind_root_id: root,
6968                inserted_new_actor: false,
6969                cancelled: false,
6970                configure_request_id: "subc-bind-7".to_string(),
6971                started_at: Instant::now(),
6972                warned_half_deadline: false,
6973                deadline_reported: false,
6974                corr: 91,
6975                ver: PROTOCOL_VERSION,
6976                flags: control_flags(),
6977                cancellation: crate::executor::JobCancellation::new(),
6978            },
6979        )]);
6980        let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
6981        let mut push_buffer =
6982            HashMap::from([(replay_key, VecDeque::from([completion_frame("b2-replay")]))]);
6983        let (writer_tx, mut writer_rx) = mpsc::channel(8);
6984        let metrics = Arc::new(DispatchPathMetrics::new());
6985
6986        handle_route_bind_completion(
6987            &writer_tx,
6988            completion,
6989            &mut HashMap::new(),
6990            &mut HashMap::new(),
6991            &mut HashMap::new(),
6992            &mut push_buffer,
6993            &mut HashMap::new(),
6994            &mut pending_binds,
6995            &mut installed_route_epochs,
6996            &Arc::new(Executor::new()),
6997            &Arc::new(Notify::new()),
6998            &metrics,
6999        )
7000        .await
7001        .unwrap();
7002
7003        let ack = writer_rx.try_recv().expect("RouteBindAck");
7004        assert_eq!(ack.header.ty, FrameType::Response);
7005        assert_eq!((ack.header.channel, ack.header.epoch), (0, 0));
7006        let route_frame = writer_rx.try_recv().expect("post-ack route frame");
7007        assert_eq!(route_frame.header.ty, FrameType::Push);
7008        assert_eq!(
7009            (route_frame.header.channel, route_frame.header.epoch),
7010            (route.channel, route.epoch)
7011        );
7012    }
7013}