Skip to main content

aft/subc/
mod.rs

1//! subc daemon attach — transport edge.
2//!
3//! When AFT is launched as `aft --subc <connection-file>`, it does NOT run the
4//! standalone NDJSON-over-stdin loop. Instead it connects to a running subc
5//! daemon over loopback TCP, authenticates with the pre-envelope HMAC handshake
6//! (`subc-transport`), then speaks the subc frame protocol (`subc-protocol`):
7//! ModuleHello → HelloAck (register as a tool provider), then a channel-0
8//! control loop (Ping/Pong, RouteBind) plus route-channel tool calls.
9//!
10//! Concurrency: subc routes tool calls through the executor. The tokio
11//! edge never dispatches against `AppContext` inline; per-actor executor lanes
12//! own the reader/mutator epoch, while a writer task serializes outbound frames.
13
14use std::collections::{HashMap, HashSet, VecDeque};
15use std::fmt;
16use std::io;
17use std::net::{IpAddr, SocketAddr};
18use std::ops::Deref;
19use std::path::{Path, PathBuf};
20use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
21use std::sync::{Arc, LazyLock};
22use std::time::{Duration, Instant};
23
24use serde::Deserialize;
25use serde_json::{json, Value};
26
27use crate::config::Config;
28use crate::config_resolve::ConfigTier;
29use crate::context::{App, AppContext, ProgressSender, RootHealthSnapshot};
30use crate::executor::{Executor, Lane};
31use crate::jsonc::strip_jsonc;
32use crate::log_ctx;
33use crate::path_identity::ProjectRootId;
34use crate::protocol::{ProgressKind, PushFrame, RawRequest, Response};
35use crate::run_tool_call::{
36    run_tool_call, strip_agent_preview_arg_owned, PhaseTrace, ToolCallContext, ToolCallOutcome,
37    ToolCallResult,
38};
39use crate::runtime_drain;
40
41use subc_protocol::manifest::{
42    Bindings, Concurrency, ExecutionMode, IdentityBinding, IdentityScope, ModuleManifest,
43    ProviderRole, StorageBinding, StorageKind, StorageScope, Tool, TrustTier,
44};
45use subc_protocol::session::{
46    HealthReport, HealthStatus, ModuleControlRequest, ModuleControlResponse,
47    MODULE_CONTROL_OP_HEALTH_CHECK,
48};
49use subc_protocol::{
50    ErrorBody, Flags, Frame, FrameType, ModuleHelloBody, Principal, Priority, PROTOCOL_VERSION,
51};
52use subc_transport::{authenticate_client, connection_file, read_frame, write_frame};
53use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
54use tokio::net::TcpStream;
55use tokio::sync::{mpsc, oneshot, Notify};
56use tokio::task::JoinHandle;
57
58/// Per-attempt handshake deadline. The initial attach loop has a separate total
59/// budget so a stalled peer cannot consume an unbounded supervisor launch window.
60const AUTH_DEADLINE: Duration = Duration::from_secs(5);
61const ATTACH_RETRY_BUDGET: Duration = Duration::from_secs(60);
62const ATTACH_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(250);
63const ATTACH_RETRY_MAX_BACKOFF: Duration = Duration::from_secs(5);
64const ATTACH_RETRY_JITTER_PERCENT: u64 = 20;
65
66/// Correlation id for the initial ModuleHello (channel 0).
67const HELLO_CORR: u64 = 1;
68
69/// Per-session in-memory replay cap for must-deliver Push frames. This covers
70/// detach/re-attach while AFT stays alive; cross-restart replay is phased later.
71const PUSH_BUFFER_MAX_PER_KEY: usize = 256;
72
73/// Bounded guard for control-frame sends. If the daemon stops reading and the
74/// writer queue stays full, tear the subc edge down instead of stalling the
75/// route loop indefinitely.
76const CONTROL_SEND_TIMEOUT: Duration = Duration::from_millis(250);
77
78/// Cadence for the loop's deadline-driven drain work (retry-buffer flush,
79/// bg-wake emission, maintenance submission). Checked at the top of every
80/// loop turn so busy select arms cannot starve it.
81const DRAIN_TICK_PERIOD: Duration = Duration::from_millis(250);
82
83/// Root-scoped stores and watcher runtimes are reopened lazily after this
84/// period without tool traffic. Keeping the value fixed avoids per-client
85/// eviction policies competing inside the module loop.
86const IDLE_ROOT_TTL: Duration = Duration::from_secs(30 * 60);
87
88const WRITER_QUEUE_CAPACITY: usize = 256;
89
90/// Keep reliable Push bursts from monopolizing the current-thread subc loop;
91/// any remaining must-deliver frames stay queued for the next loop turn.
92const RELIABLE_PUSH_DRAIN_BUDGET: usize = 32;
93
94/// Limit maintenance submissions per tick so background drains cannot delay
95/// control-plane work such as completed RouteBind acknowledgements.
96///
97/// The decomposed maintenance pass charges this budget by Mutating job, not by
98/// root. Size the default burst for one maintenance pass over eight live roots,
99/// while follow-up batches still re-enter the capped queue instead of bypassing
100/// the budget.
101const MAINTENANCE_SUBMIT_BUDGET: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len() * 8;
102const INITIAL_MAINTENANCE_DRAIN_KINDS: [MaintenanceDrainKind; 4] = [
103    MaintenanceDrainKind::Watcher,
104    MaintenanceDrainKind::Lsp,
105    MaintenanceDrainKind::ConfigureTail,
106    MaintenanceDrainKind::CompletionDrains,
107];
108#[cfg(test)]
109const INITIAL_MAINTENANCE_JOB_COUNT: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len();
110
111const RELIABLE_WRITER_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(10);
112const RELIABLE_WRITER_RETRY_MAX_BACKOFF: Duration = Duration::from_millis(250);
113
114const DISPATCH_PATH_BIND_WARN_AFTER: Duration = Duration::from_secs(6);
115const ROUTE_BIND_DEADLINE: Duration = Duration::from_secs(12);
116
117/// Small bounded memory of completed task ids used to suppress stale lossy
118/// long-running reminders that arrive after their reliable completion event.
119const COMPLETED_TASK_SUPPRESSION_MAX: usize = 4096;
120
121/// Bash foreground orchestration polls detached tasks with short read-lane jobs.
122/// The sleep between polls is outside the executor so no read or write worker is
123/// pinned while a foreground command is still running.
124const PENDING_POLL_INTERVAL: Duration = Duration::from_millis(100);
125
126/// Host elicitation asks fail closed if the MCP facade does not answer promptly.
127const BASH_ELICITATION_TIMEOUT: Duration = Duration::from_secs(60);
128const BASH_ELICITATION_CREATE_METHOD: &str = "elicitation/create";
129
130#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
131struct RouteChannel {
132    channel: u16,
133    epoch: u32,
134}
135
136impl fmt::Display for RouteChannel {
137    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
138        write!(f, "{}@{}", self.channel, self.epoch)
139    }
140}
141
142type PushEnvelope = (ProjectRootId, PushFrame);
143type LossyPushEnvelope = (u64, ProjectRootId, PushFrame);
144type RetryBuffer = HashMap<RouteChannel, VecDeque<(push::ReplayKey, PushFrame)>>;
145mod bash;
146mod health;
147mod manifest;
148mod push;
149mod wire;
150
151use self::health::{
152    build_health_report, warn_slow_pending_binds, DispatchPathMetrics, ResponseTaskGuard,
153};
154use self::manifest::{
155    build_manifest, command_lane, control_flags, control_ops, is_bash_family_tool,
156    is_subc_agent_core_tool, is_subc_native_plumbing_tool,
157};
158pub use self::wire::SubcError;
159
160/// Test-only view of the fail-closed tool-call gate: would `name` be admitted
161/// on a bound route (as an agent tool or native plumbing)? Used by the
162/// plugin-send drift guard in `subc_plumbing_drift_test.rs`.
163pub fn is_tool_call_admitted_for_test(name: &str) -> bool {
164    manifest::is_subc_agent_core_tool(name) || manifest::is_subc_native_plumbing_tool(name)
165}
166use self::wire::{
167    build_error_frame, build_goodbye_frame, build_tool_response_frame, decrement_counted_channel,
168    response_is_fatal_panic, response_message, send_counted_channel, send_frame,
169    send_reliable_writer_frame, send_traced_tool_response_frame, ToolResponseWriteTrace,
170    WriterFrame, WriterSender,
171};
172
173struct DecodedFrame {
174    frame: Frame,
175    phase_trace: PhaseTrace,
176}
177
178struct ToolCallCompletion {
179    text: String,
180    phase_trace: PhaseTrace,
181}
182
183#[derive(Clone)]
184struct PushSenders {
185    lossy_tx: mpsc::Sender<LossyPushEnvelope>,
186    reliable_tx: mpsc::UnboundedSender<PushEnvelope>,
187    lossy_overflow: Arc<push::LossyOverflow>,
188    lossy_seq: Arc<AtomicU64>,
189}
190
191#[derive(Clone)]
192struct PersistentCancelSignal {
193    inner: Arc<PersistentCancelInner>,
194}
195
196struct PersistentCancelInner {
197    cancelled: AtomicBool,
198    notify: Notify,
199}
200
201impl PersistentCancelSignal {
202    fn new() -> Self {
203        Self {
204            inner: Arc::new(PersistentCancelInner {
205                cancelled: AtomicBool::new(false),
206                notify: Notify::new(),
207            }),
208        }
209    }
210
211    fn cancel(&self) {
212        if !self.inner.cancelled.swap(true, Ordering::SeqCst) {
213            self.inner.notify.notify_waiters();
214        }
215    }
216
217    fn is_cancelled(&self) -> bool {
218        self.inner.cancelled.load(Ordering::SeqCst)
219    }
220
221    async fn cancelled(&self) {
222        // `enable()` REGISTERS this waiter before we read the flag, closing the
223        // lost-wakeup window: `notify_waiters()` only wakes already-registered
224        // waiters and stores no permit, so without enable() a `cancel()` firing
225        // between the flag read and `.await` would be missed and the future
226        // would park forever (cancel() fires only once). With enable(), a cancel
227        // racing the flag read still wakes the registered waiter. The loop is a
228        // belt-and-suspenders re-check on spurious wakeups.
229        loop {
230            let notified = self.inner.notify.notified();
231            tokio::pin!(notified);
232            notified.as_mut().enable();
233            if self.is_cancelled() {
234                return;
235            }
236            notified.await;
237        }
238    }
239}
240
241#[derive(Debug, Clone, Copy, PartialEq, Eq)]
242pub(crate) enum BindTrust {
243    FirstParty,
244    Untrusted,
245}
246
247impl BindTrust {
248    fn allows_bash_observation(self) -> bool {
249        matches!(self, Self::FirstParty)
250    }
251
252    fn label(self) -> &'static str {
253        match self {
254            Self::FirstParty => "first_party",
255            Self::Untrusted => "untrusted",
256        }
257    }
258}
259
260pub(super) fn trust_for_principal(principal: &Option<Principal>) -> BindTrust {
261    match principal {
262        Some(Principal::Direct) => BindTrust::FirstParty,
263        Some(Principal::Reserved { module_id })
264            if module_id == "llm-runner"
265                || module_id == "aft"
266                || module_id == "broca"
267                || module_id == "alfonso-core" =>
268        {
269            BindTrust::FirstParty
270        }
271        Some(Principal::Reserved { .. }) | Some(Principal::Unverified) | None => {
272            BindTrust::Untrusted
273        }
274    }
275}
276
277fn harness_forces_untrusted(harness: &str) -> bool {
278    harness.starts_with("fed:")
279}
280
281pub(super) fn trust_for_bind(harness: &str, principal: &Option<Principal>) -> BindTrust {
282    if harness_forces_untrusted(harness) {
283        BindTrust::Untrusted
284    } else {
285        trust_for_principal(principal)
286    }
287}
288
289fn principal_label(principal: &Option<Principal>) -> String {
290    match principal {
291        Some(Principal::Direct) => "direct".to_string(),
292        Some(Principal::Reserved { module_id }) => format!("reserved:{module_id}"),
293        Some(Principal::Unverified) => "unverified".to_string(),
294        None => "absent".to_string(),
295    }
296}
297
298#[derive(Debug)]
299/// Per-root route metadata owned by the subc loop. The `active_bash_waits` field
300/// counts detached bash processes that are still being observed for this root.
301/// Any future logic that evicts roots based on idle time must not evict a root
302/// while this count is greater than zero, because a foreground bash response may
303/// still arrive later.
304struct RootMeta {
305    maintenance_pending: bool,
306    maintenance_jobs_in_flight: usize,
307    maintenance_queued_kinds: VecDeque<MaintenanceDrainKind>,
308    maintenance_last_submitted: Option<Instant>,
309    maintenance_poisoned: bool,
310    last_touched: Instant,
311    diagnostics_on_edit: bool,
312    active_bash_waits: usize,
313    idle_artifacts_evicted: bool,
314    unbound_quiesced: bool,
315}
316
317#[derive(Debug)]
318struct PendingBind {
319    bind_root_id: ProjectRootId,
320    inserted_new_actor: bool,
321    cancelled: bool,
322    configure_request_id: String,
323    started_at: Instant,
324    warned_half_deadline: bool,
325    deadline_reported: bool,
326    corr: u64,
327    ver: u8,
328    flags: Flags,
329    /// Exact-job cancellation for the submitted configure: Goodbye and
330    /// deadline expiry cancel the executor job operationally (queued jobs are
331    /// removed, running configures return at their next checkpoint) instead of
332    /// only marking bookkeeping.
333    cancellation: crate::executor::JobCancellation,
334}
335
336struct RouteBindCompletion {
337    route: RouteChannel,
338    identity: RouteIdentity,
339    bind_root_id: ProjectRootId,
340    inserted_new_actor: bool,
341    configure_response: Response,
342    diagnostics_on_edit: bool,
343    ver: u8,
344    corr: u64,
345    flags: Flags,
346}
347
348#[derive(Debug, Clone)]
349struct RouteIdentity(Arc<RouteIdentityData>);
350
351#[derive(Debug)]
352struct RouteIdentityData {
353    root: ProjectRootId,
354    project_root: PathBuf,
355    harness: String,
356    session: String,
357    trust: BindTrust,
358    consumer_elicitation_capable: bool,
359}
360
361impl Deref for RouteIdentity {
362    type Target = RouteIdentityData;
363
364    fn deref(&self) -> &Self::Target {
365        &self.0
366    }
367}
368
369#[derive(Debug, Clone)]
370struct RetainedSessionIdentity {
371    harness: String,
372    trust: BindTrust,
373}
374
375#[derive(Clone, Copy)]
376struct BgSub {
377    corr: u64,
378    ver: u8,
379    flags: Flags,
380}
381
382struct MaintenanceCompletion {
383    root_id: ProjectRootId,
384    kind: MaintenanceDrainKind,
385    response: Response,
386    empty_bg_sessions: Vec<(String, u64)>,
387    requeue_kind: Option<MaintenanceDrainKind>,
388}
389
390#[derive(Clone, Copy, Debug, PartialEq, Eq)]
391enum MaintenanceDrainKind {
392    Watcher,
393    Lsp,
394    ConfigureTail,
395    CompletionDrains,
396}
397
398impl MaintenanceDrainKind {
399    fn label(self) -> &'static str {
400        match self {
401            Self::Watcher => "watcher",
402            Self::Lsp => "lsp",
403            Self::ConfigureTail => "configure-tail",
404            Self::CompletionDrains => "completion-drains",
405        }
406    }
407}
408
409#[derive(Debug, Default)]
410struct MaintenanceJobOutcome {
411    empty_bg_sessions: Vec<(String, u64)>,
412    requeue_kind: Option<MaintenanceDrainKind>,
413}
414
415#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
416struct ReverseCorrKey {
417    route: RouteChannel,
418    corr: u64,
419}
420
421struct PendingBashAsk {
422    route: RouteChannel,
423    tool_corr: u64,
424    tool_flags: Flags,
425    tool_ver: u8,
426    root: ProjectRootId,
427    project_root: PathBuf,
428    session_id: String,
429    request_id: String,
430    arguments: Value,
431    format_context: crate::subc_format::FormatContext,
432    cancel: bash::BashWaitCancel,
433    grants: Vec<String>,
434    expires_at: Instant,
435}
436
437impl RootMeta {
438    fn new(now: Instant) -> Self {
439        Self {
440            maintenance_pending: false,
441            maintenance_jobs_in_flight: 0,
442            maintenance_queued_kinds: VecDeque::new(),
443            maintenance_last_submitted: None,
444            maintenance_poisoned: false,
445            last_touched: now,
446            diagnostics_on_edit: false,
447            active_bash_waits: 0,
448            idle_artifacts_evicted: false,
449            unbound_quiesced: false,
450        }
451    }
452
453    fn note_activity(&mut self) {
454        self.last_touched = Instant::now();
455    }
456
457    fn reactivate_bound(&mut self) {
458        self.note_activity();
459        self.idle_artifacts_evicted = false;
460        self.unbound_quiesced = false;
461    }
462}
463
464fn due_maintenance_jobs(
465    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
466    executor: Option<&Executor>,
467    bg_sub_by_session: &HashMap<(ProjectRootId, String), RouteChannel>,
468    bg_wake_pending: &HashSet<RouteChannel>,
469    budget: usize,
470    pending_bind_roots: &HashSet<ProjectRootId>,
471) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
472    let mut jobs = Vec::new();
473    let mut deferred = false;
474    let mut roots = live_roots.keys().cloned().collect::<Vec<_>>();
475    roots.sort_by(|left, right| {
476        let left_last = live_roots
477            .get(left)
478            .and_then(|meta| meta.maintenance_last_submitted);
479        let right_last = live_roots
480            .get(right)
481            .and_then(|meta| meta.maintenance_last_submitted);
482        left_last
483            .cmp(&right_last)
484            .then_with(|| left.as_path().cmp(right.as_path()))
485    });
486
487    for root_id in roots {
488        let Some(meta) = live_roots.get_mut(&root_id) else {
489            continue;
490        };
491        if meta.maintenance_poisoned {
492            continue;
493        }
494
495        if pending_bind_roots.contains(&root_id) {
496            if meta.maintenance_pending || !meta.maintenance_queued_kinds.is_empty() {
497                deferred = true;
498            }
499            continue;
500        }
501
502        if !meta.maintenance_pending {
503            if jobs.len() >= budget {
504                deferred = true;
505                continue;
506            }
507            // Only enqueue kinds with pending work. Probes are cheap and
508            // fail-open (contended sources count as pending), so an idle root
509            // costs four probes per tick instead of four dispatched jobs.
510            let executor_actor_context =
511                executor.and_then(|executor| executor.actor_context(&root_id));
512            let root_has_pending_bg_wake =
513                bg_sub_by_session.iter().any(|((sub_root, _), channel)| {
514                    sub_root == &root_id && bg_wake_pending.contains(channel)
515                });
516            let kinds_with_work: Vec<MaintenanceDrainKind> = match executor_actor_context {
517                Some(ctx) => INITIAL_MAINTENANCE_DRAIN_KINDS
518                    .into_iter()
519                    .filter(|kind| {
520                        if meta.unbound_quiesced && !matches!(kind, MaintenanceDrainKind::Lsp) {
521                            return false;
522                        }
523                        match kind {
524                            MaintenanceDrainKind::Watcher => ctx.watcher_drain_has_work(),
525                            MaintenanceDrainKind::Lsp => ctx.lsp_drain_has_work(),
526                            MaintenanceDrainKind::ConfigureTail => ctx.configure_tail_has_work(),
527                            // Every CompletionDrains source is visible at this enqueue site:
528                            // AppContext probes completion queues, this loop owns bg wakes,
529                            // and queued continuations bypass probing via maintenance_pending.
530                            // New drain sources must expose a probe here rather than making
531                            // every subscribed root fail open again.
532                            MaintenanceDrainKind::CompletionDrains => {
533                                root_has_pending_bg_wake || ctx.completion_drains_have_work()
534                            }
535                        }
536                    })
537                    .collect(),
538                None if meta.unbound_quiesced => Vec::new(),
539                // No context handle (actor gone mid-tick): enqueue everything.
540                None => INITIAL_MAINTENANCE_DRAIN_KINDS.to_vec(),
541            };
542            if kinds_with_work.is_empty() {
543                continue;
544            }
545            meta.maintenance_pending = true;
546            meta.maintenance_queued_kinds.extend(kinds_with_work);
547        }
548
549        while let Some(kind) = meta.maintenance_queued_kinds.pop_front() {
550            if jobs.len() >= budget {
551                meta.maintenance_queued_kinds.push_front(kind);
552                deferred = true;
553                break;
554            }
555            meta.maintenance_jobs_in_flight += 1;
556            meta.maintenance_last_submitted = Some(Instant::now());
557            jobs.push((root_id.clone(), kind));
558        }
559
560        meta.maintenance_pending =
561            meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
562    }
563
564    (jobs, deferred)
565}
566
567fn eviction_estimate_label(estimate: &crate::memory::MemoryEstimate) -> String {
568    match estimate.estimated_bytes {
569        Some(bytes) => format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
570        None if estimate.status == "busy" => "busy".to_string(),
571        None => "not estimated".to_string(),
572    }
573}
574
575fn optional_memory_label(bytes: Option<u64>) -> String {
576    bytes.map_or_else(
577        || "not estimated".to_string(),
578        |bytes| format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
579    )
580}
581
582fn pressure_relief_label(relief: &crate::memory::AllocatorPressureRelief) -> String {
583    format!(
584        "; allocator pressure relief: RSS {} -> {}, in-use {} -> {}, allocated {} -> {}, slack {} -> {}, allocator reported {:.1} MB released",
585        optional_memory_label(relief.rss_before_bytes),
586        optional_memory_label(relief.rss_after_bytes),
587        optional_memory_label(relief.allocator_before.bytes_in_use),
588        optional_memory_label(relief.allocator_after.bytes_in_use),
589        optional_memory_label(relief.allocator_before.size_allocated),
590        optional_memory_label(relief.allocator_after.size_allocated),
591        optional_memory_label(relief.allocator_before.retained_slack_bytes),
592        optional_memory_label(relief.allocator_after.retained_slack_bytes),
593        relief.bytes_released as f64 / (1024.0 * 1024.0),
594    )
595}
596
597fn idle_root_eviction_message(
598    root_id: &ProjectRootId,
599    memory: &crate::memory::RootMemorySnapshot,
600    pressure_relief: Option<&crate::memory::AllocatorPressureRelief>,
601) -> String {
602    // Bash, LSP, and parser state remain resident. The freed total is deliberately
603    // only the known-byte portion of handles eviction actually drops.
604    let freed_bytes = [
605        &memory.semantic,
606        &memory.trigram,
607        &memory.symbols,
608        &memory.callgraph,
609        &memory.inspect,
610    ]
611    .iter()
612    .filter_map(|estimate| estimate.estimated_bytes)
613    .fold(0u64, u64::saturating_add);
614    let mut message = format!(
615        "evicted idle root {}: freed ~{:.1} MB (semantic {}, trigram {}, symbols {}, callgraph {}, inspect {}; retained: bash {}, lsp {}, parser_pool {})",
616        root_id.as_path().display(),
617        freed_bytes as f64 / (1024.0 * 1024.0),
618        eviction_estimate_label(&memory.semantic),
619        eviction_estimate_label(&memory.trigram),
620        eviction_estimate_label(&memory.symbols),
621        eviction_estimate_label(&memory.callgraph),
622        eviction_estimate_label(&memory.inspect),
623        eviction_estimate_label(&memory.bash),
624        eviction_estimate_label(&memory.lsp),
625        eviction_estimate_label(&memory.parser_pool),
626    );
627    if let Some(pressure_relief) = pressure_relief {
628        message.push_str(&pressure_relief_label(pressure_relief));
629    }
630    message
631}
632
633fn process_has_been_idle(now: Instant, live_roots: &HashMap<ProjectRootId, RootMeta>) -> bool {
634    !live_roots.is_empty()
635        && live_roots.values().all(|meta| {
636            now.saturating_duration_since(meta.last_touched) >= IDLE_ROOT_TTL
637                && meta.active_bash_waits == 0
638                && !meta.maintenance_pending
639                && meta.maintenance_queued_kinds.is_empty()
640        })
641}
642
643fn allocator_pressure_relief_after_idle_sweep(
644    now: Instant,
645    live_roots: &HashMap<ProjectRootId, RootMeta>,
646    executor: &Executor,
647) -> Option<crate::memory::AllocatorPressureRelief> {
648    if !process_has_been_idle(now, live_roots)
649        || live_roots.keys().any(|root_id| {
650            executor
651                .actor_context(root_id)
652                .is_some_and(|ctx| ctx.artifact_eviction_blocked())
653        })
654    {
655        return None;
656    }
657
658    #[cfg(target_os = "macos")]
659    {
660        Some(crate::memory::relieve_allocator_pressure())
661    }
662    #[cfg(not(target_os = "macos"))]
663    {
664        None
665    }
666}
667
668fn quiesce_unbound_root(
669    root_id: &ProjectRootId,
670    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
671    executor: &Arc<Executor>,
672) {
673    let Some(meta) = live_roots.get_mut(root_id) else {
674        return;
675    };
676
677    let ctx = executor.actor_context(root_id);
678    if let Some(ctx) = ctx.as_ref() {
679        // Close lifecycle admission before touching scheduler queues. A running
680        // ConfigureTail cannot release gates, install a watcher, or reserve a
681        // callgraph build after this transition becomes visible.
682        ctx.mark_subc_unbound();
683    }
684    let cancelled = executor.cancel_queued_maintenance(root_id);
685    // Transient unbind keeps the root WARM: the watcher stays running (its
686    // events accumulate and replay on rebind, so no unobserved gap exists) and
687    // resident artifacts stay resident. Host restarts unbind every root and
688    // rebind seconds later; stopping the watcher here would force strict
689    // re-verification plus a full callgraph rebuild on every restart. The
690    // expensive teardown (watcher stop + gap invalidation) belongs to the
691    // idle-TTL reaper and the root-deleted path.
692    let discarded = ctx
693        .map(|ctx| crate::commands::configure::cancel_deferred_configure_maintenance(&ctx))
694        .unwrap_or(0);
695    meta.unbound_quiesced = true;
696    meta.maintenance_queued_kinds.clear();
697    meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
698    log::debug!(
699        "subc attach: quiesced unbound root {} (cancelled {} queued maintenance job(s), cancelled {} configure maintenance job(s))",
700        root_id.as_path().display(),
701        cancelled,
702        discarded
703    );
704}
705
706#[derive(Debug, Default)]
707struct IdleReapOutcome {
708    evicted: usize,
709    forgotten_deleted_roots: Vec<ProjectRootId>,
710}
711
712fn reap_idle_roots(
713    now: Instant,
714    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
715    pending_binds: &HashMap<RouteChannel, PendingBind>,
716    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
717    executor: &Arc<Executor>,
718) -> IdleReapOutcome {
719    let pending_bind_roots = pending_binds
720        .values()
721        .map(|pending| pending.bind_root_id.clone())
722        .collect::<HashSet<_>>();
723    let candidates = live_roots
724        .iter()
725        .filter_map(|(root_id, meta)| {
726            let deleted = !root_id.as_path().exists();
727            let has_bound_route = root_channels
728                .get(root_id)
729                .is_some_and(|channels| !channels.is_empty());
730            // Route teardown marks the lifecycle admission gate before the last
731            // channel disappears. Requiring zero bound channels and a quiesced
732            // lifecycle prevents a still-bound root from losing its watcher.
733            if has_bound_route
734                || !meta.unbound_quiesced
735                || (!deleted
736                    && (meta.idle_artifacts_evicted
737                        || now.saturating_duration_since(meta.last_touched) < IDLE_ROOT_TTL))
738                || meta.active_bash_waits > 0
739                || meta.maintenance_pending
740                || !meta.maintenance_queued_kinds.is_empty()
741                || pending_bind_roots.contains(root_id)
742                || !executor.actor_is_idle(root_id)
743            {
744                return None;
745            }
746            Some((root_id.clone(), deleted))
747        })
748        .collect::<Vec<_>>();
749
750    let mut reaped = Vec::new();
751    let mut forgotten_deleted_roots = Vec::new();
752    for (root_id, deleted) in candidates {
753        let Some(ctx) = executor.actor_context(&root_id) else {
754            continue;
755        };
756        // A TTL-aged unbound root retained its watcher-derived pending paths
757        // across the transient-unbind window. Strict gap invalidation subsumes
758        // them, but every abort path must restore them because a rebind can
759        // still happen until eviction commits.
760        let taken_pending = Some(ctx.take_pending_reconciliation_state());
761        if ctx.artifact_eviction_blocked() {
762            if let Some(pending) = taken_pending {
763                ctx.restore_pending_reconciliation_state(pending);
764            }
765            continue;
766        }
767        let memory_before = ctx.memory_root_snapshot();
768        if !ctx.evict_idle_artifacts() {
769            if let Some(pending) = taken_pending {
770                ctx.restore_pending_reconciliation_state(pending);
771            }
772            continue;
773        }
774        drop(taken_pending);
775        ctx.stop_watcher_runtime_in_background();
776        // Edits during watcher downtime are unobserved. Advance publication
777        // epochs and force strict verification before any later warm reload.
778        ctx.invalidate_artifacts_after_watcher_gap();
779
780        if deleted {
781            if executor.retire_idle_actor_in_background(&root_id) {
782                live_roots.remove(&root_id);
783                forgotten_deleted_roots.push(root_id.clone());
784            }
785        } else {
786            if let Some(meta) = live_roots.get_mut(&root_id) {
787                meta.idle_artifacts_evicted = true;
788            }
789            ctx.release_idle_reopenable_resources_in_background();
790        }
791        reaped.push((root_id, memory_before));
792    }
793
794    let pressure_relief = (!reaped.is_empty())
795        .then(|| allocator_pressure_relief_after_idle_sweep(now, live_roots, executor))
796        .flatten();
797    for (root_id, memory_before) in &reaped {
798        log::info!(
799            "{}",
800            idle_root_eviction_message(root_id, memory_before, pressure_relief.as_ref())
801        );
802    }
803    IdleReapOutcome {
804        evicted: reaped.len(),
805        forgotten_deleted_roots,
806    }
807}
808
809#[allow(clippy::too_many_arguments)]
810fn purge_deleted_root_residents(
811    root_id: &ProjectRootId,
812    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
813    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
814    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
815    bg_subs: &mut HashMap<RouteChannel, BgSub>,
816    bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
817    bg_wake_pending: &mut HashSet<RouteChannel>,
818    bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
819    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
820) {
821    root_channels.remove(root_id);
822    session_identity.retain(|(root, _), _| root != root_id);
823    push_buffer.retain(|key, _| &key.root != root_id);
824    bg_wake_epoch.retain(|(root, _), _| root != root_id);
825    pending_bash_asks.retain(|_, ask| &ask.root != root_id);
826
827    let stale_subscriptions = bg_sub_by_session
828        .iter()
829        .filter_map(|((root, _), channel)| (root == root_id).then_some(*channel))
830        .collect::<Vec<_>>();
831    bg_sub_by_session.retain(|(root, _), _| root != root_id);
832    for channel in stale_subscriptions {
833        bg_subs.remove(&channel);
834        bg_wake_pending.remove(&channel);
835    }
836    log::info!(
837        "subc attach: fully forgot deleted root {}",
838        root_id.as_path().display()
839    );
840}
841
842#[allow(clippy::too_many_arguments)]
843fn submit_due_maintenance_jobs(
844    executor: &Arc<Executor>,
845    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
846    pending_binds: &HashMap<RouteChannel, PendingBind>,
847    bg_sub_by_session: &HashMap<(ProjectRootId, String), RouteChannel>,
848    bg_wake_pending: &HashSet<RouteChannel>,
849    bg_wake_epoch: &HashMap<(ProjectRootId, String), u64>,
850    maintenance_tx: &mpsc::Sender<MaintenanceCompletion>,
851    metrics: &Arc<DispatchPathMetrics>,
852) {
853    let pending_bind_roots = pending_binds
854        .values()
855        .map(|pending| pending.bind_root_id.clone())
856        .collect::<HashSet<_>>();
857    let (due_jobs, deferred_jobs) = due_maintenance_jobs(
858        live_roots,
859        Some(executor),
860        bg_sub_by_session,
861        bg_wake_pending,
862        MAINTENANCE_SUBMIT_BUDGET,
863        &pending_bind_roots,
864    );
865    if deferred_jobs {
866        metrics
867            .maintenance_budget_deferrals
868            .fetch_add(1, Ordering::Relaxed);
869    }
870    for (root_id, kind) in due_jobs {
871        let bg_sessions_to_check = if kind == MaintenanceDrainKind::CompletionDrains {
872            bg_sub_by_session
873                .iter()
874                .filter_map(|((root, session), _)| {
875                    if root == &root_id {
876                        Some((
877                            session.clone(),
878                            bg_wake_epoch
879                                .get(&(root_id.clone(), session.clone()))
880                                .copied()
881                                .unwrap_or(0),
882                        ))
883                    } else {
884                        None
885                    }
886                })
887                .collect()
888        } else {
889            Vec::new()
890        };
891        submit_maintenance_job(
892            executor,
893            root_id,
894            kind,
895            bg_sessions_to_check,
896            maintenance_tx,
897            metrics,
898        );
899    }
900}
901
902fn should_requiesce_after_maintenance(
903    meta: &RootMeta,
904    completed_kind: MaintenanceDrainKind,
905    bind_pending: bool,
906) -> bool {
907    meta.unbound_quiesced && completed_kind != MaintenanceDrainKind::Lsp && !bind_pending
908}
909
910fn note_maintenance_completion(
911    meta: &mut RootMeta,
912    requeue_kind: Option<MaintenanceDrainKind>,
913    fatal: bool,
914    defer_requeue: bool,
915) {
916    if fatal {
917        meta.maintenance_poisoned = true;
918    }
919
920    if let Some(kind) = requeue_kind.filter(|_| !meta.maintenance_poisoned && !defer_requeue) {
921        meta.maintenance_queued_kinds.push_back(kind);
922    }
923
924    meta.maintenance_jobs_in_flight = meta.maintenance_jobs_in_flight.saturating_sub(1);
925    meta.maintenance_pending =
926        meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
927}
928
929fn route_key(channel: u16, epoch: u32) -> RouteChannel {
930    RouteChannel { channel, epoch }
931}
932
933fn remove_installed_route(installed_epochs: &mut HashMap<u16, u32>, route: RouteChannel) {
934    if installed_epochs.get(&route.channel).copied() == Some(route.epoch) {
935        installed_epochs.remove(&route.channel);
936    }
937}
938
939fn ingress_route_is_current(installed_epochs: &HashMap<u16, u32>, frame: &Frame) -> bool {
940    frame.header.channel == 0
941        || installed_epochs.get(&frame.header.channel).copied() == Some(frame.header.epoch)
942}
943
944fn bash_elicitation_timeout() -> Duration {
945    if cfg!(debug_assertions) {
946        if let Ok(raw) = std::env::var("AFT_TEST_SUBC_BASH_ELICITATION_TTL_MS") {
947            if let Ok(ms) = raw.parse::<u64>() {
948                if ms > 0 {
949                    return Duration::from_millis(ms);
950                }
951            }
952        }
953    }
954    BASH_ELICITATION_TIMEOUT
955}
956
957fn allocate_reverse_corr(
958    pending_bash_asks: &HashMap<ReverseCorrKey, PendingBashAsk>,
959    route: RouteChannel,
960    next_corr: &mut u64,
961) -> u64 {
962    loop {
963        let corr = *next_corr;
964        *next_corr = (*next_corr).wrapping_add(1).max(1);
965        if !pending_bash_asks.contains_key(&ReverseCorrKey { route, corr }) {
966            return corr;
967        }
968    }
969}
970
971fn bash_permission_kind_label(kind: &crate::bash_permissions::PermissionKind) -> &'static str {
972    match kind {
973        crate::bash_permissions::PermissionKind::ExternalDirectory => "external directory",
974        crate::bash_permissions::PermissionKind::Bash => "bash",
975    }
976}
977
978fn bash_elicitation_patterns(asks: &[crate::bash_permissions::PermissionAsk]) -> Vec<String> {
979    let mut patterns = Vec::new();
980    let mut seen = HashSet::new();
981    for ask in asks {
982        for pattern in ask.patterns.iter().chain(ask.always.iter()) {
983            if seen.insert(pattern.clone()) {
984                patterns.push(pattern.clone());
985            }
986        }
987    }
988    patterns
989}
990
991fn bash_elicitation_message(
992    command: &str,
993    asks: &[crate::bash_permissions::PermissionAsk],
994) -> String {
995    let command = command.split_whitespace().collect::<Vec<_>>().join(" ");
996    let patterns = bash_elicitation_patterns(asks);
997    let pattern_text = if patterns.is_empty() {
998        "no matched permission patterns".to_string()
999    } else {
1000        patterns.join(", ")
1001    };
1002    let ask_kinds = asks
1003        .iter()
1004        .map(|ask| bash_permission_kind_label(&ask.kind))
1005        .collect::<HashSet<_>>()
1006        .into_iter()
1007        .collect::<Vec<_>>()
1008        .join(", ");
1009    if ask_kinds.is_empty() {
1010        format!("Allow bash command `{command}`? Matched patterns: {pattern_text}")
1011    } else {
1012        format!("Allow bash command `{command}`? Matched {ask_kinds} patterns: {pattern_text}")
1013    }
1014}
1015
1016fn bash_elicitation_request_body(
1017    command: &str,
1018    asks: &[crate::bash_permissions::PermissionAsk],
1019) -> Value {
1020    json!({
1021        "method": BASH_ELICITATION_CREATE_METHOD,
1022        "params": {
1023            "mode": "form",
1024            "message": bash_elicitation_message(command, asks),
1025            "requestedSchema": {
1026                "type": "object",
1027                "properties": {
1028                    "decision": {
1029                        "type": "string",
1030                        "enum": ["allow", "deny"],
1031                        "description": "Choose allow to run this bash command once, or deny to block it."
1032                    }
1033                },
1034                "required": ["decision"],
1035                "additionalProperties": false
1036            },
1037            "_meta": {
1038                "aft": {
1039                    "tool": "bash",
1040                    "command": command,
1041                    "asks": asks
1042                }
1043            }
1044        }
1045    })
1046}
1047
1048fn build_bash_elicitation_request_frame(
1049    ver: u8,
1050    route: RouteChannel,
1051    corr: u64,
1052    flags: Flags,
1053    command: &str,
1054    asks: &[crate::bash_permissions::PermissionAsk],
1055) -> Result<Frame, SubcError> {
1056    let body = bash_elicitation_request_body(command, asks);
1057    Frame::build_with_version(
1058        ver,
1059        FrameType::Request,
1060        flags,
1061        route.channel,
1062        route.epoch,
1063        corr,
1064        serde_json::to_vec(&body).map_err(SubcError::Json)?,
1065    )
1066    .map_err(SubcError::FrameBuild)
1067}
1068
1069fn bash_elicitation_reply_is_allow(body: &[u8]) -> bool {
1070    let Ok(value) = serde_json::from_slice::<Value>(body) else {
1071        return false;
1072    };
1073    flat_bash_elicitation_reply_is_allow(&value) || mcp_bash_elicitation_reply_is_allow(&value)
1074}
1075
1076fn flat_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1077    let Some(object) = value.as_object() else {
1078        return false;
1079    };
1080    object.len() == 1 && object.get("decision").and_then(Value::as_str) == Some("allow")
1081}
1082
1083fn mcp_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1084    let Some(object) = value.as_object() else {
1085        return false;
1086    };
1087    if object.len() != 2 || object.get("action").and_then(Value::as_str) != Some("accept") {
1088        return false;
1089    }
1090    let Some(content) = object.get("content").and_then(Value::as_object) else {
1091        return false;
1092    };
1093    content.len() == 1 && content.get("decision").and_then(Value::as_str) == Some("allow")
1094}
1095
1096#[allow(clippy::too_many_arguments)]
1097async fn settle_pending_bash_ask_denied(
1098    tx: &WriterSender,
1099    pending: PendingBashAsk,
1100    routes: &HashMap<RouteChannel, RouteIdentity>,
1101    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1102    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1103    shutdown: &Arc<Notify>,
1104    metrics: &DispatchPathMetrics,
1105) -> Result<(), SubcError> {
1106    let completion = bash::bash_denied_untrusted_completion(
1107        pending.route,
1108        pending.tool_corr,
1109        pending.tool_flags,
1110        pending.tool_ver,
1111        pending.root,
1112        pending.request_id,
1113        pending.format_context,
1114    );
1115    bash::handle_bash_deferred_completion(
1116        tx,
1117        completion,
1118        routes,
1119        live_roots,
1120        route_bash_cancels,
1121        shutdown,
1122        metrics,
1123    )
1124    .await
1125}
1126
1127fn take_pending_bash_asks_for_route(
1128    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1129    route: RouteChannel,
1130) -> Vec<PendingBashAsk> {
1131    let keys = pending_bash_asks
1132        .keys()
1133        .copied()
1134        .filter(|key| key.route == route)
1135        .collect::<Vec<_>>();
1136    keys.into_iter()
1137        .filter_map(|key| pending_bash_asks.remove(&key))
1138        .collect()
1139}
1140
1141#[allow(clippy::too_many_arguments)]
1142async fn settle_pending_bash_asks_for_route(
1143    tx: &WriterSender,
1144    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1145    route: RouteChannel,
1146    routes: &HashMap<RouteChannel, RouteIdentity>,
1147    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1148    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1149    shutdown: &Arc<Notify>,
1150    metrics: &DispatchPathMetrics,
1151) -> Result<(), SubcError> {
1152    for pending in take_pending_bash_asks_for_route(pending_bash_asks, route) {
1153        settle_pending_bash_ask_denied(
1154            tx,
1155            pending,
1156            routes,
1157            live_roots,
1158            route_bash_cancels,
1159            shutdown,
1160            metrics,
1161        )
1162        .await?;
1163    }
1164    Ok(())
1165}
1166
1167#[allow(clippy::too_many_arguments)]
1168async fn settle_all_pending_bash_asks(
1169    tx: &WriterSender,
1170    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1171    routes: &HashMap<RouteChannel, RouteIdentity>,
1172    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1173    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1174    shutdown: &Arc<Notify>,
1175    metrics: &DispatchPathMetrics,
1176) -> Result<(), SubcError> {
1177    let pending = pending_bash_asks
1178        .drain()
1179        .map(|(_, pending)| pending)
1180        .collect::<Vec<_>>();
1181    for pending in pending {
1182        settle_pending_bash_ask_denied(
1183            tx,
1184            pending,
1185            routes,
1186            live_roots,
1187            route_bash_cancels,
1188            shutdown,
1189            metrics,
1190        )
1191        .await?;
1192    }
1193    Ok(())
1194}
1195
1196#[allow(clippy::too_many_arguments)]
1197async fn expire_pending_bash_asks(
1198    tx: &WriterSender,
1199    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1200    routes: &HashMap<RouteChannel, RouteIdentity>,
1201    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1202    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1203    shutdown: &Arc<Notify>,
1204    metrics: &DispatchPathMetrics,
1205) -> Result<(), SubcError> {
1206    let now = Instant::now();
1207    let expired = pending_bash_asks
1208        .iter()
1209        .filter_map(|(key, pending)| (pending.expires_at <= now).then_some(*key))
1210        .collect::<Vec<_>>();
1211    for key in expired {
1212        if let Some(pending) = pending_bash_asks.remove(&key) {
1213            log::debug!(
1214                "subc attach: bash elicitation request {} on route {} expired fail-closed",
1215                key.corr,
1216                pending.route
1217            );
1218            settle_pending_bash_ask_denied(
1219                tx,
1220                pending,
1221                routes,
1222                live_roots,
1223                route_bash_cancels,
1224                shutdown,
1225                metrics,
1226            )
1227            .await?;
1228        }
1229    }
1230    Ok(())
1231}
1232
1233#[allow(clippy::too_many_arguments)]
1234async fn handle_bash_elicitation_reply(
1235    tx: &WriterSender,
1236    frame: &Frame,
1237    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1238    routes: &HashMap<RouteChannel, RouteIdentity>,
1239    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1240    executor: &Arc<Executor>,
1241    shutdown: &Arc<Notify>,
1242    bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
1243    bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
1244    metrics: &Arc<DispatchPathMetrics>,
1245    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1246    dispatch: DispatchFn,
1247) -> Result<(), SubcError> {
1248    let key = ReverseCorrKey {
1249        route: route_key(frame.header.channel, frame.header.epoch),
1250        corr: frame.header.corr,
1251    };
1252    let Some(pending) = pending_bash_asks.remove(&key) else {
1253        return Ok(());
1254    };
1255
1256    if frame.header.ty == FrameType::Response && bash_elicitation_reply_is_allow(&frame.body) {
1257        if routes.contains_key(&key.route) {
1258            bash::submit_deferred_bash(
1259                executor,
1260                bash_deferred_tx,
1261                bash_poll_touch_tx,
1262                metrics,
1263                dispatch,
1264                pending.root,
1265                pending.project_root,
1266                pending.session_id,
1267                pending.request_id,
1268                pending.route,
1269                pending.tool_corr,
1270                pending.tool_flags,
1271                pending.tool_ver,
1272                pending.arguments,
1273                pending.format_context,
1274                pending.cancel,
1275                BindTrust::Untrusted,
1276                Some(pending.grants),
1277            );
1278            return Ok(());
1279        }
1280        log::debug!(
1281            "subc attach: dropping allowed bash elicitation reply {} for unbound route {}",
1282            key.corr,
1283            pending.route
1284        );
1285    }
1286
1287    settle_pending_bash_ask_denied(
1288        tx,
1289        pending,
1290        routes,
1291        live_roots,
1292        route_bash_cancels,
1293        shutdown,
1294        metrics,
1295    )
1296    .await
1297}
1298
1299#[allow(clippy::too_many_arguments)]
1300async fn cancel_pending_bash_ask_for_tool_call(
1301    tx: &WriterSender,
1302    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1303    route: RouteChannel,
1304    tool_corr: u64,
1305    routes: &HashMap<RouteChannel, RouteIdentity>,
1306    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1307    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1308    shutdown: &Arc<Notify>,
1309    metrics: &DispatchPathMetrics,
1310) -> Result<(), SubcError> {
1311    let keys = pending_bash_asks
1312        .iter()
1313        .filter_map(|(key, pending)| {
1314            (key.route == route && pending.tool_corr == tool_corr).then_some(*key)
1315        })
1316        .collect::<Vec<_>>();
1317    for key in keys {
1318        if let Some(pending) = pending_bash_asks.remove(&key) {
1319            settle_pending_bash_ask_denied(
1320                tx,
1321                pending,
1322                routes,
1323                live_roots,
1324                route_bash_cancels,
1325                shutdown,
1326                metrics,
1327            )
1328            .await?;
1329        }
1330    }
1331    Ok(())
1332}
1333
1334fn remove_root_channel(
1335    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1336    root: &ProjectRootId,
1337    channel: RouteChannel,
1338) {
1339    let remove_root = if let Some(channels) = root_channels.get_mut(root) {
1340        channels.remove(&channel);
1341        channels.is_empty()
1342    } else {
1343        false
1344    };
1345    if remove_root {
1346        root_channels.remove(root);
1347    }
1348}
1349
1350fn remove_route_channel(
1351    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1352    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1353    channel: RouteChannel,
1354) -> Option<RouteIdentity> {
1355    let removed = routes.remove(&channel);
1356    if let Some(identity) = &removed {
1357        remove_root_channel(root_channels, &identity.root, channel);
1358    }
1359    removed
1360}
1361
1362fn insert_route_channel(
1363    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1364    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1365    channel: RouteChannel,
1366    identity: RouteIdentity,
1367) {
1368    if let Some(previous) = routes.insert(channel, identity.clone()) {
1369        remove_root_channel(root_channels, &previous.root, channel);
1370    }
1371    root_channels
1372        .entry(identity.root.clone())
1373        .or_default()
1374        .insert(channel);
1375}
1376
1377fn remove_bg_subscription_index(
1378    bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1379    channel: RouteChannel,
1380    identity: Option<&RouteIdentity>,
1381) {
1382    if let Some(identity) = identity {
1383        let key = (identity.root.clone(), identity.session.clone());
1384        if bg_sub_by_session.get(&key).copied() == Some(channel) {
1385            bg_sub_by_session.remove(&key);
1386        }
1387    } else {
1388        bg_sub_by_session.retain(|_, mapped_channel| *mapped_channel != channel);
1389    }
1390}
1391
1392fn route_removal_will_quiesce_root(
1393    root: &ProjectRootId,
1394    route: RouteChannel,
1395    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1396    has_pending_bind: bool,
1397    replacement_root: Option<&ProjectRootId>,
1398) -> bool {
1399    let removes_last_route = root_channels
1400        .get(root)
1401        .is_some_and(|channels| channels.len() == 1 && channels.contains(&route));
1402    removes_last_route && !has_pending_bind && replacement_root != Some(root)
1403}
1404
1405fn should_quiesce_removed_root(
1406    root: &ProjectRootId,
1407    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1408    has_pending_bind: bool,
1409    replacement_root: Option<&ProjectRootId>,
1410) -> bool {
1411    !root_channels.contains_key(root) && !has_pending_bind && replacement_root != Some(root)
1412}
1413
1414async fn end_bg_subscription(
1415    writer_tx: &WriterSender,
1416    metrics: &DispatchPathMetrics,
1417    bg_subs: &mut HashMap<RouteChannel, BgSub>,
1418    bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1419    bg_wake_pending: &mut HashSet<RouteChannel>,
1420    channel: RouteChannel,
1421    identity: Option<&RouteIdentity>,
1422) -> Result<(), SubcError> {
1423    if let Some(sub) = bg_subs.remove(&channel) {
1424        bg_wake_pending.remove(&channel);
1425        remove_bg_subscription_index(bg_sub_by_session, channel, identity);
1426        push::send_reliable_bg_stream_end(writer_tx, metrics, channel, &sub).await?;
1427    }
1428    Ok(())
1429}
1430
1431#[allow(clippy::too_many_arguments)]
1432async fn teardown_installed_route(
1433    tx: &WriterSender,
1434    metrics: &DispatchPathMetrics,
1435    executor: &Arc<Executor>,
1436    channel: RouteChannel,
1437    cancellation_reason: &str,
1438    replacement_root: Option<&ProjectRootId>,
1439    installed_route_epochs: &mut HashMap<u16, u32>,
1440    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1441    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1442    bg_subs: &mut HashMap<RouteChannel, BgSub>,
1443    bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1444    bg_wake_pending: &mut HashSet<RouteChannel>,
1445    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1446    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1447    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1448    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
1449    retry_buffer: &mut RetryBuffer,
1450    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1451    shutdown: &Arc<Notify>,
1452) -> Result<(), SubcError> {
1453    remove_installed_route(installed_route_epochs, channel);
1454    end_bg_subscription(
1455        tx,
1456        metrics,
1457        bg_subs,
1458        bg_sub_by_session,
1459        bg_wake_pending,
1460        channel,
1461        routes.get(&channel),
1462    )
1463    .await?;
1464    settle_pending_bash_asks_for_route(
1465        tx,
1466        pending_bash_asks,
1467        channel,
1468        routes,
1469        live_roots,
1470        route_bash_cancels,
1471        shutdown,
1472        metrics,
1473    )
1474    .await?;
1475    if let Some(cancel) = route_bash_cancels.remove(&channel) {
1476        cancel.token.cancel();
1477    }
1478    if let Some(pending) = pending_binds.get_mut(&channel) {
1479        pending.cancelled = true;
1480        let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
1481        log::debug!(
1482            "subc attach: cancelled pending RouteBind for route {} on {cancellation_reason} (configure job: {outcome:?})",
1483            channel.channel
1484        );
1485    }
1486    let migrated = push::migrate_retry_buffer_to_push_buffer(retry_buffer, channel, push_buffer);
1487    if let Some(identity) = routes.get(&channel) {
1488        let has_pending_bind = pending_binds
1489            .values()
1490            .any(|pending| pending.bind_root_id == identity.root);
1491        if route_removal_will_quiesce_root(
1492            &identity.root,
1493            channel,
1494            root_channels,
1495            has_pending_bind,
1496            replacement_root,
1497        ) {
1498            if let Some(ctx) = executor.actor_context(&identity.root) {
1499                // Fence deferred admissions before the final route disappears
1500                // from the loop-owned routing tables.
1501                ctx.mark_subc_unbound();
1502            }
1503        }
1504    }
1505    if let Some(identity) = remove_route_channel(routes, root_channels, channel) {
1506        if migrated > 0 {
1507            log::debug!(
1508                "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from route {} into detach replay",
1509                channel.channel
1510            );
1511        }
1512        if let Some(meta) = live_roots.get_mut(&identity.root) {
1513            let idle_for = meta.last_touched.elapsed();
1514            meta.note_activity();
1515            log::debug!(
1516                "subc attach: route {} torn down for root {} harness {} session {} (last touched {:?} ago)",
1517                channel.channel,
1518                identity.root.as_path().display(),
1519                identity.harness,
1520                identity.session,
1521                idle_for
1522            );
1523        } else {
1524            log::debug!(
1525                "subc attach: route {} torn down for root {} harness {} session {}",
1526                channel.channel,
1527                identity.root.as_path().display(),
1528                identity.harness,
1529                identity.session
1530            );
1531        }
1532        let has_pending_bind = pending_binds
1533            .values()
1534            .any(|pending| pending.bind_root_id == identity.root);
1535        if should_quiesce_removed_root(
1536            &identity.root,
1537            root_channels,
1538            has_pending_bind,
1539            replacement_root,
1540        ) {
1541            quiesce_unbound_root(&identity.root, live_roots, executor);
1542        }
1543    } else {
1544        if migrated > 0 {
1545            log::debug!(
1546                "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from unbound route {} into detach replay",
1547                channel.channel
1548            );
1549        }
1550        log::debug!("subc attach: unbound route {} torn down", channel.channel);
1551    }
1552    Ok(())
1553}
1554
1555fn remember_session_identity(
1556    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1557    identity: &RouteIdentity,
1558) {
1559    let key = (identity.root.clone(), identity.session.clone());
1560    if matches!(identity.trust, BindTrust::Untrusted)
1561        && session_identity
1562            .get(&key)
1563            .is_some_and(|retained| matches!(retained.trust, BindTrust::FirstParty))
1564    {
1565        return;
1566    }
1567
1568    // Retained after route Goodbye so reliable session-scoped frames emitted while
1569    // the session is detached can still be keyed by the full (root,harness,session)
1570    // replay triple. Untrusted binds never overwrite a retained first-party
1571    // session identity, because bash completion replay is an observation channel.
1572    session_identity.insert(
1573        key,
1574        RetainedSessionIdentity {
1575            harness: identity.harness.clone(),
1576            trust: identity.trust,
1577        },
1578    );
1579}
1580
1581fn replay_key_for_session(
1582    session_identity: &HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1583    root: &ProjectRootId,
1584    session: &str,
1585) -> Option<(push::ReplayKey, BindTrust)> {
1586    let retained = session_identity.get(&(root.clone(), session.to_string()))?;
1587    Some((
1588        push::ReplayKey {
1589            root: root.clone(),
1590            harness: retained.harness.clone(),
1591            session: session.to_string(),
1592        },
1593        retained.trust,
1594    ))
1595}
1596/// Sync command dispatch, passed in from `main` (the binary owns the command
1597/// table). Invoked only inside executor jobs in subc mode.
1598pub type DispatchFn = fn(RawRequest, &AppContext) -> Response;
1599
1600#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1601enum ModuleLoopExit {
1602    Graceful,
1603    SkipSearchFlush,
1604}
1605
1606/// Entry point for `aft --subc <connection-file>`. Synchronous on the outside;
1607/// owns an isolated current-thread tokio runtime for the async transport.
1608/// Returns `Err` (fail-loud) on any connect/auth/protocol failure — we never
1609/// fall back to the standalone loop, to avoid split-brain index state.
1610pub fn run_subc_mode(
1611    connection_file_path: &Path,
1612    ctx: Arc<AppContext>,
1613    executor: Arc<Executor>,
1614    dispatch: DispatchFn,
1615    user_config_path: Option<PathBuf>,
1616) -> Result<(), SubcError> {
1617    // Production NEVER allows non-manifest tool names on route channels: AFT
1618    // fails closed and does not trust subc to enforce the manifest. The
1619    // test-only harness sets this through `run_subc_mode_for_test`.
1620    run_subc_mode_inner(
1621        connection_file_path,
1622        ctx,
1623        executor,
1624        dispatch,
1625        user_config_path,
1626        false,
1627    )
1628}
1629
1630fn run_subc_mode_inner(
1631    connection_file_path: &Path,
1632    ctx: Arc<AppContext>,
1633    executor: Arc<Executor>,
1634    dispatch: DispatchFn,
1635    user_config_path: Option<PathBuf>,
1636    allow_native_passthrough: bool,
1637) -> Result<(), SubcError> {
1638    let runtime = tokio::runtime::Builder::new_current_thread()
1639        .enable_all()
1640        .build()
1641        .map_err(SubcError::Runtime)?;
1642
1643    let executor_for_loop = Arc::clone(&executor);
1644    let loop_result = runtime.block_on(async move {
1645        let shared_app = ctx.app();
1646        drop(ctx);
1647        let stream = connect_and_authenticate(connection_file_path).await?;
1648        log::info!(
1649            "subc attach: authenticated to daemon via {}",
1650            connection_file_path.display()
1651        );
1652        let (read_half, write_half) = tokio::io::split(stream);
1653        run_module_loop(
1654            read_half,
1655            write_half,
1656            shared_app,
1657            executor_for_loop,
1658            dispatch,
1659            user_config_path,
1660            allow_native_passthrough,
1661        )
1662        .await
1663    });
1664
1665    let actor_contexts = executor.actor_contexts();
1666    if matches!(loop_result, Ok(ModuleLoopExit::Graceful)) {
1667        // EOF/Goodbye teardown flushes each root's index deltas and queued
1668        // callgraph refreshes. Fatal/panic teardown skips this best-effort work.
1669        flush_actor_indexes_on_graceful_shutdown(&actor_contexts);
1670    }
1671    for actor_ctx in &actor_contexts {
1672        actor_ctx.lsp().shutdown_all();
1673        actor_ctx.bash_background().detach();
1674    }
1675
1676    loop_result.map(|_| ())
1677}
1678
1679fn flush_actor_indexes_on_graceful_shutdown(actor_contexts: &[Arc<AppContext>]) {
1680    for actor_ctx in actor_contexts {
1681        let _ = actor_ctx.flush_search_index_on_graceful_shutdown();
1682    }
1683    let _ = crate::callgraph_store::flush_callgraph_store_refreshes_on_graceful_shutdown();
1684}
1685
1686/// Test-only entry that enables the non-manifest native-command passthrough on
1687/// route channels. Integration tests drive synthetic native commands (`glob`,
1688/// `callers`, `subc_test_echo_session`, …) through the executor to exercise
1689/// mechanics; production callers use [`run_subc_mode`], which fails closed.
1690#[doc(hidden)]
1691pub fn run_subc_mode_for_test(
1692    connection_file_path: &Path,
1693    ctx: Arc<AppContext>,
1694    executor: Arc<Executor>,
1695    dispatch: DispatchFn,
1696    user_config_path: Option<PathBuf>,
1697) -> Result<(), SubcError> {
1698    run_subc_mode_inner(
1699        connection_file_path,
1700        ctx,
1701        executor,
1702        dispatch,
1703        user_config_path,
1704        true,
1705    )
1706}
1707
1708#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1709enum AttachErrorClass {
1710    Transient,
1711    Permanent,
1712}
1713
1714impl fmt::Display for AttachErrorClass {
1715    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1716        match self {
1717            Self::Transient => f.write_str("transient"),
1718            Self::Permanent => f.write_str("permanent"),
1719        }
1720    }
1721}
1722
1723#[derive(Clone, Copy)]
1724struct AttachRetryPolicy {
1725    budget: Duration,
1726    initial_backoff: Duration,
1727    max_backoff: Duration,
1728    jitter_percent: u64,
1729}
1730
1731const ATTACH_RETRY_POLICY: AttachRetryPolicy = AttachRetryPolicy {
1732    budget: ATTACH_RETRY_BUDGET,
1733    initial_backoff: ATTACH_RETRY_INITIAL_BACKOFF,
1734    max_backoff: ATTACH_RETRY_MAX_BACKOFF,
1735    jitter_percent: ATTACH_RETRY_JITTER_PERCENT,
1736};
1737
1738/// Retry only failures that can be caused by a daemon bounce or an interrupted
1739/// handshake. Protocol and credential failures are permanent for this process.
1740fn classify_attach_error(error: &SubcError) -> AttachErrorClass {
1741    let transient = match error {
1742        SubcError::Connect { source, .. } => is_transient_attach_io(source.kind()),
1743        SubcError::Auth { source, .. } => match source {
1744            subc_transport::AuthError::Timeout { .. }
1745            | subc_transport::AuthError::UnexpectedEof { .. } => true,
1746            subc_transport::AuthError::Io { source, .. } => is_transient_attach_io(source.kind()),
1747            _ => false,
1748        },
1749        _ => false,
1750    };
1751    if transient {
1752        AttachErrorClass::Transient
1753    } else {
1754        AttachErrorClass::Permanent
1755    }
1756}
1757
1758fn is_transient_attach_io(kind: io::ErrorKind) -> bool {
1759    matches!(
1760        kind,
1761        io::ErrorKind::ConnectionRefused
1762            | io::ErrorKind::TimedOut
1763            | io::ErrorKind::ConnectionReset
1764            | io::ErrorKind::ConnectionAborted
1765            | io::ErrorKind::BrokenPipe
1766            | io::ErrorKind::UnexpectedEof
1767    )
1768}
1769
1770/// Read the connection file → resolve the first endpoint → TCP connect → HMAC
1771/// handshake. Transient initial-attach failures retry on fresh sockets and reread
1772/// the file so a daemon bounce can publish a new endpoint or authentication key.
1773async fn connect_and_authenticate(connection_file_path: &Path) -> Result<TcpStream, SubcError> {
1774    connect_and_authenticate_with_policy(connection_file_path, ATTACH_RETRY_POLICY).await
1775}
1776
1777async fn connect_and_authenticate_with_policy(
1778    connection_file_path: &Path,
1779    policy: AttachRetryPolicy,
1780) -> Result<TcpStream, SubcError> {
1781    let started_at = Instant::now();
1782    let deadline = started_at + policy.budget;
1783    let mut attempt = 0_u32;
1784    let mut backoff = policy.initial_backoff;
1785    let mut history = Vec::new();
1786
1787    loop {
1788        attempt = attempt.saturating_add(1);
1789        let error = match connect_and_authenticate_once(connection_file_path, deadline).await {
1790            Ok(stream) => return Ok(stream),
1791            Err(error) => error,
1792        };
1793        let class = classify_attach_error(&error);
1794        let error_text = error.to_string().lines().collect::<Vec<_>>().join(" ");
1795        history.push(format!("attempt {attempt} [{class}]: {error_text}"));
1796
1797        if class == AttachErrorClass::Permanent {
1798            log_attach_final_failure(started_at.elapsed(), &history);
1799            return Err(error);
1800        }
1801
1802        let remaining = deadline.saturating_duration_since(Instant::now());
1803        if remaining.is_zero() {
1804            log_attach_final_failure(started_at.elapsed(), &history);
1805            return Err(error);
1806        }
1807
1808        let delay = jittered_attach_delay(backoff, policy.jitter_percent, attempt).min(remaining);
1809        log::info!(
1810            "subc attach retry: attempt {attempt} failed; error_class={class}; error={error_text}; next_delay={delay:?}"
1811        );
1812        tokio::time::sleep(delay).await;
1813
1814        if Instant::now() >= deadline {
1815            log_attach_final_failure(started_at.elapsed(), &history);
1816            return Err(error);
1817        }
1818        backoff = backoff.saturating_mul(2).min(policy.max_backoff);
1819    }
1820}
1821
1822fn jittered_attach_delay(base: Duration, jitter_percent: u64, attempt: u32) -> Duration {
1823    let jitter_percent = jitter_percent.min(100);
1824    if jitter_percent == 0 {
1825        return base;
1826    }
1827
1828    let mut random_bytes = [0_u8; 8];
1829    let random = if getrandom::fill(&mut random_bytes).is_ok() {
1830        u64::from_le_bytes(random_bytes)
1831    } else {
1832        let timestamp = std::time::SystemTime::now()
1833            .duration_since(std::time::UNIX_EPOCH)
1834            .unwrap_or_default()
1835            .subsec_nanos();
1836        u64::from(timestamp) ^ u64::from(attempt)
1837    };
1838    let span = jitter_percent.saturating_mul(2).saturating_add(1);
1839    let multiplier_percent = 100 - jitter_percent + random % span;
1840    let base_millis = u64::try_from(base.as_millis()).unwrap_or(u64::MAX);
1841    Duration::from_millis(base_millis.saturating_mul(multiplier_percent) / 100)
1842}
1843
1844fn log_attach_final_failure(elapsed: Duration, history: &[String]) {
1845    log::error!(
1846        "subc initial attach failed after {} attempt(s) in {elapsed:?}; attempt history: {}",
1847        history.len(),
1848        history.join(" | ")
1849    );
1850}
1851
1852async fn connect_and_authenticate_once(
1853    connection_file_path: &Path,
1854    deadline: Instant,
1855) -> Result<TcpStream, SubcError> {
1856    // This read intentionally lives inside the per-attempt function. The daemon
1857    // publishes connection files atomically and may change both port and key.
1858    let conn = connection_file::read_for_client(connection_file_path).map_err(|source| {
1859        SubcError::ConnectionFile {
1860            path: connection_file_path.to_path_buf(),
1861            source,
1862        }
1863    })?;
1864
1865    let endpoint = conn
1866        .endpoints
1867        .first()
1868        .ok_or_else(|| SubcError::NoEndpoint {
1869            path: connection_file_path.to_path_buf(),
1870        })?;
1871    let endpoint_label = format!("{}:{}", endpoint.host, endpoint.port);
1872    let ip = endpoint
1873        .host
1874        .parse::<IpAddr>()
1875        .map_err(|_| SubcError::InvalidEndpoint {
1876            path: connection_file_path.to_path_buf(),
1877            endpoint: endpoint_label.clone(),
1878        })?;
1879    let addr = SocketAddr::new(ip, endpoint.port);
1880
1881    let connect_budget = deadline.saturating_duration_since(Instant::now());
1882    let mut stream = tokio::time::timeout(connect_budget, TcpStream::connect(addr))
1883        .await
1884        .map_err(|_| SubcError::Connect {
1885            endpoint: endpoint_label.clone(),
1886            source: io::Error::new(
1887                io::ErrorKind::TimedOut,
1888                "initial subc attach retry budget elapsed during TCP connect",
1889            ),
1890        })?
1891        .map_err(|source| SubcError::Connect {
1892            endpoint: endpoint_label.clone(),
1893            source,
1894        })?;
1895    stream
1896        .set_nodelay(true)
1897        .map_err(|source| SubcError::Connect {
1898            endpoint: endpoint_label.clone(),
1899            source,
1900        })?;
1901
1902    let auth_budget = AUTH_DEADLINE.min(deadline.saturating_duration_since(Instant::now()));
1903    authenticate_client(&mut stream, &conn, auth_budget)
1904        .await
1905        .map_err(|source| SubcError::Auth {
1906            endpoint: endpoint_label,
1907            source,
1908        })?;
1909
1910    Ok(stream)
1911}
1912
1913#[allow(clippy::too_many_arguments)]
1914async fn process_route_bind_completion(
1915    writer_tx: &WriterSender,
1916    completion: RouteBindCompletion,
1917    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1918    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1919    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1920    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1921    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1922    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
1923    installed_route_epochs: &mut HashMap<u16, u32>,
1924    executor: &Arc<Executor>,
1925    shutdown: &Arc<Notify>,
1926    metrics: &Arc<DispatchPathMetrics>,
1927) -> Result<(), SubcError> {
1928    decrement_counted_channel(&metrics.control_completion_queued);
1929    handle_route_bind_completion(
1930        writer_tx,
1931        completion,
1932        routes,
1933        root_channels,
1934        session_identity,
1935        push_buffer,
1936        live_roots,
1937        pending_binds,
1938        installed_route_epochs,
1939        executor,
1940        shutdown,
1941        metrics,
1942    )
1943    .await
1944}
1945
1946#[allow(clippy::too_many_arguments)]
1947async fn drain_pending_route_bind_completions(
1948    control_completion_rx: &mut mpsc::Receiver<RouteBindCompletion>,
1949    writer_tx: &WriterSender,
1950    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1951    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1952    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1953    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1954    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1955    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
1956    installed_route_epochs: &mut HashMap<u16, u32>,
1957    executor: &Arc<Executor>,
1958    shutdown: &Arc<Notify>,
1959    metrics: &Arc<DispatchPathMetrics>,
1960) -> Result<usize, SubcError> {
1961    let mut drained = 0;
1962    while let Ok(completion) = control_completion_rx.try_recv() {
1963        process_route_bind_completion(
1964            writer_tx,
1965            completion,
1966            routes,
1967            root_channels,
1968            session_identity,
1969            push_buffer,
1970            live_roots,
1971            pending_binds,
1972            installed_route_epochs,
1973            executor,
1974            shutdown,
1975            metrics,
1976        )
1977        .await?;
1978        drained += 1;
1979    }
1980    Ok(drained)
1981}
1982
1983/// ModuleHello → HelloAck → control/route loop. Runs until the daemon closes
1984/// the connection (EOF), sends channel-0 Goodbye, or a fatal mutating executor
1985/// response requests whole-connection teardown.
1986async fn run_module_loop<R, W>(
1987    mut read: R,
1988    mut write: W,
1989    shared_app: Arc<App>,
1990    executor: Arc<Executor>,
1991    dispatch: DispatchFn,
1992    user_config_path: Option<PathBuf>,
1993    allow_native_passthrough: bool,
1994) -> Result<ModuleLoopExit, SubcError>
1995where
1996    R: AsyncRead + Unpin + Send + 'static,
1997    W: AsyncWrite + Unpin + Send + 'static,
1998{
1999    // ModuleHello: register as a tool provider and advertise the supported control-plane operations.
2000    // Echo the one-time launch nonce the daemon injected via SUBC_LAUNCH_NONCE so a
2001    // reserved module_id's HELLO is accepted; absent for non-reserved/self-connect.
2002    let hello = ModuleHelloBody {
2003        manifest: build_manifest(),
2004        protocol_ver: PROTOCOL_VERSION,
2005        control_ops: control_ops(),
2006        launch_nonce: std::env::var("SUBC_LAUNCH_NONCE").ok(),
2007    };
2008    let hello_frame = Frame::build(
2009        FrameType::Hello,
2010        control_flags(),
2011        0,
2012        0,
2013        HELLO_CORR,
2014        serde_json::to_vec(&hello).map_err(SubcError::Json)?,
2015    )
2016    .map_err(SubcError::FrameBuild)?;
2017    write_frame(&mut write, &hello_frame)
2018        .await
2019        .map_err(SubcError::FrameIo)?;
2020
2021    // Expect HelloAck (registered) or a channel-0 Error (manifest/version reject).
2022    match read_frame(&mut read).await.map_err(SubcError::FrameIo)? {
2023        None => return Err(SubcError::ClosedBeforeHelloAck),
2024        Some(frame) => match frame.header.ty {
2025            FrameType::HelloAck => {
2026                log::info!("subc attach: registered (HelloAck received)");
2027            }
2028            FrameType::Error => {
2029                let body = serde_json::from_slice::<ErrorBody>(&frame.body).ok();
2030                return Err(SubcError::HelloRejected { body });
2031            }
2032            other => return Err(SubcError::UnexpectedFrame { ty: other }),
2033        },
2034    }
2035
2036    let dispatch_path_metrics = Arc::new(DispatchPathMetrics::new());
2037    let (writer_tx, writer_rx) = mpsc::channel::<WriterFrame>(WRITER_QUEUE_CAPACITY);
2038    let writer_task = spawn_writer_task(write, writer_rx, Arc::clone(&dispatch_path_metrics));
2039    // `read_frame` is NOT cancellation-safe, so it must never sit directly inside
2040    // the `select!` below: a drain-interval tick (or shutdown) firing while a
2041    // frame is mid-transit would drop the partially-consumed bytes and desync the
2042    // stream (the next read would parse a body byte as a frame header). A
2043    // dedicated reader task owns the socket, reads whole frames sequentially, and
2044    // forwards them over a channel; the loop selects on the cancel-safe `recv()`.
2045    let (reader_tx, mut reader_rx) = mpsc::channel::<Result<DecodedFrame, SubcError>>(256);
2046    let reader_task = spawn_reader_task(read, reader_tx);
2047    let shutdown = Arc::new(Notify::new());
2048    // Drain-tick deadline is tracked manually and checked at the TOP of every
2049    // loop turn rather than as an Interval select arm: the select below is
2050    // `biased` (bind completions first), and biased polling means a saturated
2051    // higher arm (sustained lossy push traffic keeps lossy_rx always-ready)
2052    // would starve every arm below it, including a timer arm — leaving
2053    // backpressured reliable frames parked in the retry buffer past their
2054    // delivery deadline. The pre-turn check cannot be starved by arm order;
2055    // the sleep_until arm below only exists to wake an otherwise-idle loop.
2056    let mut next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2057    let mut next_maintenance_at = next_drain_at;
2058    let (maintenance_tx, mut maintenance_rx) = mpsc::channel::<MaintenanceCompletion>(256);
2059    let (bash_deferred_tx, mut bash_deferred_rx) =
2060        mpsc::channel::<bash::BashDeferredCompletion>(256);
2061    let (bash_poll_touch_tx, mut bash_poll_touch_rx) = mpsc::channel::<ProjectRootId>(256);
2062    let (control_completion_tx, mut control_completion_rx) =
2063        mpsc::channel::<RouteBindCompletion>(256);
2064    let (lossy_tx, mut lossy_rx) = mpsc::channel::<LossyPushEnvelope>(1024);
2065    let lossy_overflow = Arc::new(push::LossyOverflow::default());
2066    let lossy_seq = Arc::new(AtomicU64::new(0));
2067    let (reliable_tx, mut reliable_rx) = mpsc::unbounded_channel::<PushEnvelope>();
2068    let push_senders = PushSenders {
2069        lossy_tx,
2070        reliable_tx,
2071        lossy_overflow: Arc::clone(&lossy_overflow),
2072        lossy_seq,
2073    };
2074    let connection_cancel = PersistentCancelSignal::new();
2075    let mut installed_route_epochs: HashMap<u16, u32> = HashMap::new();
2076    let mut routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
2077    let mut bg_subs: HashMap<RouteChannel, BgSub> = HashMap::new();
2078    let mut bg_sub_by_session: HashMap<(ProjectRootId, String), RouteChannel> = HashMap::new();
2079    let mut bg_wake_pending: HashSet<RouteChannel> = HashSet::new();
2080    let mut bg_wake_epoch: HashMap<(ProjectRootId, String), u64> = HashMap::new();
2081    let mut root_channels: HashMap<ProjectRootId, HashSet<RouteChannel>> = HashMap::new();
2082    let mut session_identity: HashMap<(ProjectRootId, String), RetainedSessionIdentity> =
2083        HashMap::new();
2084    let mut push_buffer: HashMap<push::ReplayKey, VecDeque<PushFrame>> = HashMap::new();
2085    let mut retry_buffer: RetryBuffer = HashMap::new();
2086    let mut completed_tasks = push::CompletedTaskIds::default();
2087    let mut live_roots: HashMap<ProjectRootId, RootMeta> = HashMap::new();
2088    let mut pending_binds: HashMap<RouteChannel, PendingBind> = HashMap::new();
2089    let mut pending_bash_asks: HashMap<ReverseCorrKey, PendingBashAsk> = HashMap::new();
2090    let mut next_bash_ask_corr: u64 = 1;
2091    let mut route_bash_cancels: HashMap<RouteChannel, bash::RouteBashCancel> = HashMap::new();
2092
2093    let loop_result: Result<ModuleLoopExit, SubcError> = loop {
2094        shared_app.set_open_route_count(routes.len());
2095        crate::logging::perf_tick(Some(&executor));
2096        dispatch_path_metrics.mark_frame_loop_tick();
2097        if let Err(error) = expire_pending_bash_asks(
2098            &writer_tx,
2099            &mut pending_bash_asks,
2100            &routes,
2101            &mut live_roots,
2102            &mut route_bash_cancels,
2103            &shutdown,
2104            &dispatch_path_metrics,
2105        )
2106        .await
2107        {
2108            break Err(error);
2109        }
2110
2111        // RouteBind completions are control-plane unblockers. Drain any completed
2112        // binds before entering other branch work so Push and maintenance bursts
2113        // can only add one loop-turn of latency.
2114        match drain_pending_route_bind_completions(
2115            &mut control_completion_rx,
2116            &writer_tx,
2117            &mut routes,
2118            &mut root_channels,
2119            &mut session_identity,
2120            &mut push_buffer,
2121            &mut live_roots,
2122            &mut pending_binds,
2123            &mut installed_route_epochs,
2124            &executor,
2125            &shutdown,
2126            &dispatch_path_metrics,
2127        )
2128        .await
2129        {
2130            Ok(drained) => {
2131                if drained > 0 {
2132                    next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2133                }
2134            }
2135            Err(error) => break Err(error),
2136        }
2137
2138        if tokio::time::Instant::now() >= next_drain_at {
2139            push::emit_bg_event_wakes(
2140                &writer_tx,
2141                &dispatch_path_metrics,
2142                &bg_subs,
2143                &mut bg_wake_pending,
2144            );
2145            warn_slow_pending_binds(&mut pending_binds, &executor);
2146            if let Err(error) = expire_overdue_route_binds(
2147                &writer_tx,
2148                &executor,
2149                &mut pending_binds,
2150                &mut installed_route_epochs,
2151                &dispatch_path_metrics,
2152            )
2153            .await
2154            {
2155                break Err(error);
2156            }
2157
2158            let retried = push::drain_retry_buffers_for_bound_routes(
2159                &writer_tx,
2160                &dispatch_path_metrics,
2161                &routes,
2162                &mut retry_buffer,
2163            );
2164            if retried > 0 {
2165                log::debug!(
2166                    "subc attach: retried {retried} reliable Push frame(s) after writer backpressure"
2167                );
2168            }
2169
2170            next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2171        }
2172
2173        // A lossy emitter may place its newest update in the overflow buffer
2174        // when the bounded channel is full, while this receive loop is draining
2175        // the channel. Drain overflow before selecting again so that raced
2176        // update is delivered on the next timer tick instead of waiting for
2177        // another lossy enqueue.
2178        let overflow_batch = lossy_overflow.drain();
2179        if !overflow_batch.is_empty() {
2180            let (_, deferred) = push::drain_reliable_push_turn(
2181                &writer_tx,
2182                &dispatch_path_metrics,
2183                &routes,
2184                &root_channels,
2185                &session_identity,
2186                &mut retry_buffer,
2187                &mut push_buffer,
2188                &mut completed_tasks,
2189                &bg_sub_by_session,
2190                &mut bg_wake_pending,
2191                &mut bg_wake_epoch,
2192                &mut reliable_rx,
2193                None,
2194            );
2195            if deferred {
2196                tokio::task::yield_now().await;
2197            }
2198
2199            let mut batch = Vec::new();
2200            while let Ok(item) = lossy_rx.try_recv() {
2201                batch.push(item);
2202            }
2203            batch.extend(overflow_batch);
2204            push::process_lossy_push_envelope_batch(
2205                &writer_tx,
2206                &dispatch_path_metrics,
2207                &routes,
2208                &root_channels,
2209                &completed_tasks,
2210                batch,
2211            );
2212        }
2213
2214        tokio::select! {
2215            biased;
2216            Some(completion) = control_completion_rx.recv() => {
2217                if let Err(error) = process_route_bind_completion(
2218                    &writer_tx,
2219                    completion,
2220                    &mut routes,
2221                    &mut root_channels,
2222                    &mut session_identity,
2223                    &mut push_buffer,
2224                    &mut live_roots,
2225                    &mut pending_binds,
2226                    &mut installed_route_epochs,
2227                    &executor,
2228                    &shutdown,
2229                    &dispatch_path_metrics,
2230                )
2231                .await
2232                {
2233                    break Err(error);
2234                }
2235                next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2236            }
2237            _ = shutdown.notified() => {
2238                log::warn!("subc attach: fatal executor response requested teardown");
2239                break Ok(ModuleLoopExit::SkipSearchFlush);
2240            }
2241            maybe_frame = reader_rx.recv() => {
2242                let frame = match maybe_frame {
2243                    None => {
2244                        log::info!("subc attach: daemon closed connection");
2245                        break Ok(ModuleLoopExit::Graceful);
2246                    }
2247                    Some(Err(error)) => break Err(error),
2248                    Some(Ok(frame)) => frame,
2249                };
2250                let phase_trace = frame.phase_trace;
2251                let frame = frame.frame;
2252
2253                if !ingress_route_is_current(&installed_route_epochs, &frame) {
2254                    log::debug!(
2255                        "subc attach: silently dropping {:?} for uninstalled route {}@{}",
2256                        frame.header.ty,
2257                        frame.header.channel,
2258                        frame.header.epoch
2259                    );
2260                    continue;
2261                }
2262
2263                match frame.header.ty {
2264                    FrameType::Ping if frame.header.channel == 0 => {
2265                        let pong = match Frame::build_with_version(
2266                            frame.header.ver,
2267                            FrameType::Pong,
2268                            frame.header.flags,
2269                            0,
2270                            0,
2271                            frame.header.corr,
2272                            Vec::new(),
2273                        ) {
2274                            Ok(pong) => pong,
2275                            Err(error) => break Err(SubcError::FrameBuild(error)),
2276                        };
2277                        if let Err(error) = send_frame(&writer_tx, &dispatch_path_metrics, pong).await {
2278                            break Err(error);
2279                        }
2280                    }
2281                    FrameType::Goodbye if frame.header.channel == 0 => {
2282                        log::info!("subc attach: received channel-0 Goodbye");
2283                        break Ok(ModuleLoopExit::Graceful);
2284                    }
2285                    FrameType::Goodbye => {
2286                        let channel = route_key(frame.header.channel, frame.header.epoch);
2287                        if let Err(error) = teardown_installed_route(
2288                            &writer_tx,
2289                            &dispatch_path_metrics,
2290                            &executor,
2291                            channel,
2292                            "Goodbye",
2293                            None,
2294                            &mut installed_route_epochs,
2295                            &mut routes,
2296                            &mut root_channels,
2297                            &mut bg_subs,
2298                            &mut bg_sub_by_session,
2299                            &mut bg_wake_pending,
2300                            &mut pending_bash_asks,
2301                            &mut live_roots,
2302                            &mut route_bash_cancels,
2303                            &mut pending_binds,
2304                            &mut retry_buffer,
2305                            &mut push_buffer,
2306                            &shutdown,
2307                        )
2308                        .await
2309                        {
2310                            break Err(error);
2311                        }
2312                    }
2313                    FrameType::Response | FrameType::Error if frame.header.channel != 0 => {
2314                        if let Err(error) = handle_bash_elicitation_reply(
2315                            &writer_tx,
2316                            &frame,
2317                            &mut pending_bash_asks,
2318                            &routes,
2319                            &mut live_roots,
2320                            &executor,
2321                            &shutdown,
2322                            &bash_deferred_tx,
2323                            &bash_poll_touch_tx,
2324                            &dispatch_path_metrics,
2325                            &mut route_bash_cancels,
2326                            dispatch,
2327                        )
2328                        .await
2329                        {
2330                            break Err(error);
2331                        }
2332                    }
2333                    FrameType::Request if frame.header.channel == 0 => {
2334                        if let Err(error) = handle_control_request(
2335                            &writer_tx,
2336                            &frame,
2337                            &shared_app,
2338                            &executor,
2339                            &mut live_roots,
2340                            &mut pending_binds,
2341                            &mut installed_route_epochs,
2342                            &mut routes,
2343                            &mut root_channels,
2344                            &mut bg_subs,
2345                            &mut bg_sub_by_session,
2346                            &mut bg_wake_pending,
2347                            &mut pending_bash_asks,
2348                            &mut route_bash_cancels,
2349                            &mut retry_buffer,
2350                            &mut push_buffer,
2351                            &shutdown,
2352                            &control_completion_tx,
2353                            &dispatch_path_metrics,
2354                            &push_senders,
2355                            dispatch,
2356                            user_config_path.as_deref(),
2357                        )
2358                        .await
2359                        {
2360                            break Err(error);
2361                        }
2362                    }
2363                    FrameType::Request => {
2364                        if let Err(error) = handle_tool_call(
2365                            &writer_tx,
2366                            &frame,
2367                            phase_trace,
2368                            &routes,
2369                            &pending_binds,
2370                            &mut live_roots,
2371                            &executor,
2372                            &shutdown,
2373                            &connection_cancel,
2374                            &bash_deferred_tx,
2375                            &bash_poll_touch_tx,
2376                            &dispatch_path_metrics,
2377                            &mut route_bash_cancels,
2378                            &mut pending_bash_asks,
2379                            &mut next_bash_ask_corr,
2380                            &mut bg_subs,
2381                            &mut bg_sub_by_session,
2382                            &mut bg_wake_pending,
2383                            &mut bg_wake_epoch,
2384                            dispatch,
2385                            allow_native_passthrough,
2386                        )
2387                        .await
2388                        {
2389                            break Err(error);
2390                        }
2391                    }
2392                    FrameType::Cancel => {
2393                        let channel = route_key(frame.header.channel, frame.header.epoch);
2394                        if bg_subs.contains_key(&channel) {
2395                            if let Err(error) = end_bg_subscription(
2396                                &writer_tx,
2397                                &dispatch_path_metrics,
2398                                &mut bg_subs,
2399                                &mut bg_sub_by_session,
2400                                &mut bg_wake_pending,
2401                                channel,
2402                                routes.get(&channel),
2403                            )
2404                            .await
2405                            {
2406                                break Err(error);
2407                            }
2408                        }
2409                        if let Err(error) = cancel_pending_bash_ask_for_tool_call(
2410                            &writer_tx,
2411                            &mut pending_bash_asks,
2412                            channel,
2413                            frame.header.corr,
2414                            &routes,
2415                            &mut live_roots,
2416                            &mut route_bash_cancels,
2417                            &shutdown,
2418                            &dispatch_path_metrics,
2419                        )
2420                        .await
2421                        {
2422                            break Err(error);
2423                        }
2424                    }
2425                    // Incoming push messages are ignored here. Cancel frames only
2426                    // stop pending bash elicitation requests; executor-level
2427                    // cancellation for tool calls that are already running is not
2428                    // implemented.
2429                    _ => {}
2430                }
2431            }
2432            Some((root_id, frame)) = reliable_rx.recv() => {
2433                // Reliable Push frames are FIFO and must-deliver, but draining an
2434                // unbounded burst in one current-thread turn can starve RouteBind
2435                // completions. The budget defers excess frames, never drops them.
2436                let (_, deferred) = push::drain_reliable_push_turn(
2437                    &writer_tx,
2438                    &dispatch_path_metrics,
2439                    &routes,
2440                    &root_channels,
2441                    &session_identity,
2442                    &mut retry_buffer,
2443                    &mut push_buffer,
2444                    &mut completed_tasks,
2445                    &bg_sub_by_session,
2446                    &mut bg_wake_pending,
2447                    &mut bg_wake_epoch,
2448                    &mut reliable_rx,
2449                    Some((root_id, frame)),
2450                );
2451                if deferred {
2452                    tokio::task::yield_now().await;
2453                }
2454            }
2455            Some((order, root_id, frame)) = lossy_rx.recv() => {
2456                // When both push lanes have work, handle a small reliable slice before lossy work.
2457                // That ordering lets completed task ids suppress stale BashLongRunning frames.
2458                // The slice stays bounded so reliable bursts cannot monopolize this loop turn.
2459                let (_, deferred) = push::drain_reliable_push_turn(
2460                    &writer_tx,
2461                    &dispatch_path_metrics,
2462                    &routes,
2463                    &root_channels,
2464                    &session_identity,
2465                    &mut retry_buffer,
2466                    &mut push_buffer,
2467                    &mut completed_tasks,
2468                    &bg_sub_by_session,
2469                    &mut bg_wake_pending,
2470                    &mut bg_wake_epoch,
2471                    &mut reliable_rx,
2472                    None,
2473                );
2474                if deferred {
2475                    tokio::task::yield_now().await;
2476                }
2477
2478                // Drain the currently queued burst in one loop turn so lossy
2479                // status/progress updates can be merged before reaching subc's
2480                // shared egress queue. Each lossy frame gets a sequence number
2481                // before it goes to the channel or overflow buffer, so the
2482                // combined batch is sorted back into producer order before
2483                // coalescing drops stale updates for the same key.
2484                let mut batch = vec![(order, root_id, frame)];
2485                while let Ok(item) = lossy_rx.try_recv() {
2486                    batch.push(item);
2487                }
2488                batch.extend(lossy_overflow.drain());
2489                push::process_lossy_push_envelope_batch(
2490                    &writer_tx,
2491                    &dispatch_path_metrics,
2492                    &routes,
2493                    &root_channels,
2494                    &completed_tasks,
2495                    batch,
2496                );
2497            }
2498            Some(done) = bash_deferred_rx.recv() => {
2499                decrement_counted_channel(&dispatch_path_metrics.bash_deferred_queued);
2500                if let Err(error) = bash::handle_bash_deferred_completion(
2501                    &writer_tx,
2502                    done,
2503                    &routes,
2504                    &mut live_roots,
2505                    &mut route_bash_cancels,
2506                    &shutdown,
2507                    &dispatch_path_metrics,
2508                )
2509                .await
2510                {
2511                    break Err(error);
2512                }
2513            }
2514            Some(root_id) = bash_poll_touch_rx.recv() => {
2515                decrement_counted_channel(&dispatch_path_metrics.bash_poll_touch_queued);
2516                if let Some(meta) = live_roots.get_mut(&root_id) {
2517                    meta.note_activity();
2518                }
2519            }
2520            Some(completion) = maintenance_rx.recv() => {
2521                decrement_counted_channel(&dispatch_path_metrics.maintenance_queued);
2522                let root_id = completion.root_id.clone();
2523                let response = completion.response;
2524                let response_is_fatal = response_is_fatal_panic(&response);
2525                let bind_pending = pending_binds
2526                    .values()
2527                    .any(|pending| pending.bind_root_id == root_id);
2528                let requiesce = if let Some(meta) = live_roots.get_mut(&root_id) {
2529                    let defer_requeue = meta.unbound_quiesced || bind_pending;
2530                    note_maintenance_completion(
2531                        meta,
2532                        completion.requeue_kind,
2533                        response_is_fatal,
2534                        defer_requeue,
2535                    );
2536                    should_requiesce_after_maintenance(meta, completion.kind, bind_pending)
2537                } else {
2538                    false
2539                };
2540                if requiesce {
2541                    quiesce_unbound_root(&root_id, &mut live_roots, &executor);
2542                }
2543                push::clear_stale_bg_wakes_for_empty_sessions(
2544                    &root_id,
2545                    &completion.empty_bg_sessions,
2546                    &bg_sub_by_session,
2547                    &mut bg_wake_pending,
2548                    &bg_wake_epoch,
2549                );
2550                if response_is_fatal {
2551                    if let Some(meta) = live_roots.get_mut(&root_id) {
2552                        meta.maintenance_poisoned = true;
2553                    }
2554                    log::warn!(
2555                        "subc attach: maintenance drain observed a fatal actor; deferring teardown until a route request can receive actor_fatal"
2556                    );
2557                }
2558            }
2559            _ = tokio::time::sleep_until(next_drain_at) => {
2560                // Wakes an otherwise-idle loop so the pre-turn drain check
2561                // above runs on schedule; the drain work itself lives there.
2562            }
2563            _ = tokio::time::sleep_until(next_maintenance_at) => {
2564                // Delay cache-draining maintenance until any already-ready
2565                // inbound route/control messages and push completions have run,
2566                // so maintenance does not block the actor from handling the
2567                // first request that arrives after a route bind is acknowledged.
2568                let reap = reap_idle_roots(
2569                    Instant::now(),
2570                    &mut live_roots,
2571                    &pending_binds,
2572                    &root_channels,
2573                    &executor,
2574                );
2575                for root_id in &reap.forgotten_deleted_roots {
2576                    purge_deleted_root_residents(
2577                        root_id,
2578                        &mut root_channels,
2579                        &mut session_identity,
2580                        &mut push_buffer,
2581                        &mut bg_subs,
2582                        &mut bg_sub_by_session,
2583                        &mut bg_wake_pending,
2584                        &mut bg_wake_epoch,
2585                        &mut pending_bash_asks,
2586                    );
2587                }
2588                if reap.evicted > 0 {
2589                    log::debug!("subc attach: reaped {} idle root(s)", reap.evicted);
2590                }
2591                submit_due_maintenance_jobs(
2592                    &executor,
2593                    &mut live_roots,
2594                    &pending_binds,
2595                    &bg_sub_by_session,
2596                    &bg_wake_pending,
2597                    &bg_wake_epoch,
2598                    &maintenance_tx,
2599                    &dispatch_path_metrics,
2600                );
2601                next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2602            }
2603        }
2604    };
2605
2606    shared_app.set_open_route_count(0);
2607
2608    // Sweep pending binds on EVERY loop exit (channel-0 Goodbye, EOF, fatal,
2609    // error): ExecutorInner::drop joins worker threads, so a configure still
2610    // parked at a cancellation checkpoint would wedge module shutdown.
2611    for pending in pending_binds.values() {
2612        let _ = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
2613    }
2614
2615    let mut loop_result = loop_result;
2616    if !pending_bash_asks.is_empty() {
2617        let no_routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
2618        if let Err(error) = settle_all_pending_bash_asks(
2619            &writer_tx,
2620            &mut pending_bash_asks,
2621            &no_routes,
2622            &mut live_roots,
2623            &mut route_bash_cancels,
2624            &shutdown,
2625            &dispatch_path_metrics,
2626        )
2627        .await
2628        {
2629            loop_result = loop_result.and(Err(error));
2630        }
2631    }
2632
2633    // The reader task may be parked on `read_frame`; abort it (we are done with
2634    // the connection) and flush the writer.
2635    connection_cancel.cancel();
2636    reader_task.abort();
2637    drop(writer_tx);
2638    let writer_result = finish_writer_task(writer_task).await;
2639    loop_result.and_then(|exit| writer_result.map(|_| exit))
2640}
2641
2642fn spawn_writer_task<W>(
2643    mut write: W,
2644    mut rx: mpsc::Receiver<WriterFrame>,
2645    metrics: Arc<DispatchPathMetrics>,
2646) -> JoinHandle<Result<(), subc_transport::FrameIoError>>
2647where
2648    W: AsyncWrite + Unpin + Send + 'static,
2649{
2650    tokio::spawn(async move {
2651        let mut write_buffer = Vec::new();
2652        while let Some(mut queued) = rx.recv().await {
2653            let measure = queued.tool_response_trace.is_some();
2654            let dequeued = measure.then(Instant::now);
2655            metrics.writer_active.store(true, Ordering::Relaxed);
2656            decrement_counted_channel(&metrics.writer_queued);
2657            let write_timing =
2658                write_frame_contiguous(&mut write, &queued.frame, &mut write_buffer, measure).await;
2659            metrics.writer_active.store(false, Ordering::Relaxed);
2660            let write_timing = write_timing?;
2661
2662            if let (Some(trace), Some(dequeued), Some(write_timing)) =
2663                (queued.tool_response_trace.take(), dequeued, write_timing)
2664            {
2665                if let Some(completed) = trace.finish(
2666                    dequeued,
2667                    write_timing.write_started,
2668                    write_timing.write_finished,
2669                    write_timing.frame_bytes,
2670                ) {
2671                    log_ctx::with_session(Some(completed.session), || {
2672                        crate::logging::note_tool_call_trace(
2673                            &completed.name,
2674                            &completed.root,
2675                            completed.channel,
2676                            completed.corr,
2677                            completed.phases,
2678                        );
2679                    });
2680                }
2681            }
2682        }
2683        Ok(())
2684    })
2685}
2686
2687struct FrameWriteTiming {
2688    write_started: Instant,
2689    write_finished: Instant,
2690    frame_bytes: usize,
2691}
2692
2693/// Encode one complete frame into the existing reusable buffer and write it
2694/// without interleaving bytes from another channel. Timing is collected only
2695/// for tool responses, so Push and control frames add no clock reads.
2696async fn write_frame_contiguous<W>(
2697    writer: &mut W,
2698    frame: &Frame,
2699    buffer: &mut Vec<u8>,
2700    measure: bool,
2701) -> Result<Option<FrameWriteTiming>, subc_transport::FrameIoError>
2702where
2703    W: AsyncWrite + Unpin,
2704{
2705    if frame.header.len as usize != frame.body.len() {
2706        return Err(subc_transport::FrameIoError::BodyLengthMismatch {
2707            header_len: frame.header.len,
2708            body_len: frame.body.len(),
2709        });
2710    }
2711
2712    let header = frame.header.encode();
2713    buffer.clear();
2714    buffer.reserve(header.len() + frame.body.len());
2715    buffer.extend_from_slice(&header);
2716    buffer.extend_from_slice(&frame.body);
2717    let write_started = measure.then(Instant::now);
2718    writer
2719        .write_all(buffer)
2720        .await
2721        .map_err(subc_transport::FrameIoError::Io)?;
2722    Ok(write_started.map(|write_started| FrameWriteTiming {
2723        write_started,
2724        write_finished: Instant::now(),
2725        frame_bytes: buffer.len(),
2726    }))
2727}
2728
2729fn spawn_reader_task<R>(
2730    mut read: R,
2731    tx: mpsc::Sender<Result<DecodedFrame, SubcError>>,
2732) -> JoinHandle<()>
2733where
2734    R: AsyncRead + Unpin + Send + 'static,
2735{
2736    tokio::spawn(async move {
2737        loop {
2738            match read_frame(&mut read).await {
2739                Ok(Some(frame)) => {
2740                    let decoded = DecodedFrame {
2741                        frame,
2742                        phase_trace: PhaseTrace::new(Instant::now()),
2743                    };
2744                    if tx.send(Ok(decoded)).await.is_err() {
2745                        return;
2746                    }
2747                }
2748                Ok(None) => {
2749                    // EOF: let the loop observe channel close as "daemon closed".
2750                    return;
2751                }
2752                Err(error) => {
2753                    // A killed daemon surfaces as ConnectionReset (RST) on
2754                    // Windows where Unix delivers a clean EOF (FIN); a
2755                    // mid-teardown daemon can also abort the socket. Both mean
2756                    // "daemon went away", not a wire fault — normalize them to
2757                    // the clean-close path so module exit behavior matches
2758                    // across platforms (same class subc-core fixed in d33d9a71).
2759                    if let subc_transport::FrameIoError::Io(io_error) = &error {
2760                        if matches!(
2761                            io_error.kind(),
2762                            std::io::ErrorKind::ConnectionReset
2763                                | std::io::ErrorKind::ConnectionAborted
2764                        ) {
2765                            log::info!(
2766                                "subc attach: connection reset by daemon; treating as close"
2767                            );
2768                            return;
2769                        }
2770                    }
2771                    let _ = tx.send(Err(SubcError::FrameIo(error))).await;
2772                    return;
2773                }
2774            }
2775        }
2776    })
2777}
2778
2779async fn finish_writer_task(
2780    mut writer_task: JoinHandle<Result<(), subc_transport::FrameIoError>>,
2781) -> Result<(), SubcError> {
2782    match tokio::time::timeout(Duration::from_millis(100), &mut writer_task).await {
2783        Ok(Ok(Ok(()))) => Ok(()),
2784        Ok(Ok(Err(error))) => Err(SubcError::FrameIo(error)),
2785        Ok(Err(error)) => Err(SubcError::WriterJoin(error)),
2786        Err(_) => {
2787            writer_task.abort();
2788            Ok(())
2789        }
2790    }
2791}
2792
2793fn register_actor_for_bind(
2794    shared_app: &Arc<App>,
2795    executor: &Arc<Executor>,
2796    push_senders: &PushSenders,
2797    bind_root_id: &ProjectRootId,
2798    route_channel: u16,
2799    root_was_live: bool,
2800) -> bool {
2801    if executor.actor_registered(bind_root_id) {
2802        log::debug!(
2803            "subc attach: reusing actor for route {} root {}",
2804            route_channel,
2805            bind_root_id.as_path().display()
2806        );
2807        return false;
2808    }
2809
2810    if root_was_live {
2811        log::warn!(
2812            "subc attach: recreating missing actor for live root {} on route {}",
2813            bind_root_id.as_path().display(),
2814            route_channel
2815        );
2816    }
2817
2818    let actor_ctx = Arc::new(AppContext::from_app(
2819        Arc::clone(shared_app),
2820        Config::default(),
2821    ));
2822    install_bash_compressor(&actor_ctx);
2823    actor_ctx.set_progress_sender(Some(push::progress_sender_for_root(
2824        push_senders.clone(),
2825        bind_root_id.clone(),
2826    )));
2827    let inserted = executor.register_actor(bind_root_id.clone(), Arc::clone(&actor_ctx));
2828    drop(actor_ctx);
2829    if inserted {
2830        // Do not insert into live_roots until configure succeeds: live_roots
2831        // drives maintenance, and a half-configured new actor must not be
2832        // maintenance-eligible before its route/session identity exists.
2833        log::debug!(
2834            "subc attach: registered actor for route {} root {}",
2835            route_channel,
2836            bind_root_id.as_path().display()
2837        );
2838    } else {
2839        log::debug!(
2840            "subc attach: actor appeared while binding route {} root {}; reusing it",
2841            route_channel,
2842            bind_root_id.as_path().display()
2843        );
2844    }
2845    inserted
2846}
2847
2848fn rollback_pending_bind_actor(
2849    executor: &Arc<Executor>,
2850    live_roots: &HashMap<ProjectRootId, RootMeta>,
2851    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2852    root_id: &ProjectRootId,
2853    inserted_new_actor: bool,
2854) {
2855    if !inserted_new_actor || live_roots.contains_key(root_id) {
2856        return;
2857    }
2858
2859    if let Some((route, pending)) = pending_binds
2860        .iter_mut()
2861        .find(|(_, pending)| &pending.bind_root_id == root_id)
2862    {
2863        pending.inserted_new_actor = true;
2864        log::debug!(
2865            "subc attach: transferred rollback ownership for root {} to pending route {}",
2866            root_id.as_path().display(),
2867            route
2868        );
2869        return;
2870    }
2871
2872    executor.remove_actor(root_id);
2873}
2874
2875fn route_bind_error_code_for_configure_response(response: &Response) -> &'static str {
2876    match response.data.get("code").and_then(|code| code.as_str()) {
2877        // Preserve typed configure rejections across the bind boundary: a
2878        // malformed fed fingerprint means a federation-module bug or
2879        // fingerprint-format drift, and the fed side matches on the code rather
2880        // than parsing prose.
2881        Some("bad_harness_fingerprint") => "bad_harness_fingerprint",
2882        // Cache-key probe failures are transient (fd pressure, git spawn
2883        // contention); the client retries the bind rather than treating the
2884        // root as permanently divergent.
2885        Some("cache_key_probe_failed") => "cache_key_probe_failed",
2886        // Actor lifecycle gaps are transient from the daemon/client viewpoint:
2887        // a fresh bind can create or join a healthy actor, so do not classify
2888        // them as permanent config divergence.
2889        Some("actor_not_registered" | "actor_fatal") => "actor_not_ready",
2890        _ => "config_divergence",
2891    }
2892}
2893
2894fn queue_post_bind_configure_and_completion_maintenance(
2895    root_id: &ProjectRootId,
2896    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2897) {
2898    let Some(meta) = live_roots.get_mut(root_id) else {
2899        return;
2900    };
2901    if meta.maintenance_poisoned || meta.maintenance_pending {
2902        return;
2903    }
2904
2905    meta.maintenance_pending = true;
2906    meta.maintenance_queued_kinds
2907        .push_back(MaintenanceDrainKind::ConfigureTail);
2908    meta.maintenance_queued_kinds
2909        .push_back(MaintenanceDrainKind::CompletionDrains);
2910}
2911
2912#[allow(clippy::too_many_arguments)]
2913async fn handle_route_bind_completion(
2914    tx: &WriterSender,
2915    completion: RouteBindCompletion,
2916    routes: &mut HashMap<RouteChannel, RouteIdentity>,
2917    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2918    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2919    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2920    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2921    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2922    installed_route_epochs: &mut HashMap<u16, u32>,
2923    executor: &Arc<Executor>,
2924    shutdown: &Arc<Notify>,
2925    metrics: &Arc<DispatchPathMetrics>,
2926) -> Result<(), SubcError> {
2927    let route_id = completion.route;
2928    let Some(pending) = pending_binds.remove(&route_id) else {
2929        log::warn!(
2930            "subc attach: dropping RouteBind completion for non-pending route {}",
2931            completion.route
2932        );
2933        rollback_pending_bind_actor(
2934            executor,
2935            live_roots,
2936            pending_binds,
2937            &completion.bind_root_id,
2938            completion.inserted_new_actor,
2939        );
2940        let has_pending_bind = pending_binds
2941            .values()
2942            .any(|pending| pending.bind_root_id == completion.bind_root_id);
2943        if !root_channels
2944            .get(&completion.bind_root_id)
2945            .is_some_and(|channels| !channels.is_empty())
2946            && !has_pending_bind
2947        {
2948            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
2949        }
2950        remove_installed_route(installed_route_epochs, route_id);
2951        return Ok(());
2952    };
2953
2954    if pending.bind_root_id != completion.bind_root_id {
2955        log::warn!(
2956            "subc attach: pending RouteBind root mismatch for route {} (pending {} completion {})",
2957            completion.route,
2958            pending.bind_root_id.as_path().display(),
2959            completion.bind_root_id.as_path().display()
2960        );
2961    }
2962
2963    let inserted_new_actor = pending.inserted_new_actor || completion.inserted_new_actor;
2964    if pending.cancelled {
2965        rollback_pending_bind_actor(
2966            executor,
2967            live_roots,
2968            pending_binds,
2969            &completion.bind_root_id,
2970            inserted_new_actor,
2971        );
2972        let has_pending_bind = pending_binds
2973            .values()
2974            .any(|pending| pending.bind_root_id == completion.bind_root_id);
2975        if !root_channels
2976            .get(&completion.bind_root_id)
2977            .is_some_and(|channels| !channels.is_empty())
2978            && !has_pending_bind
2979        {
2980            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
2981        }
2982        log::debug!(
2983            "subc attach: discarded completed RouteBind for cancelled route {} root {}",
2984            completion.route,
2985            completion.bind_root_id.as_path().display()
2986        );
2987        remove_installed_route(installed_route_epochs, route_id);
2988        return Ok(());
2989    }
2990
2991    let failure = if !completion.configure_response.success {
2992        Some((
2993            &completion.configure_response,
2994            "configure failed during route bind",
2995        ))
2996    } else {
2997        None
2998    };
2999
3000    if let Some((response, fallback)) = failure {
3001        rollback_pending_bind_actor(
3002            executor,
3003            live_roots,
3004            pending_binds,
3005            &completion.bind_root_id,
3006            inserted_new_actor,
3007        );
3008        let has_pending_bind = pending_binds
3009            .values()
3010            .any(|pending| pending.bind_root_id == completion.bind_root_id);
3011        if !root_channels
3012            .get(&completion.bind_root_id)
3013            .is_some_and(|channels| !channels.is_empty())
3014            && !has_pending_bind
3015        {
3016            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3017        }
3018        let message = response_message(response, fallback);
3019        let fatal = response_is_fatal_panic(response);
3020        let error_code = route_bind_error_code_for_configure_response(response);
3021        send_route_bind_error_parts(
3022            tx,
3023            completion.ver,
3024            completion.corr,
3025            completion.flags,
3026            error_code,
3027            &message,
3028            metrics,
3029        )
3030        .await?;
3031        remove_installed_route(installed_route_epochs, route_id);
3032        if fatal {
3033            signal_fatal_teardown(
3034                tx,
3035                Some(completion.route),
3036                completion.ver,
3037                completion.corr,
3038                shutdown,
3039                metrics,
3040            )
3041            .await;
3042        }
3043        return Ok(());
3044    }
3045
3046    remember_session_identity(session_identity, &completion.identity);
3047    let replay_key = push::ReplayKey::from_identity(&completion.identity);
3048    let bind_trust = completion.identity.trust;
3049    insert_route_channel(routes, root_channels, route_id, completion.identity);
3050    let restore_watcher = live_roots
3051        .get(&completion.bind_root_id)
3052        .is_some_and(|meta| meta.idle_artifacts_evicted || meta.unbound_quiesced);
3053    live_roots
3054        .entry(completion.bind_root_id.clone())
3055        .and_modify(|meta| {
3056            meta.reactivate_bound();
3057            meta.diagnostics_on_edit = completion.diagnostics_on_edit;
3058            meta.maintenance_poisoned = false;
3059        })
3060        .or_insert_with(|| RootMeta::new(Instant::now()));
3061    if let Some(meta) = live_roots.get_mut(&completion.bind_root_id) {
3062        meta.diagnostics_on_edit = completion.diagnostics_on_edit;
3063        meta.maintenance_poisoned = false;
3064    }
3065    if let Some(ctx) = executor.actor_context(&completion.bind_root_id) {
3066        ctx.mark_subc_bound();
3067        if restore_watcher {
3068            crate::commands::configure::ensure_project_watcher(&ctx);
3069        }
3070    }
3071
3072    let ack =
3073        serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).map_err(SubcError::Json)?;
3074    let response = Frame::build_with_version(
3075        completion.ver,
3076        FrameType::Response,
3077        control_flags(),
3078        0,
3079        0,
3080        completion.corr,
3081        ack,
3082    )
3083    .map_err(SubcError::FrameBuild)?;
3084    send_reliable_writer_frame(tx, metrics, response, "RouteBindAck").await?;
3085    queue_post_bind_configure_and_completion_maintenance(&completion.bind_root_id, live_roots);
3086    let replayed = push::replay_buffered_push_frames(
3087        tx,
3088        metrics,
3089        route_id,
3090        push_buffer,
3091        &replay_key,
3092        bind_trust,
3093    );
3094    if replayed > 0 {
3095        log::debug!(
3096            "subc attach: replayed {} buffered Push frame(s) to route {} root {} harness {} session {}",
3097            replayed,
3098            completion.route,
3099            replay_key.root.as_path().display(),
3100            replay_key.harness,
3101            replay_key.session
3102        );
3103    }
3104    log::info!(
3105        "subc attach: route {} bound to root {}",
3106        completion.route,
3107        completion.bind_root_id.as_path().display()
3108    );
3109    Ok(())
3110}
3111
3112async fn expire_overdue_route_binds(
3113    tx: &WriterSender,
3114    executor: &Arc<Executor>,
3115    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3116    installed_route_epochs: &mut HashMap<u16, u32>,
3117    metrics: &DispatchPathMetrics,
3118) -> Result<(), SubcError> {
3119    let now = Instant::now();
3120    let expired: Vec<_> = pending_binds
3121        .iter()
3122        .filter_map(|(route, pending)| {
3123            let age = now.saturating_duration_since(pending.started_at);
3124            (!pending.deadline_reported && age >= ROUTE_BIND_DEADLINE).then(|| {
3125                (
3126                    *route,
3127                    pending.corr,
3128                    pending.ver,
3129                    pending.flags,
3130                    pending.bind_root_id.clone(),
3131                    pending.configure_request_id.clone(),
3132                    age,
3133                )
3134            })
3135        })
3136        .collect();
3137
3138    for (route, corr, ver, flags, root_id, configure_request_id, age) in expired {
3139        if let Some(pending) = pending_binds.get_mut(&route) {
3140            pending.cancelled = true;
3141            pending.deadline_reported = true;
3142            let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
3143            log::debug!(
3144                "subc attach: cancelled overdue RouteBind configure for route {route} ({outcome:?})"
3145            );
3146        }
3147        remove_installed_route(installed_route_epochs, route);
3148        let age_ms = age.as_millis().min(u128::from(u64::MAX)) as u64;
3149        let deadline_ms = ROUTE_BIND_DEADLINE.as_millis();
3150        send_route_bind_error_parts(
3151            tx,
3152            ver,
3153            corr,
3154            flags,
3155            "actor_not_ready",
3156            &format!("route bind deadline exceeded after {age_ms}ms (deadline {deadline_ms}ms)"),
3157            metrics,
3158        )
3159        .await?;
3160        log::warn!(
3161            "subc attach: route {} bind for root {} exceeded {}ms deadline (configure_request_id={})",
3162            route,
3163            root_id.as_path().display(),
3164            deadline_ms,
3165            configure_request_id
3166        );
3167    }
3168
3169    Ok(())
3170}
3171
3172/// channel-0 control requests: RouteBind plus the cached health probe. RouteBind
3173/// still reconciles the route's RootConfig through the executor's Mutating lane
3174/// and resolves completion on a loop-owned control-completion channel so slow
3175/// configure jobs do not block the transport loop.
3176#[allow(clippy::too_many_arguments)]
3177async fn handle_control_request(
3178    tx: &WriterSender,
3179    frame: &Frame,
3180    shared_app: &Arc<App>,
3181    executor: &Arc<Executor>,
3182    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3183    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3184    installed_route_epochs: &mut HashMap<u16, u32>,
3185    routes: &mut HashMap<RouteChannel, RouteIdentity>,
3186    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
3187    bg_subs: &mut HashMap<RouteChannel, BgSub>,
3188    bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
3189    bg_wake_pending: &mut HashSet<RouteChannel>,
3190    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
3191    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
3192    retry_buffer: &mut RetryBuffer,
3193    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
3194    shutdown: &Arc<Notify>,
3195    control_completion_tx: &mpsc::Sender<RouteBindCompletion>,
3196    metrics: &Arc<DispatchPathMetrics>,
3197    push_senders: &PushSenders,
3198    dispatch: DispatchFn,
3199    user_config_path: Option<&Path>,
3200) -> Result<(), SubcError> {
3201    let request =
3202        serde_json::from_slice::<ModuleControlRequest>(&frame.body).map_err(SubcError::Json)?;
3203    match request {
3204        ModuleControlRequest::RouteBind {
3205            route_channel,
3206            epoch,
3207            target: _,
3208            identity,
3209            principal,
3210            consumer_capabilities,
3211        } => {
3212            let route_id = route_key(route_channel, epoch);
3213            if epoch == 0 {
3214                return send_route_bind_error(
3215                    tx,
3216                    frame,
3217                    "config_divergence",
3218                    "route bind uses an invalid channel generation",
3219                    metrics,
3220                )
3221                .await;
3222            }
3223            let mut bind_root_id = None;
3224            if let Some(installed_epoch) = installed_route_epochs.get(&route_channel).copied() {
3225                if installed_epoch >= epoch {
3226                    return send_route_bind_error(
3227                        tx,
3228                        frame,
3229                        "config_divergence",
3230                        "route bind generation is not newer than the installed generation",
3231                        metrics,
3232                    )
3233                    .await;
3234                }
3235
3236                let replacement_root = match ProjectRootId::from_path(&identity.project_root) {
3237                    Ok(root_id) => root_id,
3238                    Err(error) => {
3239                        return send_route_bind_error(
3240                            tx,
3241                            frame,
3242                            "config_divergence",
3243                            &format!("invalid route project root: {error}"),
3244                            metrics,
3245                        )
3246                        .await;
3247                    }
3248                };
3249                teardown_installed_route(
3250                    tx,
3251                    metrics,
3252                    executor,
3253                    route_key(route_channel, installed_epoch),
3254                    "higher-epoch RouteBind",
3255                    Some(&replacement_root),
3256                    installed_route_epochs,
3257                    routes,
3258                    root_channels,
3259                    bg_subs,
3260                    bg_sub_by_session,
3261                    bg_wake_pending,
3262                    pending_bash_asks,
3263                    live_roots,
3264                    route_bash_cancels,
3265                    pending_binds,
3266                    retry_buffer,
3267                    push_buffer,
3268                    shutdown,
3269                )
3270                .await?;
3271                bind_root_id = Some(replacement_root);
3272            }
3273            if pending_binds.contains_key(&route_id) {
3274                return send_route_bind_error(
3275                    tx,
3276                    frame,
3277                    "config_divergence",
3278                    "route bind is already pending for channel",
3279                    metrics,
3280                )
3281                .await;
3282            }
3283            let bind_root_id = match bind_root_id {
3284                Some(root_id) => root_id,
3285                None => match ProjectRootId::from_path(&identity.project_root) {
3286                    Ok(root_id) => root_id,
3287                    Err(error) => {
3288                        return send_route_bind_error(
3289                            tx,
3290                            frame,
3291                            "config_divergence",
3292                            &format!("invalid route project root: {error}"),
3293                            metrics,
3294                        )
3295                        .await;
3296                    }
3297                },
3298            };
3299
3300            // Reconcile RootConfig: build a configure request from the bind
3301            // identity + forwarded config tiers and run it through the executor.
3302            let request_id = format!("subc-bind-{route_channel}");
3303            let bind_project_root = identity.project_root.clone();
3304            let bind_harness = identity.harness.clone();
3305            let bind_session = identity.session.clone();
3306            let bind_trust = trust_for_bind(&bind_harness, &principal);
3307            // Typed capability declaration from the consumer: the facade stamps it
3308            // from the MCP host's initialize-advertised capabilities. Absent
3309            // means no reverse-request capability — flat deny, fail-closed. A
3310            // consumer over-declaring only earns asks that TTL-deny.
3311            let consumer_elicitation_capable = consumer_capabilities
3312                .as_ref()
3313                .is_some_and(|capabilities| capabilities.iter().any(|c| c == "elicitation"));
3314            log::info!(
3315                "subc attach: route {} harness={} principal={} trust={} elicitation={}",
3316                route_channel,
3317                bind_harness,
3318                principal_label(&principal),
3319                bind_trust.label(),
3320                consumer_elicitation_capable
3321            );
3322
3323            // Config is single-per-project, read by AFT directly from the
3324            // CortexKit config files (user: ~/.config/cortexkit/aft.jsonc,
3325            // project: <root>/.cortexkit/aft.jsonc). Wire-relayed config tiers are
3326            // IGNORED entirely: a front (runner, mcp:*, or fed:*) cannot push config over
3327            // the wire. This is what makes config harness-INDEPENDENT — every
3328            // harness binding a project gets the identical on-disk config, so two
3329            // trust domains sharing the per-root actor can never diverge or
3330            // inherit each other's capabilities (the cross-bind escalation class).
3331            // Wire-relayed config tiers (if the protocol still carries them) are
3332            // ignored entirely; the per-tier trust boundary (user trusted, project
3333            // privileged-dropped) is applied to the FILE tiers in handle_configure.
3334            let local_tiers = crate::subc_config::read_local_cortexkit_config_tiers(
3335                user_config_path,
3336                Path::new(&bind_project_root),
3337            );
3338            let config_tiers: Vec<Value> = local_tiers
3339                .iter()
3340                .map(|t| json!({ "tier": t.tier, "source": t.source, "doc": t.doc }))
3341                .collect();
3342            let diagnostics_on_edit = diagnostics_on_edit_from_tiers(&local_tiers);
3343            let configure_json = json!({
3344                "id": request_id,
3345                "command": "configure",
3346                "project_root": bind_project_root,
3347                "harness": bind_harness,
3348                "session_id": bind_session.clone(),
3349                "config": config_tiers,
3350            });
3351            let configure_req = match serde_json::from_value::<RawRequest>(configure_json) {
3352                Ok(req) => req,
3353                Err(error) => {
3354                    return send_route_bind_error(
3355                        tx,
3356                        frame,
3357                        "config_divergence",
3358                        &format!("failed to build configure request: {error}"),
3359                        metrics,
3360                    )
3361                    .await;
3362                }
3363            };
3364
3365            let route_identity = RouteIdentity(Arc::new(RouteIdentityData {
3366                root: bind_root_id.clone(),
3367                project_root: PathBuf::from(&bind_project_root),
3368                harness: bind_harness.clone(),
3369                session: bind_session.clone(),
3370                trust: bind_trust,
3371                consumer_elicitation_capable,
3372            }));
3373            let configure_session = route_identity.session.clone();
3374            let root_was_live = live_roots.contains_key(&bind_root_id);
3375            let inserted_new_actor = register_actor_for_bind(
3376                shared_app,
3377                executor,
3378                push_senders,
3379                &bind_root_id,
3380                route_channel,
3381                root_was_live,
3382            );
3383
3384            let configure_request_id = configure_req.id.clone();
3385            installed_route_epochs.insert(route_channel, epoch);
3386            if let Some(meta) = live_roots.get_mut(&bind_root_id) {
3387                meta.maintenance_queued_kinds.clear();
3388                meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
3389            }
3390            let (configure_rx, configure_cancellation) = executor.submit_cancellable_async(
3391                bind_root_id.clone(),
3392                Lane::Mutating,
3393                configure_request_id.clone(),
3394                Box::new(move |ctx| {
3395                    log_ctx::with_session(Some(configure_session.clone()), || {
3396                        dispatch(configure_req, ctx)
3397                    })
3398                }),
3399            );
3400            pending_binds.insert(
3401                route_id,
3402                PendingBind {
3403                    bind_root_id: bind_root_id.clone(),
3404                    inserted_new_actor,
3405                    cancelled: false,
3406                    configure_request_id: configure_request_id.clone(),
3407                    started_at: Instant::now(),
3408                    warned_half_deadline: false,
3409                    deadline_reported: false,
3410                    corr: frame.header.corr,
3411                    ver: frame.header.ver,
3412                    flags: frame.header.flags,
3413                    cancellation: configure_cancellation,
3414                },
3415            );
3416
3417            let completion_tx = control_completion_tx.clone();
3418            let completion_identity = route_identity;
3419            let completion_root = bind_root_id.clone();
3420            let completion_route_channel = route_channel;
3421            let completion_ver = frame.header.ver;
3422            let completion_corr = frame.header.corr;
3423            let completion_flags = frame.header.flags;
3424            let completion_metrics = Arc::clone(metrics);
3425            tokio::spawn(async move {
3426                let _response_task = ResponseTaskGuard::new(&completion_metrics);
3427                let configure_response =
3428                    await_executor_response(configure_rx, configure_request_id.clone()).await;
3429                // Send the route-bind acknowledgment as soon as configure succeeds.
3430                // Installing completed search or callgraph builds only refreshes cached
3431                // read data, so a later maintenance pass can do it without delaying the
3432                // daemon's confirmation that the route is usable.
3433                let completion = RouteBindCompletion {
3434                    route: route_key(completion_route_channel, epoch),
3435                    identity: completion_identity,
3436                    bind_root_id: completion_root,
3437                    inserted_new_actor,
3438                    configure_response,
3439                    diagnostics_on_edit,
3440                    ver: completion_ver,
3441                    corr: completion_corr,
3442                    flags: completion_flags,
3443                };
3444                if send_counted_channel(
3445                    &completion_tx,
3446                    &completion_metrics.control_completion_queued,
3447                    completion,
3448                )
3449                .await
3450                .is_err()
3451                {
3452                    log::debug!(
3453                        "subc attach: dropped RouteBind completion for route {} after loop exit",
3454                        completion_route_channel
3455                    );
3456                }
3457            });
3458
3459            Ok(())
3460        }
3461        ModuleControlRequest::HealthCheck {} => {
3462            let report = build_health_report(executor, pending_binds, metrics);
3463            let body = serde_json::to_vec(&ModuleControlResponse::from(report))
3464                .map_err(SubcError::Json)?;
3465            let response = Frame::build_with_version(
3466                frame.header.ver,
3467                FrameType::Response,
3468                frame.header.flags,
3469                0,
3470                0,
3471                frame.header.corr,
3472                body,
3473            )
3474            .map_err(SubcError::FrameBuild)?;
3475            send_frame(tx, metrics, response).await
3476        }
3477    }
3478}
3479
3480fn install_bash_compressor(ctx: &AppContext) {
3481    // Mirrors main.rs per-actor compressor installation for subc-created actors.
3482    let filter_registry_handle = ctx.shared_filter_registry();
3483    let compress_flag = ctx.bash_compress_flag();
3484    ctx.bash_background().set_compressor_with_exit_code(
3485        move |command: &str, output: String, exit_code: Option<i32>| {
3486            if !compress_flag.load(std::sync::atomic::Ordering::Relaxed) {
3487                return crate::compress::CompressionResult::new(output);
3488            }
3489            let registry_guard = match filter_registry_handle.read() {
3490                Ok(g) => g,
3491                Err(poisoned) => poisoned.into_inner(),
3492            };
3493            crate::compress::compress_with_registry_exit_code(
3494                command,
3495                &output,
3496                exit_code,
3497                &registry_guard,
3498            )
3499        },
3500    );
3501}
3502
3503fn diagnostics_on_edit_from_tiers(tiers: &[ConfigTier]) -> bool {
3504    let mut diagnostics_on_edit = false;
3505    for tier in tiers {
3506        if let Some(value) = diagnostics_on_edit_from_doc(&tier.doc) {
3507            diagnostics_on_edit = value;
3508        }
3509    }
3510    diagnostics_on_edit
3511}
3512
3513fn diagnostics_on_edit_from_doc(doc: &str) -> Option<bool> {
3514    let stripped = strip_jsonc(doc);
3515    let value = serde_json::from_str::<Value>(&stripped).ok()?;
3516    value
3517        .get("lsp")
3518        .and_then(Value::as_object)?
3519        .get("diagnostics_on_edit")
3520        .and_then(Value::as_bool)
3521}
3522
3523async fn send_route_bind_error(
3524    tx: &WriterSender,
3525    frame: &Frame,
3526    code: &str,
3527    message: &str,
3528    metrics: &DispatchPathMetrics,
3529) -> Result<(), SubcError> {
3530    send_route_bind_error_parts(
3531        tx,
3532        frame.header.ver,
3533        frame.header.corr,
3534        frame.header.flags,
3535        code,
3536        message,
3537        metrics,
3538    )
3539    .await
3540}
3541
3542async fn send_route_bind_error_parts(
3543    tx: &WriterSender,
3544    ver: u8,
3545    corr: u64,
3546    flags: Flags,
3547    code: &str,
3548    message: &str,
3549    metrics: &DispatchPathMetrics,
3550) -> Result<(), SubcError> {
3551    let response = build_error_frame(ver, 0, 0, corr, flags, code, message)?;
3552    send_reliable_writer_frame(tx, metrics, response, "RouteBind error").await?;
3553    log::warn!("subc attach: route bind rejected ({code}): {message}");
3554    Ok(())
3555}
3556
3557/// Route-channel tool call: `{name, arguments}` → executor lane → dispatch to
3558/// the sync command core → wrap the structured Response in a CallToolResult
3559/// `{content, isError}`. Tool-result mapping: the whole `{success, ...}` Response
3560/// serialized into ONE text block; `isError` carries `success == false`.
3561async fn handle_tool_call(
3562    tx: &WriterSender,
3563    frame: &Frame,
3564    mut phase_trace: PhaseTrace,
3565    routes: &HashMap<RouteChannel, RouteIdentity>,
3566    pending_binds: &HashMap<RouteChannel, PendingBind>,
3567    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3568    executor: &Arc<Executor>,
3569    shutdown: &Arc<Notify>,
3570    connection_cancel: &PersistentCancelSignal,
3571    bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
3572    bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
3573    metrics: &Arc<DispatchPathMetrics>,
3574    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
3575    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
3576    next_bash_ask_corr: &mut u64,
3577    bg_subs: &mut HashMap<RouteChannel, BgSub>,
3578    bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
3579    bg_wake_pending: &mut HashSet<RouteChannel>,
3580    bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
3581    dispatch: DispatchFn,
3582    allow_native_passthrough: bool,
3583) -> Result<(), SubcError> {
3584    let route_id = route_key(frame.header.channel, frame.header.epoch);
3585    if pending_binds.contains_key(&route_id) {
3586        let error = build_error_frame(
3587            frame.header.ver,
3588            frame.header.channel,
3589            frame.header.epoch,
3590            frame.header.corr,
3591            frame.header.flags,
3592            "route_not_bound",
3593            "route is not bound before tool call",
3594        )?;
3595        return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
3596    }
3597
3598    let Some(identity) = routes.get(&route_id).cloned() else {
3599        let error = build_error_frame(
3600            frame.header.ver,
3601            frame.header.channel,
3602            frame.header.epoch,
3603            frame.header.corr,
3604            frame.header.flags,
3605            "route_not_bound",
3606            "route is not bound before tool call",
3607        )?;
3608        return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
3609    };
3610    let restore_watcher = live_roots
3611        .get(&identity.root)
3612        .is_some_and(|meta| meta.idle_artifacts_evicted);
3613    if let Some(meta) = live_roots.get_mut(&identity.root) {
3614        meta.reactivate_bound();
3615    }
3616    if restore_watcher {
3617        if let Some(ctx) = executor.actor_context(&identity.root) {
3618            crate::commands::configure::ensure_project_watcher(&ctx);
3619        }
3620    }
3621
3622    let route_request =
3623        serde_json::from_slice::<RouteRequest>(&frame.body).map_err(SubcError::Json)?;
3624    if matches!(
3625        route_request,
3626        RouteRequest::BgEvents(BgEventsRequest {
3627            op: BgEventsOp::BgEvents
3628        })
3629    ) {
3630        if let Some(old_sub) = bg_subs.get(&route_id).copied() {
3631            push::send_reliable_bg_stream_end(tx, metrics, route_id, &old_sub).await?;
3632        }
3633        if !identity.trust.allows_bash_observation() {
3634            bg_subs.remove(&route_id);
3635            bg_wake_pending.remove(&route_id);
3636            remove_bg_subscription_index(bg_sub_by_session, route_id, Some(&identity));
3637            let denied_sub = BgSub {
3638                corr: frame.header.corr,
3639                ver: frame.header.ver,
3640                flags: frame.header.flags,
3641            };
3642            push::send_reliable_bg_stream_end(tx, metrics, route_id, &denied_sub).await?;
3643            return Ok(());
3644        }
3645        bg_subs.insert(
3646            route_id,
3647            BgSub {
3648                corr: frame.header.corr,
3649                ver: frame.header.ver,
3650                flags: frame.header.flags,
3651            },
3652        );
3653        bg_sub_by_session.insert((identity.root.clone(), identity.session.clone()), route_id);
3654        push::arm_bg_wake(
3655            identity.root.clone(),
3656            identity.session.clone(),
3657            route_id,
3658            bg_wake_pending,
3659            bg_wake_epoch,
3660        );
3661        return Ok(());
3662    }
3663
3664    let RouteRequest::ToolCall(call) = route_request else {
3665        unreachable!("background event subscription returned above")
3666    };
3667    let bare_name = call.name;
3668    let arguments = strip_agent_preview_arg_owned(call.arguments);
3669    let format_context = crate::subc_format::FormatContext::from_tool_call(
3670        &bare_name,
3671        &arguments,
3672        identity.project_root.as_path(),
3673    );
3674
3675    let request_id = format!("subc-{}-{}", frame.header.channel, frame.header.corr);
3676    let bind_trust = identity.trust;
3677    let diagnostics_on_edit = live_roots
3678        .get(&identity.root)
3679        .map(|meta| meta.diagnostics_on_edit)
3680        .unwrap_or(false);
3681
3682    if matches!(bind_trust, BindTrust::Untrusted)
3683        && is_bash_family_tool(&bare_name)
3684        && (bare_name != "bash" || !identity.consumer_elicitation_capable)
3685    {
3686        let response = bash::bash_denied_untrusted_response(request_id.clone());
3687        let text = crate::subc_format::format_response_with_context(
3688            &bare_name,
3689            &response,
3690            &format_context,
3691        );
3692        let result = ToolCallResult { text, response };
3693        let response_frame = build_tool_response_frame(
3694            frame.header.ver,
3695            route_id,
3696            frame.header.corr,
3697            frame.header.flags,
3698            &result,
3699            bind_trust,
3700        )?;
3701        return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
3702    }
3703
3704    // A non-core name is NOT in the tool manifest. AFT fails closed and
3705    // does not trust subc to enforce the manifest: rejecting here is the
3706    // defense-in-depth backstop that prevents a forwarded native command
3707    // (e.g. `configure`, which would reach handle_configure and bypass
3708    // the RouteBind config-trust cap) from ever reaching dispatch. Only
3709    // the integration-test harness (run_subc_mode_for_test) opens this to
3710    // drive synthetic native commands through the executor.
3711    if !is_subc_agent_core_tool(&bare_name)
3712        && !is_subc_native_plumbing_tool(&bare_name)
3713        && !allow_native_passthrough
3714    {
3715        log::warn!(
3716            "subc tool call: rejecting non-manifest tool name {:?} on route {} (fail-closed)",
3717            bare_name,
3718            frame.header.channel
3719        );
3720        let response = Response::error(
3721            request_id.clone(),
3722            "unknown_tool",
3723            format!("tool {:?} is not in the AFT tool manifest", bare_name),
3724        );
3725        let text = crate::subc_format::format_response_with_context(
3726            &bare_name,
3727            &response,
3728            &format_context,
3729        );
3730        let result = ToolCallResult { text, response };
3731        let response_frame = build_tool_response_frame(
3732            frame.header.ver,
3733            route_id,
3734            frame.header.corr,
3735            frame.header.flags,
3736            &result,
3737            bind_trust,
3738        )?;
3739        return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
3740    }
3741
3742    if bare_name == "bash" {
3743        if matches!(bind_trust, BindTrust::Untrusted) {
3744            let plan = match bash::prepare_bash_elicitation_plan(
3745                &arguments,
3746                identity.project_root.as_path(),
3747            ) {
3748                Ok(plan) => plan,
3749                Err(error) => {
3750                    let response = Response::error(request_id.clone(), error.code, error.message);
3751                    let text = crate::subc_format::format_response_with_context(
3752                        &bare_name,
3753                        &response,
3754                        &format_context,
3755                    );
3756                    let result = ToolCallResult { text, response };
3757                    let response_frame = build_tool_response_frame(
3758                        frame.header.ver,
3759                        route_id,
3760                        frame.header.corr,
3761                        frame.header.flags,
3762                        &result,
3763                        bind_trust,
3764                    )?;
3765                    return send_reliable_writer_frame(
3766                        tx,
3767                        metrics,
3768                        response_frame,
3769                        "tool response",
3770                    )
3771                    .await;
3772                }
3773            };
3774
3775            let meta = live_roots
3776                .entry(identity.root.clone())
3777                .or_insert_with(|| RootMeta::new(Instant::now()));
3778            meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
3779            meta.reactivate_bound();
3780
3781            let route_cancel =
3782                route_bash_cancels
3783                    .entry(route_id)
3784                    .or_insert_with(|| bash::RouteBashCancel {
3785                        token: PersistentCancelSignal::new(),
3786                        active_waits: 0,
3787                    });
3788            route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
3789            let cancel = bash::BashWaitCancel {
3790                connection: connection_cancel.clone(),
3791                route: route_cancel.token.clone(),
3792            };
3793
3794            let reverse_corr =
3795                allocate_reverse_corr(pending_bash_asks, route_id, next_bash_ask_corr);
3796            let ask_frame = build_bash_elicitation_request_frame(
3797                frame.header.ver,
3798                route_id,
3799                reverse_corr,
3800                frame.header.flags,
3801                &plan.command,
3802                &plan.asks,
3803            )?;
3804            pending_bash_asks.insert(
3805                ReverseCorrKey {
3806                    route: route_id,
3807                    corr: reverse_corr,
3808                },
3809                PendingBashAsk {
3810                    route: route_id,
3811                    tool_corr: frame.header.corr,
3812                    tool_flags: frame.header.flags,
3813                    tool_ver: frame.header.ver,
3814                    root: identity.root.clone(),
3815                    project_root: identity.project_root.clone(),
3816                    session_id: identity.session.clone(),
3817                    request_id,
3818                    arguments,
3819                    format_context,
3820                    cancel,
3821                    grants: plan.grants,
3822                    expires_at: Instant::now() + bash_elicitation_timeout(),
3823                },
3824            );
3825            return send_reliable_writer_frame(tx, metrics, ask_frame, "bash elicitation request")
3826                .await;
3827        }
3828
3829        let meta = live_roots
3830            .entry(identity.root.clone())
3831            .or_insert_with(|| RootMeta::new(Instant::now()));
3832        meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
3833        meta.reactivate_bound();
3834
3835        let route_cancel =
3836            route_bash_cancels
3837                .entry(route_id)
3838                .or_insert_with(|| bash::RouteBashCancel {
3839                    token: PersistentCancelSignal::new(),
3840                    active_waits: 0,
3841                });
3842        route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
3843        let cancel = bash::BashWaitCancel {
3844            connection: connection_cancel.clone(),
3845            route: route_cancel.token.clone(),
3846        };
3847
3848        bash::submit_deferred_bash(
3849            executor,
3850            bash_deferred_tx,
3851            bash_poll_touch_tx,
3852            metrics,
3853            dispatch,
3854            identity.root.clone(),
3855            identity.project_root.clone(),
3856            identity.session.clone(),
3857            request_id,
3858            route_id,
3859            frame.header.corr,
3860            frame.header.flags,
3861            frame.header.ver,
3862            arguments,
3863            format_context,
3864            cancel,
3865            bind_trust,
3866            None,
3867        );
3868        return Ok(());
3869    }
3870
3871    let lane = command_lane(&bare_name);
3872    let tool_call_context = ToolCallContext {
3873        project_root: identity.project_root.clone(),
3874        session_id: Some(identity.session.clone()),
3875        request_id: request_id.clone(),
3876        diagnostics_on_edit,
3877        preview: call.preview,
3878    };
3879    let bare_name_for_frame = bare_name.clone();
3880    let identity_for_run = identity.clone();
3881    let completion_session = identity.session.clone();
3882    let completion_root = identity.project_root.clone();
3883    let request_id_for_force = request_id.clone();
3884    let format_context_for_frame = format_context.clone();
3885    let (tool_call_tx, tool_call_rx) = oneshot::channel::<ToolCallCompletion>();
3886    phase_trace.mark_executor_submitted();
3887    let rx = executor.submit_async(
3888        identity.root.clone(),
3889        lane,
3890        request_id.clone(),
3891        Box::new(move |ctx| {
3892            phase_trace.mark_job_admitted();
3893            log_ctx::with_session(Some(identity_for_run.session.clone()), || {
3894                let run = || {
3895                    let finalizer = |response: &mut Response| {
3896                        crate::response_finalize::finalize_response_with_bg_completions(
3897                            response,
3898                            ctx,
3899                            &identity_for_run.session,
3900                            &bare_name,
3901                            bind_trust.allows_bash_observation(),
3902                        );
3903                    };
3904                    match run_tool_call(
3905                        &bare_name,
3906                        &arguments,
3907                        &format_context,
3908                        &tool_call_context,
3909                        ctx,
3910                        &dispatch,
3911                        Some(&finalizer),
3912                        Some(&mut phase_trace),
3913                    ) {
3914                        ToolCallOutcome::Unary(result) => {
3915                            let response = result.response;
3916                            let _ = tool_call_tx.send(ToolCallCompletion {
3917                                text: result.text,
3918                                phase_trace,
3919                            });
3920                            response
3921                        }
3922                    }
3923                };
3924                if matches!(bind_trust, BindTrust::Untrusted) {
3925                    ctx.with_force_restrict(&request_id_for_force, run)
3926                } else {
3927                    run()
3928                }
3929            })
3930        }),
3931    );
3932    let completion_tx = tx.clone();
3933    let completion_shutdown = Arc::clone(shutdown);
3934    let route = route_id;
3935    let corr = frame.header.corr;
3936    let flags = frame.header.flags;
3937    let ver = frame.header.ver;
3938    let completion_metrics = Arc::clone(metrics);
3939    tokio::spawn(async move {
3940        let _response_task = ResponseTaskGuard::new(&completion_metrics);
3941        let response = await_executor_response(rx, request_id.clone()).await;
3942        let (text, phase_trace) = match tool_call_rx.await {
3943            Ok(completion) => (completion.text, Some(completion.phase_trace)),
3944            Err(_) => (
3945                crate::subc_format::format_response_with_context(
3946                    &bare_name_for_frame,
3947                    &response,
3948                    &format_context_for_frame,
3949                ),
3950                None,
3951            ),
3952        };
3953        let result = ToolCallResult { text, response };
3954        let fatal = response_is_fatal_panic(&result.response);
3955        match build_tool_response_frame(ver, route, corr, flags, &result, bind_trust) {
3956            Ok(response_frame) => {
3957                let send_result = if let Some(phase_trace) = phase_trace {
3958                    let trace = ToolResponseWriteTrace::new(
3959                        phase_trace,
3960                        bare_name_for_frame,
3961                        completion_root,
3962                        completion_session,
3963                        route.channel,
3964                        corr,
3965                    );
3966                    send_traced_tool_response_frame(
3967                        &completion_tx,
3968                        &completion_metrics,
3969                        response_frame,
3970                        trace,
3971                    )
3972                    .await
3973                } else {
3974                    send_reliable_writer_frame(
3975                        &completion_tx,
3976                        &completion_metrics,
3977                        response_frame,
3978                        "tool response",
3979                    )
3980                    .await
3981                };
3982                if let Err(error) = send_result {
3983                    log::warn!("subc attach: failed to queue tool response frame: {error}");
3984                }
3985            }
3986            Err(error) => {
3987                log::error!("subc attach: failed to build tool response frame: {error}");
3988            }
3989        }
3990        if fatal {
3991            signal_fatal_teardown(
3992                &completion_tx,
3993                Some(route),
3994                ver,
3995                corr,
3996                &completion_shutdown,
3997                &completion_metrics,
3998            )
3999            .await;
4000        }
4001    });
4002    Ok(())
4003}
4004
4005fn submit_maintenance_job(
4006    executor: &Arc<Executor>,
4007    root_id: ProjectRootId,
4008    kind: MaintenanceDrainKind,
4009    bg_sessions_to_check: Vec<(String, u64)>,
4010    completion_tx: &mpsc::Sender<MaintenanceCompletion>,
4011    metrics: &Arc<DispatchPathMetrics>,
4012) {
4013    let request_id = format!(
4014        "subc-maintenance-drain-{}-{}",
4015        kind.label(),
4016        root_id.as_path().to_string_lossy()
4017    );
4018    let response_id = request_id.clone();
4019    let completion_root_id = root_id.clone();
4020    let maintenance_generation = executor
4021        .actor_context(&root_id)
4022        .map(|ctx| ctx.configure_generation())
4023        .unwrap_or(0);
4024    let (outcome_tx, outcome_rx) = oneshot::channel::<MaintenanceJobOutcome>();
4025    // ConfigureTail runs deferred configure mutations and needs the actor
4026    // epoch write gate. The other drain kinds only mutate subsystem state
4027    // behind that subsystem's own lock, so they run on MaintenanceCommit and
4028    // overlap interactive reads instead of excluding them.
4029    let lane = match kind {
4030        MaintenanceDrainKind::ConfigureTail => Lane::Mutating,
4031        MaintenanceDrainKind::Watcher
4032        | MaintenanceDrainKind::Lsp
4033        | MaintenanceDrainKind::CompletionDrains => Lane::MaintenanceCommit,
4034    };
4035    let rx = executor.submit_maintenance_async(
4036        root_id,
4037        lane,
4038        request_id.clone(),
4039        Box::new(move |ctx| {
4040            let outcome = match kind {
4041                MaintenanceDrainKind::Watcher => {
4042                    let drained = runtime_drain::drain_watcher_events_bounded(
4043                        ctx,
4044                        runtime_drain::WATCHER_PATH_DRAIN_BATCH_CAP,
4045                    );
4046                    MaintenanceJobOutcome {
4047                        empty_bg_sessions: Vec::new(),
4048                        requeue_kind: drained.has_more.then_some(kind),
4049                    }
4050                }
4051                MaintenanceDrainKind::Lsp => {
4052                    let drained = runtime_drain::drain_lsp_events_bounded(
4053                        ctx,
4054                        runtime_drain::LSP_EVENT_DRAIN_BATCH_CAP,
4055                    );
4056                    MaintenanceJobOutcome {
4057                        empty_bg_sessions: Vec::new(),
4058                        requeue_kind: drained.has_more.then_some(kind),
4059                    }
4060                }
4061                MaintenanceDrainKind::ConfigureTail => {
4062                    runtime_drain::drain_deferred_configure_maintenance(ctx);
4063                    runtime_drain::drain_configure_warning_events(ctx);
4064                    MaintenanceJobOutcome::default()
4065                }
4066                MaintenanceDrainKind::CompletionDrains => {
4067                    runtime_drain::drain_search_index_events(ctx);
4068                    runtime_drain::drain_callgraph_store_events(ctx);
4069                    runtime_drain::drain_semantic_index_events(ctx);
4070                    runtime_drain::drain_semantic_refresh_events(ctx);
4071                    runtime_drain::drain_inspect_events_for_generation(ctx, maintenance_generation);
4072                    let empty_bg_sessions = bg_sessions_to_check
4073                        .into_iter()
4074                        .filter(|(session, _)| {
4075                            !ctx.bash_background()
4076                                .has_completions_for_session(Some(session.as_str()))
4077                        })
4078                        .collect();
4079                    MaintenanceJobOutcome {
4080                        empty_bg_sessions,
4081                        requeue_kind: None,
4082                    }
4083                }
4084            };
4085            let requeued = outcome.requeue_kind.is_some();
4086            let _ = outcome_tx.send(outcome);
4087            Response::success(
4088                response_id,
4089                json!({ "drained": true, "kind": kind.label(), "requeued": requeued }),
4090            )
4091        }),
4092    );
4093    let completion_tx = completion_tx.clone();
4094    let completion_metrics = Arc::clone(metrics);
4095    tokio::spawn(async move {
4096        let _response_task = ResponseTaskGuard::new(&completion_metrics);
4097        let response = await_executor_response(rx, request_id).await;
4098        let outcome = outcome_rx.await.unwrap_or_default();
4099        let _ = send_counted_channel(
4100            &completion_tx,
4101            &completion_metrics.maintenance_queued,
4102            MaintenanceCompletion {
4103                root_id: completion_root_id,
4104                kind,
4105                response,
4106                empty_bg_sessions: outcome.empty_bg_sessions,
4107                requeue_kind: outcome.requeue_kind,
4108            },
4109        )
4110        .await;
4111    });
4112}
4113
4114async fn await_executor_response(rx: oneshot::Receiver<Response>, request_id: String) -> Response {
4115    rx.await
4116        .unwrap_or_else(|_| Response::error(request_id, "internal_error", "executor dropped"))
4117}
4118async fn signal_fatal_teardown(
4119    tx: &WriterSender,
4120    route: Option<RouteChannel>,
4121    ver: u8,
4122    corr: u64,
4123    shutdown: &Arc<Notify>,
4124    metrics: &DispatchPathMetrics,
4125) {
4126    if let Some(route) = route {
4127        if let Ok(frame) = build_goodbye_frame(ver, route.channel, route.epoch, corr) {
4128            if let Err(error) = send_frame(tx, metrics, frame).await {
4129                log::warn!(
4130                    "subc attach: failed to queue fatal route Goodbye for route {route}: {error}"
4131                );
4132            }
4133        }
4134    }
4135    if let Ok(frame) = build_goodbye_frame(ver, 0, 0, 0) {
4136        if let Err(error) = send_frame(tx, metrics, frame).await {
4137            log::warn!("subc attach: failed to queue fatal channel-0 Goodbye: {error}");
4138        }
4139    }
4140    shutdown.notify_one();
4141}
4142#[derive(Debug, Deserialize)]
4143#[serde(untagged)]
4144enum RouteRequest {
4145    BgEvents(BgEventsRequest),
4146    ToolCall(ToolCallRequest),
4147}
4148
4149#[derive(Debug, Deserialize)]
4150struct BgEventsRequest {
4151    op: BgEventsOp,
4152}
4153
4154#[derive(Debug, Deserialize)]
4155#[serde(rename_all = "snake_case")]
4156enum BgEventsOp {
4157    BgEvents,
4158}
4159
4160#[derive(Debug, Deserialize)]
4161struct ToolCallRequest {
4162    name: String,
4163    #[serde(default)]
4164    arguments: Value,
4165    /// Server-owned preview control (B1c-0): the plugin's mutation flow is
4166    /// preview -> permission ask -> apply. Dropping this field made "preview"
4167    /// calls mutate disk before the permission prompt and the subsequent
4168    /// apply fail with not-found.
4169    #[serde(default)]
4170    preview: bool,
4171}
4172
4173#[cfg(test)]
4174pub(crate) mod test_support {
4175    use super::*;
4176    use crate::bash_background::BgTaskStatus;
4177    use crate::protocol::{
4178        BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, ConfigureWarningsFrame,
4179        ProgressFrame, StatusChangedFrame,
4180    };
4181    use serde_json::json;
4182
4183    pub(super) fn test_root(name: &str) -> (tempfile::TempDir, ProjectRootId) {
4184        let dir = tempfile::Builder::new()
4185            .prefix(name)
4186            .tempdir()
4187            .expect("temp root");
4188        let root = ProjectRootId::from_path(dir.path()).expect("project root id");
4189        (dir, root)
4190    }
4191
4192    pub(super) fn test_ctx() -> Arc<AppContext> {
4193        Arc::new(AppContext::new(
4194            Box::new(crate::parser::TreeSitterProvider::new()),
4195            crate::config::Config::default(),
4196        ))
4197    }
4198
4199    pub(super) fn wait_for_watcher_count(ctx: &AppContext, expected: usize) {
4200        let deadline = Instant::now() + Duration::from_secs(30);
4201        loop {
4202            let observed = ctx.watcher_registry_count();
4203            if observed == expected {
4204                return;
4205            }
4206            assert!(
4207                Instant::now() < deadline,
4208                "watcher count did not settle before deadline: expected={expected}, observed={observed}"
4209            );
4210            std::thread::sleep(Duration::from_millis(50));
4211        }
4212    }
4213
4214    pub(super) fn wait_for_actor_root_count(app: &App, expected: usize) {
4215        let deadline = Instant::now() + Duration::from_secs(30);
4216        loop {
4217            let observed = app.actor_root_count();
4218            if observed == expected {
4219                return;
4220            }
4221            assert!(
4222                Instant::now() < deadline,
4223                "actor root count did not settle before deadline: expected={expected}, observed={observed}"
4224            );
4225            std::thread::sleep(Duration::from_millis(50));
4226        }
4227    }
4228
4229    pub(super) fn status_frame(seq: u64) -> PushFrame {
4230        status_frame_with_session(seq, None)
4231    }
4232
4233    pub(super) fn status_frame_with_session(seq: u64, session_id: Option<&str>) -> PushFrame {
4234        PushFrame::StatusChanged(StatusChangedFrame {
4235            frame_type: "status_changed",
4236            session_id: session_id.map(str::to_string),
4237            snapshot: json!({ "seq": seq }),
4238        })
4239    }
4240
4241    pub(super) fn completion_frame(task_id: &str) -> PushFrame {
4242        completion_frame_with_session(task_id, "session-1")
4243    }
4244
4245    pub(super) fn completion_frame_with_session(task_id: &str, session_id: &str) -> PushFrame {
4246        PushFrame::BashCompleted(BashCompletedFrame {
4247            frame_type: "bash_completed",
4248            task_id: task_id.to_string(),
4249            session_id: session_id.to_string(),
4250            status: BgTaskStatus::Completed,
4251            exit_code: Some(0),
4252            command: format!("echo {task_id}"),
4253            output_preview: String::new(),
4254            output_truncated: false,
4255            original_tokens: None,
4256            compressed_tokens: None,
4257            tokens_skipped: false,
4258        })
4259    }
4260
4261    pub(super) fn long_running_frame(task_id: &str, elapsed_ms: u64) -> PushFrame {
4262        long_running_frame_with_session(task_id, "session-1", elapsed_ms)
4263    }
4264
4265    pub(super) fn long_running_frame_with_session(
4266        task_id: &str,
4267        session_id: &str,
4268        elapsed_ms: u64,
4269    ) -> PushFrame {
4270        PushFrame::BashLongRunning(BashLongRunningFrame {
4271            frame_type: "bash_long_running",
4272            task_id: task_id.to_string(),
4273            session_id: session_id.to_string(),
4274            command: format!("sleep {elapsed_ms}"),
4275            elapsed_ms,
4276        })
4277    }
4278
4279    pub(super) fn pattern_match_frame(session_id: &str) -> PushFrame {
4280        PushFrame::BashPatternMatch(BashPatternMatchFrame {
4281            frame_type: "bash_pattern_match",
4282            task_id: "task-pattern".to_string(),
4283            session_id: session_id.to_string(),
4284            watch_id: "watch-1".to_string(),
4285            match_text: "needle".to_string(),
4286            match_offset: 7,
4287            context: "haystack needle".to_string(),
4288            once: true,
4289            reason: "pattern_match",
4290        })
4291    }
4292
4293    pub(super) fn configure_warnings_frame(session_id: Option<&str>) -> PushFrame {
4294        PushFrame::ConfigureWarnings(ConfigureWarningsFrame {
4295            frame_type: "configure_warnings",
4296            session_id: session_id.map(str::to_string),
4297            project_root: "/tmp/subc-test".to_string(),
4298            warnings: Vec::new(),
4299        })
4300    }
4301
4302    pub(super) fn route_identity(root: &ProjectRootId, session_id: &str) -> RouteIdentity {
4303        route_identity_with_trust(root, session_id, BindTrust::FirstParty)
4304    }
4305
4306    pub(super) fn route_identity_with_trust(
4307        root: &ProjectRootId,
4308        session_id: &str,
4309        trust: BindTrust,
4310    ) -> RouteIdentity {
4311        RouteIdentity(Arc::new(RouteIdentityData {
4312            root: root.clone(),
4313            project_root: root.as_path().to_path_buf(),
4314            harness: "opencode".to_string(),
4315            session: session_id.to_string(),
4316            trust,
4317            consumer_elicitation_capable: false,
4318        }))
4319    }
4320
4321    pub(super) fn progress_frame(request_id: &str, kind: ProgressKind, chunk: &str) -> PushFrame {
4322        PushFrame::Progress(ProgressFrame::new(request_id, kind, chunk))
4323    }
4324
4325    pub(super) fn status_seq(frame: &PushFrame) -> Option<u64> {
4326        match frame {
4327            PushFrame::StatusChanged(status) => status.snapshot.get("seq").and_then(|v| v.as_u64()),
4328            _ => None,
4329        }
4330    }
4331
4332    pub(super) fn completion_task(frame: &PushFrame) -> Option<&str> {
4333        match frame {
4334            PushFrame::BashCompleted(completion) => Some(completion.task_id.as_str()),
4335            _ => None,
4336        }
4337    }
4338
4339    pub(super) fn push_frame_task_id(frame: &Frame) -> Option<String> {
4340        let body: serde_json::Value = serde_json::from_slice(&frame.body).expect("push body");
4341        body.get("task_id")
4342            .and_then(serde_json::Value::as_str)
4343            .map(str::to_string)
4344    }
4345}
4346
4347#[cfg(test)]
4348mod tests {
4349    use super::test_support::{
4350        completion_frame, test_ctx, test_root, wait_for_actor_root_count, wait_for_watcher_count,
4351    };
4352    use super::*;
4353
4354    fn attach_error(kind: io::ErrorKind) -> SubcError {
4355        SubcError::Connect {
4356            endpoint: "127.0.0.1:1".to_string(),
4357            source: io::Error::new(kind, "constructed attach failure"),
4358        }
4359    }
4360
4361    fn auth_io_error(kind: io::ErrorKind) -> SubcError {
4362        SubcError::Auth {
4363            endpoint: "127.0.0.1:1".to_string(),
4364            source: subc_transport::AuthError::Io {
4365                stage: subc_transport::AuthStage::ServerProof,
4366                source: io::Error::new(kind, "constructed auth failure"),
4367            },
4368        }
4369    }
4370
4371    #[test]
4372    fn initial_attach_error_classifier_distinguishes_transient_and_permanent_failures() {
4373        let transient_errors = vec![
4374            attach_error(io::ErrorKind::ConnectionRefused),
4375            attach_error(io::ErrorKind::TimedOut),
4376            attach_error(io::ErrorKind::ConnectionReset),
4377            auth_io_error(io::ErrorKind::ConnectionAborted),
4378            auth_io_error(io::ErrorKind::BrokenPipe),
4379            SubcError::Auth {
4380                endpoint: "127.0.0.1:1".to_string(),
4381                source: subc_transport::AuthError::UnexpectedEof {
4382                    stage: subc_transport::AuthStage::ServerProof,
4383                    expected: 4,
4384                    actual: 0,
4385                },
4386            },
4387            SubcError::Auth {
4388                endpoint: "127.0.0.1:1".to_string(),
4389                source: subc_transport::AuthError::Timeout {
4390                    stage: subc_transport::AuthStage::ServerProof,
4391                    deadline: AUTH_DEADLINE,
4392                },
4393            },
4394        ];
4395        for error in &transient_errors {
4396            assert_eq!(
4397                classify_attach_error(error),
4398                AttachErrorClass::Transient,
4399                "expected transient: {error}"
4400            );
4401        }
4402
4403        let permanent_errors = vec![
4404            attach_error(io::ErrorKind::PermissionDenied),
4405            auth_io_error(io::ErrorKind::InvalidData),
4406            SubcError::Auth {
4407                endpoint: "127.0.0.1:1".to_string(),
4408                source: subc_transport::AuthError::InvalidServerProof,
4409            },
4410            SubcError::Auth {
4411                endpoint: "127.0.0.1:1".to_string(),
4412                source: subc_transport::AuthError::DaemonIdMismatch,
4413            },
4414            SubcError::ConnectionFile {
4415                path: PathBuf::from("subc-connection.json"),
4416                source: subc_transport::ConnectionFileError::Invalid {
4417                    reason: "constructed invalid file".to_string(),
4418                },
4419            },
4420            SubcError::NoEndpoint {
4421                path: PathBuf::from("subc-connection.json"),
4422            },
4423            SubcError::InvalidEndpoint {
4424                path: PathBuf::from("subc-connection.json"),
4425                endpoint: "not-an-ip:1234".to_string(),
4426            },
4427        ];
4428        for error in &permanent_errors {
4429            assert_eq!(
4430                classify_attach_error(error),
4431                AttachErrorClass::Permanent,
4432                "expected permanent: {error}"
4433            );
4434        }
4435    }
4436
4437    #[test]
4438    fn incompatible_wire_version_is_rejected_before_tcp_connect() {
4439        let conn_dir = tempfile::tempdir().expect("connection tempdir");
4440        let conn_path = conn_dir.path().join("subc-connection.json");
4441        let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind listener");
4442        listener
4443            .set_nonblocking(true)
4444            .expect("set listener nonblocking");
4445        let port = listener.local_addr().expect("listener addr").port();
4446        connection_file::write_atomic(
4447            &conn_path,
4448            &connection_file::ConnectionInfo {
4449                schema: connection_file::SCHEMA_VERSION,
4450                wire_version: Some(PROTOCOL_VERSION.wrapping_add(1)),
4451                endpoints: vec![connection_file::Endpoint {
4452                    host: "127.0.0.1".to_string(),
4453                    port,
4454                }],
4455                key: vec![0x42; subc_transport::KEY_LEN],
4456                daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
4457                pid: std::process::id(),
4458                daemon_ver: "subc-test".to_string(),
4459            },
4460        )
4461        .expect("write connection file");
4462
4463        let runtime = tokio::runtime::Builder::new_current_thread()
4464            .enable_all()
4465            .build()
4466            .expect("test runtime");
4467        let result = runtime.block_on(connect_and_authenticate_with_policy(
4468            &conn_path,
4469            AttachRetryPolicy {
4470                budget: Duration::from_secs(1),
4471                initial_backoff: Duration::from_millis(5),
4472                max_backoff: Duration::from_millis(10),
4473                jitter_percent: 0,
4474            },
4475        ));
4476        assert!(matches!(
4477            result,
4478            Err(SubcError::ConnectionFile {
4479                source: connection_file::ConnectionFileError::WireVersionMismatch { .. },
4480                ..
4481            })
4482        ));
4483        assert!(matches!(
4484            listener.accept(),
4485            Err(error) if error.kind() == std::io::ErrorKind::WouldBlock
4486        ));
4487    }
4488
4489    #[test]
4490    fn initial_attach_unreachable_endpoint_retries_until_budget_then_fails_loud() {
4491        let conn_dir = tempfile::tempdir().expect("connection tempdir");
4492        let conn_path = conn_dir.path().join("subc-connection.json");
4493        let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve port");
4494        let port = listener.local_addr().expect("reserved addr").port();
4495        drop(listener);
4496        connection_file::write_atomic(
4497            &conn_path,
4498            &connection_file::ConnectionInfo {
4499                schema: connection_file::SCHEMA_VERSION,
4500                wire_version: Some(PROTOCOL_VERSION),
4501                endpoints: vec![connection_file::Endpoint {
4502                    host: "127.0.0.1".to_string(),
4503                    port,
4504                }],
4505                key: vec![0x42; subc_transport::KEY_LEN],
4506                daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
4507                pid: std::process::id(),
4508                daemon_ver: "subc-test".to_string(),
4509            },
4510        )
4511        .expect("write connection file");
4512
4513        let policy = AttachRetryPolicy {
4514            budget: Duration::from_millis(40),
4515            initial_backoff: Duration::from_millis(5),
4516            max_backoff: Duration::from_millis(10),
4517            jitter_percent: 0,
4518        };
4519        let runtime = tokio::runtime::Builder::new_current_thread()
4520            .enable_all()
4521            .build()
4522            .expect("test runtime");
4523        let started_at = Instant::now();
4524        let result = runtime.block_on(connect_and_authenticate_with_policy(&conn_path, policy));
4525        let elapsed = started_at.elapsed();
4526        let error = match result {
4527            Ok(_) => panic!("unreachable endpoint unexpectedly attached"),
4528            Err(error) => error,
4529        };
4530
4531        assert!(matches!(error, SubcError::Connect { .. }), "{error}");
4532        assert!(
4533            elapsed >= Duration::from_millis(35),
4534            "retry budget ended too early: {elapsed:?}"
4535        );
4536        assert!(
4537            elapsed < Duration::from_secs(1),
4538            "retry budget was not bounded: {elapsed:?}"
4539        );
4540    }
4541
4542    fn due_maintenance_jobs_without_actor_context(
4543        live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4544        budget: usize,
4545        pending_bind_roots: &HashSet<ProjectRootId>,
4546    ) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
4547        due_maintenance_jobs(
4548            live_roots,
4549            None,
4550            &HashMap::new(),
4551            &HashSet::new(),
4552            budget,
4553            pending_bind_roots,
4554        )
4555    }
4556
4557    fn actor_ctx_with_dirty_search_index(
4558        root: &Path,
4559        storage: &Path,
4560        file_name: &str,
4561        old_contents: &str,
4562        new_contents: &str,
4563    ) -> (Arc<AppContext>, PathBuf, PathBuf) {
4564        let file = root.join(file_name);
4565        std::fs::write(&file, old_contents).expect("write source");
4566        let canonical_root = std::fs::canonicalize(root).expect("canonical root");
4567        let ctx = Arc::new(AppContext::new(
4568            Box::new(crate::parser::TreeSitterProvider::new()),
4569            Config {
4570                project_root: Some(root.to_path_buf()),
4571                storage_dir: Some(storage.to_path_buf()),
4572                ..Config::default()
4573            },
4574        ));
4575        ctx.set_canonical_cache_root(canonical_root.clone());
4576
4577        let cache_dir = crate::search_index::resolve_cache_dir(&canonical_root, Some(storage));
4578        let mut index = crate::search_index::SearchIndex::build(&canonical_root);
4579        let git_head = index.stored_git_head().map(str::to_owned);
4580        index.write_to_disk(&cache_dir, git_head.as_deref());
4581
4582        std::fs::write(&file, new_contents).expect("edit source");
4583        index.update_file(&file);
4584        *ctx.search_index()
4585            .write()
4586            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
4587        (ctx, canonical_root, cache_dir)
4588    }
4589
4590    #[test]
4591    fn graceful_shutdown_flushes_every_actor_search_index() {
4592        let storage = tempfile::tempdir().expect("storage tempdir");
4593        let (root1_dir, root1) = test_root("shutdown-flush-root-1");
4594        let (root2_dir, root2) = test_root("shutdown-flush-root-2");
4595        let (ctx1, canonical_root1, cache_dir1) = actor_ctx_with_dirty_search_index(
4596            root1_dir.path(),
4597            storage.path(),
4598            "alpha.txt",
4599            "old actor one token\n",
4600            "new actor one token\n",
4601        );
4602        let (ctx2, canonical_root2, cache_dir2) = actor_ctx_with_dirty_search_index(
4603            root2_dir.path(),
4604            storage.path(),
4605            "beta.txt",
4606            "old actor two token\n",
4607            "new actor two token\n",
4608        );
4609
4610        let executor = Executor::new();
4611        assert!(executor.register_actor(root1.clone(), Arc::clone(&ctx1)));
4612        assert!(executor.register_actor(root2.clone(), Arc::clone(&ctx2)));
4613
4614        flush_actor_indexes_on_graceful_shutdown(&executor.actor_contexts());
4615
4616        let mut restored1 =
4617            crate::search_index::SearchIndex::read_from_disk(&cache_dir1, &canonical_root1)
4618                .expect("load flushed root one index");
4619        restored1.ready = true;
4620        assert_eq!(
4621            restored1
4622                .grep("new actor one token", true, &[], &[], &canonical_root1, 10)
4623                .matches
4624                .len(),
4625            1,
4626            "graceful subc shutdown should flush the first root's trigram delta"
4627        );
4628
4629        let mut restored2 =
4630            crate::search_index::SearchIndex::read_from_disk(&cache_dir2, &canonical_root2)
4631                .expect("load flushed root two index");
4632        restored2.ready = true;
4633        assert_eq!(
4634            restored2
4635                .grep("new actor two token", true, &[], &[], &canonical_root2, 10)
4636                .matches
4637                .len(),
4638            1,
4639            "graceful subc shutdown should flush every registered root"
4640        );
4641    }
4642
4643    #[test]
4644    fn idle_root_reaper_closes_artifacts_and_stops_watcher() {
4645        let _ = env_logger::builder().is_test(true).try_init();
4646        let (root_dir, root) = test_root("idle-root-reaper");
4647        let storage = tempfile::tempdir().expect("storage tempdir");
4648        std::fs::write(
4649            root_dir.path().join("main.rs"),
4650            "fn entry() { leaf(); }\nfn leaf() {}\n",
4651        )
4652        .expect("source file");
4653        let canonical_root = std::fs::canonicalize(root_dir.path()).expect("canonical root");
4654        let app = App::default_shared();
4655        let ctx = Arc::new(AppContext::from_app(
4656            Arc::clone(&app),
4657            Config {
4658                project_root: Some(canonical_root.clone()),
4659                storage_dir: Some(storage.path().to_path_buf()),
4660                callgraph_store: true,
4661                search_index: true,
4662                ..Config::default()
4663            },
4664        ));
4665        ctx.set_canonical_cache_root(canonical_root.clone());
4666        assert!(ctx
4667            .ensure_callgraph_store()
4668            .expect("build callgraph store")
4669            .is_some());
4670
4671        let cache_dir =
4672            crate::search_index::resolve_cache_dir(&canonical_root, Some(storage.path()));
4673        let mut index = crate::search_index::SearchIndex::build(&canonical_root);
4674        let git_head = index.stored_git_head().map(str::to_owned);
4675        index.write_to_disk(&cache_dir, git_head.as_deref());
4676        *ctx.search_index()
4677            .write()
4678            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
4679        // Seed a completed warm verification so the test can prove eviction
4680        // downgrades it to Strict rather than passing vacuously.
4681        let seeded_generation =
4682            crate::cache_freshness::artifact_generation(&cache_dir.join("cache.bin"))
4683                .expect("seeded artifact generation");
4684        crate::cache_freshness::record_verify_completed(
4685            &canonical_root,
4686            crate::cache_freshness::VerifyArtifact::Search,
4687            Some(seeded_generation),
4688        );
4689        assert!(
4690            matches!(
4691                crate::cache_freshness::warm_verify_plan(
4692                    canonical_root.as_path(),
4693                    crate::cache_freshness::VerifyArtifact::Search,
4694                    Some(seeded_generation),
4695                ),
4696                crate::cache_freshness::WarmVerifyPlan::Skip
4697            ),
4698            "memo must be warm before eviction for the downgrade assertion to bite"
4699        );
4700
4701        let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
4702        let _dispatch_tx = dispatch_tx;
4703        let shutdown = Arc::new(AtomicBool::new(false));
4704        let thread_shutdown = Arc::clone(&shutdown);
4705        let join = std::thread::spawn(move || {
4706            while !thread_shutdown.load(Ordering::SeqCst) {
4707                std::thread::yield_now();
4708            }
4709        });
4710        ctx.install_watcher_runtime(
4711            dispatch_rx,
4712            crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
4713        );
4714        wait_for_watcher_count(&ctx, 1);
4715
4716        let executor = Arc::new(Executor::new());
4717        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
4718        ctx.mark_subc_unbound();
4719        let mut live_roots = HashMap::new();
4720        let mut meta = RootMeta::new(Instant::now());
4721        meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
4722        meta.unbound_quiesced = true;
4723        live_roots.insert(root.clone(), meta);
4724
4725        let message = idle_root_eviction_message(&root, &ctx.memory_root_snapshot(), None);
4726        assert!(message.contains("evicted idle root"));
4727        assert!(message.contains("freed ~"));
4728        assert!(message.contains("semantic"));
4729        assert!(!message.contains("semantic not estimated retained"));
4730        assert!(message.contains("trigram"));
4731        assert!(message.contains("retained: bash"));
4732        assert!(message.contains("parser_pool"));
4733
4734        assert_eq!(
4735            reap_idle_roots(
4736                Instant::now(),
4737                &mut live_roots,
4738                &HashMap::new(),
4739                &HashMap::new(),
4740                &executor,
4741            )
4742            .evicted,
4743            1
4744        );
4745        assert!(ctx.search_index().read().unwrap().is_none());
4746        wait_for_watcher_count(&ctx, 0);
4747        // The watcher stopped with the eviction, so the idle interval is
4748        // unobserved: the pre-seeded warm-verify memo (Skip) must fall back to
4749        // strict content verification (stat-first would miss same-size,
4750        // preserved-mtime edits made while nobody was watching).
4751        assert!(
4752            matches!(
4753                crate::cache_freshness::warm_verify_plan(
4754                    canonical_root.as_path(),
4755                    crate::cache_freshness::VerifyArtifact::Search,
4756                    Some(seeded_generation),
4757                ),
4758                crate::cache_freshness::WarmVerifyPlan::Strict
4759            ),
4760            "idle eviction must force strict re-verification"
4761        );
4762        assert!(
4763            crate::search_index::SearchIndex::read_from_disk(&cache_dir, &canonical_root).is_some()
4764        );
4765        ctx.mark_subc_bound();
4766        assert!(ctx
4767            .ensure_callgraph_store()
4768            .expect("reopen callgraph store")
4769            .is_some());
4770        assert!(live_roots[&root].idle_artifacts_evicted);
4771    }
4772
4773    #[test]
4774    fn idle_root_reaper_applies_ttl_to_unbound_roots() {
4775        let (_root_dir, root) = test_root("idle-root-ttl-gate");
4776        let ctx = test_ctx();
4777        let executor = Arc::new(Executor::new());
4778        assert!(executor.register_actor(root.clone(), ctx));
4779        let ctx = executor.actor_context(&root).expect("actor context");
4780        ctx.mark_subc_unbound();
4781        let now = Instant::now();
4782        let mut meta = RootMeta::new(now);
4783        meta.unbound_quiesced = true;
4784        let mut live_roots = HashMap::from([(root.clone(), meta)]);
4785
4786        // A recently-unbound root stays warm: a transient unbind (host
4787        // restart) must not pay the strict-verify + forced-rebuild teardown
4788        // on the next maintenance sweep.
4789        assert_eq!(
4790            reap_idle_roots(
4791                now,
4792                &mut live_roots,
4793                &HashMap::new(),
4794                &HashMap::new(),
4795                &executor,
4796            )
4797            .evicted,
4798            0
4799        );
4800        assert!(!live_roots[&root].idle_artifacts_evicted);
4801
4802        // Past the TTL the same unbound root pays the full teardown. The
4803        // pending reconciliation paths retained across the transient-unbind
4804        // window would block eviction forever through
4805        // `artifact_eviction_blocked`; the reaper disposes them because the
4806        // strict gap invalidation subsumes their purpose.
4807        ctx.add_pending_search_index_paths([root.as_path().join("retained.rs")]);
4808        assert_eq!(
4809            reap_idle_roots(
4810                now + IDLE_ROOT_TTL,
4811                &mut live_roots,
4812                &HashMap::new(),
4813                &HashMap::new(),
4814                &executor,
4815            )
4816            .evicted,
4817            1
4818        );
4819        assert!(live_roots[&root].idle_artifacts_evicted);
4820        assert!(
4821            ctx.take_pending_search_index_paths().is_empty(),
4822            "TTL eviction must dispose retained pending reconciliation paths"
4823        );
4824    }
4825
4826    #[test]
4827    fn blocked_ttl_eviction_restores_taken_pending_reconciliation_state() {
4828        let (_root_dir, root) = test_root("ttl-eviction-blocked-restore");
4829        let ctx = test_ctx();
4830        let executor = Arc::new(Executor::new());
4831        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
4832        ctx.mark_subc_unbound();
4833
4834        // Retained pending path from the transient-unbind window plus a
4835        // SECONDARY eviction blocker (a non-ready resident search index, the
4836        // dirty-index blocker in artifact_eviction_blocked). Disposal must be
4837        // transactional: the blocked eviction may be followed by a rebind, and
4838        // the path is the only repair record for its consumed watcher event.
4839        let pending = root.as_path().join("edited-while-unbound.rs");
4840        ctx.add_pending_search_index_paths([pending.clone()]);
4841        let dirty_source = root.as_path().join("dirty.rs");
4842        std::fs::write(&dirty_source, "fn dirty() {}\n").expect("dirty source");
4843        let mut dirty = crate::search_index::SearchIndex::new();
4844        dirty.ready = true;
4845        dirty.update_file(&dirty_source);
4846        *ctx.search_index()
4847            .write()
4848            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(dirty);
4849        assert!(ctx.artifact_eviction_blocked());
4850
4851        let mut live_roots = HashMap::new();
4852        let mut meta = RootMeta::new(Instant::now());
4853        meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
4854        meta.unbound_quiesced = true;
4855        live_roots.insert(root.clone(), meta);
4856
4857        assert_eq!(
4858            reap_idle_roots(
4859                Instant::now(),
4860                &mut live_roots,
4861                &HashMap::new(),
4862                &HashMap::new(),
4863                &executor,
4864            )
4865            .evicted,
4866            0,
4867            "the dirty index must still block this eviction"
4868        );
4869        assert_eq!(
4870            ctx.take_pending_search_index_paths(),
4871            vec![pending],
4872            "a blocked eviction must restore the taken pending paths"
4873        );
4874    }
4875
4876    #[test]
4877    fn idle_reap_with_bound_route_keeps_watcher_running() {
4878        let (_root_dir, root) = test_root("bound-root-reap-gate");
4879        let ctx = test_ctx();
4880        let executor = Arc::new(Executor::new());
4881        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
4882
4883        let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
4884        let _dispatch_tx = dispatch_tx;
4885        let shutdown = Arc::new(AtomicBool::new(false));
4886        let thread_shutdown = Arc::clone(&shutdown);
4887        let join = std::thread::spawn(move || {
4888            while !thread_shutdown.load(Ordering::SeqCst) {
4889                std::thread::yield_now();
4890            }
4891        });
4892        ctx.install_watcher_runtime(
4893            dispatch_rx,
4894            crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
4895        );
4896
4897        let mut meta = RootMeta::new(Instant::now());
4898        meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
4899        let mut live_roots = HashMap::from([(root.clone(), meta)]);
4900        let bound = HashMap::from([(root, HashSet::from([route_key(7, 1)]))]);
4901        assert_eq!(
4902            reap_idle_roots(
4903                Instant::now(),
4904                &mut live_roots,
4905                &HashMap::new(),
4906                &bound,
4907                &executor,
4908            )
4909            .evicted,
4910            0
4911        );
4912        wait_for_watcher_count(&ctx, 1);
4913        ctx.stop_watcher_runtime_in_background();
4914        wait_for_watcher_count(&ctx, 0);
4915    }
4916
4917    #[test]
4918    fn deleted_idle_root_is_fully_forgotten_and_status_counts_drop() {
4919        let (root_dir, root) = test_root("deleted-root-reap");
4920        let app = App::default_shared();
4921        let ctx = Arc::new(AppContext::from_app(
4922            Arc::clone(&app),
4923            Config {
4924                project_root: Some(root.as_path().to_path_buf()),
4925                ..Config::default()
4926            },
4927        ));
4928        ctx.set_canonical_cache_root(root.as_path().to_path_buf());
4929        ctx.mark_subc_unbound();
4930        let executor = Arc::new(Executor::new());
4931        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
4932        assert_eq!(app.actor_root_count(), 1);
4933        drop(ctx);
4934        root_dir.close().expect("delete project root");
4935
4936        let mut meta = RootMeta::new(Instant::now());
4937        meta.unbound_quiesced = true;
4938        let mut live_roots = HashMap::from([(root.clone(), meta)]);
4939        let outcome = reap_idle_roots(
4940            Instant::now(),
4941            &mut live_roots,
4942            &HashMap::new(),
4943            &HashMap::new(),
4944            &executor,
4945        );
4946        assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
4947        assert!(!executor.actor_registered(&root));
4948        assert!(!live_roots.contains_key(&root));
4949        wait_for_actor_root_count(&app, 0);
4950
4951        let status_ctx = AppContext::from_app(app, Config::default());
4952        let status = status_ctx.build_status_snapshot();
4953        assert_eq!(status["runtime"]["live_actor_roots"], 0);
4954        assert_eq!(status["runtime"]["open_routes"], 0);
4955    }
4956
4957    #[test]
4958    fn unbound_root_quiesces_maintenance_without_removing_actor() {
4959        let (_root_dir, root) = test_root("unbound-root-quiesce");
4960        let ctx = test_ctx();
4961        let executor = Arc::new(Executor::new());
4962        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
4963        let mut meta = RootMeta::new(Instant::now());
4964        meta.maintenance_pending = true;
4965        meta.maintenance_jobs_in_flight = 1;
4966        meta.maintenance_queued_kinds
4967            .push_back(MaintenanceDrainKind::ConfigureTail);
4968        let mut live_roots = HashMap::from([(root.clone(), meta)]);
4969        // Warm state planted before the unbind: quiesce must keep it.
4970        *ctx.search_index()
4971            .write()
4972            .unwrap_or_else(std::sync::PoisonError::into_inner) =
4973            Some(crate::search_index::SearchIndex::new());
4974        ctx.set_cache_writer_capabilities(true, true);
4975        let pending = root.as_path().join("pending.rs");
4976        ctx.add_pending_search_index_paths([pending.clone()]);
4977        // A warm verify memo must survive the transient unbind: the watcher
4978        // keeps running, so no unobserved window exists and the next warm
4979        // reload must not pay a strict full-corpus re-hash.
4980        let canonical_root = root.as_path().to_path_buf();
4981        let artifact = canonical_root.join("cache.bin");
4982        std::fs::write(&artifact, b"warm-artifact").expect("write artifact");
4983        let seeded_generation = crate::cache_freshness::artifact_generation(&artifact);
4984        crate::cache_freshness::record_verify_completed(
4985            &canonical_root,
4986            crate::cache_freshness::VerifyArtifact::Search,
4987            seeded_generation,
4988        );
4989        assert!(matches!(
4990            crate::cache_freshness::warm_verify_plan(
4991                &canonical_root,
4992                crate::cache_freshness::VerifyArtifact::Search,
4993                seeded_generation,
4994            ),
4995            crate::cache_freshness::WarmVerifyPlan::Skip
4996        ));
4997        // A live watcher runtime must survive quiesce (its events accumulate
4998        // for the rebind replay).
4999        let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
5000        let _dispatch_tx = dispatch_tx;
5001        let shutdown = Arc::new(AtomicBool::new(false));
5002        let thread_shutdown = Arc::clone(&shutdown);
5003        let join = std::thread::spawn(move || {
5004            while !thread_shutdown.load(Ordering::SeqCst) {
5005                std::thread::yield_now();
5006            }
5007        });
5008        ctx.install_watcher_runtime(
5009            dispatch_rx,
5010            crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
5011        );
5012        assert!(ctx.watcher_runtime_active());
5013
5014        quiesce_unbound_root(&root, &mut live_roots, &executor);
5015        let meta = &live_roots[&root];
5016        assert!(meta.unbound_quiesced);
5017        assert!(ctx.subc_unbound_quiesced());
5018        assert!(meta.maintenance_pending);
5019        assert!(meta.maintenance_queued_kinds.is_empty());
5020        assert!(executor.actor_registered(&root));
5021        // Transient unbind keeps the root warm: resident artifacts stay
5022        // resident, no forced callgraph rebuild is planted, and pending
5023        // reconciliation paths survive for the rebind replay.
5024        assert!(
5025            ctx.search_index()
5026                .read()
5027                .unwrap_or_else(std::sync::PoisonError::into_inner)
5028                .is_some(),
5029            "quiesce must not evict resident artifacts"
5030        );
5031        assert_eq!(
5032            ctx.pending_callgraph_store_force_token(),
5033            None,
5034            "quiesce must not force a callgraph rebuild"
5035        );
5036        assert_eq!(
5037            ctx.take_pending_search_index_paths(),
5038            vec![pending],
5039            "quiesce must retain pending watcher-derived paths"
5040        );
5041        assert!(
5042            matches!(
5043                crate::cache_freshness::warm_verify_plan(
5044                    &canonical_root,
5045                    crate::cache_freshness::VerifyArtifact::Search,
5046                    seeded_generation,
5047                ),
5048                crate::cache_freshness::WarmVerifyPlan::Skip
5049            ),
5050            "quiesce must not invalidate the warm verify memo"
5051        );
5052        assert!(
5053            ctx.watcher_runtime_active(),
5054            "quiesce must not stop a running watcher"
5055        );
5056        ctx.stop_watcher_runtime();
5057
5058        let meta = live_roots.get_mut(&root).expect("root metadata");
5059        note_maintenance_completion(
5060            meta,
5061            Some(MaintenanceDrainKind::ConfigureTail),
5062            false,
5063            meta.unbound_quiesced,
5064        );
5065        assert!(!meta.maintenance_pending);
5066        assert!(meta.maintenance_queued_kinds.is_empty());
5067    }
5068
5069    #[test]
5070    fn same_root_higher_epoch_replacement_does_not_quiesce_between_generations() {
5071        let (_dir, root) = test_root("same-root-replacement");
5072        let route = route_key(7, 1);
5073        let installed_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
5074        let root_channels = HashMap::new();
5075
5076        assert!(!route_removal_will_quiesce_root(
5077            &root,
5078            route,
5079            &installed_channels,
5080            false,
5081            Some(&root),
5082        ));
5083        assert!(route_removal_will_quiesce_root(
5084            &root,
5085            route,
5086            &installed_channels,
5087            false,
5088            None,
5089        ));
5090        assert!(!should_quiesce_removed_root(
5091            &root,
5092            &root_channels,
5093            false,
5094            Some(&root),
5095        ));
5096        assert!(should_quiesce_removed_root(
5097            &root,
5098            &root_channels,
5099            false,
5100            None,
5101        ));
5102        assert!(!should_quiesce_removed_root(
5103            &root,
5104            &root_channels,
5105            true,
5106            None,
5107        ));
5108    }
5109
5110    #[test]
5111    fn root_quiesces_only_after_its_last_route_is_removed_and_reactivates_on_bind() {
5112        let (_root_dir, root) = test_root("unbound-root-route-count");
5113        let executor = Arc::new(Executor::new());
5114        assert!(executor.register_actor(root.clone(), test_ctx()));
5115        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
5116        let mut root_channels = HashMap::from([(
5117            root.clone(),
5118            HashSet::from([route_key(7, 1), route_key(8, 1)]),
5119        )]);
5120
5121        remove_root_channel(&mut root_channels, &root, route_key(7, 1));
5122        if !root_channels.contains_key(&root) {
5123            quiesce_unbound_root(&root, &mut live_roots, &executor);
5124        }
5125        assert!(!live_roots[&root].unbound_quiesced);
5126
5127        remove_root_channel(&mut root_channels, &root, route_key(8, 1));
5128        if !root_channels.contains_key(&root) {
5129            quiesce_unbound_root(&root, &mut live_roots, &executor);
5130        }
5131        assert!(live_roots[&root].unbound_quiesced);
5132
5133        live_roots
5134            .get_mut(&root)
5135            .expect("root metadata")
5136            .note_activity();
5137        assert!(
5138            live_roots[&root].unbound_quiesced,
5139            "late asynchronous activity must not reactivate an unbound root"
5140        );
5141
5142        live_roots
5143            .get_mut(&root)
5144            .expect("root metadata")
5145            .reactivate_bound();
5146        assert!(!live_roots[&root].unbound_quiesced);
5147    }
5148
5149    #[test]
5150    fn allocator_pressure_relief_requires_every_root_to_be_idle() {
5151        let (_idle_dir, idle_root) = test_root("allocator-relief-idle");
5152        let (_active_dir, active_root) = test_root("allocator-relief-active");
5153        let now = Instant::now();
5154        let mut live_roots = HashMap::new();
5155        let mut idle = RootMeta::new(now);
5156        idle.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
5157        live_roots.insert(idle_root, idle);
5158        assert!(process_has_been_idle(now, &live_roots));
5159
5160        live_roots.insert(active_root.clone(), RootMeta::new(now));
5161        assert!(!process_has_been_idle(now, &live_roots));
5162
5163        let active = live_roots
5164            .get_mut(&active_root)
5165            .expect("active root metadata");
5166        active.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
5167        active.active_bash_waits = 1;
5168        assert!(!process_has_been_idle(now, &live_roots));
5169    }
5170
5171    #[test]
5172    fn pressure_relief_log_reports_before_and_after_measurements() {
5173        let allocator = crate::memory::AllocatorMemorySnapshot {
5174            status: "measured",
5175            bytes_in_use: Some(8 * 1024 * 1024),
5176            size_allocated: Some(12 * 1024 * 1024),
5177            retained_slack_bytes: Some(4 * 1024 * 1024),
5178            not_estimated: None,
5179        };
5180        let relief = crate::memory::AllocatorPressureRelief {
5181            bytes_released: 3 * 1024 * 1024,
5182            rss_before_bytes: Some(20 * 1024 * 1024),
5183            rss_after_bytes: Some(17 * 1024 * 1024),
5184            allocator_before: allocator.clone(),
5185            allocator_after: crate::memory::AllocatorMemorySnapshot {
5186                size_allocated: Some(9 * 1024 * 1024),
5187                retained_slack_bytes: Some(1024 * 1024),
5188                ..allocator
5189            },
5190        };
5191        let message = pressure_relief_label(&relief);
5192        assert!(message.contains("RSS 20.0 MB -> 17.0 MB"));
5193        assert!(message.contains("allocated 12.0 MB -> 9.0 MB"));
5194        assert!(message.contains("slack 4.0 MB -> 1.0 MB"));
5195        assert!(message.contains("reported 3.0 MB released"));
5196    }
5197
5198    #[test]
5199    fn due_maintenance_jobs_skip_poisoned_roots() {
5200        let (_healthy_dir, healthy_root) = test_root("maintenance-healthy");
5201        let (_poisoned_dir, poisoned_root) = test_root("maintenance-poisoned");
5202        let mut live_roots = HashMap::new();
5203        live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
5204        let mut poisoned_meta = RootMeta::new(Instant::now());
5205        poisoned_meta.maintenance_poisoned = true;
5206        live_roots.insert(poisoned_root.clone(), poisoned_meta);
5207
5208        let (due, deferred) = due_maintenance_jobs_without_actor_context(
5209            &mut live_roots,
5210            MAINTENANCE_SUBMIT_BUDGET,
5211            &HashSet::new(),
5212        );
5213
5214        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
5215        assert!(due.iter().all(|(root, _)| root == &healthy_root));
5216        assert!(!deferred);
5217        assert!(live_roots[&healthy_root].maintenance_pending);
5218        assert_eq!(
5219            live_roots[&healthy_root].maintenance_jobs_in_flight,
5220            INITIAL_MAINTENANCE_JOB_COUNT
5221        );
5222        assert!(!live_roots[&poisoned_root].maintenance_pending);
5223    }
5224
5225    #[test]
5226    fn due_maintenance_jobs_do_not_restart_quiesced_root_work() {
5227        let (_dir, root) = test_root("maintenance-unbound");
5228        let mut meta = RootMeta::new(Instant::now());
5229        meta.unbound_quiesced = true;
5230        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5231
5232        let (due, deferred) = due_maintenance_jobs_without_actor_context(
5233            &mut live_roots,
5234            MAINTENANCE_SUBMIT_BUDGET,
5235            &HashSet::new(),
5236        );
5237
5238        assert!(due.is_empty());
5239        assert!(!deferred);
5240        assert!(!live_roots[&root].maintenance_pending);
5241    }
5242
5243    #[test]
5244    fn idle_bg_subscription_queues_no_jobs_until_a_wake_arrives() {
5245        let (_dir, root) = test_root("maintenance-idle-bg-subscription");
5246        let ctx = test_ctx();
5247        assert!(!ctx.completion_drains_have_work());
5248
5249        let executor = Executor::new();
5250        assert!(executor.register_actor(root.clone(), ctx));
5251        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
5252        let session = "idle-session".to_string();
5253        let channel = route_key(17, 1);
5254        let bg_sub_by_session = HashMap::from([((root.clone(), session.clone()), channel)]);
5255        let mut bg_wake_pending = HashSet::new();
5256
5257        let (idle_tick_jobs, deferred) = due_maintenance_jobs(
5258            &mut live_roots,
5259            Some(&executor),
5260            &bg_sub_by_session,
5261            &bg_wake_pending,
5262            MAINTENANCE_SUBMIT_BUDGET,
5263            &HashSet::new(),
5264        );
5265        assert!(idle_tick_jobs.is_empty());
5266        assert!(!deferred);
5267        assert!(!live_roots[&root].maintenance_pending);
5268
5269        // A completion can arm its wake after the idle tick's probes. The wake
5270        // remains loop-owned state, so the following tick must observe it.
5271        let mut bg_wake_epoch = HashMap::new();
5272        push::arm_bg_wake(
5273            root.clone(),
5274            session,
5275            channel,
5276            &mut bg_wake_pending,
5277            &mut bg_wake_epoch,
5278        );
5279        let (next_tick_jobs, deferred) = due_maintenance_jobs(
5280            &mut live_roots,
5281            Some(&executor),
5282            &bg_sub_by_session,
5283            &bg_wake_pending,
5284            MAINTENANCE_SUBMIT_BUDGET,
5285            &HashSet::new(),
5286        );
5287        assert_eq!(
5288            next_tick_jobs,
5289            vec![(root, MaintenanceDrainKind::CompletionDrains)]
5290        );
5291        assert!(!deferred);
5292    }
5293
5294    #[tokio::test]
5295    async fn subc_configure_tail_precedes_completed_search_install() {
5296        let root_dir = tempfile::tempdir().unwrap();
5297        let storage = tempfile::tempdir().unwrap();
5298        let root = ProjectRootId::from_path(root_dir.path()).unwrap();
5299        let (ctx, ignored_path) =
5300            runtime_drain::configure_search_order_context_for_test(root_dir.path(), storage.path());
5301        let ctx = Arc::new(ctx);
5302        assert!(!runtime_drain::watcher_path_is_ignored_by_current_matcher(
5303            &ctx,
5304            &ignored_path
5305        ));
5306
5307        let executor = Arc::new(Executor::new());
5308        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5309        let metrics = Arc::new(DispatchPathMetrics::new());
5310        let (completion_tx, mut completion_rx) = mpsc::channel(4);
5311        submit_maintenance_job(
5312            &executor,
5313            root.clone(),
5314            MaintenanceDrainKind::ConfigureTail,
5315            Vec::new(),
5316            &completion_tx,
5317            &metrics,
5318        );
5319        submit_maintenance_job(
5320            &executor,
5321            root,
5322            MaintenanceDrainKind::CompletionDrains,
5323            Vec::new(),
5324            &completion_tx,
5325            &metrics,
5326        );
5327
5328        let first = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
5329            .await
5330            .expect("configure-tail completion timed out")
5331            .expect("configure-tail completion channel closed");
5332        let second = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
5333            .await
5334            .expect("completion-drains completion timed out")
5335            .expect("completion-drains completion channel closed");
5336        assert!(first.response.id.contains("configure-tail"));
5337        assert!(second.response.id.contains("completion-drains"));
5338        assert!(runtime_drain::watcher_path_is_ignored_by_current_matcher(
5339            &ctx,
5340            &ignored_path
5341        ));
5342        assert_eq!(
5343            ctx.search_index()
5344                .read()
5345                .unwrap_or_else(std::sync::PoisonError::into_inner)
5346                .as_ref()
5347                .expect("completed search index installed")
5348                .file_count(),
5349            0,
5350            "configure must install the ignore matcher before pending paths replay"
5351        );
5352        ctx.stop_watcher_runtime();
5353    }
5354
5355    #[test]
5356    fn post_bind_configure_and_completion_jobs_are_queued_in_order() {
5357        let (_dir, root) = test_root("maintenance-post-bind");
5358        let mut live_roots = HashMap::new();
5359        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
5360
5361        queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
5362        queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
5363
5364        let meta = live_roots.get(&root).expect("root metadata");
5365        assert!(meta.maintenance_pending);
5366        assert_eq!(meta.maintenance_jobs_in_flight, 0);
5367        assert_eq!(
5368            meta.maintenance_queued_kinds
5369                .iter()
5370                .copied()
5371                .collect::<Vec<_>>(),
5372            vec![
5373                MaintenanceDrainKind::ConfigureTail,
5374                MaintenanceDrainKind::CompletionDrains,
5375            ]
5376        );
5377
5378        let (due, deferred) = due_maintenance_jobs_without_actor_context(
5379            &mut live_roots,
5380            MAINTENANCE_SUBMIT_BUDGET,
5381            &HashSet::new(),
5382        );
5383
5384        assert_eq!(
5385            due,
5386            vec![
5387                (root.clone(), MaintenanceDrainKind::ConfigureTail),
5388                (root.clone(), MaintenanceDrainKind::CompletionDrains),
5389            ]
5390        );
5391        assert!(!deferred);
5392        assert_eq!(live_roots[&root].maintenance_jobs_in_flight, 2);
5393        assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
5394    }
5395
5396    #[test]
5397    fn due_maintenance_jobs_defers_unsubmitted_roots_without_marking_pending() {
5398        let mut live_roots = HashMap::new();
5399        let mut root_ids = Vec::new();
5400        let mut _dirs = Vec::new();
5401        for index in 0..4 {
5402            let (dir, root_id) = test_root(&format!("maintenance-budget-{index}"));
5403            live_roots.insert(root_id.clone(), RootMeta::new(Instant::now()));
5404            root_ids.push(root_id);
5405            _dirs.push(dir);
5406        }
5407
5408        let small_budget = INITIAL_MAINTENANCE_JOB_COUNT + 1;
5409        let (first_due, first_deferred) = due_maintenance_jobs_without_actor_context(
5410            &mut live_roots,
5411            small_budget,
5412            &HashSet::new(),
5413        );
5414
5415        assert_eq!(first_due.len(), small_budget);
5416        assert!(first_deferred);
5417        let first_due_set: HashSet<_> = first_due.into_iter().map(|(root, _)| root).collect();
5418        assert!(first_due_set
5419            .iter()
5420            .all(|root| live_roots[root].maintenance_pending));
5421        assert!(first_due_set
5422            .iter()
5423            .any(|root| !live_roots[root].maintenance_queued_kinds.is_empty()));
5424
5425        let all_roots: HashSet<_> = root_ids.into_iter().collect();
5426        let deferred_roots: HashSet<_> = all_roots.difference(&first_due_set).cloned().collect();
5427        assert!(deferred_roots
5428            .iter()
5429            .all(|root| !live_roots[root].maintenance_pending));
5430    }
5431
5432    #[test]
5433    fn due_maintenance_jobs_defers_pending_bind_roots() {
5434        let (_bind_dir, bind_root) = test_root("maintenance-pending-bind");
5435        let (_healthy_dir, healthy_root) = test_root("maintenance-no-bind");
5436        let mut live_roots = HashMap::new();
5437        live_roots.insert(bind_root.clone(), RootMeta::new(Instant::now()));
5438        live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
5439        let pending_bind_roots = HashSet::from([bind_root.clone()]);
5440
5441        let (due, deferred) = due_maintenance_jobs_without_actor_context(
5442            &mut live_roots,
5443            usize::MAX,
5444            &pending_bind_roots,
5445        );
5446
5447        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
5448        assert!(due.iter().all(|(root, _)| root == &healthy_root));
5449        assert!(!deferred);
5450        assert!(!live_roots[&bind_root].maintenance_pending);
5451        assert!(live_roots[&bind_root].maintenance_queued_kinds.is_empty());
5452    }
5453
5454    #[test]
5455    fn maintenance_pending_survives_requeue_and_clears_after_final_batch() {
5456        let (_dir, root) = test_root("maintenance-requeue");
5457        let mut live_roots = HashMap::new();
5458        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
5459        let (due, deferred) = due_maintenance_jobs_without_actor_context(
5460            &mut live_roots,
5461            usize::MAX,
5462            &HashSet::new(),
5463        );
5464        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
5465        assert!(due.iter().all(|(due_root, _)| due_root == &root));
5466        assert!(!deferred);
5467
5468        let meta = live_roots.get_mut(&root).unwrap();
5469        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, false);
5470        assert!(meta.maintenance_pending);
5471        assert_eq!(
5472            meta.maintenance_jobs_in_flight,
5473            INITIAL_MAINTENANCE_JOB_COUNT - 1
5474        );
5475        assert_eq!(meta.maintenance_queued_kinds.len(), 1);
5476
5477        let (requeued, deferred) =
5478            due_maintenance_jobs_without_actor_context(&mut live_roots, 1, &HashSet::new());
5479        assert_eq!(
5480            requeued,
5481            vec![(root.clone(), MaintenanceDrainKind::Watcher)]
5482        );
5483        assert!(!deferred);
5484        let meta = live_roots.get_mut(&root).unwrap();
5485        assert_eq!(
5486            meta.maintenance_jobs_in_flight,
5487            INITIAL_MAINTENANCE_JOB_COUNT
5488        );
5489        assert!(meta.maintenance_queued_kinds.is_empty());
5490
5491        for _ in 0..INITIAL_MAINTENANCE_JOB_COUNT {
5492            note_maintenance_completion(meta, None, false, false);
5493        }
5494        assert!(!meta.maintenance_pending);
5495        assert_eq!(meta.maintenance_jobs_in_flight, 0);
5496    }
5497
5498    #[test]
5499    fn maintenance_requeue_drops_while_bind_is_pending() {
5500        let (_dir, root) = test_root("maintenance-bind-requeue");
5501        let mut live_roots = HashMap::new();
5502        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
5503        let (due, _) = due_maintenance_jobs_without_actor_context(
5504            &mut live_roots,
5505            usize::MAX,
5506            &HashSet::new(),
5507        );
5508        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
5509
5510        let meta = live_roots.get_mut(&root).unwrap();
5511        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, true);
5512
5513        assert_eq!(
5514            meta.maintenance_jobs_in_flight,
5515            INITIAL_MAINTENANCE_JOB_COUNT - 1
5516        );
5517        assert!(meta.maintenance_queued_kinds.is_empty());
5518        assert!(meta.maintenance_pending);
5519    }
5520
5521    #[test]
5522    fn parked_lsp_completion_never_requiesces_or_cancels_a_pending_bind() {
5523        let mut meta = RootMeta::new(Instant::now());
5524        meta.unbound_quiesced = true;
5525
5526        assert!(!should_requiesce_after_maintenance(
5527            &meta,
5528            MaintenanceDrainKind::Lsp,
5529            false,
5530        ));
5531        assert!(!should_requiesce_after_maintenance(
5532            &meta,
5533            MaintenanceDrainKind::ConfigureTail,
5534            true,
5535        ));
5536        assert!(should_requiesce_after_maintenance(
5537            &meta,
5538            MaintenanceDrainKind::ConfigureTail,
5539            false,
5540        ));
5541    }
5542
5543    #[test]
5544    fn maintenance_pending_clears_and_poison_stops_requeue_after_fatal() {
5545        let (_dir, root) = test_root("maintenance-fatal");
5546        let mut live_roots = HashMap::new();
5547        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
5548        let (due, _) = due_maintenance_jobs_without_actor_context(
5549            &mut live_roots,
5550            usize::MAX,
5551            &HashSet::new(),
5552        );
5553        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
5554
5555        let meta = live_roots.get_mut(&root).unwrap();
5556        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), true, false);
5557        assert!(meta.maintenance_poisoned);
5558        assert!(meta.maintenance_queued_kinds.is_empty());
5559
5560        for _ in 1..INITIAL_MAINTENANCE_JOB_COUNT {
5561            note_maintenance_completion(meta, None, false, false);
5562        }
5563        assert!(!meta.maintenance_pending);
5564        assert_eq!(meta.maintenance_jobs_in_flight, 0);
5565    }
5566
5567    #[test]
5568    fn trust_for_principal_matrix() {
5569        assert_eq!(
5570            trust_for_principal(&Some(Principal::Direct)),
5571            BindTrust::FirstParty
5572        );
5573        assert_eq!(
5574            trust_for_principal(&Some(Principal::Reserved {
5575                module_id: "llm-runner".to_string(),
5576            })),
5577            BindTrust::FirstParty
5578        );
5579        assert_eq!(
5580            trust_for_principal(&Some(Principal::Reserved {
5581                module_id: "aft".to_string(),
5582            })),
5583            BindTrust::FirstParty
5584        );
5585        assert_eq!(
5586            trust_for_principal(&Some(Principal::Reserved {
5587                module_id: "subc-mcp".to_string(),
5588            })),
5589            BindTrust::Untrusted
5590        );
5591        assert_eq!(
5592            trust_for_principal(&Some(Principal::Reserved {
5593                module_id: "anything-unknown".to_string(),
5594            })),
5595            BindTrust::Untrusted
5596        );
5597        assert_eq!(
5598            trust_for_principal(&Some(Principal::Unverified)),
5599            BindTrust::Untrusted
5600        );
5601        assert_eq!(trust_for_principal(&None), BindTrust::Untrusted);
5602    }
5603
5604    #[test]
5605    fn fed_harness_class_maps_to_untrusted_regardless_of_fingerprint_value() {
5606        let principal = Some(Principal::Direct);
5607        let fingerprint_a = "fed:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
5608        let fingerprint_b = "fed:0123456789abcdef111111111111111111111111111111111111111111111111";
5609
5610        assert_eq!(
5611            trust_for_bind(fingerprint_a, &principal),
5612            BindTrust::Untrusted
5613        );
5614        assert_eq!(
5615            trust_for_bind(fingerprint_b, &principal),
5616            BindTrust::Untrusted
5617        );
5618    }
5619
5620    #[tokio::test]
5621    async fn persistent_cancel_resolves_when_fired_before_await() {
5622        // The lost-wakeup guard: cancel() fires exactly once via notify_waiters()
5623        // (no stored permit). A waiter that registers AFTER the cancel must still
5624        // observe it via the flag; a waiter racing the cancel must still be woken.
5625        let signal = PersistentCancelSignal::new();
5626        signal.cancel();
5627        // Fired before we ever call cancelled() — must return immediately, not park.
5628        tokio::time::timeout(Duration::from_secs(1), signal.cancelled())
5629            .await
5630            .expect("cancelled() must resolve when cancel fired beforehand");
5631
5632        // A fresh signal cancelled concurrently with an in-flight cancelled().
5633        let racing = PersistentCancelSignal::new();
5634        let racing_for_task = racing.clone();
5635        let waiter = tokio::spawn(async move { racing_for_task.cancelled().await });
5636        racing.cancel();
5637        tokio::time::timeout(Duration::from_secs(1), waiter)
5638            .await
5639            .expect("cancelled() must resolve when cancel races the await")
5640            .expect("waiter task panicked");
5641    }
5642
5643    #[test]
5644    fn ingress_epoch_validation_silently_drops_every_stale_route_frame() {
5645        let installed = HashMap::from([(7, 9)]);
5646        for ty in [
5647            FrameType::Request,
5648            FrameType::Response,
5649            FrameType::Error,
5650            FrameType::Push,
5651            FrameType::Cancel,
5652            FrameType::Goodbye,
5653        ] {
5654            let body = if ty.is_pure_header() {
5655                Vec::new()
5656            } else {
5657                br#"{}"#.to_vec()
5658            };
5659            let stale = Frame::build(ty, control_flags(), 7, 8, 41, body).unwrap();
5660            assert!(!ingress_route_is_current(&installed, &stale), "{ty:?}");
5661        }
5662
5663        let uninstalled = Frame::build(
5664            FrameType::Request,
5665            control_flags(),
5666            8,
5667            1,
5668            42,
5669            br#"{}"#.to_vec(),
5670        )
5671        .unwrap();
5672        assert!(!ingress_route_is_current(&installed, &uninstalled));
5673
5674        let current = Frame::build(
5675            FrameType::Request,
5676            control_flags(),
5677            7,
5678            9,
5679            43,
5680            br#"{}"#.to_vec(),
5681        )
5682        .unwrap();
5683        let control = Frame::build(FrameType::Ping, control_flags(), 0, 0, 44, Vec::new()).unwrap();
5684        assert!(ingress_route_is_current(&installed, &current));
5685        assert!(ingress_route_is_current(&installed, &control));
5686        assert_eq!(installed, HashMap::from([(7, 9)]));
5687    }
5688
5689    #[tokio::test]
5690    async fn route_bind_ack_precedes_route_egress_in_writer_queue() {
5691        let (_dir, root) = test_root("route-bind-b2-ordering");
5692        let route = route_key(7, 3);
5693        let identity = RouteIdentity(Arc::new(RouteIdentityData {
5694            root: root.clone(),
5695            project_root: root.as_path().to_path_buf(),
5696            harness: "opencode".to_string(),
5697            session: "b2-session".to_string(),
5698            trust: BindTrust::FirstParty,
5699            consumer_elicitation_capable: false,
5700        }));
5701        let replay_key = push::ReplayKey::from_identity(&identity);
5702        let completion = RouteBindCompletion {
5703            route,
5704            identity,
5705            bind_root_id: root.clone(),
5706            inserted_new_actor: false,
5707            configure_response: Response::success("subc-bind-7", json!({})),
5708            diagnostics_on_edit: false,
5709            ver: PROTOCOL_VERSION,
5710            corr: 91,
5711            flags: control_flags(),
5712        };
5713        let mut pending_binds = HashMap::from([(
5714            route,
5715            PendingBind {
5716                bind_root_id: root,
5717                inserted_new_actor: false,
5718                cancelled: false,
5719                configure_request_id: "subc-bind-7".to_string(),
5720                started_at: Instant::now(),
5721                warned_half_deadline: false,
5722                deadline_reported: false,
5723                corr: 91,
5724                ver: PROTOCOL_VERSION,
5725                flags: control_flags(),
5726                cancellation: crate::executor::JobCancellation::new(),
5727            },
5728        )]);
5729        let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
5730        let mut push_buffer =
5731            HashMap::from([(replay_key, VecDeque::from([completion_frame("b2-replay")]))]);
5732        let (writer_tx, mut writer_rx) = mpsc::channel(8);
5733        let metrics = Arc::new(DispatchPathMetrics::new());
5734
5735        handle_route_bind_completion(
5736            &writer_tx,
5737            completion,
5738            &mut HashMap::new(),
5739            &mut HashMap::new(),
5740            &mut HashMap::new(),
5741            &mut push_buffer,
5742            &mut HashMap::new(),
5743            &mut pending_binds,
5744            &mut installed_route_epochs,
5745            &Arc::new(Executor::new()),
5746            &Arc::new(Notify::new()),
5747            &metrics,
5748        )
5749        .await
5750        .unwrap();
5751
5752        let ack = writer_rx.try_recv().expect("RouteBindAck");
5753        assert_eq!(ack.header.ty, FrameType::Response);
5754        assert_eq!((ack.header.channel, ack.header.epoch), (0, 0));
5755        let route_frame = writer_rx.try_recv().expect("post-ack route frame");
5756        assert_eq!(route_frame.header.ty, FrameType::Push);
5757        assert_eq!(
5758            (route_frame.header.channel, route_frame.header.epoch),
5759            (route.channel, route.epoch)
5760        );
5761    }
5762}