Skip to main content

aft/subc/
mod.rs

1//! subc daemon attach — transport edge.
2//!
3//! When AFT is launched as `aft --subc <connection-file>`, it does NOT run the
4//! standalone NDJSON-over-stdin loop. Instead it connects to a running subc
5//! daemon over loopback TCP, authenticates with the pre-envelope HMAC handshake
6//! (`subc-transport`), then speaks the subc frame protocol (`subc-protocol`):
7//! ModuleHello → HelloAck (register as a tool provider), then a channel-0
8//! control loop (Ping/Pong, RouteBind) plus route-channel tool calls.
9//!
10//! Concurrency: subc routes tool calls through the executor. The tokio
11//! edge never dispatches against `AppContext` inline; per-actor executor lanes
12//! own the reader/mutator epoch, while a writer task serializes outbound frames.
13
14use std::collections::{HashMap, HashSet, VecDeque};
15use std::fmt;
16use std::io;
17use std::net::{IpAddr, SocketAddr};
18use std::ops::Deref;
19use std::path::{Path, PathBuf};
20use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
21use std::sync::{Arc, LazyLock};
22use std::time::{Duration, Instant};
23
24use serde::Deserialize;
25use serde_json::{json, Value};
26
27use crate::config::Config;
28use crate::config_resolve::ConfigTier;
29use crate::context::{App, AppContext, ProgressSender, RootHealthSnapshot};
30use crate::executor::{Executor, Lane};
31use crate::jsonc::strip_jsonc;
32use crate::log_ctx;
33use crate::path_identity::ProjectRootId;
34use crate::protocol::{ProgressKind, PushFrame, RawRequest, Response};
35use crate::run_tool_call::{
36    run_tool_call, strip_agent_preview_arg_owned, PhaseTrace, ToolCallContext, ToolCallOutcome,
37    ToolCallResult,
38};
39use crate::runtime_drain;
40use crate::sandbox_spawn::{AuthenticatedPrincipal, PrincipalTrust};
41
42use subc_protocol::manifest::{
43    Bindings, Concurrency, ExecutionMode, IdentityBinding, IdentityScope, ModuleManifest,
44    ProviderRole, StorageBinding, StorageKind, StorageScope, Tool, TrustTier,
45};
46use subc_protocol::session::{
47    HealthReport, HealthStatus, ModuleControlRequest, ModuleControlResponse,
48    MODULE_CONTROL_OP_HEALTH_CHECK,
49};
50use subc_protocol::{
51    ErrorBody, Flags, Frame, FrameType, ModuleHelloBody, Principal, Priority, PROTOCOL_VERSION,
52};
53use subc_transport::{authenticate_client, connection_file, read_frame, write_frame};
54use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
55use tokio::net::TcpStream;
56use tokio::sync::{mpsc, oneshot, Notify};
57use tokio::task::JoinHandle;
58
59/// Per-attempt handshake deadline. The initial attach loop has a separate total
60/// budget so a stalled peer cannot consume an unbounded supervisor launch window.
61const AUTH_DEADLINE: Duration = Duration::from_secs(5);
62const ATTACH_RETRY_BUDGET: Duration = Duration::from_secs(60);
63const ATTACH_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(250);
64const ATTACH_RETRY_MAX_BACKOFF: Duration = Duration::from_secs(5);
65const ATTACH_RETRY_JITTER_PERCENT: u64 = 20;
66
67/// Correlation id for the initial ModuleHello (channel 0).
68const HELLO_CORR: u64 = 1;
69
70/// Per-session in-memory replay cap for must-deliver Push frames. This covers
71/// detach/re-attach while AFT stays alive; cross-restart replay is phased later.
72const PUSH_BUFFER_MAX_PER_KEY: usize = 256;
73
74/// Bounded guard for control-frame sends. If the daemon stops reading and the
75/// writer queue stays full, tear the subc edge down instead of stalling the
76/// route loop indefinitely.
77const CONTROL_SEND_TIMEOUT: Duration = Duration::from_millis(250);
78
79/// Cadence for the loop's deadline-driven drain work (retry-buffer flush,
80/// bg-wake emission, maintenance submission). Checked at the top of every
81/// loop turn so busy select arms cannot starve it.
82const DRAIN_TICK_PERIOD: Duration = Duration::from_millis(250);
83
84/// Root-scoped stores and watcher runtimes are reopened lazily after this
85/// period without tool traffic. Keeping the value fixed avoids per-client
86/// eviction policies competing inside the module loop.
87const IDLE_ROOT_TTL: Duration = Duration::from_secs(30 * 60);
88
89const WRITER_QUEUE_CAPACITY: usize = 256;
90
91/// Keep reliable Push bursts from monopolizing the current-thread subc loop;
92/// any remaining must-deliver frames stay queued for the next loop turn.
93const RELIABLE_PUSH_DRAIN_BUDGET: usize = 32;
94
95/// Limit maintenance submissions per tick so background drains cannot delay
96/// control-plane work such as completed RouteBind acknowledgements.
97///
98/// The decomposed maintenance pass charges this budget by Mutating job, not by
99/// root. Size the default burst for one maintenance pass over eight live roots,
100/// while follow-up batches still re-enter the capped queue instead of bypassing
101/// the budget.
102const MAINTENANCE_SUBMIT_BUDGET: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len() * 8;
103const INITIAL_MAINTENANCE_DRAIN_KINDS: [MaintenanceDrainKind; 4] = [
104    MaintenanceDrainKind::Watcher,
105    MaintenanceDrainKind::Lsp,
106    MaintenanceDrainKind::ConfigureTail,
107    MaintenanceDrainKind::CompletionDrains,
108];
109#[cfg(test)]
110const INITIAL_MAINTENANCE_JOB_COUNT: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len();
111
112const RELIABLE_WRITER_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(10);
113const RELIABLE_WRITER_RETRY_MAX_BACKOFF: Duration = Duration::from_millis(250);
114
115const DISPATCH_PATH_BIND_WARN_AFTER: Duration = Duration::from_secs(6);
116const ROUTE_BIND_DEADLINE: Duration = Duration::from_secs(12);
117
118/// Small bounded memory of completed task ids used to suppress stale lossy
119/// long-running reminders that arrive after their reliable completion event.
120const COMPLETED_TASK_SUPPRESSION_MAX: usize = 4096;
121
122/// Bash foreground orchestration polls detached tasks with short read-lane jobs.
123/// The sleep between polls is outside the executor so no read or write worker is
124/// pinned while a foreground command is still running.
125const PENDING_POLL_INTERVAL: Duration = Duration::from_millis(100);
126
127/// Host elicitation asks fail closed if the MCP facade does not answer promptly.
128const BASH_ELICITATION_TIMEOUT: Duration = Duration::from_secs(60);
129const BASH_ELICITATION_CREATE_METHOD: &str = "elicitation/create";
130
131#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
132struct RouteChannel {
133    channel: u16,
134    epoch: u32,
135}
136
137impl fmt::Display for RouteChannel {
138    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
139        write!(f, "{}@{}", self.channel, self.epoch)
140    }
141}
142
143type PushEnvelope = (ProjectRootId, PushFrame);
144type LossyPushEnvelope = (u64, ProjectRootId, PushFrame);
145type RetryBuffer = HashMap<RouteChannel, VecDeque<(push::ReplayKey, PushFrame)>>;
146mod bash;
147mod health;
148mod manifest;
149mod push;
150mod wire;
151
152use self::health::{
153    build_health_report, warn_slow_pending_binds, DispatchPathMetrics, ReapBlockerCensus,
154    ResponseTaskGuard,
155};
156use self::manifest::{
157    build_manifest, command_lane, control_flags, control_ops, is_bash_family_tool,
158    is_subc_agent_core_tool, is_subc_native_plumbing_tool,
159};
160pub use self::wire::SubcError;
161
162/// Test-only view of the fail-closed tool-call gate: would `name` be admitted
163/// on a bound route (as an agent tool or native plumbing)? Used by the
164/// plugin-send drift guard in `subc_plumbing_drift_test.rs`.
165pub fn is_tool_call_admitted_for_test(name: &str) -> bool {
166    manifest::is_subc_agent_core_tool(name) || manifest::is_subc_native_plumbing_tool(name)
167}
168use self::wire::{
169    build_error_frame, build_goodbye_frame, build_tool_response_frame, decrement_counted_channel,
170    response_is_fatal_panic, response_message, send_counted_channel, send_frame,
171    send_reliable_writer_frame, send_traced_tool_response_frame, ToolResponseWriteTrace,
172    WriterFrame, WriterSender,
173};
174
175struct DecodedFrame {
176    frame: Frame,
177    phase_trace: PhaseTrace,
178}
179
180struct ToolCallCompletion {
181    text: String,
182    phase_trace: PhaseTrace,
183}
184
185#[derive(Clone)]
186struct PushSenders {
187    lossy_tx: mpsc::Sender<LossyPushEnvelope>,
188    reliable_tx: mpsc::UnboundedSender<PushEnvelope>,
189    lossy_overflow: Arc<push::LossyOverflow>,
190    lossy_seq: Arc<AtomicU64>,
191}
192
193#[derive(Clone)]
194struct PersistentCancelSignal {
195    inner: Arc<PersistentCancelInner>,
196}
197
198struct PersistentCancelInner {
199    cancelled: AtomicBool,
200    notify: Notify,
201}
202
203impl PersistentCancelSignal {
204    fn new() -> Self {
205        Self {
206            inner: Arc::new(PersistentCancelInner {
207                cancelled: AtomicBool::new(false),
208                notify: Notify::new(),
209            }),
210        }
211    }
212
213    fn cancel(&self) {
214        if !self.inner.cancelled.swap(true, Ordering::SeqCst) {
215            self.inner.notify.notify_waiters();
216        }
217    }
218
219    fn is_cancelled(&self) -> bool {
220        self.inner.cancelled.load(Ordering::SeqCst)
221    }
222
223    async fn cancelled(&self) {
224        // `enable()` REGISTERS this waiter before we read the flag, closing the
225        // lost-wakeup window: `notify_waiters()` only wakes already-registered
226        // waiters and stores no permit, so without enable() a `cancel()` firing
227        // between the flag read and `.await` would be missed and the future
228        // would park forever (cancel() fires only once). With enable(), a cancel
229        // racing the flag read still wakes the registered waiter. The loop is a
230        // belt-and-suspenders re-check on spurious wakeups.
231        loop {
232            let notified = self.inner.notify.notified();
233            tokio::pin!(notified);
234            notified.as_mut().enable();
235            if self.is_cancelled() {
236                return;
237            }
238            notified.await;
239        }
240    }
241}
242
243#[derive(Debug, Clone, Copy, PartialEq, Eq)]
244pub(crate) enum BindTrust {
245    FirstParty,
246    Untrusted,
247}
248
249impl BindTrust {
250    fn allows_bash_observation(self) -> bool {
251        matches!(self, Self::FirstParty)
252    }
253
254    fn label(self) -> &'static str {
255        match self {
256            Self::FirstParty => "first_party",
257            Self::Untrusted => "untrusted",
258        }
259    }
260
261    fn sandbox_trust(self) -> PrincipalTrust {
262        match self {
263            Self::FirstParty => PrincipalTrust::FirstParty,
264            Self::Untrusted => PrincipalTrust::Untrusted,
265        }
266    }
267}
268
269pub(super) fn trust_for_principal(principal: &Option<Principal>) -> BindTrust {
270    match principal {
271        Some(Principal::Direct) => BindTrust::FirstParty,
272        Some(Principal::Reserved { module_id })
273            if module_id == "llm-runner"
274                || module_id == "aft"
275                || module_id == "broca"
276                || module_id == "alfonso-core" =>
277        {
278            BindTrust::FirstParty
279        }
280        Some(Principal::Reserved { .. }) | Some(Principal::Unverified) | None => {
281            BindTrust::Untrusted
282        }
283    }
284}
285
286fn harness_forces_untrusted(harness: &str) -> bool {
287    harness.starts_with("fed:")
288}
289
290pub(super) fn trust_for_bind(harness: &str, principal: &Option<Principal>) -> BindTrust {
291    if harness_forces_untrusted(harness) {
292        BindTrust::Untrusted
293    } else {
294        trust_for_principal(principal)
295    }
296}
297
298fn principal_id(principal: &Option<Principal>) -> Option<String> {
299    match principal {
300        Some(Principal::Direct) => Some("direct".to_string()),
301        Some(Principal::Reserved { module_id }) => Some(format!("reserved:{module_id}")),
302        Some(Principal::Unverified) => Some("unverified".to_string()),
303        None => None,
304    }
305}
306
307fn principal_label(principal: &Option<Principal>) -> String {
308    principal_id(principal).unwrap_or_else(|| "absent".to_string())
309}
310
311#[derive(Debug)]
312/// Per-root route metadata owned by the subc loop. The `active_bash_waits` field
313/// counts detached bash processes that are still being observed for this root.
314/// Any future logic that evicts roots based on idle time must not evict a root
315/// while this count is greater than zero, because a foreground bash response may
316/// still arrive later.
317struct RootMeta {
318    maintenance_pending: bool,
319    maintenance_jobs_in_flight: usize,
320    maintenance_queued_kinds: VecDeque<MaintenanceDrainKind>,
321    maintenance_last_submitted: Option<Instant>,
322    maintenance_poisoned: bool,
323    last_touched: Instant,
324    diagnostics_on_edit: bool,
325    active_bash_waits: usize,
326    idle_artifacts_evicted: bool,
327    unbound_quiesced: bool,
328}
329
330#[derive(Debug)]
331struct PendingBind {
332    bind_root_id: ProjectRootId,
333    inserted_new_actor: bool,
334    cancelled: bool,
335    configure_request_id: String,
336    started_at: Instant,
337    warned_half_deadline: bool,
338    deadline_reported: bool,
339    corr: u64,
340    ver: u8,
341    flags: Flags,
342    /// Exact-job cancellation for the submitted configure: Goodbye and
343    /// deadline expiry cancel the executor job operationally (queued jobs are
344    /// removed, running configures return at their next checkpoint) instead of
345    /// only marking bookkeeping.
346    cancellation: crate::executor::JobCancellation,
347}
348
349struct RouteBindCompletion {
350    route: RouteChannel,
351    identity: RouteIdentity,
352    bind_root_id: ProjectRootId,
353    inserted_new_actor: bool,
354    configure_response: Response,
355    diagnostics_on_edit: bool,
356    ver: u8,
357    corr: u64,
358    flags: Flags,
359}
360
361#[derive(Debug, Clone)]
362struct RouteIdentity(Arc<RouteIdentityData>);
363
364#[derive(Debug)]
365struct RouteIdentityData {
366    root: ProjectRootId,
367    project_root: PathBuf,
368    harness: String,
369    session: String,
370    trust: BindTrust,
371    spawn_principal: AuthenticatedPrincipal,
372    consumer_elicitation_capable: bool,
373}
374
375impl Deref for RouteIdentity {
376    type Target = RouteIdentityData;
377
378    fn deref(&self) -> &Self::Target {
379        &self.0
380    }
381}
382
383#[derive(Debug, Clone)]
384struct RetainedSessionIdentity {
385    harness: String,
386    trust: BindTrust,
387}
388
389#[derive(Clone, Copy)]
390struct BgSub {
391    corr: u64,
392    ver: u8,
393    flags: Flags,
394}
395
396struct MaintenanceCompletion {
397    root_id: ProjectRootId,
398    kind: MaintenanceDrainKind,
399    response: Response,
400    empty_bg_sessions: Vec<(String, u64)>,
401    requeue_kind: Option<MaintenanceDrainKind>,
402}
403
404#[derive(Clone, Copy, Debug, PartialEq, Eq)]
405enum MaintenanceDrainKind {
406    Watcher,
407    Lsp,
408    ConfigureTail,
409    CompletionDrains,
410}
411
412impl MaintenanceDrainKind {
413    fn label(self) -> &'static str {
414        match self {
415            Self::Watcher => "watcher",
416            Self::Lsp => "lsp",
417            Self::ConfigureTail => "configure-tail",
418            Self::CompletionDrains => "completion-drains",
419        }
420    }
421}
422
423#[derive(Debug, Default)]
424struct MaintenanceJobOutcome {
425    empty_bg_sessions: Vec<(String, u64)>,
426    requeue_kind: Option<MaintenanceDrainKind>,
427}
428
429#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
430struct ReverseCorrKey {
431    route: RouteChannel,
432    corr: u64,
433}
434
435struct PendingBashAsk {
436    route: RouteChannel,
437    tool_corr: u64,
438    tool_flags: Flags,
439    tool_ver: u8,
440    root: ProjectRootId,
441    project_root: PathBuf,
442    session_id: String,
443    spawn_principal: AuthenticatedPrincipal,
444    request_id: String,
445    arguments: Value,
446    format_context: crate::subc_format::FormatContext,
447    cancel: bash::BashWaitCancel,
448    grants: Vec<String>,
449    expires_at: Instant,
450}
451
452impl RootMeta {
453    fn new(now: Instant) -> Self {
454        Self {
455            maintenance_pending: false,
456            maintenance_jobs_in_flight: 0,
457            maintenance_queued_kinds: VecDeque::new(),
458            maintenance_last_submitted: None,
459            maintenance_poisoned: false,
460            last_touched: now,
461            diagnostics_on_edit: false,
462            active_bash_waits: 0,
463            idle_artifacts_evicted: false,
464            unbound_quiesced: false,
465        }
466    }
467
468    fn note_activity(&mut self) {
469        self.last_touched = Instant::now();
470    }
471
472    fn reactivate_bound(&mut self) {
473        self.note_activity();
474        self.idle_artifacts_evicted = false;
475        self.unbound_quiesced = false;
476    }
477}
478
479fn due_maintenance_jobs(
480    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
481    executor: Option<&Executor>,
482    bg_sub_by_session: &HashMap<(ProjectRootId, String), RouteChannel>,
483    bg_wake_pending: &HashSet<RouteChannel>,
484    budget: usize,
485    pending_bind_roots: &HashSet<ProjectRootId>,
486) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
487    let mut jobs = Vec::new();
488    let mut deferred = false;
489    let mut roots = live_roots.keys().cloned().collect::<Vec<_>>();
490    roots.sort_by(|left, right| {
491        let left_last = live_roots
492            .get(left)
493            .and_then(|meta| meta.maintenance_last_submitted);
494        let right_last = live_roots
495            .get(right)
496            .and_then(|meta| meta.maintenance_last_submitted);
497        left_last
498            .cmp(&right_last)
499            .then_with(|| left.as_path().cmp(right.as_path()))
500    });
501
502    for root_id in roots {
503        let Some(meta) = live_roots.get_mut(&root_id) else {
504            continue;
505        };
506        if meta.maintenance_poisoned {
507            continue;
508        }
509
510        if pending_bind_roots.contains(&root_id) {
511            if meta.maintenance_pending || !meta.maintenance_queued_kinds.is_empty() {
512                deferred = true;
513            }
514            continue;
515        }
516
517        if !meta.maintenance_pending {
518            if jobs.len() >= budget {
519                deferred = true;
520                continue;
521            }
522            // Only enqueue kinds with pending work. Probes are cheap and
523            // fail-open (contended sources count as pending), so an idle root
524            // costs four probes per tick instead of four dispatched jobs.
525            let executor_actor_context =
526                executor.and_then(|executor| executor.actor_context(&root_id));
527            let root_has_pending_bg_wake =
528                bg_sub_by_session.iter().any(|((sub_root, _), channel)| {
529                    sub_root == &root_id && bg_wake_pending.contains(channel)
530                });
531            let kinds_with_work: Vec<MaintenanceDrainKind> = match executor_actor_context {
532                Some(ctx) => INITIAL_MAINTENANCE_DRAIN_KINDS
533                    .into_iter()
534                    .filter(|kind| {
535                        if meta.unbound_quiesced && !matches!(kind, MaintenanceDrainKind::Lsp) {
536                            return false;
537                        }
538                        match kind {
539                            MaintenanceDrainKind::Watcher => ctx.watcher_drain_has_work(),
540                            MaintenanceDrainKind::Lsp => ctx.lsp_drain_has_work(),
541                            MaintenanceDrainKind::ConfigureTail => ctx.configure_tail_has_work(),
542                            // Every CompletionDrains source is visible at this enqueue site:
543                            // AppContext probes completion queues, this loop owns bg wakes,
544                            // and queued continuations bypass probing via maintenance_pending.
545                            // New drain sources must expose a probe here rather than making
546                            // every subscribed root fail open again.
547                            MaintenanceDrainKind::CompletionDrains => {
548                                root_has_pending_bg_wake || ctx.completion_drains_have_work()
549                            }
550                        }
551                    })
552                    .collect(),
553                None if meta.unbound_quiesced => Vec::new(),
554                // No context handle (actor gone mid-tick): enqueue everything.
555                None => INITIAL_MAINTENANCE_DRAIN_KINDS.to_vec(),
556            };
557            if kinds_with_work.is_empty() {
558                continue;
559            }
560            meta.maintenance_pending = true;
561            meta.maintenance_queued_kinds.extend(kinds_with_work);
562        }
563
564        while let Some(kind) = meta.maintenance_queued_kinds.pop_front() {
565            if jobs.len() >= budget {
566                meta.maintenance_queued_kinds.push_front(kind);
567                deferred = true;
568                break;
569            }
570            meta.maintenance_jobs_in_flight += 1;
571            meta.maintenance_last_submitted = Some(Instant::now());
572            jobs.push((root_id.clone(), kind));
573        }
574
575        meta.maintenance_pending =
576            meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
577    }
578
579    (jobs, deferred)
580}
581
582fn eviction_estimate_label(estimate: &crate::memory::MemoryEstimate) -> String {
583    match estimate.estimated_bytes {
584        Some(bytes) => format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
585        None if estimate.status == "busy" => "busy".to_string(),
586        None => "not estimated".to_string(),
587    }
588}
589
590fn optional_memory_label(bytes: Option<u64>) -> String {
591    bytes.map_or_else(
592        || "not estimated".to_string(),
593        |bytes| format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
594    )
595}
596
597fn pressure_relief_label(relief: &crate::memory::AllocatorPressureRelief) -> String {
598    format!(
599        "; allocator pressure relief: RSS {} -> {}, in-use {} -> {}, allocated {} -> {}, slack {} -> {}, allocator reported {:.1} MB released",
600        optional_memory_label(relief.rss_before_bytes),
601        optional_memory_label(relief.rss_after_bytes),
602        optional_memory_label(relief.allocator_before.bytes_in_use),
603        optional_memory_label(relief.allocator_after.bytes_in_use),
604        optional_memory_label(relief.allocator_before.size_allocated),
605        optional_memory_label(relief.allocator_after.size_allocated),
606        optional_memory_label(relief.allocator_before.retained_slack_bytes),
607        optional_memory_label(relief.allocator_after.retained_slack_bytes),
608        relief.bytes_released as f64 / (1024.0 * 1024.0),
609    )
610}
611
612fn idle_root_eviction_message(
613    root_id: &ProjectRootId,
614    memory: &crate::memory::RootMemorySnapshot,
615    pressure_relief: Option<&crate::memory::AllocatorPressureRelief>,
616) -> String {
617    // Bash, LSP, and parser state remain resident. The freed total is deliberately
618    // only the known-byte portion of handles eviction actually drops.
619    let freed_bytes = [
620        &memory.semantic,
621        &memory.trigram,
622        &memory.symbols,
623        &memory.callgraph,
624        &memory.inspect,
625    ]
626    .iter()
627    .filter_map(|estimate| estimate.estimated_bytes)
628    .fold(0u64, u64::saturating_add);
629    let mut message = format!(
630        "evicted idle root {}: freed ~{:.1} MB (semantic {}, trigram {}, symbols {}, callgraph {}, inspect {}; retained: bash {}, lsp {}, parser_pool {})",
631        root_id.as_path().display(),
632        freed_bytes as f64 / (1024.0 * 1024.0),
633        eviction_estimate_label(&memory.semantic),
634        eviction_estimate_label(&memory.trigram),
635        eviction_estimate_label(&memory.symbols),
636        eviction_estimate_label(&memory.callgraph),
637        eviction_estimate_label(&memory.inspect),
638        eviction_estimate_label(&memory.bash),
639        eviction_estimate_label(&memory.lsp),
640        eviction_estimate_label(&memory.parser_pool),
641    );
642    if let Some(pressure_relief) = pressure_relief {
643        message.push_str(&pressure_relief_label(pressure_relief));
644    }
645    message
646}
647
648fn process_has_been_idle(now: Instant, live_roots: &HashMap<ProjectRootId, RootMeta>) -> bool {
649    !live_roots.is_empty()
650        && live_roots.values().all(|meta| {
651            now.saturating_duration_since(meta.last_touched) >= IDLE_ROOT_TTL
652                && meta.active_bash_waits == 0
653                && !meta.maintenance_pending
654                && meta.maintenance_queued_kinds.is_empty()
655        })
656}
657
658fn allocator_pressure_relief_after_idle_sweep(
659    now: Instant,
660    live_roots: &HashMap<ProjectRootId, RootMeta>,
661    executor: &Executor,
662) -> Option<crate::memory::AllocatorPressureRelief> {
663    if !process_has_been_idle(now, live_roots)
664        || live_roots.keys().any(|root_id| {
665            executor
666                .actor_context(root_id)
667                .is_some_and(|ctx| ctx.artifact_eviction_blocked())
668        })
669    {
670        return None;
671    }
672
673    #[cfg(target_os = "macos")]
674    {
675        Some(crate::memory::relieve_allocator_pressure())
676    }
677    #[cfg(not(target_os = "macos"))]
678    {
679        None
680    }
681}
682
683fn quiesce_unbound_root(
684    root_id: &ProjectRootId,
685    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
686    executor: &Arc<Executor>,
687) {
688    let Some(meta) = live_roots.get_mut(root_id) else {
689        return;
690    };
691
692    let ctx = executor.actor_context(root_id);
693    if let Some(ctx) = ctx.as_ref() {
694        // Close lifecycle admission before touching scheduler queues. A running
695        // ConfigureTail cannot release gates, install a watcher, or reserve a
696        // callgraph build after this transition becomes visible.
697        ctx.mark_subc_unbound();
698    }
699    let cancelled = executor.cancel_queued_maintenance(root_id);
700    // Transient unbind keeps the root WARM: the watcher stays running (its
701    // events accumulate and replay on rebind, so no unobserved gap exists) and
702    // resident artifacts stay resident. Host restarts unbind every root and
703    // rebind seconds later; stopping the watcher here would force strict
704    // re-verification plus a full callgraph rebuild on every restart. The
705    // expensive teardown (watcher stop + gap invalidation) belongs to the
706    // idle-TTL reaper and the root-deleted path.
707    let discarded = ctx
708        .map(|ctx| crate::commands::configure::cancel_deferred_configure_maintenance(&ctx))
709        .unwrap_or(0);
710    meta.unbound_quiesced = true;
711    meta.maintenance_queued_kinds.clear();
712    meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
713    log::debug!(
714        "subc attach: quiesced unbound root {} (cancelled {} queued maintenance job(s), cancelled {} configure maintenance job(s))",
715        root_id.as_path().display(),
716        cancelled,
717        discarded
718    );
719}
720
721#[allow(clippy::too_many_arguments)]
722fn quiesce_connection_roots(
723    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
724    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
725    routes: &mut HashMap<RouteChannel, RouteIdentity>,
726    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
727    installed_route_epochs: &mut HashMap<u16, u32>,
728    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
729    executor: &Arc<Executor>,
730) {
731    for cancel in route_bash_cancels.values() {
732        cancel.token.cancel();
733    }
734    route_bash_cancels.clear();
735
736    let mut roots = live_roots.keys().cloned().collect::<HashSet<_>>();
737    for pending in pending_binds.values_mut() {
738        pending.cancelled = true;
739        roots.insert(pending.bind_root_id.clone());
740        let _ = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
741    }
742
743    // A connection exit abandons every installed route at once. Close lifecycle
744    // admission before cancelling maintenance so no deferred worker can restore
745    // root activity after the loop-owned route tables disappear.
746    for root_id in roots {
747        if live_roots.contains_key(&root_id) {
748            quiesce_unbound_root(&root_id, live_roots, executor);
749        } else if let Some(ctx) = executor.actor_context(&root_id) {
750            ctx.mark_subc_unbound();
751            executor.cancel_queued_maintenance(&root_id);
752            crate::commands::configure::cancel_deferred_configure_maintenance(&ctx);
753        }
754    }
755
756    routes.clear();
757    root_channels.clear();
758    installed_route_epochs.clear();
759}
760
761#[derive(Debug, Default)]
762struct IdleReapOutcome {
763    evicted: usize,
764    forgotten_deleted_roots: Vec<ProjectRootId>,
765}
766
767fn reap_idle_roots(
768    now: Instant,
769    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
770    pending_binds: &HashMap<RouteChannel, PendingBind>,
771    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
772    executor: &Arc<Executor>,
773    metrics: &DispatchPathMetrics,
774) -> IdleReapOutcome {
775    let pending_bind_roots = pending_binds
776        .values()
777        .map(|pending| pending.bind_root_id.clone())
778        .collect::<HashSet<_>>();
779    let mut census = ReapBlockerCensus::default();
780    let mut candidates = Vec::new();
781
782    for (root_id, meta) in live_roots.iter() {
783        let deleted = !root_id.as_path().exists();
784        let has_bound_route = root_channels
785            .get(root_id)
786            .is_some_and(|channels| !channels.is_empty());
787        let has_pending_bind = pending_bind_roots.contains(root_id);
788
789        if deleted {
790            let mut retained = false;
791            if has_bound_route {
792                census.bound_routes += 1;
793                retained = true;
794            }
795            // Count roots whose unbound state has not yet been quiesced.
796            if !meta.unbound_quiesced {
797                census.unbound_quiesced += 1;
798                retained = true;
799            }
800            if meta.active_bash_waits > 0 {
801                census.bash_waits += 1;
802                retained = true;
803            }
804            if meta.maintenance_pending {
805                census.maintenance_pending += 1;
806                retained = true;
807            }
808            if !meta.maintenance_queued_kinds.is_empty() {
809                census.maintenance_queued += 1;
810                retained = true;
811            }
812            if has_pending_bind {
813                census.pending_binds += 1;
814                retained = true;
815            }
816            match executor.try_actor_is_idle(root_id) {
817                Some(true) => {}
818                Some(false) => {
819                    census.actor_busy += 1;
820                    retained = true;
821                }
822                None => {
823                    census.actor_state_busy += 1;
824                    retained = true;
825                }
826            }
827            if retained {
828                census.deleted_retained += 1;
829                continue;
830            }
831        } else {
832            // Route teardown marks the lifecycle admission gate before the last
833            // channel disappears. Requiring zero bound channels and a quiesced
834            // lifecycle prevents a still-bound root from losing its watcher.
835            if has_bound_route
836                || !meta.unbound_quiesced
837                || meta.idle_artifacts_evicted
838                || now.saturating_duration_since(meta.last_touched) < IDLE_ROOT_TTL
839                || meta.active_bash_waits > 0
840                || meta.maintenance_pending
841                || !meta.maintenance_queued_kinds.is_empty()
842                || has_pending_bind
843                || !executor.actor_is_idle(root_id)
844            {
845                continue;
846            }
847        }
848        candidates.push((root_id.clone(), deleted));
849    }
850
851    let mut reaped = Vec::new();
852    let mut forgotten_deleted_roots = Vec::new();
853    for (root_id, deleted) in candidates {
854        let Some(ctx) = executor.actor_context(&root_id) else {
855            if deleted {
856                census.deleted_retained += 1;
857                census.actor_busy += 1;
858            }
859            continue;
860        };
861        // A TTL-aged unbound root retained its watcher-derived pending paths
862        // across the transient-unbind window. Strict gap invalidation subsumes
863        // them, but every abort path must restore them because a rebind can
864        // still happen until eviction commits.
865        let taken_pending = Some(ctx.take_pending_reconciliation_state());
866        if ctx.artifact_eviction_blocked() {
867            if let Some(pending) = taken_pending {
868                ctx.restore_pending_reconciliation_state(pending);
869            }
870            if deleted {
871                census.deleted_retained += 1;
872                census.artifact_eviction_blocked += 1;
873            }
874            continue;
875        }
876        let memory_before = ctx.memory_root_snapshot();
877        if !ctx.evict_idle_artifacts() {
878            if let Some(pending) = taken_pending {
879                ctx.restore_pending_reconciliation_state(pending);
880            }
881            if deleted {
882                census.deleted_retained += 1;
883                census.artifact_eviction_failed += 1;
884            }
885            continue;
886        }
887        drop(taken_pending);
888        ctx.stop_watcher_runtime_in_background();
889        // Edits during watcher downtime are unobserved. Advance publication
890        // epochs and force strict verification before any later warm reload.
891        ctx.invalidate_artifacts_after_watcher_gap();
892
893        if deleted {
894            if executor.retire_idle_actor_in_background(&root_id) {
895                live_roots.remove(&root_id);
896                forgotten_deleted_roots.push(root_id.clone());
897            } else {
898                census.deleted_retained += 1;
899                census.actor_busy += 1;
900            }
901        } else {
902            if let Some(meta) = live_roots.get_mut(&root_id) {
903                meta.idle_artifacts_evicted = true;
904            }
905            ctx.release_idle_reopenable_resources_in_background();
906        }
907        reaped.push((root_id, memory_before));
908    }
909
910    metrics.record_reap(census);
911    if census.deleted_retained > 0 {
912        log::info!(
913            "subc attach: retained {} deleted root(s) during idle reap; blockers={}",
914            census.deleted_retained,
915            census.blocker_histogram()
916        );
917    }
918
919    let pressure_relief = (!reaped.is_empty())
920        .then(|| allocator_pressure_relief_after_idle_sweep(now, live_roots, executor))
921        .flatten();
922    for (root_id, memory_before) in &reaped {
923        log::info!(
924            "{}",
925            idle_root_eviction_message(root_id, memory_before, pressure_relief.as_ref())
926        );
927    }
928    IdleReapOutcome {
929        evicted: reaped.len(),
930        forgotten_deleted_roots,
931    }
932}
933
934#[allow(clippy::too_many_arguments)]
935fn purge_deleted_root_residents(
936    root_id: &ProjectRootId,
937    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
938    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
939    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
940    bg_subs: &mut HashMap<RouteChannel, BgSub>,
941    bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
942    bg_wake_pending: &mut HashSet<RouteChannel>,
943    bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
944    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
945) {
946    root_channels.remove(root_id);
947    session_identity.retain(|(root, _), _| root != root_id);
948    push_buffer.retain(|key, _| &key.root != root_id);
949    bg_wake_epoch.retain(|(root, _), _| root != root_id);
950    pending_bash_asks.retain(|_, ask| &ask.root != root_id);
951
952    let stale_subscriptions = bg_sub_by_session
953        .iter()
954        .filter_map(|((root, _), channel)| (root == root_id).then_some(*channel))
955        .collect::<Vec<_>>();
956    bg_sub_by_session.retain(|(root, _), _| root != root_id);
957    for channel in stale_subscriptions {
958        bg_subs.remove(&channel);
959        bg_wake_pending.remove(&channel);
960    }
961    log::info!(
962        "subc attach: fully forgot deleted root {}",
963        root_id.as_path().display()
964    );
965}
966
967#[allow(clippy::too_many_arguments)]
968fn submit_due_maintenance_jobs(
969    executor: &Arc<Executor>,
970    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
971    pending_binds: &HashMap<RouteChannel, PendingBind>,
972    bg_sub_by_session: &HashMap<(ProjectRootId, String), RouteChannel>,
973    bg_wake_pending: &HashSet<RouteChannel>,
974    bg_wake_epoch: &HashMap<(ProjectRootId, String), u64>,
975    maintenance_tx: &mpsc::Sender<MaintenanceCompletion>,
976    metrics: &Arc<DispatchPathMetrics>,
977) {
978    let pending_bind_roots = pending_binds
979        .values()
980        .map(|pending| pending.bind_root_id.clone())
981        .collect::<HashSet<_>>();
982    let (due_jobs, deferred_jobs) = due_maintenance_jobs(
983        live_roots,
984        Some(executor),
985        bg_sub_by_session,
986        bg_wake_pending,
987        MAINTENANCE_SUBMIT_BUDGET,
988        &pending_bind_roots,
989    );
990    if deferred_jobs {
991        metrics
992            .maintenance_budget_deferrals
993            .fetch_add(1, Ordering::Relaxed);
994    }
995    for (root_id, kind) in due_jobs {
996        let bg_sessions_to_check = if kind == MaintenanceDrainKind::CompletionDrains {
997            bg_sub_by_session
998                .iter()
999                .filter_map(|((root, session), _)| {
1000                    if root == &root_id {
1001                        Some((
1002                            session.clone(),
1003                            bg_wake_epoch
1004                                .get(&(root_id.clone(), session.clone()))
1005                                .copied()
1006                                .unwrap_or(0),
1007                        ))
1008                    } else {
1009                        None
1010                    }
1011                })
1012                .collect()
1013        } else {
1014            Vec::new()
1015        };
1016        submit_maintenance_job(
1017            executor,
1018            root_id,
1019            kind,
1020            bg_sessions_to_check,
1021            maintenance_tx,
1022            metrics,
1023        );
1024    }
1025}
1026
1027fn should_requiesce_after_maintenance(
1028    meta: &RootMeta,
1029    completed_kind: MaintenanceDrainKind,
1030    bind_pending: bool,
1031) -> bool {
1032    meta.unbound_quiesced && completed_kind != MaintenanceDrainKind::Lsp && !bind_pending
1033}
1034
1035fn note_maintenance_completion(
1036    meta: &mut RootMeta,
1037    requeue_kind: Option<MaintenanceDrainKind>,
1038    fatal: bool,
1039    defer_requeue: bool,
1040) {
1041    if fatal {
1042        meta.maintenance_poisoned = true;
1043    }
1044
1045    if let Some(kind) = requeue_kind.filter(|_| !meta.maintenance_poisoned && !defer_requeue) {
1046        meta.maintenance_queued_kinds.push_back(kind);
1047    }
1048
1049    meta.maintenance_jobs_in_flight = meta.maintenance_jobs_in_flight.saturating_sub(1);
1050    meta.maintenance_pending =
1051        meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
1052}
1053
1054fn route_key(channel: u16, epoch: u32) -> RouteChannel {
1055    RouteChannel { channel, epoch }
1056}
1057
1058fn remove_installed_route(installed_epochs: &mut HashMap<u16, u32>, route: RouteChannel) {
1059    if installed_epochs.get(&route.channel).copied() == Some(route.epoch) {
1060        installed_epochs.remove(&route.channel);
1061    }
1062}
1063
1064fn ingress_route_is_current(installed_epochs: &HashMap<u16, u32>, frame: &Frame) -> bool {
1065    frame.header.channel == 0
1066        || installed_epochs.get(&frame.header.channel).copied() == Some(frame.header.epoch)
1067}
1068
1069fn bash_elicitation_timeout() -> Duration {
1070    if cfg!(debug_assertions) {
1071        if let Ok(raw) = std::env::var("AFT_TEST_SUBC_BASH_ELICITATION_TTL_MS") {
1072            if let Ok(ms) = raw.parse::<u64>() {
1073                if ms > 0 {
1074                    return Duration::from_millis(ms);
1075                }
1076            }
1077        }
1078    }
1079    BASH_ELICITATION_TIMEOUT
1080}
1081
1082fn allocate_reverse_corr(
1083    pending_bash_asks: &HashMap<ReverseCorrKey, PendingBashAsk>,
1084    route: RouteChannel,
1085    next_corr: &mut u64,
1086) -> u64 {
1087    loop {
1088        let corr = *next_corr;
1089        *next_corr = (*next_corr).wrapping_add(1).max(1);
1090        if !pending_bash_asks.contains_key(&ReverseCorrKey { route, corr }) {
1091            return corr;
1092        }
1093    }
1094}
1095
1096fn bash_permission_kind_label(kind: &crate::bash_permissions::PermissionKind) -> &'static str {
1097    match kind {
1098        crate::bash_permissions::PermissionKind::ExternalDirectory => "external directory",
1099        crate::bash_permissions::PermissionKind::Bash => "bash",
1100    }
1101}
1102
1103fn bash_elicitation_patterns(asks: &[crate::bash_permissions::PermissionAsk]) -> Vec<String> {
1104    let mut patterns = Vec::new();
1105    let mut seen = HashSet::new();
1106    for ask in asks {
1107        for pattern in ask.patterns.iter().chain(ask.always.iter()) {
1108            if seen.insert(pattern.clone()) {
1109                patterns.push(pattern.clone());
1110            }
1111        }
1112    }
1113    patterns
1114}
1115
1116fn bash_elicitation_message(
1117    command: &str,
1118    asks: &[crate::bash_permissions::PermissionAsk],
1119) -> String {
1120    let command = command.split_whitespace().collect::<Vec<_>>().join(" ");
1121    let patterns = bash_elicitation_patterns(asks);
1122    let pattern_text = if patterns.is_empty() {
1123        "no matched permission patterns".to_string()
1124    } else {
1125        patterns.join(", ")
1126    };
1127    let ask_kinds = asks
1128        .iter()
1129        .map(|ask| bash_permission_kind_label(&ask.kind))
1130        .collect::<HashSet<_>>()
1131        .into_iter()
1132        .collect::<Vec<_>>()
1133        .join(", ");
1134    if ask_kinds.is_empty() {
1135        format!("Allow bash command `{command}`? Matched patterns: {pattern_text}")
1136    } else {
1137        format!("Allow bash command `{command}`? Matched {ask_kinds} patterns: {pattern_text}")
1138    }
1139}
1140
1141fn bash_elicitation_request_body(
1142    command: &str,
1143    asks: &[crate::bash_permissions::PermissionAsk],
1144) -> Value {
1145    json!({
1146        "method": BASH_ELICITATION_CREATE_METHOD,
1147        "params": {
1148            "mode": "form",
1149            "message": bash_elicitation_message(command, asks),
1150            "requestedSchema": {
1151                "type": "object",
1152                "properties": {
1153                    "decision": {
1154                        "type": "string",
1155                        "enum": ["allow", "deny"],
1156                        "description": "Choose allow to run this bash command once, or deny to block it."
1157                    }
1158                },
1159                "required": ["decision"],
1160                "additionalProperties": false
1161            },
1162            "_meta": {
1163                "aft": {
1164                    "tool": "bash",
1165                    "command": command,
1166                    "asks": asks
1167                }
1168            }
1169        }
1170    })
1171}
1172
1173fn build_bash_elicitation_request_frame(
1174    ver: u8,
1175    route: RouteChannel,
1176    corr: u64,
1177    flags: Flags,
1178    command: &str,
1179    asks: &[crate::bash_permissions::PermissionAsk],
1180) -> Result<Frame, SubcError> {
1181    let body = bash_elicitation_request_body(command, asks);
1182    Frame::build_with_version(
1183        ver,
1184        FrameType::Request,
1185        flags,
1186        route.channel,
1187        route.epoch,
1188        corr,
1189        serde_json::to_vec(&body).map_err(SubcError::Json)?,
1190    )
1191    .map_err(SubcError::FrameBuild)
1192}
1193
1194fn bash_elicitation_reply_is_allow(body: &[u8]) -> bool {
1195    let Ok(value) = serde_json::from_slice::<Value>(body) else {
1196        return false;
1197    };
1198    flat_bash_elicitation_reply_is_allow(&value) || mcp_bash_elicitation_reply_is_allow(&value)
1199}
1200
1201fn flat_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1202    let Some(object) = value.as_object() else {
1203        return false;
1204    };
1205    object.len() == 1 && object.get("decision").and_then(Value::as_str) == Some("allow")
1206}
1207
1208fn mcp_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1209    let Some(object) = value.as_object() else {
1210        return false;
1211    };
1212    if object.len() != 2 || object.get("action").and_then(Value::as_str) != Some("accept") {
1213        return false;
1214    }
1215    let Some(content) = object.get("content").and_then(Value::as_object) else {
1216        return false;
1217    };
1218    content.len() == 1 && content.get("decision").and_then(Value::as_str) == Some("allow")
1219}
1220
1221#[allow(clippy::too_many_arguments)]
1222async fn settle_pending_bash_ask_denied(
1223    tx: &WriterSender,
1224    pending: PendingBashAsk,
1225    routes: &HashMap<RouteChannel, RouteIdentity>,
1226    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1227    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1228    shutdown: &Arc<Notify>,
1229    metrics: &DispatchPathMetrics,
1230) -> Result<(), SubcError> {
1231    let completion = bash::bash_denied_untrusted_completion(
1232        pending.route,
1233        pending.tool_corr,
1234        pending.tool_flags,
1235        pending.tool_ver,
1236        pending.root,
1237        pending.request_id,
1238        pending.format_context,
1239    );
1240    bash::handle_bash_deferred_completion(
1241        tx,
1242        completion,
1243        routes,
1244        live_roots,
1245        route_bash_cancels,
1246        shutdown,
1247        metrics,
1248    )
1249    .await
1250}
1251
1252fn take_pending_bash_asks_for_route(
1253    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1254    route: RouteChannel,
1255) -> Vec<PendingBashAsk> {
1256    let keys = pending_bash_asks
1257        .keys()
1258        .copied()
1259        .filter(|key| key.route == route)
1260        .collect::<Vec<_>>();
1261    keys.into_iter()
1262        .filter_map(|key| pending_bash_asks.remove(&key))
1263        .collect()
1264}
1265
1266#[allow(clippy::too_many_arguments)]
1267async fn settle_pending_bash_asks_for_route(
1268    tx: &WriterSender,
1269    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1270    route: RouteChannel,
1271    routes: &HashMap<RouteChannel, RouteIdentity>,
1272    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1273    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1274    shutdown: &Arc<Notify>,
1275    metrics: &DispatchPathMetrics,
1276) -> Result<(), SubcError> {
1277    for pending in take_pending_bash_asks_for_route(pending_bash_asks, route) {
1278        settle_pending_bash_ask_denied(
1279            tx,
1280            pending,
1281            routes,
1282            live_roots,
1283            route_bash_cancels,
1284            shutdown,
1285            metrics,
1286        )
1287        .await?;
1288    }
1289    Ok(())
1290}
1291
1292#[allow(clippy::too_many_arguments)]
1293async fn settle_all_pending_bash_asks(
1294    tx: &WriterSender,
1295    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1296    routes: &HashMap<RouteChannel, RouteIdentity>,
1297    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1298    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1299    shutdown: &Arc<Notify>,
1300    metrics: &DispatchPathMetrics,
1301) -> Result<(), SubcError> {
1302    let pending = pending_bash_asks
1303        .drain()
1304        .map(|(_, pending)| pending)
1305        .collect::<Vec<_>>();
1306    for pending in pending {
1307        settle_pending_bash_ask_denied(
1308            tx,
1309            pending,
1310            routes,
1311            live_roots,
1312            route_bash_cancels,
1313            shutdown,
1314            metrics,
1315        )
1316        .await?;
1317    }
1318    Ok(())
1319}
1320
1321#[allow(clippy::too_many_arguments)]
1322async fn expire_pending_bash_asks(
1323    tx: &WriterSender,
1324    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1325    routes: &HashMap<RouteChannel, RouteIdentity>,
1326    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1327    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1328    shutdown: &Arc<Notify>,
1329    metrics: &DispatchPathMetrics,
1330) -> Result<(), SubcError> {
1331    let now = Instant::now();
1332    let expired = pending_bash_asks
1333        .iter()
1334        .filter_map(|(key, pending)| (pending.expires_at <= now).then_some(*key))
1335        .collect::<Vec<_>>();
1336    for key in expired {
1337        if let Some(pending) = pending_bash_asks.remove(&key) {
1338            log::debug!(
1339                "subc attach: bash elicitation request {} on route {} expired fail-closed",
1340                key.corr,
1341                pending.route
1342            );
1343            settle_pending_bash_ask_denied(
1344                tx,
1345                pending,
1346                routes,
1347                live_roots,
1348                route_bash_cancels,
1349                shutdown,
1350                metrics,
1351            )
1352            .await?;
1353        }
1354    }
1355    Ok(())
1356}
1357
1358#[allow(clippy::too_many_arguments)]
1359async fn handle_bash_elicitation_reply(
1360    tx: &WriterSender,
1361    frame: &Frame,
1362    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1363    routes: &HashMap<RouteChannel, RouteIdentity>,
1364    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1365    executor: &Arc<Executor>,
1366    shutdown: &Arc<Notify>,
1367    bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
1368    bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
1369    metrics: &Arc<DispatchPathMetrics>,
1370    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1371    dispatch: DispatchFn,
1372) -> Result<(), SubcError> {
1373    let key = ReverseCorrKey {
1374        route: route_key(frame.header.channel, frame.header.epoch),
1375        corr: frame.header.corr,
1376    };
1377    let Some(pending) = pending_bash_asks.remove(&key) else {
1378        return Ok(());
1379    };
1380
1381    if frame.header.ty == FrameType::Response && bash_elicitation_reply_is_allow(&frame.body) {
1382        if routes.contains_key(&key.route) {
1383            bash::submit_deferred_bash(
1384                executor,
1385                bash_deferred_tx,
1386                bash_poll_touch_tx,
1387                metrics,
1388                dispatch,
1389                pending.root,
1390                pending.project_root,
1391                pending.session_id,
1392                pending.request_id,
1393                pending.route,
1394                pending.tool_corr,
1395                pending.tool_flags,
1396                pending.tool_ver,
1397                pending.arguments,
1398                pending.format_context,
1399                pending.cancel,
1400                BindTrust::Untrusted,
1401                pending.spawn_principal,
1402                Some(pending.grants),
1403            );
1404            return Ok(());
1405        }
1406        log::debug!(
1407            "subc attach: dropping allowed bash elicitation reply {} for unbound route {}",
1408            key.corr,
1409            pending.route
1410        );
1411    }
1412
1413    settle_pending_bash_ask_denied(
1414        tx,
1415        pending,
1416        routes,
1417        live_roots,
1418        route_bash_cancels,
1419        shutdown,
1420        metrics,
1421    )
1422    .await
1423}
1424
1425#[allow(clippy::too_many_arguments)]
1426async fn cancel_pending_bash_ask_for_tool_call(
1427    tx: &WriterSender,
1428    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1429    route: RouteChannel,
1430    tool_corr: u64,
1431    routes: &HashMap<RouteChannel, RouteIdentity>,
1432    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1433    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1434    shutdown: &Arc<Notify>,
1435    metrics: &DispatchPathMetrics,
1436) -> Result<(), SubcError> {
1437    let keys = pending_bash_asks
1438        .iter()
1439        .filter_map(|(key, pending)| {
1440            (key.route == route && pending.tool_corr == tool_corr).then_some(*key)
1441        })
1442        .collect::<Vec<_>>();
1443    for key in keys {
1444        if let Some(pending) = pending_bash_asks.remove(&key) {
1445            settle_pending_bash_ask_denied(
1446                tx,
1447                pending,
1448                routes,
1449                live_roots,
1450                route_bash_cancels,
1451                shutdown,
1452                metrics,
1453            )
1454            .await?;
1455        }
1456    }
1457    Ok(())
1458}
1459
1460fn remove_root_channel(
1461    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1462    root: &ProjectRootId,
1463    channel: RouteChannel,
1464) {
1465    let remove_root = if let Some(channels) = root_channels.get_mut(root) {
1466        channels.remove(&channel);
1467        channels.is_empty()
1468    } else {
1469        false
1470    };
1471    if remove_root {
1472        root_channels.remove(root);
1473    }
1474}
1475
1476fn remove_route_channel(
1477    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1478    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1479    channel: RouteChannel,
1480) -> Option<RouteIdentity> {
1481    let removed = routes.remove(&channel);
1482    if let Some(identity) = &removed {
1483        remove_root_channel(root_channels, &identity.root, channel);
1484    }
1485    removed
1486}
1487
1488fn insert_route_channel(
1489    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1490    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1491    channel: RouteChannel,
1492    identity: RouteIdentity,
1493) {
1494    if let Some(previous) = routes.insert(channel, identity.clone()) {
1495        remove_root_channel(root_channels, &previous.root, channel);
1496    }
1497    root_channels
1498        .entry(identity.root.clone())
1499        .or_default()
1500        .insert(channel);
1501}
1502
1503fn remove_bg_subscription_index(
1504    bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1505    channel: RouteChannel,
1506    identity: Option<&RouteIdentity>,
1507) {
1508    if let Some(identity) = identity {
1509        let key = (identity.root.clone(), identity.session.clone());
1510        if bg_sub_by_session.get(&key).copied() == Some(channel) {
1511            bg_sub_by_session.remove(&key);
1512        }
1513    } else {
1514        bg_sub_by_session.retain(|_, mapped_channel| *mapped_channel != channel);
1515    }
1516}
1517
1518fn route_removal_will_quiesce_root(
1519    root: &ProjectRootId,
1520    route: RouteChannel,
1521    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1522    has_pending_bind: bool,
1523    replacement_root: Option<&ProjectRootId>,
1524) -> bool {
1525    let removes_last_route = root_channels
1526        .get(root)
1527        .is_some_and(|channels| channels.len() == 1 && channels.contains(&route));
1528    removes_last_route && !has_pending_bind && replacement_root != Some(root)
1529}
1530
1531fn should_quiesce_removed_root(
1532    root: &ProjectRootId,
1533    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1534    has_pending_bind: bool,
1535    replacement_root: Option<&ProjectRootId>,
1536) -> bool {
1537    !root_channels.contains_key(root) && !has_pending_bind && replacement_root != Some(root)
1538}
1539
1540async fn end_bg_subscription(
1541    writer_tx: &WriterSender,
1542    metrics: &DispatchPathMetrics,
1543    bg_subs: &mut HashMap<RouteChannel, BgSub>,
1544    bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1545    bg_wake_pending: &mut HashSet<RouteChannel>,
1546    channel: RouteChannel,
1547    identity: Option<&RouteIdentity>,
1548) -> Result<(), SubcError> {
1549    if let Some(sub) = bg_subs.remove(&channel) {
1550        bg_wake_pending.remove(&channel);
1551        remove_bg_subscription_index(bg_sub_by_session, channel, identity);
1552        push::send_reliable_bg_stream_end(writer_tx, metrics, channel, &sub).await?;
1553    }
1554    Ok(())
1555}
1556
1557#[allow(clippy::too_many_arguments)]
1558async fn teardown_installed_route(
1559    tx: &WriterSender,
1560    metrics: &DispatchPathMetrics,
1561    executor: &Arc<Executor>,
1562    channel: RouteChannel,
1563    cancellation_reason: &str,
1564    replacement_root: Option<&ProjectRootId>,
1565    installed_route_epochs: &mut HashMap<u16, u32>,
1566    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1567    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1568    bg_subs: &mut HashMap<RouteChannel, BgSub>,
1569    bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1570    bg_wake_pending: &mut HashSet<RouteChannel>,
1571    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1572    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1573    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1574    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
1575    retry_buffer: &mut RetryBuffer,
1576    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1577    shutdown: &Arc<Notify>,
1578) -> Result<(), SubcError> {
1579    remove_installed_route(installed_route_epochs, channel);
1580    end_bg_subscription(
1581        tx,
1582        metrics,
1583        bg_subs,
1584        bg_sub_by_session,
1585        bg_wake_pending,
1586        channel,
1587        routes.get(&channel),
1588    )
1589    .await?;
1590    settle_pending_bash_asks_for_route(
1591        tx,
1592        pending_bash_asks,
1593        channel,
1594        routes,
1595        live_roots,
1596        route_bash_cancels,
1597        shutdown,
1598        metrics,
1599    )
1600    .await?;
1601    if let Some(cancel) = route_bash_cancels.remove(&channel) {
1602        cancel.token.cancel();
1603    }
1604    if let Some(pending) = pending_binds.get_mut(&channel) {
1605        pending.cancelled = true;
1606        let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
1607        log::debug!(
1608            "subc attach: cancelled pending RouteBind for route {} on {cancellation_reason} (configure job: {outcome:?})",
1609            channel.channel
1610        );
1611    }
1612    let migrated = push::migrate_retry_buffer_to_push_buffer(retry_buffer, channel, push_buffer);
1613    if let Some(identity) = routes.get(&channel) {
1614        let has_pending_bind = pending_binds
1615            .values()
1616            .any(|pending| pending.bind_root_id == identity.root);
1617        if route_removal_will_quiesce_root(
1618            &identity.root,
1619            channel,
1620            root_channels,
1621            has_pending_bind,
1622            replacement_root,
1623        ) {
1624            if let Some(ctx) = executor.actor_context(&identity.root) {
1625                // Fence deferred admissions before the final route disappears
1626                // from the loop-owned routing tables.
1627                ctx.mark_subc_unbound();
1628            }
1629        }
1630    }
1631    if let Some(identity) = remove_route_channel(routes, root_channels, channel) {
1632        if migrated > 0 {
1633            log::debug!(
1634                "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from route {} into detach replay",
1635                channel.channel
1636            );
1637        }
1638        if let Some(meta) = live_roots.get_mut(&identity.root) {
1639            let idle_for = meta.last_touched.elapsed();
1640            meta.note_activity();
1641            log::debug!(
1642                "subc attach: route {} torn down for root {} harness {} session {} (last touched {:?} ago)",
1643                channel.channel,
1644                identity.root.as_path().display(),
1645                identity.harness,
1646                identity.session,
1647                idle_for
1648            );
1649        } else {
1650            log::debug!(
1651                "subc attach: route {} torn down for root {} harness {} session {}",
1652                channel.channel,
1653                identity.root.as_path().display(),
1654                identity.harness,
1655                identity.session
1656            );
1657        }
1658        let has_pending_bind = pending_binds
1659            .values()
1660            .any(|pending| pending.bind_root_id == identity.root);
1661        if should_quiesce_removed_root(
1662            &identity.root,
1663            root_channels,
1664            has_pending_bind,
1665            replacement_root,
1666        ) {
1667            quiesce_unbound_root(&identity.root, live_roots, executor);
1668        }
1669    } else {
1670        if migrated > 0 {
1671            log::debug!(
1672                "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from unbound route {} into detach replay",
1673                channel.channel
1674            );
1675        }
1676        log::debug!("subc attach: unbound route {} torn down", channel.channel);
1677    }
1678    Ok(())
1679}
1680
1681fn remember_session_identity(
1682    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1683    identity: &RouteIdentity,
1684) {
1685    let key = (identity.root.clone(), identity.session.clone());
1686    if matches!(identity.trust, BindTrust::Untrusted)
1687        && session_identity
1688            .get(&key)
1689            .is_some_and(|retained| matches!(retained.trust, BindTrust::FirstParty))
1690    {
1691        return;
1692    }
1693
1694    // Retained after route Goodbye so reliable session-scoped frames emitted while
1695    // the session is detached can still be keyed by the full (root,harness,session)
1696    // replay triple. Untrusted binds never overwrite a retained first-party
1697    // session identity, because bash completion replay is an observation channel.
1698    session_identity.insert(
1699        key,
1700        RetainedSessionIdentity {
1701            harness: identity.harness.clone(),
1702            trust: identity.trust,
1703        },
1704    );
1705}
1706
1707fn replay_key_for_session(
1708    session_identity: &HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1709    root: &ProjectRootId,
1710    session: &str,
1711) -> Option<(push::ReplayKey, BindTrust)> {
1712    let retained = session_identity.get(&(root.clone(), session.to_string()))?;
1713    Some((
1714        push::ReplayKey {
1715            root: root.clone(),
1716            harness: retained.harness.clone(),
1717            session: session.to_string(),
1718        },
1719        retained.trust,
1720    ))
1721}
1722/// Sync command dispatch, passed in from `main` (the binary owns the command
1723/// table). Invoked only inside executor jobs in subc mode.
1724pub type DispatchFn = fn(RawRequest, &AppContext) -> Response;
1725
1726#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1727enum ModuleLoopExit {
1728    Graceful,
1729    SkipSearchFlush,
1730}
1731
1732/// Entry point for `aft --subc <connection-file>`. Synchronous on the outside;
1733/// owns an isolated current-thread tokio runtime for the async transport.
1734/// Returns `Err` (fail-loud) on any connect/auth/protocol failure — we never
1735/// fall back to the standalone loop, to avoid split-brain index state.
1736pub fn run_subc_mode(
1737    connection_file_path: &Path,
1738    ctx: Arc<AppContext>,
1739    executor: Arc<Executor>,
1740    dispatch: DispatchFn,
1741    user_config_path: Option<PathBuf>,
1742) -> Result<(), SubcError> {
1743    // Production NEVER allows non-manifest tool names on route channels: AFT
1744    // fails closed and does not trust subc to enforce the manifest. The
1745    // test-only harness sets this through `run_subc_mode_for_test`.
1746    run_subc_mode_inner(
1747        connection_file_path,
1748        ctx,
1749        executor,
1750        dispatch,
1751        user_config_path,
1752        false,
1753    )
1754}
1755
1756fn run_subc_mode_inner(
1757    connection_file_path: &Path,
1758    ctx: Arc<AppContext>,
1759    executor: Arc<Executor>,
1760    dispatch: DispatchFn,
1761    user_config_path: Option<PathBuf>,
1762    allow_native_passthrough: bool,
1763) -> Result<(), SubcError> {
1764    let runtime = tokio::runtime::Builder::new_current_thread()
1765        .enable_all()
1766        .build()
1767        .map_err(SubcError::Runtime)?;
1768
1769    let executor_for_loop = Arc::clone(&executor);
1770    let loop_result = runtime.block_on(async move {
1771        let shared_app = ctx.app();
1772        drop(ctx);
1773        let stream = connect_and_authenticate(connection_file_path).await?;
1774        log::info!(
1775            "subc attach: authenticated to daemon via {}",
1776            connection_file_path.display()
1777        );
1778        let (read_half, write_half) = tokio::io::split(stream);
1779        run_module_loop(
1780            read_half,
1781            write_half,
1782            shared_app,
1783            executor_for_loop,
1784            dispatch,
1785            user_config_path,
1786            allow_native_passthrough,
1787        )
1788        .await
1789    });
1790
1791    let actor_contexts = executor.actor_contexts();
1792    if matches!(loop_result, Ok(ModuleLoopExit::Graceful)) {
1793        // EOF/Goodbye teardown flushes each root's index deltas and queued
1794        // callgraph refreshes. Fatal/panic teardown skips this best-effort work.
1795        flush_actor_indexes_on_graceful_shutdown(&actor_contexts);
1796    }
1797    for actor_ctx in &actor_contexts {
1798        actor_ctx.lsp().shutdown_all();
1799        actor_ctx.bash_background().detach();
1800    }
1801
1802    loop_result.map(|_| ())
1803}
1804
1805fn flush_actor_indexes_on_graceful_shutdown(actor_contexts: &[Arc<AppContext>]) {
1806    for actor_ctx in actor_contexts {
1807        let _ = actor_ctx.flush_search_index_on_graceful_shutdown();
1808    }
1809    let _ = crate::callgraph_store::flush_callgraph_store_refreshes_on_graceful_shutdown();
1810}
1811
1812/// Test-only entry that enables the non-manifest native-command passthrough on
1813/// route channels. Integration tests drive synthetic native commands (`glob`,
1814/// `callers`, `subc_test_echo_session`, …) through the executor to exercise
1815/// mechanics; production callers use [`run_subc_mode`], which fails closed.
1816#[doc(hidden)]
1817pub fn run_subc_mode_for_test(
1818    connection_file_path: &Path,
1819    ctx: Arc<AppContext>,
1820    executor: Arc<Executor>,
1821    dispatch: DispatchFn,
1822    user_config_path: Option<PathBuf>,
1823) -> Result<(), SubcError> {
1824    run_subc_mode_inner(
1825        connection_file_path,
1826        ctx,
1827        executor,
1828        dispatch,
1829        user_config_path,
1830        true,
1831    )
1832}
1833
1834#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1835enum AttachErrorClass {
1836    Transient,
1837    Permanent,
1838}
1839
1840impl fmt::Display for AttachErrorClass {
1841    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1842        match self {
1843            Self::Transient => f.write_str("transient"),
1844            Self::Permanent => f.write_str("permanent"),
1845        }
1846    }
1847}
1848
1849#[derive(Clone, Copy)]
1850struct AttachRetryPolicy {
1851    budget: Duration,
1852    initial_backoff: Duration,
1853    max_backoff: Duration,
1854    jitter_percent: u64,
1855}
1856
1857const ATTACH_RETRY_POLICY: AttachRetryPolicy = AttachRetryPolicy {
1858    budget: ATTACH_RETRY_BUDGET,
1859    initial_backoff: ATTACH_RETRY_INITIAL_BACKOFF,
1860    max_backoff: ATTACH_RETRY_MAX_BACKOFF,
1861    jitter_percent: ATTACH_RETRY_JITTER_PERCENT,
1862};
1863
1864/// Retry only failures that can be caused by a daemon bounce or an interrupted
1865/// handshake. Protocol and credential failures are permanent for this process.
1866fn classify_attach_error(error: &SubcError) -> AttachErrorClass {
1867    let transient = match error {
1868        SubcError::Connect { source, .. } => is_transient_attach_io(source.kind()),
1869        SubcError::Auth { source, .. } => match source {
1870            subc_transport::AuthError::Timeout { .. }
1871            | subc_transport::AuthError::UnexpectedEof { .. } => true,
1872            subc_transport::AuthError::Io { source, .. } => is_transient_attach_io(source.kind()),
1873            _ => false,
1874        },
1875        _ => false,
1876    };
1877    if transient {
1878        AttachErrorClass::Transient
1879    } else {
1880        AttachErrorClass::Permanent
1881    }
1882}
1883
1884fn is_transient_attach_io(kind: io::ErrorKind) -> bool {
1885    matches!(
1886        kind,
1887        io::ErrorKind::ConnectionRefused
1888            | io::ErrorKind::TimedOut
1889            | io::ErrorKind::ConnectionReset
1890            | io::ErrorKind::ConnectionAborted
1891            | io::ErrorKind::BrokenPipe
1892            | io::ErrorKind::UnexpectedEof
1893    )
1894}
1895
1896/// Read the connection file → resolve the first endpoint → TCP connect → HMAC
1897/// handshake. Transient initial-attach failures retry on fresh sockets and reread
1898/// the file so a daemon bounce can publish a new endpoint or authentication key.
1899async fn connect_and_authenticate(connection_file_path: &Path) -> Result<TcpStream, SubcError> {
1900    connect_and_authenticate_with_policy(connection_file_path, ATTACH_RETRY_POLICY).await
1901}
1902
1903async fn connect_and_authenticate_with_policy(
1904    connection_file_path: &Path,
1905    policy: AttachRetryPolicy,
1906) -> Result<TcpStream, SubcError> {
1907    let started_at = Instant::now();
1908    let deadline = started_at + policy.budget;
1909    let mut attempt = 0_u32;
1910    let mut backoff = policy.initial_backoff;
1911    let mut history = Vec::new();
1912
1913    loop {
1914        attempt = attempt.saturating_add(1);
1915        let error = match connect_and_authenticate_once(connection_file_path, deadline).await {
1916            Ok(stream) => return Ok(stream),
1917            Err(error) => error,
1918        };
1919        let class = classify_attach_error(&error);
1920        let error_text = error.to_string().lines().collect::<Vec<_>>().join(" ");
1921        history.push(format!("attempt {attempt} [{class}]: {error_text}"));
1922
1923        if class == AttachErrorClass::Permanent {
1924            log_attach_final_failure(started_at.elapsed(), &history);
1925            return Err(error);
1926        }
1927
1928        let remaining = deadline.saturating_duration_since(Instant::now());
1929        if remaining.is_zero() {
1930            log_attach_final_failure(started_at.elapsed(), &history);
1931            return Err(error);
1932        }
1933
1934        let delay = jittered_attach_delay(backoff, policy.jitter_percent, attempt).min(remaining);
1935        log::info!(
1936            "subc attach retry: attempt {attempt} failed; error_class={class}; error={error_text}; next_delay={delay:?}"
1937        );
1938        tokio::time::sleep(delay).await;
1939
1940        if Instant::now() >= deadline {
1941            log_attach_final_failure(started_at.elapsed(), &history);
1942            return Err(error);
1943        }
1944        backoff = backoff.saturating_mul(2).min(policy.max_backoff);
1945    }
1946}
1947
1948fn jittered_attach_delay(base: Duration, jitter_percent: u64, attempt: u32) -> Duration {
1949    let jitter_percent = jitter_percent.min(100);
1950    if jitter_percent == 0 {
1951        return base;
1952    }
1953
1954    let mut random_bytes = [0_u8; 8];
1955    let random = if getrandom::fill(&mut random_bytes).is_ok() {
1956        u64::from_le_bytes(random_bytes)
1957    } else {
1958        let timestamp = std::time::SystemTime::now()
1959            .duration_since(std::time::UNIX_EPOCH)
1960            .unwrap_or_default()
1961            .subsec_nanos();
1962        u64::from(timestamp) ^ u64::from(attempt)
1963    };
1964    let span = jitter_percent.saturating_mul(2).saturating_add(1);
1965    let multiplier_percent = 100 - jitter_percent + random % span;
1966    let base_millis = u64::try_from(base.as_millis()).unwrap_or(u64::MAX);
1967    Duration::from_millis(base_millis.saturating_mul(multiplier_percent) / 100)
1968}
1969
1970fn log_attach_final_failure(elapsed: Duration, history: &[String]) {
1971    log::error!(
1972        "subc initial attach failed after {} attempt(s) in {elapsed:?}; attempt history: {}",
1973        history.len(),
1974        history.join(" | ")
1975    );
1976}
1977
1978async fn connect_and_authenticate_once(
1979    connection_file_path: &Path,
1980    deadline: Instant,
1981) -> Result<TcpStream, SubcError> {
1982    // This read intentionally lives inside the per-attempt function. The daemon
1983    // publishes connection files atomically and may change both port and key.
1984    let conn = connection_file::read_for_client(connection_file_path).map_err(|source| {
1985        SubcError::ConnectionFile {
1986            path: connection_file_path.to_path_buf(),
1987            source,
1988        }
1989    })?;
1990
1991    let endpoint = conn
1992        .endpoints
1993        .first()
1994        .ok_or_else(|| SubcError::NoEndpoint {
1995            path: connection_file_path.to_path_buf(),
1996        })?;
1997    let endpoint_label = format!("{}:{}", endpoint.host, endpoint.port);
1998    let ip = endpoint
1999        .host
2000        .parse::<IpAddr>()
2001        .map_err(|_| SubcError::InvalidEndpoint {
2002            path: connection_file_path.to_path_buf(),
2003            endpoint: endpoint_label.clone(),
2004        })?;
2005    let addr = SocketAddr::new(ip, endpoint.port);
2006
2007    let connect_budget = deadline.saturating_duration_since(Instant::now());
2008    let mut stream = tokio::time::timeout(connect_budget, TcpStream::connect(addr))
2009        .await
2010        .map_err(|_| SubcError::Connect {
2011            endpoint: endpoint_label.clone(),
2012            source: io::Error::new(
2013                io::ErrorKind::TimedOut,
2014                "initial subc attach retry budget elapsed during TCP connect",
2015            ),
2016        })?
2017        .map_err(|source| SubcError::Connect {
2018            endpoint: endpoint_label.clone(),
2019            source,
2020        })?;
2021    stream
2022        .set_nodelay(true)
2023        .map_err(|source| SubcError::Connect {
2024            endpoint: endpoint_label.clone(),
2025            source,
2026        })?;
2027
2028    let auth_budget = AUTH_DEADLINE.min(deadline.saturating_duration_since(Instant::now()));
2029    authenticate_client(&mut stream, &conn, auth_budget)
2030        .await
2031        .map_err(|source| SubcError::Auth {
2032            endpoint: endpoint_label,
2033            source,
2034        })?;
2035
2036    Ok(stream)
2037}
2038
2039#[allow(clippy::too_many_arguments)]
2040async fn process_route_bind_completion(
2041    writer_tx: &WriterSender,
2042    completion: RouteBindCompletion,
2043    routes: &mut HashMap<RouteChannel, RouteIdentity>,
2044    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2045    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2046    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2047    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2048    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2049    installed_route_epochs: &mut HashMap<u16, u32>,
2050    executor: &Arc<Executor>,
2051    shutdown: &Arc<Notify>,
2052    metrics: &Arc<DispatchPathMetrics>,
2053) -> Result<(), SubcError> {
2054    decrement_counted_channel(&metrics.control_completion_queued);
2055    handle_route_bind_completion(
2056        writer_tx,
2057        completion,
2058        routes,
2059        root_channels,
2060        session_identity,
2061        push_buffer,
2062        live_roots,
2063        pending_binds,
2064        installed_route_epochs,
2065        executor,
2066        shutdown,
2067        metrics,
2068    )
2069    .await
2070}
2071
2072#[allow(clippy::too_many_arguments)]
2073async fn drain_pending_route_bind_completions(
2074    control_completion_rx: &mut mpsc::Receiver<RouteBindCompletion>,
2075    writer_tx: &WriterSender,
2076    routes: &mut HashMap<RouteChannel, RouteIdentity>,
2077    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2078    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2079    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2080    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2081    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2082    installed_route_epochs: &mut HashMap<u16, u32>,
2083    executor: &Arc<Executor>,
2084    shutdown: &Arc<Notify>,
2085    metrics: &Arc<DispatchPathMetrics>,
2086) -> Result<usize, SubcError> {
2087    let mut drained = 0;
2088    while let Ok(completion) = control_completion_rx.try_recv() {
2089        process_route_bind_completion(
2090            writer_tx,
2091            completion,
2092            routes,
2093            root_channels,
2094            session_identity,
2095            push_buffer,
2096            live_roots,
2097            pending_binds,
2098            installed_route_epochs,
2099            executor,
2100            shutdown,
2101            metrics,
2102        )
2103        .await?;
2104        drained += 1;
2105    }
2106    Ok(drained)
2107}
2108
2109/// ModuleHello → HelloAck → control/route loop. Runs until the daemon closes
2110/// the connection (EOF), sends channel-0 Goodbye, or a fatal mutating executor
2111/// response requests whole-connection teardown.
2112async fn run_module_loop<R, W>(
2113    mut read: R,
2114    mut write: W,
2115    shared_app: Arc<App>,
2116    executor: Arc<Executor>,
2117    dispatch: DispatchFn,
2118    user_config_path: Option<PathBuf>,
2119    allow_native_passthrough: bool,
2120) -> Result<ModuleLoopExit, SubcError>
2121where
2122    R: AsyncRead + Unpin + Send + 'static,
2123    W: AsyncWrite + Unpin + Send + 'static,
2124{
2125    // ModuleHello: register as a tool provider and advertise the supported control-plane operations.
2126    // Echo the one-time launch nonce the daemon injected via SUBC_LAUNCH_NONCE so a
2127    // reserved module_id's HELLO is accepted; absent for non-reserved/self-connect.
2128    let hello = ModuleHelloBody {
2129        manifest: build_manifest(),
2130        protocol_ver: PROTOCOL_VERSION,
2131        control_ops: control_ops(),
2132        launch_nonce: std::env::var("SUBC_LAUNCH_NONCE").ok(),
2133    };
2134    let hello_frame = Frame::build(
2135        FrameType::Hello,
2136        control_flags(),
2137        0,
2138        0,
2139        HELLO_CORR,
2140        serde_json::to_vec(&hello).map_err(SubcError::Json)?,
2141    )
2142    .map_err(SubcError::FrameBuild)?;
2143    write_frame(&mut write, &hello_frame)
2144        .await
2145        .map_err(SubcError::FrameIo)?;
2146
2147    // Expect HelloAck (registered) or a channel-0 Error (manifest/version reject).
2148    match read_frame(&mut read).await.map_err(SubcError::FrameIo)? {
2149        None => return Err(SubcError::ClosedBeforeHelloAck),
2150        Some(frame) => match frame.header.ty {
2151            FrameType::HelloAck => {
2152                log::info!("subc attach: registered (HelloAck received)");
2153            }
2154            FrameType::Error => {
2155                let body = serde_json::from_slice::<ErrorBody>(&frame.body).ok();
2156                return Err(SubcError::HelloRejected { body });
2157            }
2158            other => return Err(SubcError::UnexpectedFrame { ty: other }),
2159        },
2160    }
2161
2162    let dispatch_path_metrics = Arc::new(DispatchPathMetrics::new());
2163    let (writer_tx, writer_rx) = mpsc::channel::<WriterFrame>(WRITER_QUEUE_CAPACITY);
2164    let writer_task = spawn_writer_task(write, writer_rx, Arc::clone(&dispatch_path_metrics));
2165    // `read_frame` is NOT cancellation-safe, so it must never sit directly inside
2166    // the `select!` below: a drain-interval tick (or shutdown) firing while a
2167    // frame is mid-transit would drop the partially-consumed bytes and desync the
2168    // stream (the next read would parse a body byte as a frame header). A
2169    // dedicated reader task owns the socket, reads whole frames sequentially, and
2170    // forwards them over a channel; the loop selects on the cancel-safe `recv()`.
2171    let (reader_tx, mut reader_rx) = mpsc::channel::<Result<DecodedFrame, SubcError>>(256);
2172    let reader_task = spawn_reader_task(read, reader_tx);
2173    let shutdown = Arc::new(Notify::new());
2174    // Drain-tick deadline is tracked manually and checked at the TOP of every
2175    // loop turn rather than as an Interval select arm: the select below is
2176    // `biased` (bind completions first), and biased polling means a saturated
2177    // higher arm (sustained lossy push traffic keeps lossy_rx always-ready)
2178    // would starve every arm below it, including a timer arm — leaving
2179    // backpressured reliable frames parked in the retry buffer past their
2180    // delivery deadline. The pre-turn check cannot be starved by arm order;
2181    // the sleep_until arm below only exists to wake an otherwise-idle loop.
2182    let mut next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2183    let mut next_maintenance_at = next_drain_at;
2184    let (maintenance_tx, mut maintenance_rx) = mpsc::channel::<MaintenanceCompletion>(256);
2185    let (bash_deferred_tx, mut bash_deferred_rx) =
2186        mpsc::channel::<bash::BashDeferredCompletion>(256);
2187    let (bash_poll_touch_tx, mut bash_poll_touch_rx) = mpsc::channel::<ProjectRootId>(256);
2188    let (control_completion_tx, mut control_completion_rx) =
2189        mpsc::channel::<RouteBindCompletion>(256);
2190    let (lossy_tx, mut lossy_rx) = mpsc::channel::<LossyPushEnvelope>(1024);
2191    let lossy_overflow = Arc::new(push::LossyOverflow::default());
2192    let lossy_seq = Arc::new(AtomicU64::new(0));
2193    let (reliable_tx, mut reliable_rx) = mpsc::unbounded_channel::<PushEnvelope>();
2194    let push_senders = PushSenders {
2195        lossy_tx,
2196        reliable_tx,
2197        lossy_overflow: Arc::clone(&lossy_overflow),
2198        lossy_seq,
2199    };
2200    let connection_cancel = PersistentCancelSignal::new();
2201    let mut installed_route_epochs: HashMap<u16, u32> = HashMap::new();
2202    let mut routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
2203    let mut bg_subs: HashMap<RouteChannel, BgSub> = HashMap::new();
2204    let mut bg_sub_by_session: HashMap<(ProjectRootId, String), RouteChannel> = HashMap::new();
2205    let mut bg_wake_pending: HashSet<RouteChannel> = HashSet::new();
2206    let mut bg_wake_epoch: HashMap<(ProjectRootId, String), u64> = HashMap::new();
2207    let mut root_channels: HashMap<ProjectRootId, HashSet<RouteChannel>> = HashMap::new();
2208    let mut session_identity: HashMap<(ProjectRootId, String), RetainedSessionIdentity> =
2209        HashMap::new();
2210    let mut push_buffer: HashMap<push::ReplayKey, VecDeque<PushFrame>> = HashMap::new();
2211    let mut retry_buffer: RetryBuffer = HashMap::new();
2212    let mut completed_tasks = push::CompletedTaskIds::default();
2213    let mut live_roots: HashMap<ProjectRootId, RootMeta> = HashMap::new();
2214    let mut pending_binds: HashMap<RouteChannel, PendingBind> = HashMap::new();
2215    let mut pending_bash_asks: HashMap<ReverseCorrKey, PendingBashAsk> = HashMap::new();
2216    let mut next_bash_ask_corr: u64 = 1;
2217    let mut route_bash_cancels: HashMap<RouteChannel, bash::RouteBashCancel> = HashMap::new();
2218
2219    let loop_result: Result<ModuleLoopExit, SubcError> = loop {
2220        shared_app.set_open_route_count(routes.len());
2221        crate::logging::perf_tick(Some(&executor));
2222        dispatch_path_metrics.mark_frame_loop_tick();
2223        if let Err(error) = expire_pending_bash_asks(
2224            &writer_tx,
2225            &mut pending_bash_asks,
2226            &routes,
2227            &mut live_roots,
2228            &mut route_bash_cancels,
2229            &shutdown,
2230            &dispatch_path_metrics,
2231        )
2232        .await
2233        {
2234            break Err(error);
2235        }
2236
2237        // RouteBind completions are control-plane unblockers. Drain any completed
2238        // binds before entering other branch work so Push and maintenance bursts
2239        // can only add one loop-turn of latency.
2240        match drain_pending_route_bind_completions(
2241            &mut control_completion_rx,
2242            &writer_tx,
2243            &mut routes,
2244            &mut root_channels,
2245            &mut session_identity,
2246            &mut push_buffer,
2247            &mut live_roots,
2248            &mut pending_binds,
2249            &mut installed_route_epochs,
2250            &executor,
2251            &shutdown,
2252            &dispatch_path_metrics,
2253        )
2254        .await
2255        {
2256            Ok(drained) => {
2257                if drained > 0 {
2258                    next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2259                }
2260            }
2261            Err(error) => break Err(error),
2262        }
2263
2264        if tokio::time::Instant::now() >= next_drain_at {
2265            push::emit_bg_event_wakes(
2266                &writer_tx,
2267                &dispatch_path_metrics,
2268                &bg_subs,
2269                &mut bg_wake_pending,
2270            );
2271            warn_slow_pending_binds(&mut pending_binds, &executor);
2272            if let Err(error) = expire_overdue_route_binds(
2273                &writer_tx,
2274                &executor,
2275                &mut pending_binds,
2276                &mut installed_route_epochs,
2277                &dispatch_path_metrics,
2278            )
2279            .await
2280            {
2281                break Err(error);
2282            }
2283
2284            let retried = push::drain_retry_buffers_for_bound_routes(
2285                &writer_tx,
2286                &dispatch_path_metrics,
2287                &routes,
2288                &mut retry_buffer,
2289            );
2290            if retried > 0 {
2291                log::debug!(
2292                    "subc attach: retried {retried} reliable Push frame(s) after writer backpressure"
2293                );
2294            }
2295
2296            next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2297        }
2298
2299        // A lossy emitter may place its newest update in the overflow buffer
2300        // when the bounded channel is full, while this receive loop is draining
2301        // the channel. Drain overflow before selecting again so that raced
2302        // update is delivered on the next timer tick instead of waiting for
2303        // another lossy enqueue.
2304        let overflow_batch = lossy_overflow.drain();
2305        if !overflow_batch.is_empty() {
2306            let (_, deferred) = push::drain_reliable_push_turn(
2307                &writer_tx,
2308                &dispatch_path_metrics,
2309                &routes,
2310                &root_channels,
2311                &session_identity,
2312                &mut retry_buffer,
2313                &mut push_buffer,
2314                &mut completed_tasks,
2315                &bg_sub_by_session,
2316                &mut bg_wake_pending,
2317                &mut bg_wake_epoch,
2318                &mut reliable_rx,
2319                None,
2320            );
2321            if deferred {
2322                tokio::task::yield_now().await;
2323            }
2324
2325            let mut batch = Vec::new();
2326            while let Ok(item) = lossy_rx.try_recv() {
2327                batch.push(item);
2328            }
2329            batch.extend(overflow_batch);
2330            push::process_lossy_push_envelope_batch(
2331                &writer_tx,
2332                &dispatch_path_metrics,
2333                &routes,
2334                &root_channels,
2335                &completed_tasks,
2336                batch,
2337            );
2338        }
2339
2340        tokio::select! {
2341            biased;
2342            Some(completion) = control_completion_rx.recv() => {
2343                if let Err(error) = process_route_bind_completion(
2344                    &writer_tx,
2345                    completion,
2346                    &mut routes,
2347                    &mut root_channels,
2348                    &mut session_identity,
2349                    &mut push_buffer,
2350                    &mut live_roots,
2351                    &mut pending_binds,
2352                    &mut installed_route_epochs,
2353                    &executor,
2354                    &shutdown,
2355                    &dispatch_path_metrics,
2356                )
2357                .await
2358                {
2359                    break Err(error);
2360                }
2361                next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2362            }
2363            _ = shutdown.notified() => {
2364                log::warn!("subc attach: fatal executor response requested teardown");
2365                break Ok(ModuleLoopExit::SkipSearchFlush);
2366            }
2367            maybe_frame = reader_rx.recv() => {
2368                let frame = match maybe_frame {
2369                    None => {
2370                        log::info!("subc attach: daemon closed connection");
2371                        break Ok(ModuleLoopExit::Graceful);
2372                    }
2373                    Some(Err(error)) => break Err(error),
2374                    Some(Ok(frame)) => frame,
2375                };
2376                let phase_trace = frame.phase_trace;
2377                let frame = frame.frame;
2378
2379                if !ingress_route_is_current(&installed_route_epochs, &frame) {
2380                    log::debug!(
2381                        "subc attach: silently dropping {:?} for uninstalled route {}@{}",
2382                        frame.header.ty,
2383                        frame.header.channel,
2384                        frame.header.epoch
2385                    );
2386                    continue;
2387                }
2388
2389                match frame.header.ty {
2390                    FrameType::Ping if frame.header.channel == 0 => {
2391                        let pong = match Frame::build_with_version(
2392                            frame.header.ver,
2393                            FrameType::Pong,
2394                            frame.header.flags,
2395                            0,
2396                            0,
2397                            frame.header.corr,
2398                            Vec::new(),
2399                        ) {
2400                            Ok(pong) => pong,
2401                            Err(error) => break Err(SubcError::FrameBuild(error)),
2402                        };
2403                        if let Err(error) = send_frame(&writer_tx, &dispatch_path_metrics, pong).await {
2404                            break Err(error);
2405                        }
2406                    }
2407                    FrameType::Goodbye if frame.header.channel == 0 => {
2408                        log::info!("subc attach: received channel-0 Goodbye");
2409                        break Ok(ModuleLoopExit::Graceful);
2410                    }
2411                    FrameType::Goodbye => {
2412                        let channel = route_key(frame.header.channel, frame.header.epoch);
2413                        if let Err(error) = teardown_installed_route(
2414                            &writer_tx,
2415                            &dispatch_path_metrics,
2416                            &executor,
2417                            channel,
2418                            "Goodbye",
2419                            None,
2420                            &mut installed_route_epochs,
2421                            &mut routes,
2422                            &mut root_channels,
2423                            &mut bg_subs,
2424                            &mut bg_sub_by_session,
2425                            &mut bg_wake_pending,
2426                            &mut pending_bash_asks,
2427                            &mut live_roots,
2428                            &mut route_bash_cancels,
2429                            &mut pending_binds,
2430                            &mut retry_buffer,
2431                            &mut push_buffer,
2432                            &shutdown,
2433                        )
2434                        .await
2435                        {
2436                            break Err(error);
2437                        }
2438                    }
2439                    FrameType::Response | FrameType::Error if frame.header.channel != 0 => {
2440                        if let Err(error) = handle_bash_elicitation_reply(
2441                            &writer_tx,
2442                            &frame,
2443                            &mut pending_bash_asks,
2444                            &routes,
2445                            &mut live_roots,
2446                            &executor,
2447                            &shutdown,
2448                            &bash_deferred_tx,
2449                            &bash_poll_touch_tx,
2450                            &dispatch_path_metrics,
2451                            &mut route_bash_cancels,
2452                            dispatch,
2453                        )
2454                        .await
2455                        {
2456                            break Err(error);
2457                        }
2458                    }
2459                    FrameType::Request if frame.header.channel == 0 => {
2460                        if let Err(error) = handle_control_request(
2461                            &writer_tx,
2462                            &frame,
2463                            &shared_app,
2464                            &executor,
2465                            &mut live_roots,
2466                            &mut pending_binds,
2467                            &mut installed_route_epochs,
2468                            &mut routes,
2469                            &mut root_channels,
2470                            &mut bg_subs,
2471                            &mut bg_sub_by_session,
2472                            &mut bg_wake_pending,
2473                            &mut pending_bash_asks,
2474                            &mut route_bash_cancels,
2475                            &mut retry_buffer,
2476                            &mut push_buffer,
2477                            &shutdown,
2478                            &control_completion_tx,
2479                            &dispatch_path_metrics,
2480                            &push_senders,
2481                            dispatch,
2482                            user_config_path.as_deref(),
2483                        )
2484                        .await
2485                        {
2486                            break Err(error);
2487                        }
2488                    }
2489                    FrameType::Request => {
2490                        if let Err(error) = handle_tool_call(
2491                            &writer_tx,
2492                            &frame,
2493                            phase_trace,
2494                            &routes,
2495                            &pending_binds,
2496                            &mut live_roots,
2497                            &executor,
2498                            &shutdown,
2499                            &connection_cancel,
2500                            &bash_deferred_tx,
2501                            &bash_poll_touch_tx,
2502                            &dispatch_path_metrics,
2503                            &mut route_bash_cancels,
2504                            &mut pending_bash_asks,
2505                            &mut next_bash_ask_corr,
2506                            &mut bg_subs,
2507                            &mut bg_sub_by_session,
2508                            &mut bg_wake_pending,
2509                            &mut bg_wake_epoch,
2510                            dispatch,
2511                            allow_native_passthrough,
2512                        )
2513                        .await
2514                        {
2515                            break Err(error);
2516                        }
2517                    }
2518                    FrameType::Cancel => {
2519                        let channel = route_key(frame.header.channel, frame.header.epoch);
2520                        if bg_subs.contains_key(&channel) {
2521                            if let Err(error) = end_bg_subscription(
2522                                &writer_tx,
2523                                &dispatch_path_metrics,
2524                                &mut bg_subs,
2525                                &mut bg_sub_by_session,
2526                                &mut bg_wake_pending,
2527                                channel,
2528                                routes.get(&channel),
2529                            )
2530                            .await
2531                            {
2532                                break Err(error);
2533                            }
2534                        }
2535                        if let Err(error) = cancel_pending_bash_ask_for_tool_call(
2536                            &writer_tx,
2537                            &mut pending_bash_asks,
2538                            channel,
2539                            frame.header.corr,
2540                            &routes,
2541                            &mut live_roots,
2542                            &mut route_bash_cancels,
2543                            &shutdown,
2544                            &dispatch_path_metrics,
2545                        )
2546                        .await
2547                        {
2548                            break Err(error);
2549                        }
2550                    }
2551                    // Incoming push messages are ignored here. Cancel frames only
2552                    // stop pending bash elicitation requests; executor-level
2553                    // cancellation for tool calls that are already running is not
2554                    // implemented.
2555                    _ => {}
2556                }
2557            }
2558            Some((root_id, frame)) = reliable_rx.recv() => {
2559                // Reliable Push frames are FIFO and must-deliver, but draining an
2560                // unbounded burst in one current-thread turn can starve RouteBind
2561                // completions. The budget defers excess frames, never drops them.
2562                let (_, deferred) = push::drain_reliable_push_turn(
2563                    &writer_tx,
2564                    &dispatch_path_metrics,
2565                    &routes,
2566                    &root_channels,
2567                    &session_identity,
2568                    &mut retry_buffer,
2569                    &mut push_buffer,
2570                    &mut completed_tasks,
2571                    &bg_sub_by_session,
2572                    &mut bg_wake_pending,
2573                    &mut bg_wake_epoch,
2574                    &mut reliable_rx,
2575                    Some((root_id, frame)),
2576                );
2577                if deferred {
2578                    tokio::task::yield_now().await;
2579                }
2580            }
2581            Some((order, root_id, frame)) = lossy_rx.recv() => {
2582                // When both push lanes have work, handle a small reliable slice before lossy work.
2583                // That ordering lets completed task ids suppress stale BashLongRunning frames.
2584                // The slice stays bounded so reliable bursts cannot monopolize this loop turn.
2585                let (_, deferred) = push::drain_reliable_push_turn(
2586                    &writer_tx,
2587                    &dispatch_path_metrics,
2588                    &routes,
2589                    &root_channels,
2590                    &session_identity,
2591                    &mut retry_buffer,
2592                    &mut push_buffer,
2593                    &mut completed_tasks,
2594                    &bg_sub_by_session,
2595                    &mut bg_wake_pending,
2596                    &mut bg_wake_epoch,
2597                    &mut reliable_rx,
2598                    None,
2599                );
2600                if deferred {
2601                    tokio::task::yield_now().await;
2602                }
2603
2604                // Drain the currently queued burst in one loop turn so lossy
2605                // status/progress updates can be merged before reaching subc's
2606                // shared egress queue. Each lossy frame gets a sequence number
2607                // before it goes to the channel or overflow buffer, so the
2608                // combined batch is sorted back into producer order before
2609                // coalescing drops stale updates for the same key.
2610                let mut batch = vec![(order, root_id, frame)];
2611                while let Ok(item) = lossy_rx.try_recv() {
2612                    batch.push(item);
2613                }
2614                batch.extend(lossy_overflow.drain());
2615                push::process_lossy_push_envelope_batch(
2616                    &writer_tx,
2617                    &dispatch_path_metrics,
2618                    &routes,
2619                    &root_channels,
2620                    &completed_tasks,
2621                    batch,
2622                );
2623            }
2624            Some(done) = bash_deferred_rx.recv() => {
2625                decrement_counted_channel(&dispatch_path_metrics.bash_deferred_queued);
2626                if let Err(error) = bash::handle_bash_deferred_completion(
2627                    &writer_tx,
2628                    done,
2629                    &routes,
2630                    &mut live_roots,
2631                    &mut route_bash_cancels,
2632                    &shutdown,
2633                    &dispatch_path_metrics,
2634                )
2635                .await
2636                {
2637                    break Err(error);
2638                }
2639            }
2640            Some(root_id) = bash_poll_touch_rx.recv() => {
2641                decrement_counted_channel(&dispatch_path_metrics.bash_poll_touch_queued);
2642                if let Some(meta) = live_roots.get_mut(&root_id) {
2643                    meta.note_activity();
2644                }
2645            }
2646            Some(completion) = maintenance_rx.recv() => {
2647                decrement_counted_channel(&dispatch_path_metrics.maintenance_queued);
2648                let root_id = completion.root_id.clone();
2649                let response = completion.response;
2650                let response_is_fatal = response_is_fatal_panic(&response);
2651                let bind_pending = pending_binds
2652                    .values()
2653                    .any(|pending| pending.bind_root_id == root_id);
2654                let requiesce = if let Some(meta) = live_roots.get_mut(&root_id) {
2655                    let defer_requeue = meta.unbound_quiesced || bind_pending;
2656                    note_maintenance_completion(
2657                        meta,
2658                        completion.requeue_kind,
2659                        response_is_fatal,
2660                        defer_requeue,
2661                    );
2662                    should_requiesce_after_maintenance(meta, completion.kind, bind_pending)
2663                } else {
2664                    false
2665                };
2666                if requiesce {
2667                    quiesce_unbound_root(&root_id, &mut live_roots, &executor);
2668                }
2669                push::clear_stale_bg_wakes_for_empty_sessions(
2670                    &root_id,
2671                    &completion.empty_bg_sessions,
2672                    &bg_sub_by_session,
2673                    &mut bg_wake_pending,
2674                    &bg_wake_epoch,
2675                );
2676                if response_is_fatal {
2677                    if let Some(meta) = live_roots.get_mut(&root_id) {
2678                        meta.maintenance_poisoned = true;
2679                    }
2680                    log::warn!(
2681                        "subc attach: maintenance drain observed a fatal actor; deferring teardown until a route request can receive actor_fatal"
2682                    );
2683                }
2684            }
2685            _ = tokio::time::sleep_until(next_drain_at) => {
2686                // Wakes an otherwise-idle loop so the pre-turn drain check
2687                // above runs on schedule; the drain work itself lives there.
2688            }
2689            _ = tokio::time::sleep_until(next_maintenance_at) => {
2690                // Delay cache-draining maintenance until any already-ready
2691                // inbound route/control messages and push completions have run,
2692                // so maintenance does not block the actor from handling the
2693                // first request that arrives after a route bind is acknowledged.
2694                let reap = reap_idle_roots(
2695                    Instant::now(),
2696                    &mut live_roots,
2697                    &pending_binds,
2698                    &root_channels,
2699                    &executor,
2700                    &dispatch_path_metrics,
2701                );
2702                for root_id in &reap.forgotten_deleted_roots {
2703                    purge_deleted_root_residents(
2704                        root_id,
2705                        &mut root_channels,
2706                        &mut session_identity,
2707                        &mut push_buffer,
2708                        &mut bg_subs,
2709                        &mut bg_sub_by_session,
2710                        &mut bg_wake_pending,
2711                        &mut bg_wake_epoch,
2712                        &mut pending_bash_asks,
2713                    );
2714                }
2715                if reap.evicted > 0 {
2716                    log::debug!("subc attach: reaped {} idle root(s)", reap.evicted);
2717                }
2718                submit_due_maintenance_jobs(
2719                    &executor,
2720                    &mut live_roots,
2721                    &pending_binds,
2722                    &bg_sub_by_session,
2723                    &bg_wake_pending,
2724                    &bg_wake_epoch,
2725                    &maintenance_tx,
2726                    &dispatch_path_metrics,
2727                );
2728                next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2729            }
2730        }
2731    };
2732
2733    shared_app.set_open_route_count(0);
2734
2735    connection_cancel.cancel();
2736    // Channel-0 Goodbye, EOF, and fatal exits bypass per-route Goodbye. Settle
2737    // their root lifecycle state before loop-owned routing metadata is dropped.
2738    quiesce_connection_roots(
2739        &mut live_roots,
2740        &mut pending_binds,
2741        &mut routes,
2742        &mut root_channels,
2743        &mut installed_route_epochs,
2744        &mut route_bash_cancels,
2745        &executor,
2746    );
2747
2748    let mut loop_result = loop_result;
2749    if !pending_bash_asks.is_empty() {
2750        let no_routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
2751        if let Err(error) = settle_all_pending_bash_asks(
2752            &writer_tx,
2753            &mut pending_bash_asks,
2754            &no_routes,
2755            &mut live_roots,
2756            &mut route_bash_cancels,
2757            &shutdown,
2758            &dispatch_path_metrics,
2759        )
2760        .await
2761        {
2762            loop_result = loop_result.and(Err(error));
2763        }
2764    }
2765
2766    // The reader task may be parked on `read_frame`; abort it (we are done with
2767    // the connection) and flush the writer.
2768    reader_task.abort();
2769    drop(writer_tx);
2770    let writer_result = finish_writer_task(writer_task).await;
2771    loop_result.and_then(|exit| writer_result.map(|_| exit))
2772}
2773
2774fn spawn_writer_task<W>(
2775    mut write: W,
2776    mut rx: mpsc::Receiver<WriterFrame>,
2777    metrics: Arc<DispatchPathMetrics>,
2778) -> JoinHandle<Result<(), subc_transport::FrameIoError>>
2779where
2780    W: AsyncWrite + Unpin + Send + 'static,
2781{
2782    tokio::spawn(async move {
2783        let mut write_buffer = Vec::new();
2784        while let Some(mut queued) = rx.recv().await {
2785            let measure = queued.tool_response_trace.is_some();
2786            let dequeued = measure.then(Instant::now);
2787            metrics.writer_active.store(true, Ordering::Relaxed);
2788            decrement_counted_channel(&metrics.writer_queued);
2789            let write_timing = write_frame_contiguous(
2790                &mut write,
2791                queued.frame(),
2792                queued.body(),
2793                &mut write_buffer,
2794                measure,
2795            )
2796            .await;
2797            metrics.writer_active.store(false, Ordering::Relaxed);
2798            let write_timing = write_timing?;
2799
2800            if let (Some(trace), Some(dequeued), Some(write_timing)) =
2801                (queued.tool_response_trace.take(), dequeued, write_timing)
2802            {
2803                if let Some(completed) = trace.finish(
2804                    dequeued,
2805                    write_timing.write_started,
2806                    write_timing.write_finished,
2807                    write_timing.frame_bytes,
2808                ) {
2809                    log_ctx::with_session(Some(completed.session), || {
2810                        crate::logging::note_tool_call_trace(
2811                            &completed.name,
2812                            &completed.root,
2813                            completed.channel,
2814                            completed.corr,
2815                            completed.phases,
2816                        );
2817                    });
2818                }
2819            }
2820        }
2821        Ok(())
2822    })
2823}
2824
2825struct FrameWriteTiming {
2826    write_started: Instant,
2827    write_finished: Instant,
2828    frame_bytes: usize,
2829}
2830
2831/// Encode one complete frame into the existing reusable buffer and write it
2832/// without interleaving bytes from another channel. Timing is collected only
2833/// for tool responses, so Push and control frames add no clock reads.
2834async fn write_frame_contiguous<W>(
2835    writer: &mut W,
2836    frame: &Frame,
2837    body: &[u8],
2838    buffer: &mut Vec<u8>,
2839    measure: bool,
2840) -> Result<Option<FrameWriteTiming>, subc_transport::FrameIoError>
2841where
2842    W: AsyncWrite + Unpin,
2843{
2844    if frame.header.len as usize != body.len() {
2845        return Err(subc_transport::FrameIoError::BodyLengthMismatch {
2846            header_len: frame.header.len,
2847            body_len: body.len(),
2848        });
2849    }
2850
2851    let header = frame.header.encode();
2852    buffer.clear();
2853    buffer.reserve(header.len() + body.len());
2854    buffer.extend_from_slice(&header);
2855    buffer.extend_from_slice(body);
2856    let write_started = measure.then(Instant::now);
2857    writer
2858        .write_all(buffer)
2859        .await
2860        .map_err(subc_transport::FrameIoError::Io)?;
2861    Ok(write_started.map(|write_started| FrameWriteTiming {
2862        write_started,
2863        write_finished: Instant::now(),
2864        frame_bytes: buffer.len(),
2865    }))
2866}
2867
2868fn spawn_reader_task<R>(
2869    mut read: R,
2870    tx: mpsc::Sender<Result<DecodedFrame, SubcError>>,
2871) -> JoinHandle<()>
2872where
2873    R: AsyncRead + Unpin + Send + 'static,
2874{
2875    tokio::spawn(async move {
2876        loop {
2877            match read_frame(&mut read).await {
2878                Ok(Some(frame)) => {
2879                    let decoded = DecodedFrame {
2880                        frame,
2881                        phase_trace: PhaseTrace::new(Instant::now()),
2882                    };
2883                    if tx.send(Ok(decoded)).await.is_err() {
2884                        return;
2885                    }
2886                }
2887                Ok(None) => {
2888                    // EOF: let the loop observe channel close as "daemon closed".
2889                    return;
2890                }
2891                Err(error) => {
2892                    // A killed daemon surfaces as ConnectionReset (RST) on
2893                    // Windows where Unix delivers a clean EOF (FIN); a
2894                    // mid-teardown daemon can also abort the socket. Both mean
2895                    // "daemon went away", not a wire fault — normalize them to
2896                    // the clean-close path so module exit behavior matches
2897                    // across platforms (same class subc-core fixed in d33d9a71).
2898                    if let subc_transport::FrameIoError::Io(io_error) = &error {
2899                        if matches!(
2900                            io_error.kind(),
2901                            std::io::ErrorKind::ConnectionReset
2902                                | std::io::ErrorKind::ConnectionAborted
2903                        ) {
2904                            log::info!(
2905                                "subc attach: connection reset by daemon; treating as close"
2906                            );
2907                            return;
2908                        }
2909                    }
2910                    let _ = tx.send(Err(SubcError::FrameIo(error))).await;
2911                    return;
2912                }
2913            }
2914        }
2915    })
2916}
2917
2918async fn finish_writer_task(
2919    mut writer_task: JoinHandle<Result<(), subc_transport::FrameIoError>>,
2920) -> Result<(), SubcError> {
2921    match tokio::time::timeout(Duration::from_millis(100), &mut writer_task).await {
2922        Ok(Ok(Ok(()))) => Ok(()),
2923        Ok(Ok(Err(error))) => Err(SubcError::FrameIo(error)),
2924        Ok(Err(error)) => Err(SubcError::WriterJoin(error)),
2925        Err(_) => {
2926            writer_task.abort();
2927            Ok(())
2928        }
2929    }
2930}
2931
2932fn register_actor_for_bind(
2933    shared_app: &Arc<App>,
2934    executor: &Arc<Executor>,
2935    push_senders: &PushSenders,
2936    bind_root_id: &ProjectRootId,
2937    route_channel: u16,
2938    root_was_live: bool,
2939) -> bool {
2940    if executor.actor_registered(bind_root_id) {
2941        log::debug!(
2942            "subc attach: reusing actor for route {} root {}",
2943            route_channel,
2944            bind_root_id.as_path().display()
2945        );
2946        return false;
2947    }
2948
2949    if root_was_live {
2950        log::warn!(
2951            "subc attach: recreating missing actor for live root {} on route {}",
2952            bind_root_id.as_path().display(),
2953            route_channel
2954        );
2955    }
2956
2957    let actor_ctx = Arc::new(AppContext::from_app(
2958        Arc::clone(shared_app),
2959        Config::default(),
2960    ));
2961    install_bash_compressor(&actor_ctx);
2962    actor_ctx.set_progress_sender(Some(push::progress_sender_for_root(
2963        push_senders.clone(),
2964        bind_root_id.clone(),
2965    )));
2966    let inserted = executor.register_actor(bind_root_id.clone(), Arc::clone(&actor_ctx));
2967    drop(actor_ctx);
2968    if inserted {
2969        // Do not insert into live_roots until configure succeeds: live_roots
2970        // drives maintenance, and a half-configured new actor must not be
2971        // maintenance-eligible before its route/session identity exists.
2972        log::debug!(
2973            "subc attach: registered actor for route {} root {}",
2974            route_channel,
2975            bind_root_id.as_path().display()
2976        );
2977    } else {
2978        log::debug!(
2979            "subc attach: actor appeared while binding route {} root {}; reusing it",
2980            route_channel,
2981            bind_root_id.as_path().display()
2982        );
2983    }
2984    inserted
2985}
2986
2987fn rollback_pending_bind_actor(
2988    executor: &Arc<Executor>,
2989    live_roots: &HashMap<ProjectRootId, RootMeta>,
2990    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2991    root_id: &ProjectRootId,
2992    inserted_new_actor: bool,
2993) {
2994    if !inserted_new_actor || live_roots.contains_key(root_id) {
2995        return;
2996    }
2997
2998    if let Some((route, pending)) = pending_binds
2999        .iter_mut()
3000        .find(|(_, pending)| &pending.bind_root_id == root_id)
3001    {
3002        pending.inserted_new_actor = true;
3003        log::debug!(
3004            "subc attach: transferred rollback ownership for root {} to pending route {}",
3005            root_id.as_path().display(),
3006            route
3007        );
3008        return;
3009    }
3010
3011    executor.remove_actor(root_id);
3012}
3013
3014fn route_bind_error_code_for_configure_response(response: &Response) -> &'static str {
3015    match response.data.get("code").and_then(|code| code.as_str()) {
3016        // Preserve typed configure rejections across the bind boundary: a
3017        // malformed fed fingerprint means a federation-module bug or
3018        // fingerprint-format drift, and the fed side matches on the code rather
3019        // than parsing prose.
3020        Some("bad_harness_fingerprint") => "bad_harness_fingerprint",
3021        // Cache-key probe failures are transient (fd pressure, git spawn
3022        // contention); the client retries the bind rather than treating the
3023        // root as permanently divergent.
3024        Some("cache_key_probe_failed") => "cache_key_probe_failed",
3025        // Actor lifecycle gaps are transient from the daemon/client viewpoint:
3026        // a fresh bind can create or join a healthy actor, so do not classify
3027        // them as permanent config divergence.
3028        Some("actor_not_registered" | "actor_fatal") => "actor_not_ready",
3029        _ => "config_divergence",
3030    }
3031}
3032
3033fn queue_post_bind_configure_and_completion_maintenance(
3034    root_id: &ProjectRootId,
3035    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3036) {
3037    let Some(meta) = live_roots.get_mut(root_id) else {
3038        return;
3039    };
3040    if meta.maintenance_poisoned || meta.maintenance_pending {
3041        return;
3042    }
3043
3044    meta.maintenance_pending = true;
3045    meta.maintenance_queued_kinds
3046        .push_back(MaintenanceDrainKind::ConfigureTail);
3047    meta.maintenance_queued_kinds
3048        .push_back(MaintenanceDrainKind::CompletionDrains);
3049}
3050
3051#[allow(clippy::too_many_arguments)]
3052async fn handle_route_bind_completion(
3053    tx: &WriterSender,
3054    completion: RouteBindCompletion,
3055    routes: &mut HashMap<RouteChannel, RouteIdentity>,
3056    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
3057    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
3058    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
3059    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3060    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3061    installed_route_epochs: &mut HashMap<u16, u32>,
3062    executor: &Arc<Executor>,
3063    shutdown: &Arc<Notify>,
3064    metrics: &Arc<DispatchPathMetrics>,
3065) -> Result<(), SubcError> {
3066    let route_id = completion.route;
3067    let Some(pending) = pending_binds.remove(&route_id) else {
3068        log::warn!(
3069            "subc attach: dropping RouteBind completion for non-pending route {}",
3070            completion.route
3071        );
3072        rollback_pending_bind_actor(
3073            executor,
3074            live_roots,
3075            pending_binds,
3076            &completion.bind_root_id,
3077            completion.inserted_new_actor,
3078        );
3079        let has_pending_bind = pending_binds
3080            .values()
3081            .any(|pending| pending.bind_root_id == completion.bind_root_id);
3082        if !root_channels
3083            .get(&completion.bind_root_id)
3084            .is_some_and(|channels| !channels.is_empty())
3085            && !has_pending_bind
3086        {
3087            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3088        }
3089        remove_installed_route(installed_route_epochs, route_id);
3090        return Ok(());
3091    };
3092
3093    if pending.bind_root_id != completion.bind_root_id {
3094        log::warn!(
3095            "subc attach: pending RouteBind root mismatch for route {} (pending {} completion {})",
3096            completion.route,
3097            pending.bind_root_id.as_path().display(),
3098            completion.bind_root_id.as_path().display()
3099        );
3100    }
3101
3102    let inserted_new_actor = pending.inserted_new_actor || completion.inserted_new_actor;
3103    if pending.cancelled {
3104        rollback_pending_bind_actor(
3105            executor,
3106            live_roots,
3107            pending_binds,
3108            &completion.bind_root_id,
3109            inserted_new_actor,
3110        );
3111        let has_pending_bind = pending_binds
3112            .values()
3113            .any(|pending| pending.bind_root_id == completion.bind_root_id);
3114        if !root_channels
3115            .get(&completion.bind_root_id)
3116            .is_some_and(|channels| !channels.is_empty())
3117            && !has_pending_bind
3118        {
3119            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3120        }
3121        log::debug!(
3122            "subc attach: discarded completed RouteBind for cancelled route {} root {}",
3123            completion.route,
3124            completion.bind_root_id.as_path().display()
3125        );
3126        remove_installed_route(installed_route_epochs, route_id);
3127        return Ok(());
3128    }
3129
3130    let failure = if !completion.configure_response.success {
3131        Some((
3132            &completion.configure_response,
3133            "configure failed during route bind",
3134        ))
3135    } else {
3136        None
3137    };
3138
3139    if let Some((response, fallback)) = failure {
3140        rollback_pending_bind_actor(
3141            executor,
3142            live_roots,
3143            pending_binds,
3144            &completion.bind_root_id,
3145            inserted_new_actor,
3146        );
3147        let has_pending_bind = pending_binds
3148            .values()
3149            .any(|pending| pending.bind_root_id == completion.bind_root_id);
3150        if !root_channels
3151            .get(&completion.bind_root_id)
3152            .is_some_and(|channels| !channels.is_empty())
3153            && !has_pending_bind
3154        {
3155            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3156        }
3157        let message = response_message(response, fallback);
3158        let fatal = response_is_fatal_panic(response);
3159        let error_code = route_bind_error_code_for_configure_response(response);
3160        send_route_bind_error_parts(
3161            tx,
3162            completion.ver,
3163            completion.corr,
3164            completion.flags,
3165            error_code,
3166            &message,
3167            metrics,
3168        )
3169        .await?;
3170        remove_installed_route(installed_route_epochs, route_id);
3171        if fatal {
3172            signal_fatal_teardown(
3173                tx,
3174                Some(completion.route),
3175                completion.ver,
3176                completion.corr,
3177                shutdown,
3178                metrics,
3179            )
3180            .await;
3181        }
3182        return Ok(());
3183    }
3184
3185    remember_session_identity(session_identity, &completion.identity);
3186    let replay_key = push::ReplayKey::from_identity(&completion.identity);
3187    let bind_trust = completion.identity.trust;
3188    insert_route_channel(routes, root_channels, route_id, completion.identity);
3189    let restore_watcher = live_roots
3190        .get(&completion.bind_root_id)
3191        .is_some_and(|meta| meta.idle_artifacts_evicted || meta.unbound_quiesced);
3192    live_roots
3193        .entry(completion.bind_root_id.clone())
3194        .and_modify(|meta| {
3195            meta.reactivate_bound();
3196            meta.diagnostics_on_edit = completion.diagnostics_on_edit;
3197            meta.maintenance_poisoned = false;
3198        })
3199        .or_insert_with(|| RootMeta::new(Instant::now()));
3200    if let Some(meta) = live_roots.get_mut(&completion.bind_root_id) {
3201        meta.diagnostics_on_edit = completion.diagnostics_on_edit;
3202        meta.maintenance_poisoned = false;
3203    }
3204    if let Some(ctx) = executor.actor_context(&completion.bind_root_id) {
3205        ctx.mark_subc_bound();
3206        if restore_watcher {
3207            crate::commands::configure::ensure_project_watcher(&ctx);
3208        }
3209    }
3210
3211    let ack =
3212        serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).map_err(SubcError::Json)?;
3213    let response = Frame::build_with_version(
3214        completion.ver,
3215        FrameType::Response,
3216        control_flags(),
3217        0,
3218        0,
3219        completion.corr,
3220        ack,
3221    )
3222    .map_err(SubcError::FrameBuild)?;
3223    send_reliable_writer_frame(tx, metrics, response, "RouteBindAck").await?;
3224    queue_post_bind_configure_and_completion_maintenance(&completion.bind_root_id, live_roots);
3225    let replayed = push::replay_buffered_push_frames(
3226        tx,
3227        metrics,
3228        route_id,
3229        push_buffer,
3230        &replay_key,
3231        bind_trust,
3232    );
3233    if replayed > 0 {
3234        log::debug!(
3235            "subc attach: replayed {} buffered Push frame(s) to route {} root {} harness {} session {}",
3236            replayed,
3237            completion.route,
3238            replay_key.root.as_path().display(),
3239            replay_key.harness,
3240            replay_key.session
3241        );
3242    }
3243    log::info!(
3244        "subc attach: route {} bound to root {}",
3245        completion.route,
3246        completion.bind_root_id.as_path().display()
3247    );
3248    Ok(())
3249}
3250
3251async fn expire_overdue_route_binds(
3252    tx: &WriterSender,
3253    executor: &Arc<Executor>,
3254    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3255    installed_route_epochs: &mut HashMap<u16, u32>,
3256    metrics: &DispatchPathMetrics,
3257) -> Result<(), SubcError> {
3258    let now = Instant::now();
3259    let expired: Vec<_> = pending_binds
3260        .iter()
3261        .filter_map(|(route, pending)| {
3262            let age = now.saturating_duration_since(pending.started_at);
3263            (!pending.deadline_reported && age >= ROUTE_BIND_DEADLINE).then(|| {
3264                (
3265                    *route,
3266                    pending.corr,
3267                    pending.ver,
3268                    pending.flags,
3269                    pending.bind_root_id.clone(),
3270                    pending.configure_request_id.clone(),
3271                    age,
3272                )
3273            })
3274        })
3275        .collect();
3276
3277    for (route, corr, ver, flags, root_id, configure_request_id, age) in expired {
3278        if let Some(pending) = pending_binds.get_mut(&route) {
3279            pending.cancelled = true;
3280            pending.deadline_reported = true;
3281            let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
3282            log::debug!(
3283                "subc attach: cancelled overdue RouteBind configure for route {route} ({outcome:?})"
3284            );
3285        }
3286        remove_installed_route(installed_route_epochs, route);
3287        let age_ms = age.as_millis().min(u128::from(u64::MAX)) as u64;
3288        let deadline_ms = ROUTE_BIND_DEADLINE.as_millis();
3289        send_route_bind_error_parts(
3290            tx,
3291            ver,
3292            corr,
3293            flags,
3294            "actor_not_ready",
3295            &format!("route bind deadline exceeded after {age_ms}ms (deadline {deadline_ms}ms)"),
3296            metrics,
3297        )
3298        .await?;
3299        log::warn!(
3300            "subc attach: route {} bind for root {} exceeded {}ms deadline (configure_request_id={})",
3301            route,
3302            root_id.as_path().display(),
3303            deadline_ms,
3304            configure_request_id
3305        );
3306    }
3307
3308    Ok(())
3309}
3310
3311/// channel-0 control requests: RouteBind plus the cached health probe. RouteBind
3312/// still reconciles the route's RootConfig through the executor's Mutating lane
3313/// and resolves completion on a loop-owned control-completion channel so slow
3314/// configure jobs do not block the transport loop.
3315#[allow(clippy::too_many_arguments)]
3316async fn handle_control_request(
3317    tx: &WriterSender,
3318    frame: &Frame,
3319    shared_app: &Arc<App>,
3320    executor: &Arc<Executor>,
3321    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3322    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3323    installed_route_epochs: &mut HashMap<u16, u32>,
3324    routes: &mut HashMap<RouteChannel, RouteIdentity>,
3325    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
3326    bg_subs: &mut HashMap<RouteChannel, BgSub>,
3327    bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
3328    bg_wake_pending: &mut HashSet<RouteChannel>,
3329    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
3330    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
3331    retry_buffer: &mut RetryBuffer,
3332    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
3333    shutdown: &Arc<Notify>,
3334    control_completion_tx: &mpsc::Sender<RouteBindCompletion>,
3335    metrics: &Arc<DispatchPathMetrics>,
3336    push_senders: &PushSenders,
3337    dispatch: DispatchFn,
3338    user_config_path: Option<&Path>,
3339) -> Result<(), SubcError> {
3340    let request =
3341        serde_json::from_slice::<ModuleControlRequest>(&frame.body).map_err(SubcError::Json)?;
3342    match request {
3343        ModuleControlRequest::RouteBind {
3344            route_channel,
3345            epoch,
3346            target: _,
3347            identity,
3348            principal,
3349            consumer_capabilities,
3350        } => {
3351            let route_id = route_key(route_channel, epoch);
3352            if epoch == 0 {
3353                return send_route_bind_error(
3354                    tx,
3355                    frame,
3356                    "config_divergence",
3357                    "route bind uses an invalid channel generation",
3358                    metrics,
3359                )
3360                .await;
3361            }
3362            let mut bind_root_id = None;
3363            if let Some(installed_epoch) = installed_route_epochs.get(&route_channel).copied() {
3364                if installed_epoch >= epoch {
3365                    return send_route_bind_error(
3366                        tx,
3367                        frame,
3368                        "config_divergence",
3369                        "route bind generation is not newer than the installed generation",
3370                        metrics,
3371                    )
3372                    .await;
3373                }
3374
3375                let replacement_root = match ProjectRootId::from_path(&identity.project_root) {
3376                    Ok(root_id) => root_id,
3377                    Err(error) => {
3378                        return send_route_bind_error(
3379                            tx,
3380                            frame,
3381                            "config_divergence",
3382                            &format!("invalid route project root: {error}"),
3383                            metrics,
3384                        )
3385                        .await;
3386                    }
3387                };
3388                teardown_installed_route(
3389                    tx,
3390                    metrics,
3391                    executor,
3392                    route_key(route_channel, installed_epoch),
3393                    "higher-epoch RouteBind",
3394                    Some(&replacement_root),
3395                    installed_route_epochs,
3396                    routes,
3397                    root_channels,
3398                    bg_subs,
3399                    bg_sub_by_session,
3400                    bg_wake_pending,
3401                    pending_bash_asks,
3402                    live_roots,
3403                    route_bash_cancels,
3404                    pending_binds,
3405                    retry_buffer,
3406                    push_buffer,
3407                    shutdown,
3408                )
3409                .await?;
3410                bind_root_id = Some(replacement_root);
3411            }
3412            if pending_binds.contains_key(&route_id) {
3413                return send_route_bind_error(
3414                    tx,
3415                    frame,
3416                    "config_divergence",
3417                    "route bind is already pending for channel",
3418                    metrics,
3419                )
3420                .await;
3421            }
3422            let bind_root_id = match bind_root_id {
3423                Some(root_id) => root_id,
3424                None => match ProjectRootId::from_path(&identity.project_root) {
3425                    Ok(root_id) => root_id,
3426                    Err(error) => {
3427                        return send_route_bind_error(
3428                            tx,
3429                            frame,
3430                            "config_divergence",
3431                            &format!("invalid route project root: {error}"),
3432                            metrics,
3433                        )
3434                        .await;
3435                    }
3436                },
3437            };
3438
3439            // Reconcile RootConfig: build a configure request from the bind
3440            // identity + forwarded config tiers and run it through the executor.
3441            let request_id = format!("subc-bind-{route_channel}");
3442            let bind_project_root = identity.project_root.clone();
3443            let bind_harness = identity.harness.clone();
3444            let bind_session = identity.session.clone();
3445            let bind_trust = trust_for_bind(&bind_harness, &principal);
3446            let bind_principal_id = principal_id(&principal);
3447            // Typed capability declaration from the consumer: the facade stamps it
3448            // from the MCP host's initialize-advertised capabilities. Absent
3449            // means no reverse-request capability — flat deny, fail-closed. A
3450            // consumer over-declaring only earns asks that TTL-deny.
3451            let consumer_elicitation_capable = consumer_capabilities
3452                .as_ref()
3453                .is_some_and(|capabilities| capabilities.iter().any(|c| c == "elicitation"));
3454            log::info!(
3455                "subc attach: route {} harness={} principal={} trust={} elicitation={}",
3456                route_channel,
3457                bind_harness,
3458                principal_label(&principal),
3459                bind_trust.label(),
3460                consumer_elicitation_capable
3461            );
3462
3463            // Config is single-per-project, read by AFT directly from the
3464            // CortexKit config files (user: ~/.config/cortexkit/aft.jsonc,
3465            // project: <root>/.cortexkit/aft.jsonc). Wire-relayed config tiers are
3466            // IGNORED entirely: a front (runner, mcp:*, or fed:*) cannot push config over
3467            // the wire. This is what makes config harness-INDEPENDENT — every
3468            // harness binding a project gets the identical on-disk config, so two
3469            // trust domains sharing the per-root actor can never diverge or
3470            // inherit each other's capabilities (the cross-bind escalation class).
3471            // Wire-relayed config tiers (if the protocol still carries them) are
3472            // ignored entirely; the per-tier trust boundary (user trusted, project
3473            // privileged-dropped) is applied to the FILE tiers in handle_configure.
3474            let local_tiers = crate::subc_config::read_local_cortexkit_config_tiers(
3475                user_config_path,
3476                Path::new(&bind_project_root),
3477            );
3478            let config_tiers: Vec<Value> = local_tiers
3479                .iter()
3480                .map(|t| json!({ "tier": t.tier, "source": t.source, "doc": t.doc }))
3481                .collect();
3482            let diagnostics_on_edit = diagnostics_on_edit_from_tiers(&local_tiers);
3483            let configure_json = json!({
3484                "id": request_id,
3485                "command": "configure",
3486                "project_root": bind_project_root,
3487                "harness": bind_harness,
3488                "session_id": bind_session.clone(),
3489                "config": config_tiers,
3490            });
3491            let configure_req = match serde_json::from_value::<RawRequest>(configure_json) {
3492                Ok(req) => req,
3493                Err(error) => {
3494                    return send_route_bind_error(
3495                        tx,
3496                        frame,
3497                        "config_divergence",
3498                        &format!("failed to build configure request: {error}"),
3499                        metrics,
3500                    )
3501                    .await;
3502                }
3503            };
3504
3505            let route_identity = RouteIdentity(Arc::new(RouteIdentityData {
3506                root: bind_root_id.clone(),
3507                project_root: PathBuf::from(&bind_project_root),
3508                harness: bind_harness.clone(),
3509                session: bind_session.clone(),
3510                trust: bind_trust,
3511                spawn_principal: AuthenticatedPrincipal::RouteBind {
3512                    trust: bind_trust.sandbox_trust(),
3513                    route_channel,
3514                    route_epoch: epoch,
3515                    project_root: PathBuf::from(&bind_project_root),
3516                    harness: bind_harness.clone(),
3517                    session_id: bind_session.clone(),
3518                    principal_id: bind_principal_id,
3519                },
3520                consumer_elicitation_capable,
3521            }));
3522            let configure_session = route_identity.session.clone();
3523            let root_was_live = live_roots.contains_key(&bind_root_id);
3524            let inserted_new_actor = register_actor_for_bind(
3525                shared_app,
3526                executor,
3527                push_senders,
3528                &bind_root_id,
3529                route_channel,
3530                root_was_live,
3531            );
3532
3533            let configure_request_id = configure_req.id.clone();
3534            installed_route_epochs.insert(route_channel, epoch);
3535            if let Some(meta) = live_roots.get_mut(&bind_root_id) {
3536                meta.maintenance_queued_kinds.clear();
3537                meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
3538            }
3539            let (configure_rx, configure_cancellation) = executor.submit_cancellable_async(
3540                bind_root_id.clone(),
3541                Lane::Mutating,
3542                configure_request_id.clone(),
3543                Box::new(move |ctx| {
3544                    log_ctx::with_session(Some(configure_session.clone()), || {
3545                        dispatch(configure_req, ctx)
3546                    })
3547                }),
3548            );
3549            pending_binds.insert(
3550                route_id,
3551                PendingBind {
3552                    bind_root_id: bind_root_id.clone(),
3553                    inserted_new_actor,
3554                    cancelled: false,
3555                    configure_request_id: configure_request_id.clone(),
3556                    started_at: Instant::now(),
3557                    warned_half_deadline: false,
3558                    deadline_reported: false,
3559                    corr: frame.header.corr,
3560                    ver: frame.header.ver,
3561                    flags: frame.header.flags,
3562                    cancellation: configure_cancellation,
3563                },
3564            );
3565
3566            let completion_tx = control_completion_tx.clone();
3567            let completion_identity = route_identity;
3568            let completion_root = bind_root_id.clone();
3569            let completion_route_channel = route_channel;
3570            let completion_ver = frame.header.ver;
3571            let completion_corr = frame.header.corr;
3572            let completion_flags = frame.header.flags;
3573            let completion_metrics = Arc::clone(metrics);
3574            tokio::spawn(async move {
3575                let _response_task = ResponseTaskGuard::new(&completion_metrics);
3576                let configure_response =
3577                    await_executor_response(configure_rx, configure_request_id.clone()).await;
3578                // Send the route-bind acknowledgment as soon as configure succeeds.
3579                // Installing completed search or callgraph builds only refreshes cached
3580                // read data, so a later maintenance pass can do it without delaying the
3581                // daemon's confirmation that the route is usable.
3582                let completion = RouteBindCompletion {
3583                    route: route_key(completion_route_channel, epoch),
3584                    identity: completion_identity,
3585                    bind_root_id: completion_root,
3586                    inserted_new_actor,
3587                    configure_response,
3588                    diagnostics_on_edit,
3589                    ver: completion_ver,
3590                    corr: completion_corr,
3591                    flags: completion_flags,
3592                };
3593                if send_counted_channel(
3594                    &completion_tx,
3595                    &completion_metrics.control_completion_queued,
3596                    completion,
3597                )
3598                .await
3599                .is_err()
3600                {
3601                    log::debug!(
3602                        "subc attach: dropped RouteBind completion for route {} after loop exit",
3603                        completion_route_channel
3604                    );
3605                }
3606            });
3607
3608            Ok(())
3609        }
3610        ModuleControlRequest::HealthCheck {} => {
3611            let report = build_health_report(executor, pending_binds, metrics, shared_app);
3612            let body = serde_json::to_vec(&ModuleControlResponse::from(report))
3613                .map_err(SubcError::Json)?;
3614            let response = Frame::build_with_version(
3615                frame.header.ver,
3616                FrameType::Response,
3617                frame.header.flags,
3618                0,
3619                0,
3620                frame.header.corr,
3621                body,
3622            )
3623            .map_err(SubcError::FrameBuild)?;
3624            send_frame(tx, metrics, response).await
3625        }
3626    }
3627}
3628
3629fn install_bash_compressor(ctx: &AppContext) {
3630    // Mirrors main.rs per-actor compressor installation for subc-created actors.
3631    let filter_registry_handle = ctx.shared_filter_registry();
3632    let compress_flag = ctx.bash_compress_flag();
3633    ctx.bash_background().set_compressor_with_exit_code(
3634        move |command: &str, output: String, exit_code: Option<i32>| {
3635            if !compress_flag.load(std::sync::atomic::Ordering::Relaxed) {
3636                return crate::compress::CompressionResult::new(output);
3637            }
3638            let registry_guard = match filter_registry_handle.read() {
3639                Ok(g) => g,
3640                Err(poisoned) => poisoned.into_inner(),
3641            };
3642            crate::compress::compress_with_registry_exit_code(
3643                command,
3644                &output,
3645                exit_code,
3646                &registry_guard,
3647            )
3648        },
3649    );
3650}
3651
3652fn diagnostics_on_edit_from_tiers(tiers: &[ConfigTier]) -> bool {
3653    let mut diagnostics_on_edit = false;
3654    for tier in tiers {
3655        if let Some(value) = diagnostics_on_edit_from_doc(&tier.doc) {
3656            diagnostics_on_edit = value;
3657        }
3658    }
3659    diagnostics_on_edit
3660}
3661
3662fn diagnostics_on_edit_from_doc(doc: &str) -> Option<bool> {
3663    let stripped = strip_jsonc(doc);
3664    let value = serde_json::from_str::<Value>(&stripped).ok()?;
3665    value
3666        .get("lsp")
3667        .and_then(Value::as_object)?
3668        .get("diagnostics_on_edit")
3669        .and_then(Value::as_bool)
3670}
3671
3672async fn send_route_bind_error(
3673    tx: &WriterSender,
3674    frame: &Frame,
3675    code: &str,
3676    message: &str,
3677    metrics: &DispatchPathMetrics,
3678) -> Result<(), SubcError> {
3679    send_route_bind_error_parts(
3680        tx,
3681        frame.header.ver,
3682        frame.header.corr,
3683        frame.header.flags,
3684        code,
3685        message,
3686        metrics,
3687    )
3688    .await
3689}
3690
3691async fn send_route_bind_error_parts(
3692    tx: &WriterSender,
3693    ver: u8,
3694    corr: u64,
3695    flags: Flags,
3696    code: &str,
3697    message: &str,
3698    metrics: &DispatchPathMetrics,
3699) -> Result<(), SubcError> {
3700    let response = build_error_frame(ver, 0, 0, corr, flags, code, message)?;
3701    send_reliable_writer_frame(tx, metrics, response, "RouteBind error").await?;
3702    log::warn!("subc attach: route bind rejected ({code}): {message}");
3703    Ok(())
3704}
3705
3706/// Route-channel tool call: `{name, arguments}` → executor lane → dispatch to
3707/// the sync command core → wrap the structured Response in a CallToolResult
3708/// `{content, isError}`. Tool-result mapping: the whole `{success, ...}` Response
3709/// serialized into ONE text block; `isError` carries `success == false`.
3710async fn handle_tool_call(
3711    tx: &WriterSender,
3712    frame: &Frame,
3713    mut phase_trace: PhaseTrace,
3714    routes: &HashMap<RouteChannel, RouteIdentity>,
3715    pending_binds: &HashMap<RouteChannel, PendingBind>,
3716    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3717    executor: &Arc<Executor>,
3718    shutdown: &Arc<Notify>,
3719    connection_cancel: &PersistentCancelSignal,
3720    bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
3721    bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
3722    metrics: &Arc<DispatchPathMetrics>,
3723    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
3724    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
3725    next_bash_ask_corr: &mut u64,
3726    bg_subs: &mut HashMap<RouteChannel, BgSub>,
3727    bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
3728    bg_wake_pending: &mut HashSet<RouteChannel>,
3729    bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
3730    dispatch: DispatchFn,
3731    allow_native_passthrough: bool,
3732) -> Result<(), SubcError> {
3733    let route_id = route_key(frame.header.channel, frame.header.epoch);
3734    if pending_binds.contains_key(&route_id) {
3735        let error = build_error_frame(
3736            frame.header.ver,
3737            frame.header.channel,
3738            frame.header.epoch,
3739            frame.header.corr,
3740            frame.header.flags,
3741            "route_not_bound",
3742            "route is not bound before tool call",
3743        )?;
3744        return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
3745    }
3746
3747    let Some(identity) = routes.get(&route_id).cloned() else {
3748        let error = build_error_frame(
3749            frame.header.ver,
3750            frame.header.channel,
3751            frame.header.epoch,
3752            frame.header.corr,
3753            frame.header.flags,
3754            "route_not_bound",
3755            "route is not bound before tool call",
3756        )?;
3757        return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
3758    };
3759    let restore_watcher = live_roots
3760        .get(&identity.root)
3761        .is_some_and(|meta| meta.idle_artifacts_evicted);
3762    if let Some(meta) = live_roots.get_mut(&identity.root) {
3763        meta.reactivate_bound();
3764    }
3765    if restore_watcher {
3766        if let Some(ctx) = executor.actor_context(&identity.root) {
3767            crate::commands::configure::ensure_project_watcher(&ctx);
3768        }
3769    }
3770
3771    let route_request =
3772        serde_json::from_slice::<RouteRequest>(&frame.body).map_err(SubcError::Json)?;
3773    if matches!(
3774        route_request,
3775        RouteRequest::BgEvents(BgEventsRequest {
3776            op: BgEventsOp::BgEvents
3777        })
3778    ) {
3779        if let Some(old_sub) = bg_subs.get(&route_id).copied() {
3780            push::send_reliable_bg_stream_end(tx, metrics, route_id, &old_sub).await?;
3781        }
3782        if !identity.trust.allows_bash_observation() {
3783            bg_subs.remove(&route_id);
3784            bg_wake_pending.remove(&route_id);
3785            remove_bg_subscription_index(bg_sub_by_session, route_id, Some(&identity));
3786            let denied_sub = BgSub {
3787                corr: frame.header.corr,
3788                ver: frame.header.ver,
3789                flags: frame.header.flags,
3790            };
3791            push::send_reliable_bg_stream_end(tx, metrics, route_id, &denied_sub).await?;
3792            return Ok(());
3793        }
3794        bg_subs.insert(
3795            route_id,
3796            BgSub {
3797                corr: frame.header.corr,
3798                ver: frame.header.ver,
3799                flags: frame.header.flags,
3800            },
3801        );
3802        bg_sub_by_session.insert((identity.root.clone(), identity.session.clone()), route_id);
3803        push::arm_bg_wake(
3804            identity.root.clone(),
3805            identity.session.clone(),
3806            route_id,
3807            bg_wake_pending,
3808            bg_wake_epoch,
3809        );
3810        return Ok(());
3811    }
3812
3813    let RouteRequest::ToolCall(call) = route_request else {
3814        unreachable!("background event subscription returned above")
3815    };
3816    let bare_name = call.name;
3817    let arguments = strip_agent_preview_arg_owned(call.arguments);
3818    let format_context = crate::subc_format::FormatContext::from_tool_call(
3819        &bare_name,
3820        &arguments,
3821        identity.project_root.as_path(),
3822    );
3823
3824    let request_id = format!("subc-{}-{}", frame.header.channel, frame.header.corr);
3825    let bind_trust = identity.trust;
3826    let diagnostics_on_edit = live_roots
3827        .get(&identity.root)
3828        .map(|meta| meta.diagnostics_on_edit)
3829        .unwrap_or(false);
3830
3831    let requests_host = bare_name == "bash"
3832        && arguments
3833            .get("sandbox")
3834            .or_else(|| {
3835                arguments
3836                    .get("params")
3837                    .and_then(|params| params.get("sandbox"))
3838            })
3839            .and_then(Value::as_str)
3840            == Some("host");
3841    if matches!(bind_trust, BindTrust::Untrusted) && requests_host {
3842        let response = Response::error(
3843            request_id.clone(),
3844            "sandbox_escalation_denied",
3845            "sandbox host escalation is unavailable to untrusted principals",
3846        );
3847        let text = crate::subc_format::format_response_with_context(
3848            &bare_name,
3849            &response,
3850            &format_context,
3851        );
3852        let result = ToolCallResult { text, response };
3853        let response_frame = build_tool_response_frame(
3854            frame.header.ver,
3855            route_id,
3856            frame.header.corr,
3857            frame.header.flags,
3858            &result,
3859            bind_trust,
3860        )?;
3861        return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
3862    }
3863
3864    if matches!(bind_trust, BindTrust::Untrusted)
3865        && is_bash_family_tool(&bare_name)
3866        && (bare_name != "bash" || !identity.consumer_elicitation_capable)
3867    {
3868        let response = bash::bash_denied_untrusted_response(request_id.clone());
3869        let text = crate::subc_format::format_response_with_context(
3870            &bare_name,
3871            &response,
3872            &format_context,
3873        );
3874        let result = ToolCallResult { text, response };
3875        let response_frame = build_tool_response_frame(
3876            frame.header.ver,
3877            route_id,
3878            frame.header.corr,
3879            frame.header.flags,
3880            &result,
3881            bind_trust,
3882        )?;
3883        return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
3884    }
3885
3886    // A non-core name is NOT in the tool manifest. AFT fails closed and
3887    // does not trust subc to enforce the manifest: rejecting here is the
3888    // defense-in-depth backstop that prevents a forwarded native command
3889    // (e.g. `configure`, which would reach handle_configure and bypass
3890    // the RouteBind config-trust cap) from ever reaching dispatch. Only
3891    // the integration-test harness (run_subc_mode_for_test) opens this to
3892    // drive synthetic native commands through the executor.
3893    if !is_subc_agent_core_tool(&bare_name)
3894        && !is_subc_native_plumbing_tool(&bare_name)
3895        && !allow_native_passthrough
3896    {
3897        log::warn!(
3898            "subc tool call: rejecting non-manifest tool name {:?} on route {} (fail-closed)",
3899            bare_name,
3900            frame.header.channel
3901        );
3902        let response = Response::error(
3903            request_id.clone(),
3904            "unknown_tool",
3905            format!("tool {:?} is not in the AFT tool manifest", bare_name),
3906        );
3907        let text = crate::subc_format::format_response_with_context(
3908            &bare_name,
3909            &response,
3910            &format_context,
3911        );
3912        let result = ToolCallResult { text, response };
3913        let response_frame = build_tool_response_frame(
3914            frame.header.ver,
3915            route_id,
3916            frame.header.corr,
3917            frame.header.flags,
3918            &result,
3919            bind_trust,
3920        )?;
3921        return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
3922    }
3923
3924    if bare_name == "bash" {
3925        if matches!(bind_trust, BindTrust::Untrusted) {
3926            let plan = match bash::prepare_bash_elicitation_plan(
3927                &arguments,
3928                identity.project_root.as_path(),
3929            ) {
3930                Ok(plan) => plan,
3931                Err(error) => {
3932                    let response = Response::error(request_id.clone(), error.code, error.message);
3933                    let text = crate::subc_format::format_response_with_context(
3934                        &bare_name,
3935                        &response,
3936                        &format_context,
3937                    );
3938                    let result = ToolCallResult { text, response };
3939                    let response_frame = build_tool_response_frame(
3940                        frame.header.ver,
3941                        route_id,
3942                        frame.header.corr,
3943                        frame.header.flags,
3944                        &result,
3945                        bind_trust,
3946                    )?;
3947                    return send_reliable_writer_frame(
3948                        tx,
3949                        metrics,
3950                        response_frame,
3951                        "tool response",
3952                    )
3953                    .await;
3954                }
3955            };
3956
3957            let reverse_corr =
3958                allocate_reverse_corr(pending_bash_asks, route_id, next_bash_ask_corr);
3959            let ask_frame = build_bash_elicitation_request_frame(
3960                frame.header.ver,
3961                route_id,
3962                reverse_corr,
3963                frame.header.flags,
3964                &plan.command,
3965                &plan.asks,
3966            )?;
3967
3968            let meta = live_roots
3969                .entry(identity.root.clone())
3970                .or_insert_with(|| RootMeta::new(Instant::now()));
3971            meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
3972            meta.reactivate_bound();
3973
3974            let route_cancel =
3975                route_bash_cancels
3976                    .entry(route_id)
3977                    .or_insert_with(|| bash::RouteBashCancel {
3978                        token: PersistentCancelSignal::new(),
3979                        active_waits: 0,
3980                    });
3981            route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
3982            let cancel = bash::BashWaitCancel {
3983                connection: connection_cancel.clone(),
3984                route: route_cancel.token.clone(),
3985            };
3986            pending_bash_asks.insert(
3987                ReverseCorrKey {
3988                    route: route_id,
3989                    corr: reverse_corr,
3990                },
3991                PendingBashAsk {
3992                    route: route_id,
3993                    tool_corr: frame.header.corr,
3994                    tool_flags: frame.header.flags,
3995                    tool_ver: frame.header.ver,
3996                    root: identity.root.clone(),
3997                    project_root: identity.project_root.clone(),
3998                    session_id: identity.session.clone(),
3999                    spawn_principal: identity.spawn_principal.clone(),
4000                    request_id,
4001                    arguments,
4002                    format_context,
4003                    cancel,
4004                    grants: plan.grants,
4005                    expires_at: Instant::now() + bash_elicitation_timeout(),
4006                },
4007            );
4008            return send_reliable_writer_frame(tx, metrics, ask_frame, "bash elicitation request")
4009                .await;
4010        }
4011
4012        let meta = live_roots
4013            .entry(identity.root.clone())
4014            .or_insert_with(|| RootMeta::new(Instant::now()));
4015        meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
4016        meta.reactivate_bound();
4017
4018        let route_cancel =
4019            route_bash_cancels
4020                .entry(route_id)
4021                .or_insert_with(|| bash::RouteBashCancel {
4022                    token: PersistentCancelSignal::new(),
4023                    active_waits: 0,
4024                });
4025        route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
4026        let cancel = bash::BashWaitCancel {
4027            connection: connection_cancel.clone(),
4028            route: route_cancel.token.clone(),
4029        };
4030
4031        bash::submit_deferred_bash(
4032            executor,
4033            bash_deferred_tx,
4034            bash_poll_touch_tx,
4035            metrics,
4036            dispatch,
4037            identity.root.clone(),
4038            identity.project_root.clone(),
4039            identity.session.clone(),
4040            request_id,
4041            route_id,
4042            frame.header.corr,
4043            frame.header.flags,
4044            frame.header.ver,
4045            arguments,
4046            format_context,
4047            cancel,
4048            bind_trust,
4049            identity.spawn_principal.clone(),
4050            None,
4051        );
4052        return Ok(());
4053    }
4054
4055    let lane = command_lane(&bare_name);
4056    let tool_call_context = ToolCallContext {
4057        project_root: identity.project_root.clone(),
4058        session_id: Some(identity.session.clone()),
4059        request_id: request_id.clone(),
4060        diagnostics_on_edit,
4061        preview: call.preview,
4062    };
4063    let bare_name_for_frame = bare_name.clone();
4064    let identity_for_run = identity.clone();
4065    let completion_session = identity.session.clone();
4066    let completion_root = identity.project_root.clone();
4067    let request_id_for_force = request_id.clone();
4068    let format_context_for_frame = format_context.clone();
4069    let (tool_call_tx, tool_call_rx) = oneshot::channel::<ToolCallCompletion>();
4070    phase_trace.mark_executor_submitted();
4071    let rx = executor.submit_async(
4072        identity.root.clone(),
4073        lane,
4074        request_id.clone(),
4075        Box::new(move |ctx| {
4076            phase_trace.mark_job_admitted();
4077            log_ctx::with_session(Some(identity_for_run.session.clone()), || {
4078                let run = || {
4079                    let finalizer = |response: &mut Response| {
4080                        crate::response_finalize::finalize_response_with_bg_completions(
4081                            response,
4082                            ctx,
4083                            &identity_for_run.session,
4084                            &bare_name,
4085                            bind_trust.allows_bash_observation(),
4086                        );
4087                    };
4088                    match run_tool_call(
4089                        &bare_name,
4090                        arguments,
4091                        &format_context,
4092                        &tool_call_context,
4093                        ctx,
4094                        &dispatch,
4095                        Some(&finalizer),
4096                        Some(&mut phase_trace),
4097                    ) {
4098                        ToolCallOutcome::Unary(result) => {
4099                            let response = result.response;
4100                            let _ = tool_call_tx.send(ToolCallCompletion {
4101                                text: result.text,
4102                                phase_trace,
4103                            });
4104                            response
4105                        }
4106                    }
4107                };
4108                if matches!(bind_trust, BindTrust::Untrusted) {
4109                    ctx.with_force_restrict(&request_id_for_force, run)
4110                } else {
4111                    run()
4112                }
4113            })
4114        }),
4115    );
4116    let completion_tx = tx.clone();
4117    let completion_shutdown = Arc::clone(shutdown);
4118    let route = route_id;
4119    let corr = frame.header.corr;
4120    let flags = frame.header.flags;
4121    let ver = frame.header.ver;
4122    let completion_metrics = Arc::clone(metrics);
4123    tokio::spawn(async move {
4124        let _response_task = ResponseTaskGuard::new(&completion_metrics);
4125        let response = await_executor_response(rx, request_id.clone()).await;
4126        let (text, phase_trace) = match tool_call_rx.await {
4127            Ok(completion) => (completion.text, Some(completion.phase_trace)),
4128            Err(_) => (
4129                crate::subc_format::format_response_with_context(
4130                    &bare_name_for_frame,
4131                    &response,
4132                    &format_context_for_frame,
4133                ),
4134                None,
4135            ),
4136        };
4137        let result = ToolCallResult { text, response };
4138        let fatal = response_is_fatal_panic(&result.response);
4139        match build_tool_response_frame(ver, route, corr, flags, &result, bind_trust) {
4140            Ok(response_frame) => {
4141                let send_result = if let Some(phase_trace) = phase_trace {
4142                    let trace = ToolResponseWriteTrace::new(
4143                        phase_trace,
4144                        bare_name_for_frame,
4145                        completion_root,
4146                        completion_session,
4147                        route.channel,
4148                        corr,
4149                    );
4150                    send_traced_tool_response_frame(
4151                        &completion_tx,
4152                        &completion_metrics,
4153                        response_frame,
4154                        trace,
4155                    )
4156                    .await
4157                } else {
4158                    send_reliable_writer_frame(
4159                        &completion_tx,
4160                        &completion_metrics,
4161                        response_frame,
4162                        "tool response",
4163                    )
4164                    .await
4165                };
4166                if let Err(error) = send_result {
4167                    log::warn!("subc attach: failed to queue tool response frame: {error}");
4168                }
4169            }
4170            Err(error) => {
4171                log::error!("subc attach: failed to build tool response frame: {error}");
4172            }
4173        }
4174        if fatal {
4175            signal_fatal_teardown(
4176                &completion_tx,
4177                Some(route),
4178                ver,
4179                corr,
4180                &completion_shutdown,
4181                &completion_metrics,
4182            )
4183            .await;
4184        }
4185    });
4186    Ok(())
4187}
4188
4189fn submit_maintenance_job(
4190    executor: &Arc<Executor>,
4191    root_id: ProjectRootId,
4192    kind: MaintenanceDrainKind,
4193    bg_sessions_to_check: Vec<(String, u64)>,
4194    completion_tx: &mpsc::Sender<MaintenanceCompletion>,
4195    metrics: &Arc<DispatchPathMetrics>,
4196) {
4197    let request_id = format!(
4198        "subc-maintenance-drain-{}-{}",
4199        kind.label(),
4200        root_id.as_path().to_string_lossy()
4201    );
4202    let response_id = request_id.clone();
4203    let completion_root_id = root_id.clone();
4204    let maintenance_generation = executor
4205        .actor_context(&root_id)
4206        .map(|ctx| ctx.configure_generation())
4207        .unwrap_or(0);
4208    let (outcome_tx, outcome_rx) = oneshot::channel::<MaintenanceJobOutcome>();
4209    // ConfigureTail runs deferred configure mutations and needs the actor
4210    // epoch write gate. The other drain kinds only mutate subsystem state
4211    // behind that subsystem's own lock, so they run on MaintenanceCommit and
4212    // overlap interactive reads instead of excluding them.
4213    let lane = match kind {
4214        MaintenanceDrainKind::ConfigureTail => Lane::Mutating,
4215        MaintenanceDrainKind::Watcher
4216        | MaintenanceDrainKind::Lsp
4217        | MaintenanceDrainKind::CompletionDrains => Lane::MaintenanceCommit,
4218    };
4219    let rx = executor.submit_maintenance_async(
4220        root_id,
4221        lane,
4222        request_id.clone(),
4223        Box::new(move |ctx| {
4224            let outcome = match kind {
4225                MaintenanceDrainKind::Watcher => {
4226                    let drained = runtime_drain::drain_watcher_events_bounded(
4227                        ctx,
4228                        runtime_drain::WATCHER_PATH_DRAIN_BATCH_CAP,
4229                    );
4230                    MaintenanceJobOutcome {
4231                        empty_bg_sessions: Vec::new(),
4232                        requeue_kind: drained.has_more.then_some(kind),
4233                    }
4234                }
4235                MaintenanceDrainKind::Lsp => {
4236                    let drained = runtime_drain::drain_lsp_events_bounded(
4237                        ctx,
4238                        runtime_drain::LSP_EVENT_DRAIN_BATCH_CAP,
4239                    );
4240                    MaintenanceJobOutcome {
4241                        empty_bg_sessions: Vec::new(),
4242                        requeue_kind: drained.has_more.then_some(kind),
4243                    }
4244                }
4245                MaintenanceDrainKind::ConfigureTail => {
4246                    runtime_drain::drain_deferred_configure_maintenance(ctx);
4247                    runtime_drain::drain_configure_warning_events(ctx);
4248                    MaintenanceJobOutcome::default()
4249                }
4250                MaintenanceDrainKind::CompletionDrains => {
4251                    runtime_drain::drain_search_index_events(ctx);
4252                    runtime_drain::drain_callgraph_store_events(ctx);
4253                    runtime_drain::drain_semantic_index_events(ctx);
4254                    runtime_drain::drain_semantic_refresh_events(ctx);
4255                    runtime_drain::drain_inspect_events_for_generation(ctx, maintenance_generation);
4256                    let empty_bg_sessions = bg_sessions_to_check
4257                        .into_iter()
4258                        .filter(|(session, _)| {
4259                            !ctx.bash_background()
4260                                .has_completions_for_session(Some(session.as_str()))
4261                        })
4262                        .collect();
4263                    MaintenanceJobOutcome {
4264                        empty_bg_sessions,
4265                        requeue_kind: None,
4266                    }
4267                }
4268            };
4269            let requeued = outcome.requeue_kind.is_some();
4270            let _ = outcome_tx.send(outcome);
4271            Response::success(
4272                response_id,
4273                json!({ "drained": true, "kind": kind.label(), "requeued": requeued }),
4274            )
4275        }),
4276    );
4277    let completion_tx = completion_tx.clone();
4278    let completion_metrics = Arc::clone(metrics);
4279    tokio::spawn(async move {
4280        let _response_task = ResponseTaskGuard::new(&completion_metrics);
4281        let response = await_executor_response(rx, request_id).await;
4282        let outcome = outcome_rx.await.unwrap_or_default();
4283        let _ = send_counted_channel(
4284            &completion_tx,
4285            &completion_metrics.maintenance_queued,
4286            MaintenanceCompletion {
4287                root_id: completion_root_id,
4288                kind,
4289                response,
4290                empty_bg_sessions: outcome.empty_bg_sessions,
4291                requeue_kind: outcome.requeue_kind,
4292            },
4293        )
4294        .await;
4295    });
4296}
4297
4298async fn await_executor_response(rx: oneshot::Receiver<Response>, request_id: String) -> Response {
4299    rx.await
4300        .unwrap_or_else(|_| Response::error(request_id, "internal_error", "executor dropped"))
4301}
4302async fn signal_fatal_teardown(
4303    tx: &WriterSender,
4304    route: Option<RouteChannel>,
4305    ver: u8,
4306    corr: u64,
4307    shutdown: &Arc<Notify>,
4308    metrics: &DispatchPathMetrics,
4309) {
4310    if let Some(route) = route {
4311        if let Ok(frame) = build_goodbye_frame(ver, route.channel, route.epoch, corr) {
4312            if let Err(error) = send_frame(tx, metrics, frame).await {
4313                log::warn!(
4314                    "subc attach: failed to queue fatal route Goodbye for route {route}: {error}"
4315                );
4316            }
4317        }
4318    }
4319    if let Ok(frame) = build_goodbye_frame(ver, 0, 0, 0) {
4320        if let Err(error) = send_frame(tx, metrics, frame).await {
4321            log::warn!("subc attach: failed to queue fatal channel-0 Goodbye: {error}");
4322        }
4323    }
4324    shutdown.notify_one();
4325}
4326#[derive(Debug, Deserialize)]
4327#[serde(untagged)]
4328enum RouteRequest {
4329    BgEvents(BgEventsRequest),
4330    ToolCall(ToolCallRequest),
4331}
4332
4333#[derive(Debug, Deserialize)]
4334struct BgEventsRequest {
4335    op: BgEventsOp,
4336}
4337
4338#[derive(Debug, Deserialize)]
4339#[serde(rename_all = "snake_case")]
4340enum BgEventsOp {
4341    BgEvents,
4342}
4343
4344#[derive(Debug, Deserialize)]
4345struct ToolCallRequest {
4346    name: String,
4347    #[serde(default)]
4348    arguments: Value,
4349    /// Server-owned preview control (B1c-0): the plugin's mutation flow is
4350    /// preview -> permission ask -> apply. Dropping this field made "preview"
4351    /// calls mutate disk before the permission prompt and the subsequent
4352    /// apply fail with not-found.
4353    #[serde(default)]
4354    preview: bool,
4355}
4356
4357#[cfg(test)]
4358pub(crate) mod test_support {
4359    use super::*;
4360    use crate::bash_background::BgTaskStatus;
4361    use crate::protocol::{
4362        BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, ConfigureWarningsFrame,
4363        ProgressFrame, StatusChangedFrame,
4364    };
4365    use serde_json::json;
4366
4367    pub(super) fn test_root(name: &str) -> (tempfile::TempDir, ProjectRootId) {
4368        let dir = tempfile::Builder::new()
4369            .prefix(name)
4370            .tempdir()
4371            .expect("temp root");
4372        let root = ProjectRootId::from_path(dir.path()).expect("project root id");
4373        (dir, root)
4374    }
4375
4376    pub(super) fn test_ctx() -> Arc<AppContext> {
4377        Arc::new(AppContext::new(
4378            Box::new(crate::parser::TreeSitterProvider::new()),
4379            crate::config::Config::default(),
4380        ))
4381    }
4382
4383    pub(super) fn wait_for_watcher_count(ctx: &AppContext, expected: usize) {
4384        let deadline = Instant::now() + Duration::from_secs(30);
4385        loop {
4386            let observed = ctx.watcher_registry_count();
4387            if observed == expected {
4388                return;
4389            }
4390            assert!(
4391                Instant::now() < deadline,
4392                "watcher count did not settle before deadline: expected={expected}, observed={observed}"
4393            );
4394            std::thread::sleep(Duration::from_millis(50));
4395        }
4396    }
4397
4398    pub(super) fn wait_for_actor_root_count(app: &App, expected: usize) {
4399        let deadline = Instant::now() + Duration::from_secs(30);
4400        loop {
4401            let observed = app.actor_root_count();
4402            if observed == expected {
4403                return;
4404            }
4405            assert!(
4406                Instant::now() < deadline,
4407                "actor root count did not settle before deadline: expected={expected}, observed={observed}"
4408            );
4409            std::thread::sleep(Duration::from_millis(50));
4410        }
4411    }
4412
4413    pub(super) fn status_frame(seq: u64) -> PushFrame {
4414        status_frame_with_session(seq, None)
4415    }
4416
4417    pub(super) fn status_frame_with_session(seq: u64, session_id: Option<&str>) -> PushFrame {
4418        PushFrame::StatusChanged(StatusChangedFrame {
4419            frame_type: "status_changed",
4420            session_id: session_id.map(str::to_string),
4421            snapshot: json!({ "seq": seq }),
4422        })
4423    }
4424
4425    pub(super) fn completion_frame(task_id: &str) -> PushFrame {
4426        completion_frame_with_session(task_id, "session-1")
4427    }
4428
4429    pub(super) fn completion_frame_with_session(task_id: &str, session_id: &str) -> PushFrame {
4430        PushFrame::BashCompleted(BashCompletedFrame {
4431            frame_type: "bash_completed",
4432            task_id: task_id.to_string(),
4433            session_id: session_id.to_string(),
4434            status: BgTaskStatus::Completed,
4435            exit_code: Some(0),
4436            command: format!("echo {task_id}"),
4437            output_preview: String::new(),
4438            output_truncated: false,
4439            original_tokens: None,
4440            compressed_tokens: None,
4441            tokens_skipped: false,
4442        })
4443    }
4444
4445    pub(super) fn long_running_frame(task_id: &str, elapsed_ms: u64) -> PushFrame {
4446        long_running_frame_with_session(task_id, "session-1", elapsed_ms)
4447    }
4448
4449    pub(super) fn long_running_frame_with_session(
4450        task_id: &str,
4451        session_id: &str,
4452        elapsed_ms: u64,
4453    ) -> PushFrame {
4454        PushFrame::BashLongRunning(BashLongRunningFrame {
4455            frame_type: "bash_long_running",
4456            task_id: task_id.to_string(),
4457            session_id: session_id.to_string(),
4458            command: format!("sleep {elapsed_ms}"),
4459            elapsed_ms,
4460        })
4461    }
4462
4463    pub(super) fn pattern_match_frame(session_id: &str) -> PushFrame {
4464        PushFrame::BashPatternMatch(BashPatternMatchFrame {
4465            frame_type: "bash_pattern_match",
4466            task_id: "task-pattern".to_string(),
4467            session_id: session_id.to_string(),
4468            watch_id: "watch-1".to_string(),
4469            match_text: "needle".to_string(),
4470            match_offset: 7,
4471            context: "haystack needle".to_string(),
4472            once: true,
4473            reason: "pattern_match",
4474        })
4475    }
4476
4477    pub(super) fn configure_warnings_frame(session_id: Option<&str>) -> PushFrame {
4478        PushFrame::ConfigureWarnings(ConfigureWarningsFrame {
4479            frame_type: "configure_warnings",
4480            session_id: session_id.map(str::to_string),
4481            project_root: "/tmp/subc-test".to_string(),
4482            warnings: Vec::new(),
4483        })
4484    }
4485
4486    pub(super) fn route_identity(root: &ProjectRootId, session_id: &str) -> RouteIdentity {
4487        route_identity_with_trust(root, session_id, BindTrust::FirstParty)
4488    }
4489
4490    pub(super) fn route_identity_with_trust(
4491        root: &ProjectRootId,
4492        session_id: &str,
4493        trust: BindTrust,
4494    ) -> RouteIdentity {
4495        RouteIdentity(Arc::new(RouteIdentityData {
4496            root: root.clone(),
4497            project_root: root.as_path().to_path_buf(),
4498            harness: "opencode".to_string(),
4499            session: session_id.to_string(),
4500            trust,
4501            spawn_principal: AuthenticatedPrincipal::RouteBind {
4502                trust: trust.sandbox_trust(),
4503                route_channel: 0,
4504                route_epoch: 0,
4505                project_root: root.as_path().to_path_buf(),
4506                harness: "opencode".to_string(),
4507                session_id: session_id.to_string(),
4508                principal_id: Some(match trust {
4509                    BindTrust::FirstParty => "direct".to_string(),
4510                    BindTrust::Untrusted => "unverified".to_string(),
4511                }),
4512            },
4513            consumer_elicitation_capable: false,
4514        }))
4515    }
4516
4517    pub(super) fn progress_frame(request_id: &str, kind: ProgressKind, chunk: &str) -> PushFrame {
4518        PushFrame::Progress(ProgressFrame::new(request_id, kind, chunk))
4519    }
4520
4521    pub(super) fn status_seq(frame: &PushFrame) -> Option<u64> {
4522        match frame {
4523            PushFrame::StatusChanged(status) => status.snapshot.get("seq").and_then(|v| v.as_u64()),
4524            _ => None,
4525        }
4526    }
4527
4528    pub(super) fn completion_task(frame: &PushFrame) -> Option<&str> {
4529        match frame {
4530            PushFrame::BashCompleted(completion) => Some(completion.task_id.as_str()),
4531            _ => None,
4532        }
4533    }
4534
4535    pub(super) fn push_frame_task_id(frame: &Frame) -> Option<String> {
4536        let body: serde_json::Value = serde_json::from_slice(&frame.body).expect("push body");
4537        body.get("task_id")
4538            .and_then(serde_json::Value::as_str)
4539            .map(str::to_string)
4540    }
4541}
4542
4543#[cfg(test)]
4544mod tests {
4545    use super::test_support::{
4546        completion_frame, route_identity, test_ctx, test_root, wait_for_actor_root_count,
4547        wait_for_watcher_count,
4548    };
4549    use super::*;
4550
4551    fn attach_error(kind: io::ErrorKind) -> SubcError {
4552        SubcError::Connect {
4553            endpoint: "127.0.0.1:1".to_string(),
4554            source: io::Error::new(kind, "constructed attach failure"),
4555        }
4556    }
4557
4558    fn auth_io_error(kind: io::ErrorKind) -> SubcError {
4559        SubcError::Auth {
4560            endpoint: "127.0.0.1:1".to_string(),
4561            source: subc_transport::AuthError::Io {
4562                stage: subc_transport::AuthStage::ServerProof,
4563                source: io::Error::new(kind, "constructed auth failure"),
4564            },
4565        }
4566    }
4567
4568    #[test]
4569    fn initial_attach_error_classifier_distinguishes_transient_and_permanent_failures() {
4570        let transient_errors = vec![
4571            attach_error(io::ErrorKind::ConnectionRefused),
4572            attach_error(io::ErrorKind::TimedOut),
4573            attach_error(io::ErrorKind::ConnectionReset),
4574            auth_io_error(io::ErrorKind::ConnectionAborted),
4575            auth_io_error(io::ErrorKind::BrokenPipe),
4576            SubcError::Auth {
4577                endpoint: "127.0.0.1:1".to_string(),
4578                source: subc_transport::AuthError::UnexpectedEof {
4579                    stage: subc_transport::AuthStage::ServerProof,
4580                    expected: 4,
4581                    actual: 0,
4582                },
4583            },
4584            SubcError::Auth {
4585                endpoint: "127.0.0.1:1".to_string(),
4586                source: subc_transport::AuthError::Timeout {
4587                    stage: subc_transport::AuthStage::ServerProof,
4588                    deadline: AUTH_DEADLINE,
4589                },
4590            },
4591        ];
4592        for error in &transient_errors {
4593            assert_eq!(
4594                classify_attach_error(error),
4595                AttachErrorClass::Transient,
4596                "expected transient: {error}"
4597            );
4598        }
4599
4600        let permanent_errors = vec![
4601            attach_error(io::ErrorKind::PermissionDenied),
4602            auth_io_error(io::ErrorKind::InvalidData),
4603            SubcError::Auth {
4604                endpoint: "127.0.0.1:1".to_string(),
4605                source: subc_transport::AuthError::InvalidServerProof,
4606            },
4607            SubcError::Auth {
4608                endpoint: "127.0.0.1:1".to_string(),
4609                source: subc_transport::AuthError::DaemonIdMismatch,
4610            },
4611            SubcError::ConnectionFile {
4612                path: PathBuf::from("subc-connection.json"),
4613                source: subc_transport::ConnectionFileError::Invalid {
4614                    reason: "constructed invalid file".to_string(),
4615                },
4616            },
4617            SubcError::NoEndpoint {
4618                path: PathBuf::from("subc-connection.json"),
4619            },
4620            SubcError::InvalidEndpoint {
4621                path: PathBuf::from("subc-connection.json"),
4622                endpoint: "not-an-ip:1234".to_string(),
4623            },
4624        ];
4625        for error in &permanent_errors {
4626            assert_eq!(
4627                classify_attach_error(error),
4628                AttachErrorClass::Permanent,
4629                "expected permanent: {error}"
4630            );
4631        }
4632    }
4633
4634    #[test]
4635    fn incompatible_wire_version_is_rejected_before_tcp_connect() {
4636        let conn_dir = tempfile::tempdir().expect("connection tempdir");
4637        let conn_path = conn_dir.path().join("subc-connection.json");
4638        let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind listener");
4639        listener
4640            .set_nonblocking(true)
4641            .expect("set listener nonblocking");
4642        let port = listener.local_addr().expect("listener addr").port();
4643        connection_file::write_atomic(
4644            &conn_path,
4645            &connection_file::ConnectionInfo {
4646                schema: connection_file::SCHEMA_VERSION,
4647                wire_version: Some(PROTOCOL_VERSION.wrapping_add(1)),
4648                endpoints: vec![connection_file::Endpoint {
4649                    host: "127.0.0.1".to_string(),
4650                    port,
4651                }],
4652                key: vec![0x42; subc_transport::KEY_LEN],
4653                daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
4654                pid: std::process::id(),
4655                daemon_ver: "subc-test".to_string(),
4656            },
4657        )
4658        .expect("write connection file");
4659
4660        let runtime = tokio::runtime::Builder::new_current_thread()
4661            .enable_all()
4662            .build()
4663            .expect("test runtime");
4664        let result = runtime.block_on(connect_and_authenticate_with_policy(
4665            &conn_path,
4666            AttachRetryPolicy {
4667                budget: Duration::from_secs(1),
4668                initial_backoff: Duration::from_millis(5),
4669                max_backoff: Duration::from_millis(10),
4670                jitter_percent: 0,
4671            },
4672        ));
4673        assert!(matches!(
4674            result,
4675            Err(SubcError::ConnectionFile {
4676                source: connection_file::ConnectionFileError::WireVersionMismatch { .. },
4677                ..
4678            })
4679        ));
4680        assert!(matches!(
4681            listener.accept(),
4682            Err(error) if error.kind() == std::io::ErrorKind::WouldBlock
4683        ));
4684    }
4685
4686    #[test]
4687    fn initial_attach_unreachable_endpoint_retries_until_budget_then_fails_loud() {
4688        let conn_dir = tempfile::tempdir().expect("connection tempdir");
4689        let conn_path = conn_dir.path().join("subc-connection.json");
4690        let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve port");
4691        let port = listener.local_addr().expect("reserved addr").port();
4692        drop(listener);
4693        connection_file::write_atomic(
4694            &conn_path,
4695            &connection_file::ConnectionInfo {
4696                schema: connection_file::SCHEMA_VERSION,
4697                wire_version: Some(PROTOCOL_VERSION),
4698                endpoints: vec![connection_file::Endpoint {
4699                    host: "127.0.0.1".to_string(),
4700                    port,
4701                }],
4702                key: vec![0x42; subc_transport::KEY_LEN],
4703                daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
4704                pid: std::process::id(),
4705                daemon_ver: "subc-test".to_string(),
4706            },
4707        )
4708        .expect("write connection file");
4709
4710        let policy = AttachRetryPolicy {
4711            budget: Duration::from_millis(40),
4712            initial_backoff: Duration::from_millis(5),
4713            max_backoff: Duration::from_millis(10),
4714            jitter_percent: 0,
4715        };
4716        let runtime = tokio::runtime::Builder::new_current_thread()
4717            .enable_all()
4718            .build()
4719            .expect("test runtime");
4720        let started_at = Instant::now();
4721        let result = runtime.block_on(connect_and_authenticate_with_policy(&conn_path, policy));
4722        let elapsed = started_at.elapsed();
4723        let error = match result {
4724            Ok(_) => panic!("unreachable endpoint unexpectedly attached"),
4725            Err(error) => error,
4726        };
4727
4728        assert!(matches!(error, SubcError::Connect { .. }), "{error}");
4729        assert!(
4730            elapsed >= Duration::from_millis(35),
4731            "retry budget ended too early: {elapsed:?}"
4732        );
4733        assert!(
4734            elapsed < Duration::from_secs(1),
4735            "retry budget was not bounded: {elapsed:?}"
4736        );
4737    }
4738
4739    fn due_maintenance_jobs_without_actor_context(
4740        live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4741        budget: usize,
4742        pending_bind_roots: &HashSet<ProjectRootId>,
4743    ) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
4744        due_maintenance_jobs(
4745            live_roots,
4746            None,
4747            &HashMap::new(),
4748            &HashSet::new(),
4749            budget,
4750            pending_bind_roots,
4751        )
4752    }
4753
4754    fn actor_ctx_with_dirty_search_index(
4755        root: &Path,
4756        storage: &Path,
4757        file_name: &str,
4758        old_contents: &str,
4759        new_contents: &str,
4760    ) -> (Arc<AppContext>, PathBuf, PathBuf) {
4761        let file = root.join(file_name);
4762        std::fs::write(&file, old_contents).expect("write source");
4763        let canonical_root = std::fs::canonicalize(root).expect("canonical root");
4764        let ctx = Arc::new(AppContext::new(
4765            Box::new(crate::parser::TreeSitterProvider::new()),
4766            Config {
4767                project_root: Some(root.to_path_buf()),
4768                storage_dir: Some(storage.to_path_buf()),
4769                ..Config::default()
4770            },
4771        ));
4772        ctx.set_canonical_cache_root(canonical_root.clone());
4773
4774        let cache_dir = crate::search_index::resolve_cache_dir(&canonical_root, Some(storage));
4775        let mut index = crate::search_index::SearchIndex::build(&canonical_root);
4776        let git_head = index.stored_git_head().map(str::to_owned);
4777        index.write_to_disk(&cache_dir, git_head.as_deref());
4778
4779        std::fs::write(&file, new_contents).expect("edit source");
4780        index.update_file(&file);
4781        *ctx.search_index()
4782            .write()
4783            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
4784        (ctx, canonical_root, cache_dir)
4785    }
4786
4787    #[test]
4788    fn graceful_shutdown_flushes_every_actor_search_index() {
4789        let storage = tempfile::tempdir().expect("storage tempdir");
4790        let (root1_dir, root1) = test_root("shutdown-flush-root-1");
4791        let (root2_dir, root2) = test_root("shutdown-flush-root-2");
4792        let (ctx1, canonical_root1, cache_dir1) = actor_ctx_with_dirty_search_index(
4793            root1_dir.path(),
4794            storage.path(),
4795            "alpha.txt",
4796            "old actor one token\n",
4797            "new actor one token\n",
4798        );
4799        let (ctx2, canonical_root2, cache_dir2) = actor_ctx_with_dirty_search_index(
4800            root2_dir.path(),
4801            storage.path(),
4802            "beta.txt",
4803            "old actor two token\n",
4804            "new actor two token\n",
4805        );
4806
4807        let executor = Executor::new();
4808        assert!(executor.register_actor(root1.clone(), Arc::clone(&ctx1)));
4809        assert!(executor.register_actor(root2.clone(), Arc::clone(&ctx2)));
4810
4811        flush_actor_indexes_on_graceful_shutdown(&executor.actor_contexts());
4812
4813        let mut restored1 =
4814            crate::search_index::SearchIndex::read_from_disk(&cache_dir1, &canonical_root1)
4815                .expect("load flushed root one index");
4816        restored1.ready = true;
4817        assert_eq!(
4818            restored1
4819                .grep("new actor one token", true, &[], &[], &canonical_root1, 10)
4820                .matches
4821                .len(),
4822            1,
4823            "graceful subc shutdown should flush the first root's trigram delta"
4824        );
4825
4826        let mut restored2 =
4827            crate::search_index::SearchIndex::read_from_disk(&cache_dir2, &canonical_root2)
4828                .expect("load flushed root two index");
4829        restored2.ready = true;
4830        assert_eq!(
4831            restored2
4832                .grep("new actor two token", true, &[], &[], &canonical_root2, 10)
4833                .matches
4834                .len(),
4835            1,
4836            "graceful subc shutdown should flush every registered root"
4837        );
4838    }
4839
4840    #[test]
4841    fn idle_root_reaper_closes_artifacts_and_stops_watcher() {
4842        let _ = env_logger::builder().is_test(true).try_init();
4843        let (root_dir, root) = test_root("idle-root-reaper");
4844        let storage = tempfile::tempdir().expect("storage tempdir");
4845        std::fs::write(
4846            root_dir.path().join("main.rs"),
4847            "fn entry() { leaf(); }\nfn leaf() {}\n",
4848        )
4849        .expect("source file");
4850        let canonical_root = std::fs::canonicalize(root_dir.path()).expect("canonical root");
4851        let app = App::default_shared();
4852        let ctx = Arc::new(AppContext::from_app(
4853            Arc::clone(&app),
4854            Config {
4855                project_root: Some(canonical_root.clone()),
4856                storage_dir: Some(storage.path().to_path_buf()),
4857                callgraph_store: true,
4858                search_index: true,
4859                ..Config::default()
4860            },
4861        ));
4862        ctx.set_canonical_cache_root(canonical_root.clone());
4863        assert!(ctx
4864            .ensure_callgraph_store()
4865            .expect("build callgraph store")
4866            .is_some());
4867
4868        let cache_dir =
4869            crate::search_index::resolve_cache_dir(&canonical_root, Some(storage.path()));
4870        let mut index = crate::search_index::SearchIndex::build(&canonical_root);
4871        let git_head = index.stored_git_head().map(str::to_owned);
4872        index.write_to_disk(&cache_dir, git_head.as_deref());
4873        *ctx.search_index()
4874            .write()
4875            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
4876        // Seed a completed warm verification so the test can prove eviction
4877        // downgrades it to Strict rather than passing vacuously.
4878        let seeded_generation =
4879            crate::cache_freshness::artifact_generation(&cache_dir.join("cache.bin"))
4880                .expect("seeded artifact generation");
4881        crate::cache_freshness::record_verify_completed(
4882            &canonical_root,
4883            crate::cache_freshness::VerifyArtifact::Search,
4884            Some(seeded_generation),
4885        );
4886        assert!(
4887            matches!(
4888                crate::cache_freshness::warm_verify_plan(
4889                    canonical_root.as_path(),
4890                    crate::cache_freshness::VerifyArtifact::Search,
4891                    Some(seeded_generation),
4892                ),
4893                crate::cache_freshness::WarmVerifyPlan::Skip
4894            ),
4895            "memo must be warm before eviction for the downgrade assertion to bite"
4896        );
4897
4898        let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
4899        let _dispatch_tx = dispatch_tx;
4900        let shutdown = Arc::new(AtomicBool::new(false));
4901        let thread_shutdown = Arc::clone(&shutdown);
4902        let join = std::thread::spawn(move || {
4903            while !thread_shutdown.load(Ordering::SeqCst) {
4904                std::thread::yield_now();
4905            }
4906        });
4907        ctx.install_watcher_runtime(
4908            dispatch_rx,
4909            crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
4910        );
4911        wait_for_watcher_count(&ctx, 1);
4912
4913        let executor = Arc::new(Executor::new());
4914        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
4915        ctx.mark_subc_unbound();
4916        let mut live_roots = HashMap::new();
4917        let mut meta = RootMeta::new(Instant::now());
4918        meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
4919        meta.unbound_quiesced = true;
4920        live_roots.insert(root.clone(), meta);
4921
4922        let message = idle_root_eviction_message(&root, &ctx.memory_root_snapshot(), None);
4923        assert!(message.contains("evicted idle root"));
4924        assert!(message.contains("freed ~"));
4925        assert!(message.contains("semantic"));
4926        assert!(!message.contains("semantic not estimated retained"));
4927        assert!(message.contains("trigram"));
4928        assert!(message.contains("retained: bash"));
4929        assert!(message.contains("parser_pool"));
4930
4931        assert_eq!(
4932            reap_idle_roots(
4933                Instant::now(),
4934                &mut live_roots,
4935                &HashMap::new(),
4936                &HashMap::new(),
4937                &executor,
4938                &DispatchPathMetrics::new(),
4939            )
4940            .evicted,
4941            1
4942        );
4943        assert!(ctx.search_index().read().unwrap().is_none());
4944        wait_for_watcher_count(&ctx, 0);
4945        // The watcher stopped with the eviction, so the idle interval is
4946        // unobserved: the pre-seeded warm-verify memo (Skip) must fall back to
4947        // strict content verification (stat-first would miss same-size,
4948        // preserved-mtime edits made while nobody was watching).
4949        assert!(
4950            matches!(
4951                crate::cache_freshness::warm_verify_plan(
4952                    canonical_root.as_path(),
4953                    crate::cache_freshness::VerifyArtifact::Search,
4954                    Some(seeded_generation),
4955                ),
4956                crate::cache_freshness::WarmVerifyPlan::Strict
4957            ),
4958            "idle eviction must force strict re-verification"
4959        );
4960        assert!(
4961            crate::search_index::SearchIndex::read_from_disk(&cache_dir, &canonical_root).is_some()
4962        );
4963        ctx.mark_subc_bound();
4964        assert!(ctx
4965            .ensure_callgraph_store()
4966            .expect("reopen callgraph store")
4967            .is_some());
4968        assert!(live_roots[&root].idle_artifacts_evicted);
4969    }
4970
4971    #[test]
4972    fn idle_root_reaper_applies_ttl_to_unbound_roots() {
4973        let (_root_dir, root) = test_root("idle-root-ttl-gate");
4974        let ctx = test_ctx();
4975        let executor = Arc::new(Executor::new());
4976        assert!(executor.register_actor(root.clone(), ctx));
4977        let ctx = executor.actor_context(&root).expect("actor context");
4978        ctx.mark_subc_unbound();
4979        let now = Instant::now();
4980        let mut meta = RootMeta::new(now);
4981        meta.unbound_quiesced = true;
4982        let mut live_roots = HashMap::from([(root.clone(), meta)]);
4983
4984        // A recently-unbound root stays warm: a transient unbind (host
4985        // restart) must not pay the strict-verify + forced-rebuild teardown
4986        // on the next maintenance sweep.
4987        assert_eq!(
4988            reap_idle_roots(
4989                now,
4990                &mut live_roots,
4991                &HashMap::new(),
4992                &HashMap::new(),
4993                &executor,
4994                &DispatchPathMetrics::new(),
4995            )
4996            .evicted,
4997            0
4998        );
4999        assert!(!live_roots[&root].idle_artifacts_evicted);
5000
5001        // Past the TTL the same unbound root pays the full teardown. The
5002        // pending reconciliation paths retained across the transient-unbind
5003        // window would block eviction forever through
5004        // `artifact_eviction_blocked`; the reaper disposes them because the
5005        // strict gap invalidation subsumes their purpose.
5006        ctx.add_pending_search_index_paths([root.as_path().join("retained.rs")]);
5007        assert_eq!(
5008            reap_idle_roots(
5009                now + IDLE_ROOT_TTL,
5010                &mut live_roots,
5011                &HashMap::new(),
5012                &HashMap::new(),
5013                &executor,
5014                &DispatchPathMetrics::new(),
5015            )
5016            .evicted,
5017            1
5018        );
5019        assert!(live_roots[&root].idle_artifacts_evicted);
5020        assert!(
5021            ctx.take_pending_search_index_paths().is_empty(),
5022            "TTL eviction must dispose retained pending reconciliation paths"
5023        );
5024    }
5025
5026    #[test]
5027    fn blocked_ttl_eviction_restores_taken_pending_reconciliation_state() {
5028        let (_root_dir, root) = test_root("ttl-eviction-blocked-restore");
5029        let ctx = test_ctx();
5030        let executor = Arc::new(Executor::new());
5031        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5032        ctx.mark_subc_unbound();
5033
5034        // Retained pending path from the transient-unbind window plus a
5035        // SECONDARY eviction blocker (a non-ready resident search index, the
5036        // dirty-index blocker in artifact_eviction_blocked). Disposal must be
5037        // transactional: the blocked eviction may be followed by a rebind, and
5038        // the path is the only repair record for its consumed watcher event.
5039        let pending = root.as_path().join("edited-while-unbound.rs");
5040        ctx.add_pending_search_index_paths([pending.clone()]);
5041        let dirty_source = root.as_path().join("dirty.rs");
5042        std::fs::write(&dirty_source, "fn dirty() {}\n").expect("dirty source");
5043        let mut dirty = crate::search_index::SearchIndex::new();
5044        dirty.ready = true;
5045        dirty.update_file(&dirty_source);
5046        *ctx.search_index()
5047            .write()
5048            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(dirty);
5049        assert!(ctx.artifact_eviction_blocked());
5050
5051        let mut live_roots = HashMap::new();
5052        let mut meta = RootMeta::new(Instant::now());
5053        meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
5054        meta.unbound_quiesced = true;
5055        live_roots.insert(root.clone(), meta);
5056
5057        assert_eq!(
5058            reap_idle_roots(
5059                Instant::now(),
5060                &mut live_roots,
5061                &HashMap::new(),
5062                &HashMap::new(),
5063                &executor,
5064                &DispatchPathMetrics::new(),
5065            )
5066            .evicted,
5067            0,
5068            "the dirty index must still block this eviction"
5069        );
5070        assert_eq!(
5071            ctx.take_pending_search_index_paths(),
5072            vec![pending],
5073            "a blocked eviction must restore the taken pending paths"
5074        );
5075    }
5076
5077    #[test]
5078    fn idle_reap_with_bound_route_keeps_watcher_running() {
5079        let (_root_dir, root) = test_root("bound-root-reap-gate");
5080        let ctx = test_ctx();
5081        let executor = Arc::new(Executor::new());
5082        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5083
5084        let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
5085        let _dispatch_tx = dispatch_tx;
5086        let shutdown = Arc::new(AtomicBool::new(false));
5087        let thread_shutdown = Arc::clone(&shutdown);
5088        let join = std::thread::spawn(move || {
5089            while !thread_shutdown.load(Ordering::SeqCst) {
5090                std::thread::yield_now();
5091            }
5092        });
5093        ctx.install_watcher_runtime(
5094            dispatch_rx,
5095            crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
5096        );
5097
5098        let mut meta = RootMeta::new(Instant::now());
5099        meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
5100        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5101        let bound = HashMap::from([(root, HashSet::from([route_key(7, 1)]))]);
5102        assert_eq!(
5103            reap_idle_roots(
5104                Instant::now(),
5105                &mut live_roots,
5106                &HashMap::new(),
5107                &bound,
5108                &executor,
5109                &DispatchPathMetrics::new(),
5110            )
5111            .evicted,
5112            0
5113        );
5114        wait_for_watcher_count(&ctx, 1);
5115        ctx.stop_watcher_runtime_in_background();
5116        wait_for_watcher_count(&ctx, 0);
5117    }
5118
5119    #[test]
5120    fn deleted_idle_root_is_fully_forgotten_and_status_counts_drop() {
5121        let (root_dir, root) = test_root("deleted-root-reap");
5122        let app = App::default_shared();
5123        let ctx = Arc::new(AppContext::from_app(
5124            Arc::clone(&app),
5125            Config {
5126                project_root: Some(root.as_path().to_path_buf()),
5127                ..Config::default()
5128            },
5129        ));
5130        ctx.set_canonical_cache_root(root.as_path().to_path_buf());
5131        ctx.mark_subc_unbound();
5132        let executor = Arc::new(Executor::new());
5133        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5134        assert_eq!(app.actor_root_count(), 1);
5135        drop(ctx);
5136        root_dir.close().expect("delete project root");
5137
5138        let mut meta = RootMeta::new(Instant::now());
5139        meta.unbound_quiesced = true;
5140        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5141        let outcome = reap_idle_roots(
5142            Instant::now(),
5143            &mut live_roots,
5144            &HashMap::new(),
5145            &HashMap::new(),
5146            &executor,
5147            &DispatchPathMetrics::new(),
5148        );
5149        assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
5150        assert!(!executor.actor_registered(&root));
5151        assert!(!live_roots.contains_key(&root));
5152        wait_for_actor_root_count(&app, 0);
5153
5154        let status_ctx = AppContext::from_app(app, Config::default());
5155        let status = status_ctx.build_status_snapshot();
5156        assert_eq!(status["runtime"]["live_actor_roots"], 0);
5157        assert_eq!(status["runtime"]["open_routes"], 0);
5158    }
5159
5160    #[test]
5161    fn deleted_root_reap_blocker_census_is_exposed_in_health_metrics() {
5162        let (root_dir, root) = test_root("deleted-root-reap-census");
5163        let executor = Arc::new(Executor::new());
5164        assert!(executor.register_actor(root.clone(), test_ctx()));
5165        root_dir.close().expect("delete project root");
5166
5167        let mut live_roots = HashMap::from([(root, RootMeta::new(Instant::now()))]);
5168        let metrics = DispatchPathMetrics::new();
5169        let outcome = reap_idle_roots(
5170            Instant::now(),
5171            &mut live_roots,
5172            &HashMap::new(),
5173            &HashMap::new(),
5174            &executor,
5175            &metrics,
5176        );
5177        assert_eq!(outcome.evicted, 0);
5178
5179        let report = build_health_report(
5180            &executor,
5181            &HashMap::new(),
5182            &metrics,
5183            &crate::context::App::default_shared(),
5184        );
5185        let reap = report
5186            .metrics
5187            .as_ref()
5188            .and_then(|metrics| metrics.get("reap"))
5189            .expect("reap health metrics");
5190        assert_eq!(reap["deleted_retained"].as_u64(), Some(1));
5191        assert_eq!(reap["blockers"]["unbound_quiesced"].as_u64(), Some(1));
5192        assert_eq!(reap["blockers"]["actor_busy"].as_u64(), Some(0));
5193    }
5194
5195    #[test]
5196    fn connection_exit_quiesces_queued_maintenance_and_deleted_root_is_purged() {
5197        let (root_dir, root) = test_root("connection-exit-deleted-root");
5198        let executor = Arc::new(Executor::new());
5199        assert!(executor.register_actor(root.clone(), test_ctx()));
5200
5201        let route = route_key(11, 1);
5202        let mut meta = RootMeta::new(Instant::now());
5203        meta.maintenance_pending = true;
5204        meta.maintenance_queued_kinds
5205            .push_back(MaintenanceDrainKind::CompletionDrains);
5206        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5207        let mut pending_binds = HashMap::new();
5208        let mut routes = HashMap::from([(route, route_identity(&root, "abandoned"))]);
5209        let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
5210        let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
5211        let mut route_bash_cancels = HashMap::new();
5212
5213        quiesce_connection_roots(
5214            &mut live_roots,
5215            &mut pending_binds,
5216            &mut routes,
5217            &mut root_channels,
5218            &mut installed_route_epochs,
5219            &mut route_bash_cancels,
5220            &executor,
5221        );
5222        assert!(live_roots[&root].unbound_quiesced);
5223        assert!(!live_roots[&root].maintenance_pending);
5224        assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
5225        assert!(routes.is_empty());
5226        assert!(root_channels.is_empty());
5227
5228        root_dir.close().expect("delete project root");
5229        let metrics = DispatchPathMetrics::new();
5230        let outcome = reap_idle_roots(
5231            Instant::now(),
5232            &mut live_roots,
5233            &pending_binds,
5234            &root_channels,
5235            &executor,
5236            &metrics,
5237        );
5238        let mut session_identity = HashMap::new();
5239        let mut push_buffer = HashMap::new();
5240        let mut bg_subs = HashMap::new();
5241        let mut bg_sub_by_session = HashMap::new();
5242        let mut bg_wake_pending = HashSet::new();
5243        let mut bg_wake_epoch = HashMap::new();
5244        let mut pending_bash_asks = HashMap::new();
5245        for forgotten in &outcome.forgotten_deleted_roots {
5246            purge_deleted_root_residents(
5247                forgotten,
5248                &mut root_channels,
5249                &mut session_identity,
5250                &mut push_buffer,
5251                &mut bg_subs,
5252                &mut bg_sub_by_session,
5253                &mut bg_wake_pending,
5254                &mut bg_wake_epoch,
5255                &mut pending_bash_asks,
5256            );
5257        }
5258
5259        assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
5260        assert!(!executor.actor_registered(&root));
5261        assert!(!live_roots.contains_key(&root));
5262    }
5263
5264    #[test]
5265    fn unbound_root_quiesces_maintenance_without_removing_actor() {
5266        let (_root_dir, root) = test_root("unbound-root-quiesce");
5267        let ctx = test_ctx();
5268        let executor = Arc::new(Executor::new());
5269        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5270        let mut meta = RootMeta::new(Instant::now());
5271        meta.maintenance_pending = true;
5272        meta.maintenance_jobs_in_flight = 1;
5273        meta.maintenance_queued_kinds
5274            .push_back(MaintenanceDrainKind::ConfigureTail);
5275        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5276        // Warm state planted before the unbind: quiesce must keep it.
5277        *ctx.search_index()
5278            .write()
5279            .unwrap_or_else(std::sync::PoisonError::into_inner) =
5280            Some(crate::search_index::SearchIndex::new());
5281        ctx.set_cache_writer_capabilities(true, true);
5282        let pending = root.as_path().join("pending.rs");
5283        ctx.add_pending_search_index_paths([pending.clone()]);
5284        // A warm verify memo must survive the transient unbind: the watcher
5285        // keeps running, so no unobserved window exists and the next warm
5286        // reload must not pay a strict full-corpus re-hash.
5287        let canonical_root = root.as_path().to_path_buf();
5288        let artifact = canonical_root.join("cache.bin");
5289        std::fs::write(&artifact, b"warm-artifact").expect("write artifact");
5290        let seeded_generation = crate::cache_freshness::artifact_generation(&artifact);
5291        crate::cache_freshness::record_verify_completed(
5292            &canonical_root,
5293            crate::cache_freshness::VerifyArtifact::Search,
5294            seeded_generation,
5295        );
5296        assert!(matches!(
5297            crate::cache_freshness::warm_verify_plan(
5298                &canonical_root,
5299                crate::cache_freshness::VerifyArtifact::Search,
5300                seeded_generation,
5301            ),
5302            crate::cache_freshness::WarmVerifyPlan::Skip
5303        ));
5304        // A live watcher runtime must survive quiesce (its events accumulate
5305        // for the rebind replay).
5306        let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
5307        let _dispatch_tx = dispatch_tx;
5308        let shutdown = Arc::new(AtomicBool::new(false));
5309        let thread_shutdown = Arc::clone(&shutdown);
5310        let join = std::thread::spawn(move || {
5311            while !thread_shutdown.load(Ordering::SeqCst) {
5312                std::thread::yield_now();
5313            }
5314        });
5315        ctx.install_watcher_runtime(
5316            dispatch_rx,
5317            crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
5318        );
5319        assert!(ctx.watcher_runtime_active());
5320
5321        quiesce_unbound_root(&root, &mut live_roots, &executor);
5322        let meta = &live_roots[&root];
5323        assert!(meta.unbound_quiesced);
5324        assert!(ctx.subc_unbound_quiesced());
5325        assert!(meta.maintenance_pending);
5326        assert!(meta.maintenance_queued_kinds.is_empty());
5327        assert!(executor.actor_registered(&root));
5328        // Transient unbind keeps the root warm: resident artifacts stay
5329        // resident, no forced callgraph rebuild is planted, and pending
5330        // reconciliation paths survive for the rebind replay.
5331        assert!(
5332            ctx.search_index()
5333                .read()
5334                .unwrap_or_else(std::sync::PoisonError::into_inner)
5335                .is_some(),
5336            "quiesce must not evict resident artifacts"
5337        );
5338        assert_eq!(
5339            ctx.pending_callgraph_store_force_token(),
5340            None,
5341            "quiesce must not force a callgraph rebuild"
5342        );
5343        assert_eq!(
5344            ctx.take_pending_search_index_paths(),
5345            vec![pending],
5346            "quiesce must retain pending watcher-derived paths"
5347        );
5348        assert!(
5349            matches!(
5350                crate::cache_freshness::warm_verify_plan(
5351                    &canonical_root,
5352                    crate::cache_freshness::VerifyArtifact::Search,
5353                    seeded_generation,
5354                ),
5355                crate::cache_freshness::WarmVerifyPlan::Skip
5356            ),
5357            "quiesce must not invalidate the warm verify memo"
5358        );
5359        assert!(
5360            ctx.watcher_runtime_active(),
5361            "quiesce must not stop a running watcher"
5362        );
5363        ctx.stop_watcher_runtime();
5364
5365        let meta = live_roots.get_mut(&root).expect("root metadata");
5366        note_maintenance_completion(
5367            meta,
5368            Some(MaintenanceDrainKind::ConfigureTail),
5369            false,
5370            meta.unbound_quiesced,
5371        );
5372        assert!(!meta.maintenance_pending);
5373        assert!(meta.maintenance_queued_kinds.is_empty());
5374    }
5375
5376    #[test]
5377    fn same_root_higher_epoch_replacement_does_not_quiesce_between_generations() {
5378        let (_dir, root) = test_root("same-root-replacement");
5379        let route = route_key(7, 1);
5380        let installed_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
5381        let root_channels = HashMap::new();
5382
5383        assert!(!route_removal_will_quiesce_root(
5384            &root,
5385            route,
5386            &installed_channels,
5387            false,
5388            Some(&root),
5389        ));
5390        assert!(route_removal_will_quiesce_root(
5391            &root,
5392            route,
5393            &installed_channels,
5394            false,
5395            None,
5396        ));
5397        assert!(!should_quiesce_removed_root(
5398            &root,
5399            &root_channels,
5400            false,
5401            Some(&root),
5402        ));
5403        assert!(should_quiesce_removed_root(
5404            &root,
5405            &root_channels,
5406            false,
5407            None,
5408        ));
5409        assert!(!should_quiesce_removed_root(
5410            &root,
5411            &root_channels,
5412            true,
5413            None,
5414        ));
5415    }
5416
5417    #[test]
5418    fn root_quiesces_only_after_its_last_route_is_removed_and_reactivates_on_bind() {
5419        let (_root_dir, root) = test_root("unbound-root-route-count");
5420        let executor = Arc::new(Executor::new());
5421        assert!(executor.register_actor(root.clone(), test_ctx()));
5422        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
5423        let mut root_channels = HashMap::from([(
5424            root.clone(),
5425            HashSet::from([route_key(7, 1), route_key(8, 1)]),
5426        )]);
5427
5428        remove_root_channel(&mut root_channels, &root, route_key(7, 1));
5429        if !root_channels.contains_key(&root) {
5430            quiesce_unbound_root(&root, &mut live_roots, &executor);
5431        }
5432        assert!(!live_roots[&root].unbound_quiesced);
5433
5434        remove_root_channel(&mut root_channels, &root, route_key(8, 1));
5435        if !root_channels.contains_key(&root) {
5436            quiesce_unbound_root(&root, &mut live_roots, &executor);
5437        }
5438        assert!(live_roots[&root].unbound_quiesced);
5439
5440        live_roots
5441            .get_mut(&root)
5442            .expect("root metadata")
5443            .note_activity();
5444        assert!(
5445            live_roots[&root].unbound_quiesced,
5446            "late asynchronous activity must not reactivate an unbound root"
5447        );
5448
5449        live_roots
5450            .get_mut(&root)
5451            .expect("root metadata")
5452            .reactivate_bound();
5453        assert!(!live_roots[&root].unbound_quiesced);
5454    }
5455
5456    #[test]
5457    fn allocator_pressure_relief_requires_every_root_to_be_idle() {
5458        let (_idle_dir, idle_root) = test_root("allocator-relief-idle");
5459        let (_active_dir, active_root) = test_root("allocator-relief-active");
5460        let now = Instant::now();
5461        let mut live_roots = HashMap::new();
5462        let mut idle = RootMeta::new(now);
5463        idle.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
5464        live_roots.insert(idle_root, idle);
5465        assert!(process_has_been_idle(now, &live_roots));
5466
5467        live_roots.insert(active_root.clone(), RootMeta::new(now));
5468        assert!(!process_has_been_idle(now, &live_roots));
5469
5470        let active = live_roots
5471            .get_mut(&active_root)
5472            .expect("active root metadata");
5473        active.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
5474        active.active_bash_waits = 1;
5475        assert!(!process_has_been_idle(now, &live_roots));
5476    }
5477
5478    #[test]
5479    fn pressure_relief_log_reports_before_and_after_measurements() {
5480        let allocator = crate::memory::AllocatorMemorySnapshot {
5481            status: "measured",
5482            bytes_in_use: Some(8 * 1024 * 1024),
5483            size_allocated: Some(12 * 1024 * 1024),
5484            retained_slack_bytes: Some(4 * 1024 * 1024),
5485            not_estimated: None,
5486        };
5487        let relief = crate::memory::AllocatorPressureRelief {
5488            bytes_released: 3 * 1024 * 1024,
5489            rss_before_bytes: Some(20 * 1024 * 1024),
5490            rss_after_bytes: Some(17 * 1024 * 1024),
5491            allocator_before: allocator.clone(),
5492            allocator_after: crate::memory::AllocatorMemorySnapshot {
5493                size_allocated: Some(9 * 1024 * 1024),
5494                retained_slack_bytes: Some(1024 * 1024),
5495                ..allocator
5496            },
5497        };
5498        let message = pressure_relief_label(&relief);
5499        assert!(message.contains("RSS 20.0 MB -> 17.0 MB"));
5500        assert!(message.contains("allocated 12.0 MB -> 9.0 MB"));
5501        assert!(message.contains("slack 4.0 MB -> 1.0 MB"));
5502        assert!(message.contains("reported 3.0 MB released"));
5503    }
5504
5505    #[test]
5506    fn due_maintenance_jobs_skip_poisoned_roots() {
5507        let (_healthy_dir, healthy_root) = test_root("maintenance-healthy");
5508        let (_poisoned_dir, poisoned_root) = test_root("maintenance-poisoned");
5509        let mut live_roots = HashMap::new();
5510        live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
5511        let mut poisoned_meta = RootMeta::new(Instant::now());
5512        poisoned_meta.maintenance_poisoned = true;
5513        live_roots.insert(poisoned_root.clone(), poisoned_meta);
5514
5515        let (due, deferred) = due_maintenance_jobs_without_actor_context(
5516            &mut live_roots,
5517            MAINTENANCE_SUBMIT_BUDGET,
5518            &HashSet::new(),
5519        );
5520
5521        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
5522        assert!(due.iter().all(|(root, _)| root == &healthy_root));
5523        assert!(!deferred);
5524        assert!(live_roots[&healthy_root].maintenance_pending);
5525        assert_eq!(
5526            live_roots[&healthy_root].maintenance_jobs_in_flight,
5527            INITIAL_MAINTENANCE_JOB_COUNT
5528        );
5529        assert!(!live_roots[&poisoned_root].maintenance_pending);
5530    }
5531
5532    #[test]
5533    fn due_maintenance_jobs_do_not_restart_quiesced_root_work() {
5534        let (_dir, root) = test_root("maintenance-unbound");
5535        let mut meta = RootMeta::new(Instant::now());
5536        meta.unbound_quiesced = true;
5537        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5538
5539        let (due, deferred) = due_maintenance_jobs_without_actor_context(
5540            &mut live_roots,
5541            MAINTENANCE_SUBMIT_BUDGET,
5542            &HashSet::new(),
5543        );
5544
5545        assert!(due.is_empty());
5546        assert!(!deferred);
5547        assert!(!live_roots[&root].maintenance_pending);
5548    }
5549
5550    #[test]
5551    fn idle_bg_subscription_queues_no_jobs_until_a_wake_arrives() {
5552        let (_dir, root) = test_root("maintenance-idle-bg-subscription");
5553        let ctx = test_ctx();
5554        assert!(!ctx.completion_drains_have_work());
5555
5556        let executor = Executor::new();
5557        assert!(executor.register_actor(root.clone(), ctx));
5558        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
5559        let session = "idle-session".to_string();
5560        let channel = route_key(17, 1);
5561        let bg_sub_by_session = HashMap::from([((root.clone(), session.clone()), channel)]);
5562        let mut bg_wake_pending = HashSet::new();
5563
5564        let (idle_tick_jobs, deferred) = due_maintenance_jobs(
5565            &mut live_roots,
5566            Some(&executor),
5567            &bg_sub_by_session,
5568            &bg_wake_pending,
5569            MAINTENANCE_SUBMIT_BUDGET,
5570            &HashSet::new(),
5571        );
5572        assert!(idle_tick_jobs.is_empty());
5573        assert!(!deferred);
5574        assert!(!live_roots[&root].maintenance_pending);
5575
5576        // A completion can arm its wake after the idle tick's probes. The wake
5577        // remains loop-owned state, so the following tick must observe it.
5578        let mut bg_wake_epoch = HashMap::new();
5579        push::arm_bg_wake(
5580            root.clone(),
5581            session,
5582            channel,
5583            &mut bg_wake_pending,
5584            &mut bg_wake_epoch,
5585        );
5586        let (next_tick_jobs, deferred) = due_maintenance_jobs(
5587            &mut live_roots,
5588            Some(&executor),
5589            &bg_sub_by_session,
5590            &bg_wake_pending,
5591            MAINTENANCE_SUBMIT_BUDGET,
5592            &HashSet::new(),
5593        );
5594        assert_eq!(
5595            next_tick_jobs,
5596            vec![(root, MaintenanceDrainKind::CompletionDrains)]
5597        );
5598        assert!(!deferred);
5599    }
5600
5601    #[tokio::test]
5602    async fn subc_configure_tail_precedes_completed_search_install() {
5603        let root_dir = tempfile::tempdir().unwrap();
5604        let storage = tempfile::tempdir().unwrap();
5605        let root = ProjectRootId::from_path(root_dir.path()).unwrap();
5606        let (ctx, ignored_path) =
5607            runtime_drain::configure_search_order_context_for_test(root_dir.path(), storage.path());
5608        let ctx = Arc::new(ctx);
5609        assert!(!runtime_drain::watcher_path_is_ignored_by_current_matcher(
5610            &ctx,
5611            &ignored_path
5612        ));
5613
5614        let executor = Arc::new(Executor::new());
5615        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5616        let metrics = Arc::new(DispatchPathMetrics::new());
5617        let (completion_tx, mut completion_rx) = mpsc::channel(4);
5618        submit_maintenance_job(
5619            &executor,
5620            root.clone(),
5621            MaintenanceDrainKind::ConfigureTail,
5622            Vec::new(),
5623            &completion_tx,
5624            &metrics,
5625        );
5626        submit_maintenance_job(
5627            &executor,
5628            root,
5629            MaintenanceDrainKind::CompletionDrains,
5630            Vec::new(),
5631            &completion_tx,
5632            &metrics,
5633        );
5634
5635        let first = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
5636            .await
5637            .expect("configure-tail completion timed out")
5638            .expect("configure-tail completion channel closed");
5639        let second = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
5640            .await
5641            .expect("completion-drains completion timed out")
5642            .expect("completion-drains completion channel closed");
5643        assert!(first.response.id.contains("configure-tail"));
5644        assert!(second.response.id.contains("completion-drains"));
5645        assert!(runtime_drain::watcher_path_is_ignored_by_current_matcher(
5646            &ctx,
5647            &ignored_path
5648        ));
5649        assert_eq!(
5650            ctx.search_index()
5651                .read()
5652                .unwrap_or_else(std::sync::PoisonError::into_inner)
5653                .as_ref()
5654                .expect("completed search index installed")
5655                .file_count(),
5656            0,
5657            "configure must install the ignore matcher before pending paths replay"
5658        );
5659        ctx.stop_watcher_runtime();
5660    }
5661
5662    #[test]
5663    fn post_bind_configure_and_completion_jobs_are_queued_in_order() {
5664        let (_dir, root) = test_root("maintenance-post-bind");
5665        let mut live_roots = HashMap::new();
5666        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
5667
5668        queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
5669        queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
5670
5671        let meta = live_roots.get(&root).expect("root metadata");
5672        assert!(meta.maintenance_pending);
5673        assert_eq!(meta.maintenance_jobs_in_flight, 0);
5674        assert_eq!(
5675            meta.maintenance_queued_kinds
5676                .iter()
5677                .copied()
5678                .collect::<Vec<_>>(),
5679            vec![
5680                MaintenanceDrainKind::ConfigureTail,
5681                MaintenanceDrainKind::CompletionDrains,
5682            ]
5683        );
5684
5685        let (due, deferred) = due_maintenance_jobs_without_actor_context(
5686            &mut live_roots,
5687            MAINTENANCE_SUBMIT_BUDGET,
5688            &HashSet::new(),
5689        );
5690
5691        assert_eq!(
5692            due,
5693            vec![
5694                (root.clone(), MaintenanceDrainKind::ConfigureTail),
5695                (root.clone(), MaintenanceDrainKind::CompletionDrains),
5696            ]
5697        );
5698        assert!(!deferred);
5699        assert_eq!(live_roots[&root].maintenance_jobs_in_flight, 2);
5700        assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
5701    }
5702
5703    #[test]
5704    fn due_maintenance_jobs_defers_unsubmitted_roots_without_marking_pending() {
5705        let mut live_roots = HashMap::new();
5706        let mut root_ids = Vec::new();
5707        let mut _dirs = Vec::new();
5708        for index in 0..4 {
5709            let (dir, root_id) = test_root(&format!("maintenance-budget-{index}"));
5710            live_roots.insert(root_id.clone(), RootMeta::new(Instant::now()));
5711            root_ids.push(root_id);
5712            _dirs.push(dir);
5713        }
5714
5715        let small_budget = INITIAL_MAINTENANCE_JOB_COUNT + 1;
5716        let (first_due, first_deferred) = due_maintenance_jobs_without_actor_context(
5717            &mut live_roots,
5718            small_budget,
5719            &HashSet::new(),
5720        );
5721
5722        assert_eq!(first_due.len(), small_budget);
5723        assert!(first_deferred);
5724        let first_due_set: HashSet<_> = first_due.into_iter().map(|(root, _)| root).collect();
5725        assert!(first_due_set
5726            .iter()
5727            .all(|root| live_roots[root].maintenance_pending));
5728        assert!(first_due_set
5729            .iter()
5730            .any(|root| !live_roots[root].maintenance_queued_kinds.is_empty()));
5731
5732        let all_roots: HashSet<_> = root_ids.into_iter().collect();
5733        let deferred_roots: HashSet<_> = all_roots.difference(&first_due_set).cloned().collect();
5734        assert!(deferred_roots
5735            .iter()
5736            .all(|root| !live_roots[root].maintenance_pending));
5737    }
5738
5739    #[test]
5740    fn due_maintenance_jobs_defers_pending_bind_roots() {
5741        let (_bind_dir, bind_root) = test_root("maintenance-pending-bind");
5742        let (_healthy_dir, healthy_root) = test_root("maintenance-no-bind");
5743        let mut live_roots = HashMap::new();
5744        live_roots.insert(bind_root.clone(), RootMeta::new(Instant::now()));
5745        live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
5746        let pending_bind_roots = HashSet::from([bind_root.clone()]);
5747
5748        let (due, deferred) = due_maintenance_jobs_without_actor_context(
5749            &mut live_roots,
5750            usize::MAX,
5751            &pending_bind_roots,
5752        );
5753
5754        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
5755        assert!(due.iter().all(|(root, _)| root == &healthy_root));
5756        assert!(!deferred);
5757        assert!(!live_roots[&bind_root].maintenance_pending);
5758        assert!(live_roots[&bind_root].maintenance_queued_kinds.is_empty());
5759    }
5760
5761    #[test]
5762    fn maintenance_pending_survives_requeue_and_clears_after_final_batch() {
5763        let (_dir, root) = test_root("maintenance-requeue");
5764        let mut live_roots = HashMap::new();
5765        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
5766        let (due, deferred) = due_maintenance_jobs_without_actor_context(
5767            &mut live_roots,
5768            usize::MAX,
5769            &HashSet::new(),
5770        );
5771        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
5772        assert!(due.iter().all(|(due_root, _)| due_root == &root));
5773        assert!(!deferred);
5774
5775        let meta = live_roots.get_mut(&root).unwrap();
5776        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, false);
5777        assert!(meta.maintenance_pending);
5778        assert_eq!(
5779            meta.maintenance_jobs_in_flight,
5780            INITIAL_MAINTENANCE_JOB_COUNT - 1
5781        );
5782        assert_eq!(meta.maintenance_queued_kinds.len(), 1);
5783
5784        let (requeued, deferred) =
5785            due_maintenance_jobs_without_actor_context(&mut live_roots, 1, &HashSet::new());
5786        assert_eq!(
5787            requeued,
5788            vec![(root.clone(), MaintenanceDrainKind::Watcher)]
5789        );
5790        assert!(!deferred);
5791        let meta = live_roots.get_mut(&root).unwrap();
5792        assert_eq!(
5793            meta.maintenance_jobs_in_flight,
5794            INITIAL_MAINTENANCE_JOB_COUNT
5795        );
5796        assert!(meta.maintenance_queued_kinds.is_empty());
5797
5798        for _ in 0..INITIAL_MAINTENANCE_JOB_COUNT {
5799            note_maintenance_completion(meta, None, false, false);
5800        }
5801        assert!(!meta.maintenance_pending);
5802        assert_eq!(meta.maintenance_jobs_in_flight, 0);
5803    }
5804
5805    #[test]
5806    fn maintenance_requeue_drops_while_bind_is_pending() {
5807        let (_dir, root) = test_root("maintenance-bind-requeue");
5808        let mut live_roots = HashMap::new();
5809        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
5810        let (due, _) = due_maintenance_jobs_without_actor_context(
5811            &mut live_roots,
5812            usize::MAX,
5813            &HashSet::new(),
5814        );
5815        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
5816
5817        let meta = live_roots.get_mut(&root).unwrap();
5818        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, true);
5819
5820        assert_eq!(
5821            meta.maintenance_jobs_in_flight,
5822            INITIAL_MAINTENANCE_JOB_COUNT - 1
5823        );
5824        assert!(meta.maintenance_queued_kinds.is_empty());
5825        assert!(meta.maintenance_pending);
5826    }
5827
5828    #[test]
5829    fn parked_lsp_completion_never_requiesces_or_cancels_a_pending_bind() {
5830        let mut meta = RootMeta::new(Instant::now());
5831        meta.unbound_quiesced = true;
5832
5833        assert!(!should_requiesce_after_maintenance(
5834            &meta,
5835            MaintenanceDrainKind::Lsp,
5836            false,
5837        ));
5838        assert!(!should_requiesce_after_maintenance(
5839            &meta,
5840            MaintenanceDrainKind::ConfigureTail,
5841            true,
5842        ));
5843        assert!(should_requiesce_after_maintenance(
5844            &meta,
5845            MaintenanceDrainKind::ConfigureTail,
5846            false,
5847        ));
5848    }
5849
5850    #[test]
5851    fn maintenance_pending_clears_and_poison_stops_requeue_after_fatal() {
5852        let (_dir, root) = test_root("maintenance-fatal");
5853        let mut live_roots = HashMap::new();
5854        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
5855        let (due, _) = due_maintenance_jobs_without_actor_context(
5856            &mut live_roots,
5857            usize::MAX,
5858            &HashSet::new(),
5859        );
5860        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
5861
5862        let meta = live_roots.get_mut(&root).unwrap();
5863        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), true, false);
5864        assert!(meta.maintenance_poisoned);
5865        assert!(meta.maintenance_queued_kinds.is_empty());
5866
5867        for _ in 1..INITIAL_MAINTENANCE_JOB_COUNT {
5868            note_maintenance_completion(meta, None, false, false);
5869        }
5870        assert!(!meta.maintenance_pending);
5871        assert_eq!(meta.maintenance_jobs_in_flight, 0);
5872    }
5873
5874    #[test]
5875    fn trust_for_principal_matrix() {
5876        assert_eq!(
5877            trust_for_principal(&Some(Principal::Direct)),
5878            BindTrust::FirstParty
5879        );
5880        assert_eq!(
5881            trust_for_principal(&Some(Principal::Reserved {
5882                module_id: "llm-runner".to_string(),
5883            })),
5884            BindTrust::FirstParty
5885        );
5886        assert_eq!(
5887            trust_for_principal(&Some(Principal::Reserved {
5888                module_id: "aft".to_string(),
5889            })),
5890            BindTrust::FirstParty
5891        );
5892        assert_eq!(
5893            trust_for_principal(&Some(Principal::Reserved {
5894                module_id: "subc-mcp".to_string(),
5895            })),
5896            BindTrust::Untrusted
5897        );
5898        assert_eq!(
5899            trust_for_principal(&Some(Principal::Reserved {
5900                module_id: "anything-unknown".to_string(),
5901            })),
5902            BindTrust::Untrusted
5903        );
5904        assert_eq!(
5905            trust_for_principal(&Some(Principal::Unverified)),
5906            BindTrust::Untrusted
5907        );
5908        assert_eq!(trust_for_principal(&None), BindTrust::Untrusted);
5909    }
5910
5911    #[test]
5912    fn fed_harness_class_maps_to_untrusted_regardless_of_fingerprint_value() {
5913        let principal = Some(Principal::Direct);
5914        let fingerprint_a = "fed:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
5915        let fingerprint_b = "fed:0123456789abcdef111111111111111111111111111111111111111111111111";
5916
5917        assert_eq!(
5918            trust_for_bind(fingerprint_a, &principal),
5919            BindTrust::Untrusted
5920        );
5921        assert_eq!(
5922            trust_for_bind(fingerprint_b, &principal),
5923            BindTrust::Untrusted
5924        );
5925    }
5926
5927    #[tokio::test]
5928    async fn persistent_cancel_resolves_when_fired_before_await() {
5929        // The lost-wakeup guard: cancel() fires exactly once via notify_waiters()
5930        // (no stored permit). A waiter that registers AFTER the cancel must still
5931        // observe it via the flag; a waiter racing the cancel must still be woken.
5932        let signal = PersistentCancelSignal::new();
5933        signal.cancel();
5934        // Fired before we ever call cancelled() — must return immediately, not park.
5935        tokio::time::timeout(Duration::from_secs(1), signal.cancelled())
5936            .await
5937            .expect("cancelled() must resolve when cancel fired beforehand");
5938
5939        // A fresh signal cancelled concurrently with an in-flight cancelled().
5940        let racing = PersistentCancelSignal::new();
5941        let racing_for_task = racing.clone();
5942        let waiter = tokio::spawn(async move { racing_for_task.cancelled().await });
5943        racing.cancel();
5944        tokio::time::timeout(Duration::from_secs(1), waiter)
5945            .await
5946            .expect("cancelled() must resolve when cancel races the await")
5947            .expect("waiter task panicked");
5948    }
5949
5950    #[test]
5951    fn ingress_epoch_validation_silently_drops_every_stale_route_frame() {
5952        let installed = HashMap::from([(7, 9)]);
5953        for ty in [
5954            FrameType::Request,
5955            FrameType::Response,
5956            FrameType::Error,
5957            FrameType::Push,
5958            FrameType::Cancel,
5959            FrameType::Goodbye,
5960        ] {
5961            let body = if ty.is_pure_header() {
5962                Vec::new()
5963            } else {
5964                br#"{}"#.to_vec()
5965            };
5966            let stale = Frame::build(ty, control_flags(), 7, 8, 41, body).unwrap();
5967            assert!(!ingress_route_is_current(&installed, &stale), "{ty:?}");
5968        }
5969
5970        let uninstalled = Frame::build(
5971            FrameType::Request,
5972            control_flags(),
5973            8,
5974            1,
5975            42,
5976            br#"{}"#.to_vec(),
5977        )
5978        .unwrap();
5979        assert!(!ingress_route_is_current(&installed, &uninstalled));
5980
5981        let current = Frame::build(
5982            FrameType::Request,
5983            control_flags(),
5984            7,
5985            9,
5986            43,
5987            br#"{}"#.to_vec(),
5988        )
5989        .unwrap();
5990        let control = Frame::build(FrameType::Ping, control_flags(), 0, 0, 44, Vec::new()).unwrap();
5991        assert!(ingress_route_is_current(&installed, &current));
5992        assert!(ingress_route_is_current(&installed, &control));
5993        assert_eq!(installed, HashMap::from([(7, 9)]));
5994    }
5995
5996    #[tokio::test]
5997    async fn route_bind_ack_precedes_route_egress_in_writer_queue() {
5998        let (_dir, root) = test_root("route-bind-b2-ordering");
5999        let route = route_key(7, 3);
6000        let identity = RouteIdentity(Arc::new(RouteIdentityData {
6001            root: root.clone(),
6002            project_root: root.as_path().to_path_buf(),
6003            harness: "opencode".to_string(),
6004            session: "b2-session".to_string(),
6005            trust: BindTrust::FirstParty,
6006            spawn_principal: AuthenticatedPrincipal::FirstParty,
6007            consumer_elicitation_capable: false,
6008        }));
6009        let replay_key = push::ReplayKey::from_identity(&identity);
6010        let completion = RouteBindCompletion {
6011            route,
6012            identity,
6013            bind_root_id: root.clone(),
6014            inserted_new_actor: false,
6015            configure_response: Response::success("subc-bind-7", json!({})),
6016            diagnostics_on_edit: false,
6017            ver: PROTOCOL_VERSION,
6018            corr: 91,
6019            flags: control_flags(),
6020        };
6021        let mut pending_binds = HashMap::from([(
6022            route,
6023            PendingBind {
6024                bind_root_id: root,
6025                inserted_new_actor: false,
6026                cancelled: false,
6027                configure_request_id: "subc-bind-7".to_string(),
6028                started_at: Instant::now(),
6029                warned_half_deadline: false,
6030                deadline_reported: false,
6031                corr: 91,
6032                ver: PROTOCOL_VERSION,
6033                flags: control_flags(),
6034                cancellation: crate::executor::JobCancellation::new(),
6035            },
6036        )]);
6037        let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
6038        let mut push_buffer =
6039            HashMap::from([(replay_key, VecDeque::from([completion_frame("b2-replay")]))]);
6040        let (writer_tx, mut writer_rx) = mpsc::channel(8);
6041        let metrics = Arc::new(DispatchPathMetrics::new());
6042
6043        handle_route_bind_completion(
6044            &writer_tx,
6045            completion,
6046            &mut HashMap::new(),
6047            &mut HashMap::new(),
6048            &mut HashMap::new(),
6049            &mut push_buffer,
6050            &mut HashMap::new(),
6051            &mut pending_binds,
6052            &mut installed_route_epochs,
6053            &Arc::new(Executor::new()),
6054            &Arc::new(Notify::new()),
6055            &metrics,
6056        )
6057        .await
6058        .unwrap();
6059
6060        let ack = writer_rx.try_recv().expect("RouteBindAck");
6061        assert_eq!(ack.header.ty, FrameType::Response);
6062        assert_eq!((ack.header.channel, ack.header.epoch), (0, 0));
6063        let route_frame = writer_rx.try_recv().expect("post-ack route frame");
6064        assert_eq!(route_frame.header.ty, FrameType::Push);
6065        assert_eq!(
6066            (route_frame.header.channel, route_frame.header.epoch),
6067            (route.channel, route.epoch)
6068        );
6069    }
6070}