Skip to main content

aft/subc/
mod.rs

1//! subc daemon attach — transport edge.
2//!
3//! When AFT is launched as `aft --subc <connection-file>`, it does NOT run the
4//! standalone NDJSON-over-stdin loop. Instead it connects to a running subc
5//! daemon over loopback TCP, authenticates with the pre-envelope HMAC handshake
6//! (`subc-transport`), then speaks the subc frame protocol (`subc-protocol`):
7//! ModuleHello → HelloAck (register as a tool provider), then a channel-0
8//! control loop (Ping/Pong, RouteBind) plus route-channel tool calls.
9//!
10//! Concurrency: subc routes tool calls through the executor. The tokio
11//! edge never dispatches against `AppContext` inline; per-actor executor lanes
12//! own the reader/mutator epoch, while a writer task serializes outbound frames.
13
14use std::collections::{HashMap, HashSet, VecDeque};
15use std::fmt;
16use std::io;
17use std::net::{IpAddr, SocketAddr};
18use std::ops::Deref;
19use std::path::{Path, PathBuf};
20use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
21use std::sync::{Arc, LazyLock};
22use std::time::{Duration, Instant};
23
24use serde::Deserialize;
25use serde_json::{json, Value};
26
27use crate::config::Config;
28use crate::config_resolve::ConfigTier;
29use crate::context::{App, AppContext, ProgressSender, RootHealthSnapshot};
30use crate::executor::{Executor, Lane};
31use crate::jsonc::strip_jsonc;
32use crate::log_ctx;
33use crate::path_identity::ProjectRootId;
34use crate::protocol::{ProgressKind, PushFrame, RawRequest, Response};
35use crate::run_tool_call::{
36    run_tool_call, strip_agent_preview_arg_owned, PhaseTrace, ToolCallContext, ToolCallOutcome,
37    ToolCallResult,
38};
39use crate::runtime_drain;
40
41use subc_protocol::manifest::{
42    Bindings, Concurrency, ExecutionMode, IdentityBinding, IdentityScope, ModuleManifest,
43    ProviderRole, StorageBinding, StorageKind, StorageScope, Tool, TrustTier,
44};
45use subc_protocol::session::{
46    HealthReport, HealthStatus, ModuleControlRequest, ModuleControlResponse,
47    MODULE_CONTROL_OP_HEALTH_CHECK,
48};
49use subc_protocol::{
50    ErrorBody, Flags, Frame, FrameType, ModuleHelloBody, Principal, Priority, PROTOCOL_VERSION,
51};
52use subc_transport::{authenticate_client, connection_file, read_frame, write_frame};
53use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
54use tokio::net::TcpStream;
55use tokio::sync::{mpsc, oneshot, Notify};
56use tokio::task::JoinHandle;
57
58/// Per-attempt handshake deadline. The initial attach loop has a separate total
59/// budget so a stalled peer cannot consume an unbounded supervisor launch window.
60const AUTH_DEADLINE: Duration = Duration::from_secs(5);
61const ATTACH_RETRY_BUDGET: Duration = Duration::from_secs(60);
62const ATTACH_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(250);
63const ATTACH_RETRY_MAX_BACKOFF: Duration = Duration::from_secs(5);
64const ATTACH_RETRY_JITTER_PERCENT: u64 = 20;
65
66/// Correlation id for the initial ModuleHello (channel 0).
67const HELLO_CORR: u64 = 1;
68
69/// Per-session in-memory replay cap for must-deliver Push frames. This covers
70/// detach/re-attach while AFT stays alive; cross-restart replay is phased later.
71const PUSH_BUFFER_MAX_PER_KEY: usize = 256;
72
73/// Bounded guard for control-frame sends. If the daemon stops reading and the
74/// writer queue stays full, tear the subc edge down instead of stalling the
75/// route loop indefinitely.
76const CONTROL_SEND_TIMEOUT: Duration = Duration::from_millis(250);
77
78/// Cadence for the loop's deadline-driven drain work (retry-buffer flush,
79/// bg-wake emission, maintenance submission). Checked at the top of every
80/// loop turn so busy select arms cannot starve it.
81const DRAIN_TICK_PERIOD: Duration = Duration::from_millis(250);
82
83/// Root-scoped stores and watcher runtimes are reopened lazily after this
84/// period without tool traffic. Keeping the value fixed avoids per-client
85/// eviction policies competing inside the module loop.
86const IDLE_ROOT_TTL: Duration = Duration::from_secs(30 * 60);
87
88const WRITER_QUEUE_CAPACITY: usize = 256;
89
90/// Keep reliable Push bursts from monopolizing the current-thread subc loop;
91/// any remaining must-deliver frames stay queued for the next loop turn.
92const RELIABLE_PUSH_DRAIN_BUDGET: usize = 32;
93
94/// Limit maintenance submissions per tick so background drains cannot delay
95/// control-plane work such as completed RouteBind acknowledgements.
96///
97/// The decomposed maintenance pass charges this budget by Mutating job, not by
98/// root. Size the default burst for one maintenance pass over eight live roots,
99/// while follow-up batches still re-enter the capped queue instead of bypassing
100/// the budget.
101const MAINTENANCE_SUBMIT_BUDGET: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len() * 8;
102const INITIAL_MAINTENANCE_DRAIN_KINDS: [MaintenanceDrainKind; 4] = [
103    MaintenanceDrainKind::Watcher,
104    MaintenanceDrainKind::Lsp,
105    MaintenanceDrainKind::ConfigureTail,
106    MaintenanceDrainKind::CompletionDrains,
107];
108#[cfg(test)]
109const INITIAL_MAINTENANCE_JOB_COUNT: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len();
110
111const RELIABLE_WRITER_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(10);
112const RELIABLE_WRITER_RETRY_MAX_BACKOFF: Duration = Duration::from_millis(250);
113
114const DISPATCH_PATH_BIND_WARN_AFTER: Duration = Duration::from_secs(6);
115const ROUTE_BIND_DEADLINE: Duration = Duration::from_secs(12);
116
117/// Small bounded memory of completed task ids used to suppress stale lossy
118/// long-running reminders that arrive after their reliable completion event.
119const COMPLETED_TASK_SUPPRESSION_MAX: usize = 4096;
120
121/// Bash foreground orchestration polls detached tasks with short read-lane jobs.
122/// The sleep between polls is outside the executor so no read or write worker is
123/// pinned while a foreground command is still running.
124const PENDING_POLL_INTERVAL: Duration = Duration::from_millis(100);
125
126/// Host elicitation asks fail closed if the MCP facade does not answer promptly.
127const BASH_ELICITATION_TIMEOUT: Duration = Duration::from_secs(60);
128const BASH_ELICITATION_CREATE_METHOD: &str = "elicitation/create";
129
130#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
131struct RouteChannel {
132    channel: u16,
133    epoch: u32,
134}
135
136impl fmt::Display for RouteChannel {
137    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
138        write!(f, "{}@{}", self.channel, self.epoch)
139    }
140}
141
142type PushEnvelope = (ProjectRootId, PushFrame);
143type LossyPushEnvelope = (u64, ProjectRootId, PushFrame);
144type RetryBuffer = HashMap<RouteChannel, VecDeque<(push::ReplayKey, PushFrame)>>;
145mod bash;
146mod health;
147mod manifest;
148mod push;
149mod wire;
150
151use self::health::{
152    build_health_report, warn_slow_pending_binds, DispatchPathMetrics, ResponseTaskGuard,
153};
154use self::manifest::{
155    build_manifest, command_lane, control_flags, control_ops, is_bash_family_tool,
156    is_subc_agent_core_tool, is_subc_native_plumbing_tool,
157};
158pub use self::wire::SubcError;
159
160/// Test-only view of the fail-closed tool-call gate: would `name` be admitted
161/// on a bound route (as an agent tool or native plumbing)? Used by the
162/// plugin-send drift guard in `subc_plumbing_drift_test.rs`.
163pub fn is_tool_call_admitted_for_test(name: &str) -> bool {
164    manifest::is_subc_agent_core_tool(name) || manifest::is_subc_native_plumbing_tool(name)
165}
166use self::wire::{
167    build_error_frame, build_goodbye_frame, build_tool_response_frame, decrement_counted_channel,
168    response_is_fatal_panic, response_message, send_counted_channel, send_frame,
169    send_reliable_writer_frame, send_traced_tool_response_frame, ToolResponseWriteTrace,
170    WriterFrame, WriterSender,
171};
172
173struct DecodedFrame {
174    frame: Frame,
175    phase_trace: PhaseTrace,
176}
177
178struct ToolCallCompletion {
179    text: String,
180    phase_trace: PhaseTrace,
181}
182
183#[derive(Clone)]
184struct PushSenders {
185    lossy_tx: mpsc::Sender<LossyPushEnvelope>,
186    reliable_tx: mpsc::UnboundedSender<PushEnvelope>,
187    lossy_overflow: Arc<push::LossyOverflow>,
188    lossy_seq: Arc<AtomicU64>,
189}
190
191#[derive(Clone)]
192struct PersistentCancelSignal {
193    inner: Arc<PersistentCancelInner>,
194}
195
196struct PersistentCancelInner {
197    cancelled: AtomicBool,
198    notify: Notify,
199}
200
201impl PersistentCancelSignal {
202    fn new() -> Self {
203        Self {
204            inner: Arc::new(PersistentCancelInner {
205                cancelled: AtomicBool::new(false),
206                notify: Notify::new(),
207            }),
208        }
209    }
210
211    fn cancel(&self) {
212        if !self.inner.cancelled.swap(true, Ordering::SeqCst) {
213            self.inner.notify.notify_waiters();
214        }
215    }
216
217    fn is_cancelled(&self) -> bool {
218        self.inner.cancelled.load(Ordering::SeqCst)
219    }
220
221    async fn cancelled(&self) {
222        // `enable()` REGISTERS this waiter before we read the flag, closing the
223        // lost-wakeup window: `notify_waiters()` only wakes already-registered
224        // waiters and stores no permit, so without enable() a `cancel()` firing
225        // between the flag read and `.await` would be missed and the future
226        // would park forever (cancel() fires only once). With enable(), a cancel
227        // racing the flag read still wakes the registered waiter. The loop is a
228        // belt-and-suspenders re-check on spurious wakeups.
229        loop {
230            let notified = self.inner.notify.notified();
231            tokio::pin!(notified);
232            notified.as_mut().enable();
233            if self.is_cancelled() {
234                return;
235            }
236            notified.await;
237        }
238    }
239}
240
241#[derive(Debug, Clone, Copy, PartialEq, Eq)]
242pub(crate) enum BindTrust {
243    FirstParty,
244    Untrusted,
245}
246
247impl BindTrust {
248    fn allows_bash_observation(self) -> bool {
249        matches!(self, Self::FirstParty)
250    }
251
252    fn label(self) -> &'static str {
253        match self {
254            Self::FirstParty => "first_party",
255            Self::Untrusted => "untrusted",
256        }
257    }
258}
259
260pub(super) fn trust_for_principal(principal: &Option<Principal>) -> BindTrust {
261    match principal {
262        Some(Principal::Direct) => BindTrust::FirstParty,
263        Some(Principal::Reserved { module_id })
264            if module_id == "llm-runner"
265                || module_id == "aft"
266                || module_id == "broca"
267                || module_id == "alfonso-core" =>
268        {
269            BindTrust::FirstParty
270        }
271        Some(Principal::Reserved { .. }) | Some(Principal::Unverified) | None => {
272            BindTrust::Untrusted
273        }
274    }
275}
276
277fn harness_forces_untrusted(harness: &str) -> bool {
278    harness.starts_with("fed:")
279}
280
281pub(super) fn trust_for_bind(harness: &str, principal: &Option<Principal>) -> BindTrust {
282    if harness_forces_untrusted(harness) {
283        BindTrust::Untrusted
284    } else {
285        trust_for_principal(principal)
286    }
287}
288
289fn principal_label(principal: &Option<Principal>) -> String {
290    match principal {
291        Some(Principal::Direct) => "direct".to_string(),
292        Some(Principal::Reserved { module_id }) => format!("reserved:{module_id}"),
293        Some(Principal::Unverified) => "unverified".to_string(),
294        None => "absent".to_string(),
295    }
296}
297
298#[derive(Debug)]
299/// Per-root route metadata owned by the subc loop. The `active_bash_waits` field
300/// counts detached bash processes that are still being observed for this root.
301/// Any future logic that evicts roots based on idle time must not evict a root
302/// while this count is greater than zero, because a foreground bash response may
303/// still arrive later.
304struct RootMeta {
305    maintenance_pending: bool,
306    maintenance_jobs_in_flight: usize,
307    maintenance_queued_kinds: VecDeque<MaintenanceDrainKind>,
308    maintenance_last_submitted: Option<Instant>,
309    maintenance_poisoned: bool,
310    last_touched: Instant,
311    diagnostics_on_edit: bool,
312    active_bash_waits: usize,
313    idle_artifacts_evicted: bool,
314    unbound_quiesced: bool,
315}
316
317#[derive(Debug)]
318struct PendingBind {
319    bind_root_id: ProjectRootId,
320    inserted_new_actor: bool,
321    cancelled: bool,
322    configure_request_id: String,
323    started_at: Instant,
324    warned_half_deadline: bool,
325    deadline_reported: bool,
326    corr: u64,
327    ver: u8,
328    flags: Flags,
329    /// Exact-job cancellation for the submitted configure: Goodbye and
330    /// deadline expiry cancel the executor job operationally (queued jobs are
331    /// removed, running configures return at their next checkpoint) instead of
332    /// only marking bookkeeping.
333    cancellation: crate::executor::JobCancellation,
334}
335
336struct RouteBindCompletion {
337    route: RouteChannel,
338    identity: RouteIdentity,
339    bind_root_id: ProjectRootId,
340    inserted_new_actor: bool,
341    configure_response: Response,
342    diagnostics_on_edit: bool,
343    ver: u8,
344    corr: u64,
345    flags: Flags,
346}
347
348#[derive(Debug, Clone)]
349struct RouteIdentity(Arc<RouteIdentityData>);
350
351#[derive(Debug)]
352struct RouteIdentityData {
353    root: ProjectRootId,
354    project_root: PathBuf,
355    harness: String,
356    session: String,
357    trust: BindTrust,
358    consumer_elicitation_capable: bool,
359}
360
361impl Deref for RouteIdentity {
362    type Target = RouteIdentityData;
363
364    fn deref(&self) -> &Self::Target {
365        &self.0
366    }
367}
368
369#[derive(Debug, Clone)]
370struct RetainedSessionIdentity {
371    harness: String,
372    trust: BindTrust,
373}
374
375#[derive(Clone, Copy)]
376struct BgSub {
377    corr: u64,
378    ver: u8,
379    flags: Flags,
380}
381
382struct MaintenanceCompletion {
383    root_id: ProjectRootId,
384    kind: MaintenanceDrainKind,
385    response: Response,
386    empty_bg_sessions: Vec<(String, u64)>,
387    requeue_kind: Option<MaintenanceDrainKind>,
388}
389
390#[derive(Clone, Copy, Debug, PartialEq, Eq)]
391enum MaintenanceDrainKind {
392    Watcher,
393    Lsp,
394    ConfigureTail,
395    CompletionDrains,
396}
397
398impl MaintenanceDrainKind {
399    fn label(self) -> &'static str {
400        match self {
401            Self::Watcher => "watcher",
402            Self::Lsp => "lsp",
403            Self::ConfigureTail => "configure-tail",
404            Self::CompletionDrains => "completion-drains",
405        }
406    }
407}
408
409#[derive(Debug, Default)]
410struct MaintenanceJobOutcome {
411    empty_bg_sessions: Vec<(String, u64)>,
412    requeue_kind: Option<MaintenanceDrainKind>,
413}
414
415#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
416struct ReverseCorrKey {
417    route: RouteChannel,
418    corr: u64,
419}
420
421struct PendingBashAsk {
422    route: RouteChannel,
423    tool_corr: u64,
424    tool_flags: Flags,
425    tool_ver: u8,
426    root: ProjectRootId,
427    project_root: PathBuf,
428    session_id: String,
429    request_id: String,
430    arguments: Value,
431    format_context: crate::subc_format::FormatContext,
432    cancel: bash::BashWaitCancel,
433    grants: Vec<String>,
434    expires_at: Instant,
435}
436
437impl RootMeta {
438    fn new(now: Instant) -> Self {
439        Self {
440            maintenance_pending: false,
441            maintenance_jobs_in_flight: 0,
442            maintenance_queued_kinds: VecDeque::new(),
443            maintenance_last_submitted: None,
444            maintenance_poisoned: false,
445            last_touched: now,
446            diagnostics_on_edit: false,
447            active_bash_waits: 0,
448            idle_artifacts_evicted: false,
449            unbound_quiesced: false,
450        }
451    }
452
453    fn note_activity(&mut self) {
454        self.last_touched = Instant::now();
455    }
456
457    fn reactivate_bound(&mut self) {
458        self.note_activity();
459        self.idle_artifacts_evicted = false;
460        self.unbound_quiesced = false;
461    }
462}
463
464fn due_maintenance_jobs(
465    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
466    executor: Option<&Executor>,
467    bg_sub_by_session: &HashMap<(ProjectRootId, String), RouteChannel>,
468    bg_wake_pending: &HashSet<RouteChannel>,
469    budget: usize,
470    pending_bind_roots: &HashSet<ProjectRootId>,
471) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
472    let mut jobs = Vec::new();
473    let mut deferred = false;
474    let mut roots = live_roots.keys().cloned().collect::<Vec<_>>();
475    roots.sort_by(|left, right| {
476        let left_last = live_roots
477            .get(left)
478            .and_then(|meta| meta.maintenance_last_submitted);
479        let right_last = live_roots
480            .get(right)
481            .and_then(|meta| meta.maintenance_last_submitted);
482        left_last
483            .cmp(&right_last)
484            .then_with(|| left.as_path().cmp(right.as_path()))
485    });
486
487    for root_id in roots {
488        let Some(meta) = live_roots.get_mut(&root_id) else {
489            continue;
490        };
491        if meta.maintenance_poisoned {
492            continue;
493        }
494
495        if pending_bind_roots.contains(&root_id) {
496            if meta.maintenance_pending || !meta.maintenance_queued_kinds.is_empty() {
497                deferred = true;
498            }
499            continue;
500        }
501
502        if !meta.maintenance_pending {
503            if jobs.len() >= budget {
504                deferred = true;
505                continue;
506            }
507            // Only enqueue kinds with pending work. Probes are cheap and
508            // fail-open (contended sources count as pending), so an idle root
509            // costs four probes per tick instead of four dispatched jobs.
510            let executor_actor_context =
511                executor.and_then(|executor| executor.actor_context(&root_id));
512            let root_has_pending_bg_wake =
513                bg_sub_by_session.iter().any(|((sub_root, _), channel)| {
514                    sub_root == &root_id && bg_wake_pending.contains(channel)
515                });
516            let kinds_with_work: Vec<MaintenanceDrainKind> = match executor_actor_context {
517                Some(ctx) => INITIAL_MAINTENANCE_DRAIN_KINDS
518                    .into_iter()
519                    .filter(|kind| {
520                        if meta.unbound_quiesced && !matches!(kind, MaintenanceDrainKind::Lsp) {
521                            return false;
522                        }
523                        match kind {
524                            MaintenanceDrainKind::Watcher => ctx.watcher_drain_has_work(),
525                            MaintenanceDrainKind::Lsp => ctx.lsp_drain_has_work(),
526                            MaintenanceDrainKind::ConfigureTail => ctx.configure_tail_has_work(),
527                            // Every CompletionDrains source is visible at this enqueue site:
528                            // AppContext probes completion queues, this loop owns bg wakes,
529                            // and queued continuations bypass probing via maintenance_pending.
530                            // New drain sources must expose a probe here rather than making
531                            // every subscribed root fail open again.
532                            MaintenanceDrainKind::CompletionDrains => {
533                                root_has_pending_bg_wake || ctx.completion_drains_have_work()
534                            }
535                        }
536                    })
537                    .collect(),
538                None if meta.unbound_quiesced => Vec::new(),
539                // No context handle (actor gone mid-tick): enqueue everything.
540                None => INITIAL_MAINTENANCE_DRAIN_KINDS.to_vec(),
541            };
542            if kinds_with_work.is_empty() {
543                continue;
544            }
545            meta.maintenance_pending = true;
546            meta.maintenance_queued_kinds.extend(kinds_with_work);
547        }
548
549        while let Some(kind) = meta.maintenance_queued_kinds.pop_front() {
550            if jobs.len() >= budget {
551                meta.maintenance_queued_kinds.push_front(kind);
552                deferred = true;
553                break;
554            }
555            meta.maintenance_jobs_in_flight += 1;
556            meta.maintenance_last_submitted = Some(Instant::now());
557            jobs.push((root_id.clone(), kind));
558        }
559
560        meta.maintenance_pending =
561            meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
562    }
563
564    (jobs, deferred)
565}
566
567fn eviction_estimate_label(estimate: &crate::memory::MemoryEstimate) -> String {
568    match estimate.estimated_bytes {
569        Some(bytes) => format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
570        None if estimate.status == "busy" => "busy".to_string(),
571        None => "not estimated".to_string(),
572    }
573}
574
575fn optional_memory_label(bytes: Option<u64>) -> String {
576    bytes.map_or_else(
577        || "not estimated".to_string(),
578        |bytes| format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
579    )
580}
581
582fn pressure_relief_label(relief: &crate::memory::AllocatorPressureRelief) -> String {
583    format!(
584        "; allocator pressure relief: RSS {} -> {}, in-use {} -> {}, allocated {} -> {}, slack {} -> {}, allocator reported {:.1} MB released",
585        optional_memory_label(relief.rss_before_bytes),
586        optional_memory_label(relief.rss_after_bytes),
587        optional_memory_label(relief.allocator_before.bytes_in_use),
588        optional_memory_label(relief.allocator_after.bytes_in_use),
589        optional_memory_label(relief.allocator_before.size_allocated),
590        optional_memory_label(relief.allocator_after.size_allocated),
591        optional_memory_label(relief.allocator_before.retained_slack_bytes),
592        optional_memory_label(relief.allocator_after.retained_slack_bytes),
593        relief.bytes_released as f64 / (1024.0 * 1024.0),
594    )
595}
596
597fn idle_root_eviction_message(
598    root_id: &ProjectRootId,
599    memory: &crate::memory::RootMemorySnapshot,
600    pressure_relief: Option<&crate::memory::AllocatorPressureRelief>,
601) -> String {
602    // Bash, LSP, and parser state remain resident. The freed total is deliberately
603    // only the known-byte portion of handles eviction actually drops.
604    let freed_bytes = [
605        &memory.semantic,
606        &memory.trigram,
607        &memory.symbols,
608        &memory.callgraph,
609        &memory.inspect,
610    ]
611    .iter()
612    .filter_map(|estimate| estimate.estimated_bytes)
613    .fold(0u64, u64::saturating_add);
614    let mut message = format!(
615        "evicted idle root {}: freed ~{:.1} MB (semantic {}, trigram {}, symbols {}, callgraph {}, inspect {}; retained: bash {}, lsp {}, parser_pool {})",
616        root_id.as_path().display(),
617        freed_bytes as f64 / (1024.0 * 1024.0),
618        eviction_estimate_label(&memory.semantic),
619        eviction_estimate_label(&memory.trigram),
620        eviction_estimate_label(&memory.symbols),
621        eviction_estimate_label(&memory.callgraph),
622        eviction_estimate_label(&memory.inspect),
623        eviction_estimate_label(&memory.bash),
624        eviction_estimate_label(&memory.lsp),
625        eviction_estimate_label(&memory.parser_pool),
626    );
627    if let Some(pressure_relief) = pressure_relief {
628        message.push_str(&pressure_relief_label(pressure_relief));
629    }
630    message
631}
632
633fn process_has_been_idle(now: Instant, live_roots: &HashMap<ProjectRootId, RootMeta>) -> bool {
634    !live_roots.is_empty()
635        && live_roots.values().all(|meta| {
636            now.saturating_duration_since(meta.last_touched) >= IDLE_ROOT_TTL
637                && meta.active_bash_waits == 0
638                && !meta.maintenance_pending
639                && meta.maintenance_queued_kinds.is_empty()
640        })
641}
642
643fn allocator_pressure_relief_after_idle_sweep(
644    now: Instant,
645    live_roots: &HashMap<ProjectRootId, RootMeta>,
646    executor: &Executor,
647) -> Option<crate::memory::AllocatorPressureRelief> {
648    if !process_has_been_idle(now, live_roots)
649        || live_roots.keys().any(|root_id| {
650            executor
651                .actor_context(root_id)
652                .is_some_and(|ctx| ctx.artifact_eviction_blocked())
653        })
654    {
655        return None;
656    }
657
658    #[cfg(target_os = "macos")]
659    {
660        Some(crate::memory::relieve_allocator_pressure())
661    }
662    #[cfg(not(target_os = "macos"))]
663    {
664        None
665    }
666}
667
668fn quiesce_unbound_root(
669    root_id: &ProjectRootId,
670    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
671    executor: &Arc<Executor>,
672) {
673    let Some(meta) = live_roots.get_mut(root_id) else {
674        return;
675    };
676
677    let ctx = executor.actor_context(root_id);
678    if let Some(ctx) = ctx.as_ref() {
679        // Close lifecycle admission before touching scheduler queues. A running
680        // ConfigureTail cannot release gates, install a watcher, or reserve a
681        // callgraph build after this transition becomes visible.
682        ctx.mark_subc_unbound();
683    }
684    let cancelled = executor.cancel_queued_maintenance(root_id);
685    // Transient unbind keeps the root WARM: the watcher stays running (its
686    // events accumulate and replay on rebind, so no unobserved gap exists) and
687    // resident artifacts stay resident. Host restarts unbind every root and
688    // rebind seconds later; stopping the watcher here would force strict
689    // re-verification plus a full callgraph rebuild on every restart. The
690    // expensive teardown (watcher stop + gap invalidation) belongs to the
691    // idle-TTL reaper and the root-deleted path.
692    let discarded = ctx
693        .map(|ctx| crate::commands::configure::cancel_deferred_configure_maintenance(&ctx))
694        .unwrap_or(0);
695    meta.unbound_quiesced = true;
696    meta.maintenance_queued_kinds.clear();
697    meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
698    log::debug!(
699        "subc attach: quiesced unbound root {} (cancelled {} queued maintenance job(s), cancelled {} configure maintenance job(s))",
700        root_id.as_path().display(),
701        cancelled,
702        discarded
703    );
704}
705
706fn reap_idle_roots(
707    now: Instant,
708    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
709    pending_binds: &HashMap<RouteChannel, PendingBind>,
710    executor: &Arc<Executor>,
711) -> usize {
712    let pending_bind_roots = pending_binds
713        .values()
714        .map(|pending| pending.bind_root_id.clone())
715        .collect::<HashSet<_>>();
716    let candidates = live_roots
717        .iter()
718        .filter_map(|(root_id, meta)| {
719            // The TTL applies to unbound roots too: a transient unbind (host
720            // restart) keeps the root warm — watcher running, artifacts
721            // resident — and rebinding within the TTL costs nothing. Only a
722            // root that stays unbound past the TTL pays the full teardown.
723            if meta.idle_artifacts_evicted
724                || now.saturating_duration_since(meta.last_touched) < IDLE_ROOT_TTL
725                || meta.active_bash_waits > 0
726                || meta.maintenance_pending
727                || !meta.maintenance_queued_kinds.is_empty()
728                || pending_bind_roots.contains(root_id)
729            {
730                return None;
731            }
732            Some(root_id.clone())
733        })
734        .collect::<Vec<_>>();
735
736    let mut reaped = Vec::new();
737    for root_id in candidates {
738        let Some(ctx) = executor.actor_context(&root_id) else {
739            continue;
740        };
741        // A TTL-aged unbound root retained its watcher-derived pending paths
742        // across the transient-unbind window (they repair artifacts a
743        // pre-unbind worker persisted). The strict gap invalidation below
744        // subsumes their purpose, and leaving them would block eviction
745        // forever through `artifact_eviction_blocked`. Disposal is
746        // TRANSACTIONAL: a secondary blocker (running bash, in-flight build)
747        // can still abort the eviction below, and the root may rebind before
748        // the next reap attempt — the taken paths are the only repair record
749        // for already-consumed watcher events, so they are restored on every
750        // abort path and dropped only after eviction succeeds.
751        let taken_pending = ctx
752            .subc_unbound_quiesced()
753            .then(|| ctx.take_pending_reconciliation_state());
754        if ctx.artifact_eviction_blocked() {
755            if let Some(pending) = taken_pending {
756                ctx.restore_pending_reconciliation_state(pending);
757            }
758            continue;
759        }
760        let memory_before = ctx.memory_root_snapshot();
761        if !ctx.evict_idle_artifacts() {
762            if let Some(pending) = taken_pending {
763                ctx.restore_pending_reconciliation_state(pending);
764            }
765            continue;
766        }
767        drop(taken_pending);
768        // The watcher backend owns the OS watcher and can block while joining
769        // on macOS. Request shutdown here, but let a dedicated reaper thread
770        // perform the join rather than holding up an executor maintenance lane.
771        ctx.stop_watcher_runtime_in_background();
772        // The watcher is now stopped: edits during the idle interval go
773        // unobserved, so a later warm reload must not trust the verify memo
774        // (same-size, preserved-mtime edits would pass stat-first) and the
775        // callgraph store must reconcile instead of reopening as fresh.
776        ctx.invalidate_artifacts_after_watcher_gap();
777        if let Some(meta) = live_roots.get_mut(&root_id) {
778            meta.idle_artifacts_evicted = true;
779        }
780        reaped.push((root_id, memory_before));
781    }
782
783    let pressure_relief = (!reaped.is_empty())
784        .then(|| allocator_pressure_relief_after_idle_sweep(now, live_roots, executor))
785        .flatten();
786    for (root_id, memory_before) in &reaped {
787        log::info!(
788            "{}",
789            idle_root_eviction_message(root_id, memory_before, pressure_relief.as_ref())
790        );
791    }
792    reaped.len()
793}
794
795#[allow(clippy::too_many_arguments)]
796fn submit_due_maintenance_jobs(
797    executor: &Arc<Executor>,
798    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
799    pending_binds: &HashMap<RouteChannel, PendingBind>,
800    bg_sub_by_session: &HashMap<(ProjectRootId, String), RouteChannel>,
801    bg_wake_pending: &HashSet<RouteChannel>,
802    bg_wake_epoch: &HashMap<(ProjectRootId, String), u64>,
803    maintenance_tx: &mpsc::Sender<MaintenanceCompletion>,
804    metrics: &Arc<DispatchPathMetrics>,
805) {
806    let pending_bind_roots = pending_binds
807        .values()
808        .map(|pending| pending.bind_root_id.clone())
809        .collect::<HashSet<_>>();
810    let (due_jobs, deferred_jobs) = due_maintenance_jobs(
811        live_roots,
812        Some(executor),
813        bg_sub_by_session,
814        bg_wake_pending,
815        MAINTENANCE_SUBMIT_BUDGET,
816        &pending_bind_roots,
817    );
818    if deferred_jobs {
819        metrics
820            .maintenance_budget_deferrals
821            .fetch_add(1, Ordering::Relaxed);
822    }
823    for (root_id, kind) in due_jobs {
824        let bg_sessions_to_check = if kind == MaintenanceDrainKind::CompletionDrains {
825            bg_sub_by_session
826                .iter()
827                .filter_map(|((root, session), _)| {
828                    if root == &root_id {
829                        Some((
830                            session.clone(),
831                            bg_wake_epoch
832                                .get(&(root_id.clone(), session.clone()))
833                                .copied()
834                                .unwrap_or(0),
835                        ))
836                    } else {
837                        None
838                    }
839                })
840                .collect()
841        } else {
842            Vec::new()
843        };
844        submit_maintenance_job(
845            executor,
846            root_id,
847            kind,
848            bg_sessions_to_check,
849            maintenance_tx,
850            metrics,
851        );
852    }
853}
854
855fn should_requiesce_after_maintenance(
856    meta: &RootMeta,
857    completed_kind: MaintenanceDrainKind,
858    bind_pending: bool,
859) -> bool {
860    meta.unbound_quiesced && completed_kind != MaintenanceDrainKind::Lsp && !bind_pending
861}
862
863fn note_maintenance_completion(
864    meta: &mut RootMeta,
865    requeue_kind: Option<MaintenanceDrainKind>,
866    fatal: bool,
867    defer_requeue: bool,
868) {
869    if fatal {
870        meta.maintenance_poisoned = true;
871    }
872
873    if let Some(kind) = requeue_kind.filter(|_| !meta.maintenance_poisoned && !defer_requeue) {
874        meta.maintenance_queued_kinds.push_back(kind);
875    }
876
877    meta.maintenance_jobs_in_flight = meta.maintenance_jobs_in_flight.saturating_sub(1);
878    meta.maintenance_pending =
879        meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
880}
881
882fn route_key(channel: u16, epoch: u32) -> RouteChannel {
883    RouteChannel { channel, epoch }
884}
885
886fn remove_installed_route(installed_epochs: &mut HashMap<u16, u32>, route: RouteChannel) {
887    if installed_epochs.get(&route.channel).copied() == Some(route.epoch) {
888        installed_epochs.remove(&route.channel);
889    }
890}
891
892fn ingress_route_is_current(installed_epochs: &HashMap<u16, u32>, frame: &Frame) -> bool {
893    frame.header.channel == 0
894        || installed_epochs.get(&frame.header.channel).copied() == Some(frame.header.epoch)
895}
896
897fn bash_elicitation_timeout() -> Duration {
898    if cfg!(debug_assertions) {
899        if let Ok(raw) = std::env::var("AFT_TEST_SUBC_BASH_ELICITATION_TTL_MS") {
900            if let Ok(ms) = raw.parse::<u64>() {
901                if ms > 0 {
902                    return Duration::from_millis(ms);
903                }
904            }
905        }
906    }
907    BASH_ELICITATION_TIMEOUT
908}
909
910fn allocate_reverse_corr(
911    pending_bash_asks: &HashMap<ReverseCorrKey, PendingBashAsk>,
912    route: RouteChannel,
913    next_corr: &mut u64,
914) -> u64 {
915    loop {
916        let corr = *next_corr;
917        *next_corr = (*next_corr).wrapping_add(1).max(1);
918        if !pending_bash_asks.contains_key(&ReverseCorrKey { route, corr }) {
919            return corr;
920        }
921    }
922}
923
924fn bash_permission_kind_label(kind: &crate::bash_permissions::PermissionKind) -> &'static str {
925    match kind {
926        crate::bash_permissions::PermissionKind::ExternalDirectory => "external directory",
927        crate::bash_permissions::PermissionKind::Bash => "bash",
928    }
929}
930
931fn bash_elicitation_patterns(asks: &[crate::bash_permissions::PermissionAsk]) -> Vec<String> {
932    let mut patterns = Vec::new();
933    let mut seen = HashSet::new();
934    for ask in asks {
935        for pattern in ask.patterns.iter().chain(ask.always.iter()) {
936            if seen.insert(pattern.clone()) {
937                patterns.push(pattern.clone());
938            }
939        }
940    }
941    patterns
942}
943
944fn bash_elicitation_message(
945    command: &str,
946    asks: &[crate::bash_permissions::PermissionAsk],
947) -> String {
948    let command = command.split_whitespace().collect::<Vec<_>>().join(" ");
949    let patterns = bash_elicitation_patterns(asks);
950    let pattern_text = if patterns.is_empty() {
951        "no matched permission patterns".to_string()
952    } else {
953        patterns.join(", ")
954    };
955    let ask_kinds = asks
956        .iter()
957        .map(|ask| bash_permission_kind_label(&ask.kind))
958        .collect::<HashSet<_>>()
959        .into_iter()
960        .collect::<Vec<_>>()
961        .join(", ");
962    if ask_kinds.is_empty() {
963        format!("Allow bash command `{command}`? Matched patterns: {pattern_text}")
964    } else {
965        format!("Allow bash command `{command}`? Matched {ask_kinds} patterns: {pattern_text}")
966    }
967}
968
969fn bash_elicitation_request_body(
970    command: &str,
971    asks: &[crate::bash_permissions::PermissionAsk],
972) -> Value {
973    json!({
974        "method": BASH_ELICITATION_CREATE_METHOD,
975        "params": {
976            "mode": "form",
977            "message": bash_elicitation_message(command, asks),
978            "requestedSchema": {
979                "type": "object",
980                "properties": {
981                    "decision": {
982                        "type": "string",
983                        "enum": ["allow", "deny"],
984                        "description": "Choose allow to run this bash command once, or deny to block it."
985                    }
986                },
987                "required": ["decision"],
988                "additionalProperties": false
989            },
990            "_meta": {
991                "aft": {
992                    "tool": "bash",
993                    "command": command,
994                    "asks": asks
995                }
996            }
997        }
998    })
999}
1000
1001fn build_bash_elicitation_request_frame(
1002    ver: u8,
1003    route: RouteChannel,
1004    corr: u64,
1005    flags: Flags,
1006    command: &str,
1007    asks: &[crate::bash_permissions::PermissionAsk],
1008) -> Result<Frame, SubcError> {
1009    let body = bash_elicitation_request_body(command, asks);
1010    Frame::build_with_version(
1011        ver,
1012        FrameType::Request,
1013        flags,
1014        route.channel,
1015        route.epoch,
1016        corr,
1017        serde_json::to_vec(&body).map_err(SubcError::Json)?,
1018    )
1019    .map_err(SubcError::FrameBuild)
1020}
1021
1022fn bash_elicitation_reply_is_allow(body: &[u8]) -> bool {
1023    let Ok(value) = serde_json::from_slice::<Value>(body) else {
1024        return false;
1025    };
1026    flat_bash_elicitation_reply_is_allow(&value) || mcp_bash_elicitation_reply_is_allow(&value)
1027}
1028
1029fn flat_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1030    let Some(object) = value.as_object() else {
1031        return false;
1032    };
1033    object.len() == 1 && object.get("decision").and_then(Value::as_str) == Some("allow")
1034}
1035
1036fn mcp_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1037    let Some(object) = value.as_object() else {
1038        return false;
1039    };
1040    if object.len() != 2 || object.get("action").and_then(Value::as_str) != Some("accept") {
1041        return false;
1042    }
1043    let Some(content) = object.get("content").and_then(Value::as_object) else {
1044        return false;
1045    };
1046    content.len() == 1 && content.get("decision").and_then(Value::as_str) == Some("allow")
1047}
1048
1049#[allow(clippy::too_many_arguments)]
1050async fn settle_pending_bash_ask_denied(
1051    tx: &WriterSender,
1052    pending: PendingBashAsk,
1053    routes: &HashMap<RouteChannel, RouteIdentity>,
1054    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1055    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1056    shutdown: &Arc<Notify>,
1057    metrics: &DispatchPathMetrics,
1058) -> Result<(), SubcError> {
1059    let completion = bash::bash_denied_untrusted_completion(
1060        pending.route,
1061        pending.tool_corr,
1062        pending.tool_flags,
1063        pending.tool_ver,
1064        pending.root,
1065        pending.request_id,
1066        pending.format_context,
1067    );
1068    bash::handle_bash_deferred_completion(
1069        tx,
1070        completion,
1071        routes,
1072        live_roots,
1073        route_bash_cancels,
1074        shutdown,
1075        metrics,
1076    )
1077    .await
1078}
1079
1080fn take_pending_bash_asks_for_route(
1081    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1082    route: RouteChannel,
1083) -> Vec<PendingBashAsk> {
1084    let keys = pending_bash_asks
1085        .keys()
1086        .copied()
1087        .filter(|key| key.route == route)
1088        .collect::<Vec<_>>();
1089    keys.into_iter()
1090        .filter_map(|key| pending_bash_asks.remove(&key))
1091        .collect()
1092}
1093
1094#[allow(clippy::too_many_arguments)]
1095async fn settle_pending_bash_asks_for_route(
1096    tx: &WriterSender,
1097    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1098    route: RouteChannel,
1099    routes: &HashMap<RouteChannel, RouteIdentity>,
1100    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1101    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1102    shutdown: &Arc<Notify>,
1103    metrics: &DispatchPathMetrics,
1104) -> Result<(), SubcError> {
1105    for pending in take_pending_bash_asks_for_route(pending_bash_asks, route) {
1106        settle_pending_bash_ask_denied(
1107            tx,
1108            pending,
1109            routes,
1110            live_roots,
1111            route_bash_cancels,
1112            shutdown,
1113            metrics,
1114        )
1115        .await?;
1116    }
1117    Ok(())
1118}
1119
1120#[allow(clippy::too_many_arguments)]
1121async fn settle_all_pending_bash_asks(
1122    tx: &WriterSender,
1123    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1124    routes: &HashMap<RouteChannel, RouteIdentity>,
1125    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1126    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1127    shutdown: &Arc<Notify>,
1128    metrics: &DispatchPathMetrics,
1129) -> Result<(), SubcError> {
1130    let pending = pending_bash_asks
1131        .drain()
1132        .map(|(_, pending)| pending)
1133        .collect::<Vec<_>>();
1134    for pending in pending {
1135        settle_pending_bash_ask_denied(
1136            tx,
1137            pending,
1138            routes,
1139            live_roots,
1140            route_bash_cancels,
1141            shutdown,
1142            metrics,
1143        )
1144        .await?;
1145    }
1146    Ok(())
1147}
1148
1149#[allow(clippy::too_many_arguments)]
1150async fn expire_pending_bash_asks(
1151    tx: &WriterSender,
1152    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1153    routes: &HashMap<RouteChannel, RouteIdentity>,
1154    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1155    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1156    shutdown: &Arc<Notify>,
1157    metrics: &DispatchPathMetrics,
1158) -> Result<(), SubcError> {
1159    let now = Instant::now();
1160    let expired = pending_bash_asks
1161        .iter()
1162        .filter_map(|(key, pending)| (pending.expires_at <= now).then_some(*key))
1163        .collect::<Vec<_>>();
1164    for key in expired {
1165        if let Some(pending) = pending_bash_asks.remove(&key) {
1166            log::debug!(
1167                "subc attach: bash elicitation request {} on route {} expired fail-closed",
1168                key.corr,
1169                pending.route
1170            );
1171            settle_pending_bash_ask_denied(
1172                tx,
1173                pending,
1174                routes,
1175                live_roots,
1176                route_bash_cancels,
1177                shutdown,
1178                metrics,
1179            )
1180            .await?;
1181        }
1182    }
1183    Ok(())
1184}
1185
1186#[allow(clippy::too_many_arguments)]
1187async fn handle_bash_elicitation_reply(
1188    tx: &WriterSender,
1189    frame: &Frame,
1190    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1191    routes: &HashMap<RouteChannel, RouteIdentity>,
1192    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1193    executor: &Arc<Executor>,
1194    shutdown: &Arc<Notify>,
1195    bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
1196    bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
1197    metrics: &Arc<DispatchPathMetrics>,
1198    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1199    dispatch: DispatchFn,
1200) -> Result<(), SubcError> {
1201    let key = ReverseCorrKey {
1202        route: route_key(frame.header.channel, frame.header.epoch),
1203        corr: frame.header.corr,
1204    };
1205    let Some(pending) = pending_bash_asks.remove(&key) else {
1206        return Ok(());
1207    };
1208
1209    if frame.header.ty == FrameType::Response && bash_elicitation_reply_is_allow(&frame.body) {
1210        if routes.contains_key(&key.route) {
1211            bash::submit_deferred_bash(
1212                executor,
1213                bash_deferred_tx,
1214                bash_poll_touch_tx,
1215                metrics,
1216                dispatch,
1217                pending.root,
1218                pending.project_root,
1219                pending.session_id,
1220                pending.request_id,
1221                pending.route,
1222                pending.tool_corr,
1223                pending.tool_flags,
1224                pending.tool_ver,
1225                pending.arguments,
1226                pending.format_context,
1227                pending.cancel,
1228                BindTrust::Untrusted,
1229                Some(pending.grants),
1230            );
1231            return Ok(());
1232        }
1233        log::debug!(
1234            "subc attach: dropping allowed bash elicitation reply {} for unbound route {}",
1235            key.corr,
1236            pending.route
1237        );
1238    }
1239
1240    settle_pending_bash_ask_denied(
1241        tx,
1242        pending,
1243        routes,
1244        live_roots,
1245        route_bash_cancels,
1246        shutdown,
1247        metrics,
1248    )
1249    .await
1250}
1251
1252#[allow(clippy::too_many_arguments)]
1253async fn cancel_pending_bash_ask_for_tool_call(
1254    tx: &WriterSender,
1255    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1256    route: RouteChannel,
1257    tool_corr: u64,
1258    routes: &HashMap<RouteChannel, RouteIdentity>,
1259    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1260    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1261    shutdown: &Arc<Notify>,
1262    metrics: &DispatchPathMetrics,
1263) -> Result<(), SubcError> {
1264    let keys = pending_bash_asks
1265        .iter()
1266        .filter_map(|(key, pending)| {
1267            (key.route == route && pending.tool_corr == tool_corr).then_some(*key)
1268        })
1269        .collect::<Vec<_>>();
1270    for key in keys {
1271        if let Some(pending) = pending_bash_asks.remove(&key) {
1272            settle_pending_bash_ask_denied(
1273                tx,
1274                pending,
1275                routes,
1276                live_roots,
1277                route_bash_cancels,
1278                shutdown,
1279                metrics,
1280            )
1281            .await?;
1282        }
1283    }
1284    Ok(())
1285}
1286
1287fn remove_root_channel(
1288    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1289    root: &ProjectRootId,
1290    channel: RouteChannel,
1291) {
1292    let remove_root = if let Some(channels) = root_channels.get_mut(root) {
1293        channels.remove(&channel);
1294        channels.is_empty()
1295    } else {
1296        false
1297    };
1298    if remove_root {
1299        root_channels.remove(root);
1300    }
1301}
1302
1303fn remove_route_channel(
1304    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1305    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1306    channel: RouteChannel,
1307) -> Option<RouteIdentity> {
1308    let removed = routes.remove(&channel);
1309    if let Some(identity) = &removed {
1310        remove_root_channel(root_channels, &identity.root, channel);
1311    }
1312    removed
1313}
1314
1315fn insert_route_channel(
1316    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1317    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1318    channel: RouteChannel,
1319    identity: RouteIdentity,
1320) {
1321    if let Some(previous) = routes.insert(channel, identity.clone()) {
1322        remove_root_channel(root_channels, &previous.root, channel);
1323    }
1324    root_channels
1325        .entry(identity.root.clone())
1326        .or_default()
1327        .insert(channel);
1328}
1329
1330fn remove_bg_subscription_index(
1331    bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1332    channel: RouteChannel,
1333    identity: Option<&RouteIdentity>,
1334) {
1335    if let Some(identity) = identity {
1336        let key = (identity.root.clone(), identity.session.clone());
1337        if bg_sub_by_session.get(&key).copied() == Some(channel) {
1338            bg_sub_by_session.remove(&key);
1339        }
1340    } else {
1341        bg_sub_by_session.retain(|_, mapped_channel| *mapped_channel != channel);
1342    }
1343}
1344
1345fn route_removal_will_quiesce_root(
1346    root: &ProjectRootId,
1347    route: RouteChannel,
1348    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1349    has_pending_bind: bool,
1350    replacement_root: Option<&ProjectRootId>,
1351) -> bool {
1352    let removes_last_route = root_channels
1353        .get(root)
1354        .is_some_and(|channels| channels.len() == 1 && channels.contains(&route));
1355    removes_last_route && !has_pending_bind && replacement_root != Some(root)
1356}
1357
1358fn should_quiesce_removed_root(
1359    root: &ProjectRootId,
1360    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1361    has_pending_bind: bool,
1362    replacement_root: Option<&ProjectRootId>,
1363) -> bool {
1364    !root_channels.contains_key(root) && !has_pending_bind && replacement_root != Some(root)
1365}
1366
1367async fn end_bg_subscription(
1368    writer_tx: &WriterSender,
1369    metrics: &DispatchPathMetrics,
1370    bg_subs: &mut HashMap<RouteChannel, BgSub>,
1371    bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1372    bg_wake_pending: &mut HashSet<RouteChannel>,
1373    channel: RouteChannel,
1374    identity: Option<&RouteIdentity>,
1375) -> Result<(), SubcError> {
1376    if let Some(sub) = bg_subs.remove(&channel) {
1377        bg_wake_pending.remove(&channel);
1378        remove_bg_subscription_index(bg_sub_by_session, channel, identity);
1379        push::send_reliable_bg_stream_end(writer_tx, metrics, channel, &sub).await?;
1380    }
1381    Ok(())
1382}
1383
1384#[allow(clippy::too_many_arguments)]
1385async fn teardown_installed_route(
1386    tx: &WriterSender,
1387    metrics: &DispatchPathMetrics,
1388    executor: &Arc<Executor>,
1389    channel: RouteChannel,
1390    cancellation_reason: &str,
1391    replacement_root: Option<&ProjectRootId>,
1392    installed_route_epochs: &mut HashMap<u16, u32>,
1393    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1394    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1395    bg_subs: &mut HashMap<RouteChannel, BgSub>,
1396    bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1397    bg_wake_pending: &mut HashSet<RouteChannel>,
1398    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1399    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1400    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1401    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
1402    retry_buffer: &mut RetryBuffer,
1403    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1404    shutdown: &Arc<Notify>,
1405) -> Result<(), SubcError> {
1406    remove_installed_route(installed_route_epochs, channel);
1407    end_bg_subscription(
1408        tx,
1409        metrics,
1410        bg_subs,
1411        bg_sub_by_session,
1412        bg_wake_pending,
1413        channel,
1414        routes.get(&channel),
1415    )
1416    .await?;
1417    settle_pending_bash_asks_for_route(
1418        tx,
1419        pending_bash_asks,
1420        channel,
1421        routes,
1422        live_roots,
1423        route_bash_cancels,
1424        shutdown,
1425        metrics,
1426    )
1427    .await?;
1428    if let Some(cancel) = route_bash_cancels.remove(&channel) {
1429        cancel.token.cancel();
1430    }
1431    if let Some(pending) = pending_binds.get_mut(&channel) {
1432        pending.cancelled = true;
1433        let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
1434        log::debug!(
1435            "subc attach: cancelled pending RouteBind for route {} on {cancellation_reason} (configure job: {outcome:?})",
1436            channel.channel
1437        );
1438    }
1439    let migrated = push::migrate_retry_buffer_to_push_buffer(retry_buffer, channel, push_buffer);
1440    if let Some(identity) = routes.get(&channel) {
1441        let has_pending_bind = pending_binds
1442            .values()
1443            .any(|pending| pending.bind_root_id == identity.root);
1444        if route_removal_will_quiesce_root(
1445            &identity.root,
1446            channel,
1447            root_channels,
1448            has_pending_bind,
1449            replacement_root,
1450        ) {
1451            if let Some(ctx) = executor.actor_context(&identity.root) {
1452                // Fence deferred admissions before the final route disappears
1453                // from the loop-owned routing tables.
1454                ctx.mark_subc_unbound();
1455            }
1456        }
1457    }
1458    if let Some(identity) = remove_route_channel(routes, root_channels, channel) {
1459        if migrated > 0 {
1460            log::debug!(
1461                "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from route {} into detach replay",
1462                channel.channel
1463            );
1464        }
1465        if let Some(meta) = live_roots.get_mut(&identity.root) {
1466            let idle_for = meta.last_touched.elapsed();
1467            meta.note_activity();
1468            log::debug!(
1469                "subc attach: route {} torn down for root {} harness {} session {} (last touched {:?} ago)",
1470                channel.channel,
1471                identity.root.as_path().display(),
1472                identity.harness,
1473                identity.session,
1474                idle_for
1475            );
1476        } else {
1477            log::debug!(
1478                "subc attach: route {} torn down for root {} harness {} session {}",
1479                channel.channel,
1480                identity.root.as_path().display(),
1481                identity.harness,
1482                identity.session
1483            );
1484        }
1485        let has_pending_bind = pending_binds
1486            .values()
1487            .any(|pending| pending.bind_root_id == identity.root);
1488        if should_quiesce_removed_root(
1489            &identity.root,
1490            root_channels,
1491            has_pending_bind,
1492            replacement_root,
1493        ) {
1494            quiesce_unbound_root(&identity.root, live_roots, executor);
1495        }
1496    } else {
1497        if migrated > 0 {
1498            log::debug!(
1499                "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from unbound route {} into detach replay",
1500                channel.channel
1501            );
1502        }
1503        log::debug!("subc attach: unbound route {} torn down", channel.channel);
1504    }
1505    Ok(())
1506}
1507
1508fn remember_session_identity(
1509    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1510    identity: &RouteIdentity,
1511) {
1512    let key = (identity.root.clone(), identity.session.clone());
1513    if matches!(identity.trust, BindTrust::Untrusted)
1514        && session_identity
1515            .get(&key)
1516            .is_some_and(|retained| matches!(retained.trust, BindTrust::FirstParty))
1517    {
1518        return;
1519    }
1520
1521    // Retained after route Goodbye so reliable session-scoped frames emitted while
1522    // the session is detached can still be keyed by the full (root,harness,session)
1523    // replay triple. Untrusted binds never overwrite a retained first-party
1524    // session identity, because bash completion replay is an observation channel.
1525    session_identity.insert(
1526        key,
1527        RetainedSessionIdentity {
1528            harness: identity.harness.clone(),
1529            trust: identity.trust,
1530        },
1531    );
1532}
1533
1534fn replay_key_for_session(
1535    session_identity: &HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1536    root: &ProjectRootId,
1537    session: &str,
1538) -> Option<(push::ReplayKey, BindTrust)> {
1539    let retained = session_identity.get(&(root.clone(), session.to_string()))?;
1540    Some((
1541        push::ReplayKey {
1542            root: root.clone(),
1543            harness: retained.harness.clone(),
1544            session: session.to_string(),
1545        },
1546        retained.trust,
1547    ))
1548}
1549/// Sync command dispatch, passed in from `main` (the binary owns the command
1550/// table). Invoked only inside executor jobs in subc mode.
1551pub type DispatchFn = fn(RawRequest, &AppContext) -> Response;
1552
1553#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1554enum ModuleLoopExit {
1555    Graceful,
1556    SkipSearchFlush,
1557}
1558
1559/// Entry point for `aft --subc <connection-file>`. Synchronous on the outside;
1560/// owns an isolated current-thread tokio runtime for the async transport.
1561/// Returns `Err` (fail-loud) on any connect/auth/protocol failure — we never
1562/// fall back to the standalone loop, to avoid split-brain index state.
1563pub fn run_subc_mode(
1564    connection_file_path: &Path,
1565    ctx: Arc<AppContext>,
1566    executor: Arc<Executor>,
1567    dispatch: DispatchFn,
1568    user_config_path: Option<PathBuf>,
1569) -> Result<(), SubcError> {
1570    // Production NEVER allows non-manifest tool names on route channels: AFT
1571    // fails closed and does not trust subc to enforce the manifest. The
1572    // test-only harness sets this through `run_subc_mode_for_test`.
1573    run_subc_mode_inner(
1574        connection_file_path,
1575        ctx,
1576        executor,
1577        dispatch,
1578        user_config_path,
1579        false,
1580    )
1581}
1582
1583fn run_subc_mode_inner(
1584    connection_file_path: &Path,
1585    ctx: Arc<AppContext>,
1586    executor: Arc<Executor>,
1587    dispatch: DispatchFn,
1588    user_config_path: Option<PathBuf>,
1589    allow_native_passthrough: bool,
1590) -> Result<(), SubcError> {
1591    let runtime = tokio::runtime::Builder::new_current_thread()
1592        .enable_all()
1593        .build()
1594        .map_err(SubcError::Runtime)?;
1595
1596    let executor_for_loop = Arc::clone(&executor);
1597    let loop_result = runtime.block_on(async move {
1598        let shared_app = ctx.app();
1599        drop(ctx);
1600        let stream = connect_and_authenticate(connection_file_path).await?;
1601        log::info!(
1602            "subc attach: authenticated to daemon via {}",
1603            connection_file_path.display()
1604        );
1605        let (read_half, write_half) = tokio::io::split(stream);
1606        run_module_loop(
1607            read_half,
1608            write_half,
1609            shared_app,
1610            executor_for_loop,
1611            dispatch,
1612            user_config_path,
1613            allow_native_passthrough,
1614        )
1615        .await
1616    });
1617
1618    let actor_contexts = executor.actor_contexts();
1619    if matches!(loop_result, Ok(ModuleLoopExit::Graceful)) {
1620        // EOF/Goodbye teardown flushes each root's index deltas and queued
1621        // callgraph refreshes. Fatal/panic teardown skips this best-effort work.
1622        flush_actor_indexes_on_graceful_shutdown(&actor_contexts);
1623    }
1624    for actor_ctx in &actor_contexts {
1625        actor_ctx.lsp().shutdown_all();
1626        actor_ctx.bash_background().detach();
1627    }
1628
1629    loop_result.map(|_| ())
1630}
1631
1632fn flush_actor_indexes_on_graceful_shutdown(actor_contexts: &[Arc<AppContext>]) {
1633    for actor_ctx in actor_contexts {
1634        let _ = actor_ctx.flush_search_index_on_graceful_shutdown();
1635    }
1636    let _ = crate::callgraph_store::flush_callgraph_store_refreshes_on_graceful_shutdown();
1637}
1638
1639/// Test-only entry that enables the non-manifest native-command passthrough on
1640/// route channels. Integration tests drive synthetic native commands (`glob`,
1641/// `callers`, `subc_test_echo_session`, …) through the executor to exercise
1642/// mechanics; production callers use [`run_subc_mode`], which fails closed.
1643#[doc(hidden)]
1644pub fn run_subc_mode_for_test(
1645    connection_file_path: &Path,
1646    ctx: Arc<AppContext>,
1647    executor: Arc<Executor>,
1648    dispatch: DispatchFn,
1649    user_config_path: Option<PathBuf>,
1650) -> Result<(), SubcError> {
1651    run_subc_mode_inner(
1652        connection_file_path,
1653        ctx,
1654        executor,
1655        dispatch,
1656        user_config_path,
1657        true,
1658    )
1659}
1660
1661#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1662enum AttachErrorClass {
1663    Transient,
1664    Permanent,
1665}
1666
1667impl fmt::Display for AttachErrorClass {
1668    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1669        match self {
1670            Self::Transient => f.write_str("transient"),
1671            Self::Permanent => f.write_str("permanent"),
1672        }
1673    }
1674}
1675
1676#[derive(Clone, Copy)]
1677struct AttachRetryPolicy {
1678    budget: Duration,
1679    initial_backoff: Duration,
1680    max_backoff: Duration,
1681    jitter_percent: u64,
1682}
1683
1684const ATTACH_RETRY_POLICY: AttachRetryPolicy = AttachRetryPolicy {
1685    budget: ATTACH_RETRY_BUDGET,
1686    initial_backoff: ATTACH_RETRY_INITIAL_BACKOFF,
1687    max_backoff: ATTACH_RETRY_MAX_BACKOFF,
1688    jitter_percent: ATTACH_RETRY_JITTER_PERCENT,
1689};
1690
1691/// Retry only failures that can be caused by a daemon bounce or an interrupted
1692/// handshake. Protocol and credential failures are permanent for this process.
1693fn classify_attach_error(error: &SubcError) -> AttachErrorClass {
1694    let transient = match error {
1695        SubcError::Connect { source, .. } => is_transient_attach_io(source.kind()),
1696        SubcError::Auth { source, .. } => match source {
1697            subc_transport::AuthError::Timeout { .. }
1698            | subc_transport::AuthError::UnexpectedEof { .. } => true,
1699            subc_transport::AuthError::Io { source, .. } => is_transient_attach_io(source.kind()),
1700            _ => false,
1701        },
1702        _ => false,
1703    };
1704    if transient {
1705        AttachErrorClass::Transient
1706    } else {
1707        AttachErrorClass::Permanent
1708    }
1709}
1710
1711fn is_transient_attach_io(kind: io::ErrorKind) -> bool {
1712    matches!(
1713        kind,
1714        io::ErrorKind::ConnectionRefused
1715            | io::ErrorKind::TimedOut
1716            | io::ErrorKind::ConnectionReset
1717            | io::ErrorKind::ConnectionAborted
1718            | io::ErrorKind::BrokenPipe
1719            | io::ErrorKind::UnexpectedEof
1720    )
1721}
1722
1723/// Read the connection file → resolve the first endpoint → TCP connect → HMAC
1724/// handshake. Transient initial-attach failures retry on fresh sockets and reread
1725/// the file so a daemon bounce can publish a new endpoint or authentication key.
1726async fn connect_and_authenticate(connection_file_path: &Path) -> Result<TcpStream, SubcError> {
1727    connect_and_authenticate_with_policy(connection_file_path, ATTACH_RETRY_POLICY).await
1728}
1729
1730async fn connect_and_authenticate_with_policy(
1731    connection_file_path: &Path,
1732    policy: AttachRetryPolicy,
1733) -> Result<TcpStream, SubcError> {
1734    let started_at = Instant::now();
1735    let deadline = started_at + policy.budget;
1736    let mut attempt = 0_u32;
1737    let mut backoff = policy.initial_backoff;
1738    let mut history = Vec::new();
1739
1740    loop {
1741        attempt = attempt.saturating_add(1);
1742        let error = match connect_and_authenticate_once(connection_file_path, deadline).await {
1743            Ok(stream) => return Ok(stream),
1744            Err(error) => error,
1745        };
1746        let class = classify_attach_error(&error);
1747        let error_text = error.to_string().lines().collect::<Vec<_>>().join(" ");
1748        history.push(format!("attempt {attempt} [{class}]: {error_text}"));
1749
1750        if class == AttachErrorClass::Permanent {
1751            log_attach_final_failure(started_at.elapsed(), &history);
1752            return Err(error);
1753        }
1754
1755        let remaining = deadline.saturating_duration_since(Instant::now());
1756        if remaining.is_zero() {
1757            log_attach_final_failure(started_at.elapsed(), &history);
1758            return Err(error);
1759        }
1760
1761        let delay = jittered_attach_delay(backoff, policy.jitter_percent, attempt).min(remaining);
1762        log::info!(
1763            "subc attach retry: attempt {attempt} failed; error_class={class}; error={error_text}; next_delay={delay:?}"
1764        );
1765        tokio::time::sleep(delay).await;
1766
1767        if Instant::now() >= deadline {
1768            log_attach_final_failure(started_at.elapsed(), &history);
1769            return Err(error);
1770        }
1771        backoff = backoff.saturating_mul(2).min(policy.max_backoff);
1772    }
1773}
1774
1775fn jittered_attach_delay(base: Duration, jitter_percent: u64, attempt: u32) -> Duration {
1776    let jitter_percent = jitter_percent.min(100);
1777    if jitter_percent == 0 {
1778        return base;
1779    }
1780
1781    let mut random_bytes = [0_u8; 8];
1782    let random = if getrandom::fill(&mut random_bytes).is_ok() {
1783        u64::from_le_bytes(random_bytes)
1784    } else {
1785        let timestamp = std::time::SystemTime::now()
1786            .duration_since(std::time::UNIX_EPOCH)
1787            .unwrap_or_default()
1788            .subsec_nanos();
1789        u64::from(timestamp) ^ u64::from(attempt)
1790    };
1791    let span = jitter_percent.saturating_mul(2).saturating_add(1);
1792    let multiplier_percent = 100 - jitter_percent + random % span;
1793    let base_millis = u64::try_from(base.as_millis()).unwrap_or(u64::MAX);
1794    Duration::from_millis(base_millis.saturating_mul(multiplier_percent) / 100)
1795}
1796
1797fn log_attach_final_failure(elapsed: Duration, history: &[String]) {
1798    log::error!(
1799        "subc initial attach failed after {} attempt(s) in {elapsed:?}; attempt history: {}",
1800        history.len(),
1801        history.join(" | ")
1802    );
1803}
1804
1805async fn connect_and_authenticate_once(
1806    connection_file_path: &Path,
1807    deadline: Instant,
1808) -> Result<TcpStream, SubcError> {
1809    // This read intentionally lives inside the per-attempt function. The daemon
1810    // publishes connection files atomically and may change both port and key.
1811    let conn = connection_file::read_for_client(connection_file_path).map_err(|source| {
1812        SubcError::ConnectionFile {
1813            path: connection_file_path.to_path_buf(),
1814            source,
1815        }
1816    })?;
1817
1818    let endpoint = conn
1819        .endpoints
1820        .first()
1821        .ok_or_else(|| SubcError::NoEndpoint {
1822            path: connection_file_path.to_path_buf(),
1823        })?;
1824    let endpoint_label = format!("{}:{}", endpoint.host, endpoint.port);
1825    let ip = endpoint
1826        .host
1827        .parse::<IpAddr>()
1828        .map_err(|_| SubcError::InvalidEndpoint {
1829            path: connection_file_path.to_path_buf(),
1830            endpoint: endpoint_label.clone(),
1831        })?;
1832    let addr = SocketAddr::new(ip, endpoint.port);
1833
1834    let connect_budget = deadline.saturating_duration_since(Instant::now());
1835    let mut stream = tokio::time::timeout(connect_budget, TcpStream::connect(addr))
1836        .await
1837        .map_err(|_| SubcError::Connect {
1838            endpoint: endpoint_label.clone(),
1839            source: io::Error::new(
1840                io::ErrorKind::TimedOut,
1841                "initial subc attach retry budget elapsed during TCP connect",
1842            ),
1843        })?
1844        .map_err(|source| SubcError::Connect {
1845            endpoint: endpoint_label.clone(),
1846            source,
1847        })?;
1848    stream
1849        .set_nodelay(true)
1850        .map_err(|source| SubcError::Connect {
1851            endpoint: endpoint_label.clone(),
1852            source,
1853        })?;
1854
1855    let auth_budget = AUTH_DEADLINE.min(deadline.saturating_duration_since(Instant::now()));
1856    authenticate_client(&mut stream, &conn, auth_budget)
1857        .await
1858        .map_err(|source| SubcError::Auth {
1859            endpoint: endpoint_label,
1860            source,
1861        })?;
1862
1863    Ok(stream)
1864}
1865
1866#[allow(clippy::too_many_arguments)]
1867async fn process_route_bind_completion(
1868    writer_tx: &WriterSender,
1869    completion: RouteBindCompletion,
1870    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1871    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1872    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1873    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1874    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1875    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
1876    installed_route_epochs: &mut HashMap<u16, u32>,
1877    executor: &Arc<Executor>,
1878    shutdown: &Arc<Notify>,
1879    metrics: &Arc<DispatchPathMetrics>,
1880) -> Result<(), SubcError> {
1881    decrement_counted_channel(&metrics.control_completion_queued);
1882    handle_route_bind_completion(
1883        writer_tx,
1884        completion,
1885        routes,
1886        root_channels,
1887        session_identity,
1888        push_buffer,
1889        live_roots,
1890        pending_binds,
1891        installed_route_epochs,
1892        executor,
1893        shutdown,
1894        metrics,
1895    )
1896    .await
1897}
1898
1899#[allow(clippy::too_many_arguments)]
1900async fn drain_pending_route_bind_completions(
1901    control_completion_rx: &mut mpsc::Receiver<RouteBindCompletion>,
1902    writer_tx: &WriterSender,
1903    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1904    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1905    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1906    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1907    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1908    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
1909    installed_route_epochs: &mut HashMap<u16, u32>,
1910    executor: &Arc<Executor>,
1911    shutdown: &Arc<Notify>,
1912    metrics: &Arc<DispatchPathMetrics>,
1913) -> Result<usize, SubcError> {
1914    let mut drained = 0;
1915    while let Ok(completion) = control_completion_rx.try_recv() {
1916        process_route_bind_completion(
1917            writer_tx,
1918            completion,
1919            routes,
1920            root_channels,
1921            session_identity,
1922            push_buffer,
1923            live_roots,
1924            pending_binds,
1925            installed_route_epochs,
1926            executor,
1927            shutdown,
1928            metrics,
1929        )
1930        .await?;
1931        drained += 1;
1932    }
1933    Ok(drained)
1934}
1935
1936/// ModuleHello → HelloAck → control/route loop. Runs until the daemon closes
1937/// the connection (EOF), sends channel-0 Goodbye, or a fatal mutating executor
1938/// response requests whole-connection teardown.
1939async fn run_module_loop<R, W>(
1940    mut read: R,
1941    mut write: W,
1942    shared_app: Arc<App>,
1943    executor: Arc<Executor>,
1944    dispatch: DispatchFn,
1945    user_config_path: Option<PathBuf>,
1946    allow_native_passthrough: bool,
1947) -> Result<ModuleLoopExit, SubcError>
1948where
1949    R: AsyncRead + Unpin + Send + 'static,
1950    W: AsyncWrite + Unpin + Send + 'static,
1951{
1952    // ModuleHello: register as a tool provider and advertise the supported control-plane operations.
1953    // Echo the one-time launch nonce the daemon injected via SUBC_LAUNCH_NONCE so a
1954    // reserved module_id's HELLO is accepted; absent for non-reserved/self-connect.
1955    let hello = ModuleHelloBody {
1956        manifest: build_manifest(),
1957        protocol_ver: PROTOCOL_VERSION,
1958        control_ops: control_ops(),
1959        launch_nonce: std::env::var("SUBC_LAUNCH_NONCE").ok(),
1960    };
1961    let hello_frame = Frame::build(
1962        FrameType::Hello,
1963        control_flags(),
1964        0,
1965        0,
1966        HELLO_CORR,
1967        serde_json::to_vec(&hello).map_err(SubcError::Json)?,
1968    )
1969    .map_err(SubcError::FrameBuild)?;
1970    write_frame(&mut write, &hello_frame)
1971        .await
1972        .map_err(SubcError::FrameIo)?;
1973
1974    // Expect HelloAck (registered) or a channel-0 Error (manifest/version reject).
1975    match read_frame(&mut read).await.map_err(SubcError::FrameIo)? {
1976        None => return Err(SubcError::ClosedBeforeHelloAck),
1977        Some(frame) => match frame.header.ty {
1978            FrameType::HelloAck => {
1979                log::info!("subc attach: registered (HelloAck received)");
1980            }
1981            FrameType::Error => {
1982                let body = serde_json::from_slice::<ErrorBody>(&frame.body).ok();
1983                return Err(SubcError::HelloRejected { body });
1984            }
1985            other => return Err(SubcError::UnexpectedFrame { ty: other }),
1986        },
1987    }
1988
1989    let dispatch_path_metrics = Arc::new(DispatchPathMetrics::new());
1990    let (writer_tx, writer_rx) = mpsc::channel::<WriterFrame>(WRITER_QUEUE_CAPACITY);
1991    let writer_task = spawn_writer_task(write, writer_rx, Arc::clone(&dispatch_path_metrics));
1992    // `read_frame` is NOT cancellation-safe, so it must never sit directly inside
1993    // the `select!` below: a drain-interval tick (or shutdown) firing while a
1994    // frame is mid-transit would drop the partially-consumed bytes and desync the
1995    // stream (the next read would parse a body byte as a frame header). A
1996    // dedicated reader task owns the socket, reads whole frames sequentially, and
1997    // forwards them over a channel; the loop selects on the cancel-safe `recv()`.
1998    let (reader_tx, mut reader_rx) = mpsc::channel::<Result<DecodedFrame, SubcError>>(256);
1999    let reader_task = spawn_reader_task(read, reader_tx);
2000    let shutdown = Arc::new(Notify::new());
2001    // Drain-tick deadline is tracked manually and checked at the TOP of every
2002    // loop turn rather than as an Interval select arm: the select below is
2003    // `biased` (bind completions first), and biased polling means a saturated
2004    // higher arm (sustained lossy push traffic keeps lossy_rx always-ready)
2005    // would starve every arm below it, including a timer arm — leaving
2006    // backpressured reliable frames parked in the retry buffer past their
2007    // delivery deadline. The pre-turn check cannot be starved by arm order;
2008    // the sleep_until arm below only exists to wake an otherwise-idle loop.
2009    let mut next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2010    let mut next_maintenance_at = next_drain_at;
2011    let (maintenance_tx, mut maintenance_rx) = mpsc::channel::<MaintenanceCompletion>(256);
2012    let (bash_deferred_tx, mut bash_deferred_rx) =
2013        mpsc::channel::<bash::BashDeferredCompletion>(256);
2014    let (bash_poll_touch_tx, mut bash_poll_touch_rx) = mpsc::channel::<ProjectRootId>(256);
2015    let (control_completion_tx, mut control_completion_rx) =
2016        mpsc::channel::<RouteBindCompletion>(256);
2017    let (lossy_tx, mut lossy_rx) = mpsc::channel::<LossyPushEnvelope>(1024);
2018    let lossy_overflow = Arc::new(push::LossyOverflow::default());
2019    let lossy_seq = Arc::new(AtomicU64::new(0));
2020    let (reliable_tx, mut reliable_rx) = mpsc::unbounded_channel::<PushEnvelope>();
2021    let push_senders = PushSenders {
2022        lossy_tx,
2023        reliable_tx,
2024        lossy_overflow: Arc::clone(&lossy_overflow),
2025        lossy_seq,
2026    };
2027    let connection_cancel = PersistentCancelSignal::new();
2028    let mut installed_route_epochs: HashMap<u16, u32> = HashMap::new();
2029    let mut routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
2030    let mut bg_subs: HashMap<RouteChannel, BgSub> = HashMap::new();
2031    let mut bg_sub_by_session: HashMap<(ProjectRootId, String), RouteChannel> = HashMap::new();
2032    let mut bg_wake_pending: HashSet<RouteChannel> = HashSet::new();
2033    let mut bg_wake_epoch: HashMap<(ProjectRootId, String), u64> = HashMap::new();
2034    let mut root_channels: HashMap<ProjectRootId, HashSet<RouteChannel>> = HashMap::new();
2035    let mut session_identity: HashMap<(ProjectRootId, String), RetainedSessionIdentity> =
2036        HashMap::new();
2037    let mut push_buffer: HashMap<push::ReplayKey, VecDeque<PushFrame>> = HashMap::new();
2038    let mut retry_buffer: RetryBuffer = HashMap::new();
2039    let mut completed_tasks = push::CompletedTaskIds::default();
2040    let mut live_roots: HashMap<ProjectRootId, RootMeta> = HashMap::new();
2041    let mut pending_binds: HashMap<RouteChannel, PendingBind> = HashMap::new();
2042    let mut pending_bash_asks: HashMap<ReverseCorrKey, PendingBashAsk> = HashMap::new();
2043    let mut next_bash_ask_corr: u64 = 1;
2044    let mut route_bash_cancels: HashMap<RouteChannel, bash::RouteBashCancel> = HashMap::new();
2045
2046    let loop_result: Result<ModuleLoopExit, SubcError> = loop {
2047        crate::logging::perf_tick(Some(&executor));
2048        dispatch_path_metrics.mark_frame_loop_tick();
2049        if let Err(error) = expire_pending_bash_asks(
2050            &writer_tx,
2051            &mut pending_bash_asks,
2052            &routes,
2053            &mut live_roots,
2054            &mut route_bash_cancels,
2055            &shutdown,
2056            &dispatch_path_metrics,
2057        )
2058        .await
2059        {
2060            break Err(error);
2061        }
2062
2063        // RouteBind completions are control-plane unblockers. Drain any completed
2064        // binds before entering other branch work so Push and maintenance bursts
2065        // can only add one loop-turn of latency.
2066        match drain_pending_route_bind_completions(
2067            &mut control_completion_rx,
2068            &writer_tx,
2069            &mut routes,
2070            &mut root_channels,
2071            &mut session_identity,
2072            &mut push_buffer,
2073            &mut live_roots,
2074            &mut pending_binds,
2075            &mut installed_route_epochs,
2076            &executor,
2077            &shutdown,
2078            &dispatch_path_metrics,
2079        )
2080        .await
2081        {
2082            Ok(drained) => {
2083                if drained > 0 {
2084                    next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2085                }
2086            }
2087            Err(error) => break Err(error),
2088        }
2089
2090        if tokio::time::Instant::now() >= next_drain_at {
2091            push::emit_bg_event_wakes(
2092                &writer_tx,
2093                &dispatch_path_metrics,
2094                &bg_subs,
2095                &mut bg_wake_pending,
2096            );
2097            warn_slow_pending_binds(&mut pending_binds, &executor);
2098            if let Err(error) = expire_overdue_route_binds(
2099                &writer_tx,
2100                &executor,
2101                &mut pending_binds,
2102                &mut installed_route_epochs,
2103                &dispatch_path_metrics,
2104            )
2105            .await
2106            {
2107                break Err(error);
2108            }
2109
2110            let retried = push::drain_retry_buffers_for_bound_routes(
2111                &writer_tx,
2112                &dispatch_path_metrics,
2113                &routes,
2114                &mut retry_buffer,
2115            );
2116            if retried > 0 {
2117                log::debug!(
2118                    "subc attach: retried {retried} reliable Push frame(s) after writer backpressure"
2119                );
2120            }
2121
2122            next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2123        }
2124
2125        // A lossy emitter may place its newest update in the overflow buffer
2126        // when the bounded channel is full, while this receive loop is draining
2127        // the channel. Drain overflow before selecting again so that raced
2128        // update is delivered on the next timer tick instead of waiting for
2129        // another lossy enqueue.
2130        let overflow_batch = lossy_overflow.drain();
2131        if !overflow_batch.is_empty() {
2132            let (_, deferred) = push::drain_reliable_push_turn(
2133                &writer_tx,
2134                &dispatch_path_metrics,
2135                &routes,
2136                &root_channels,
2137                &session_identity,
2138                &mut retry_buffer,
2139                &mut push_buffer,
2140                &mut completed_tasks,
2141                &bg_sub_by_session,
2142                &mut bg_wake_pending,
2143                &mut bg_wake_epoch,
2144                &mut reliable_rx,
2145                None,
2146            );
2147            if deferred {
2148                tokio::task::yield_now().await;
2149            }
2150
2151            let mut batch = Vec::new();
2152            while let Ok(item) = lossy_rx.try_recv() {
2153                batch.push(item);
2154            }
2155            batch.extend(overflow_batch);
2156            push::process_lossy_push_envelope_batch(
2157                &writer_tx,
2158                &dispatch_path_metrics,
2159                &routes,
2160                &root_channels,
2161                &completed_tasks,
2162                batch,
2163            );
2164        }
2165
2166        tokio::select! {
2167            biased;
2168            Some(completion) = control_completion_rx.recv() => {
2169                if let Err(error) = process_route_bind_completion(
2170                    &writer_tx,
2171                    completion,
2172                    &mut routes,
2173                    &mut root_channels,
2174                    &mut session_identity,
2175                    &mut push_buffer,
2176                    &mut live_roots,
2177                    &mut pending_binds,
2178                    &mut installed_route_epochs,
2179                    &executor,
2180                    &shutdown,
2181                    &dispatch_path_metrics,
2182                )
2183                .await
2184                {
2185                    break Err(error);
2186                }
2187                next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2188            }
2189            _ = shutdown.notified() => {
2190                log::warn!("subc attach: fatal executor response requested teardown");
2191                break Ok(ModuleLoopExit::SkipSearchFlush);
2192            }
2193            maybe_frame = reader_rx.recv() => {
2194                let frame = match maybe_frame {
2195                    None => {
2196                        log::info!("subc attach: daemon closed connection");
2197                        break Ok(ModuleLoopExit::Graceful);
2198                    }
2199                    Some(Err(error)) => break Err(error),
2200                    Some(Ok(frame)) => frame,
2201                };
2202                let phase_trace = frame.phase_trace;
2203                let frame = frame.frame;
2204
2205                if !ingress_route_is_current(&installed_route_epochs, &frame) {
2206                    log::debug!(
2207                        "subc attach: silently dropping {:?} for uninstalled route {}@{}",
2208                        frame.header.ty,
2209                        frame.header.channel,
2210                        frame.header.epoch
2211                    );
2212                    continue;
2213                }
2214
2215                match frame.header.ty {
2216                    FrameType::Ping if frame.header.channel == 0 => {
2217                        let pong = match Frame::build_with_version(
2218                            frame.header.ver,
2219                            FrameType::Pong,
2220                            frame.header.flags,
2221                            0,
2222                            0,
2223                            frame.header.corr,
2224                            Vec::new(),
2225                        ) {
2226                            Ok(pong) => pong,
2227                            Err(error) => break Err(SubcError::FrameBuild(error)),
2228                        };
2229                        if let Err(error) = send_frame(&writer_tx, &dispatch_path_metrics, pong).await {
2230                            break Err(error);
2231                        }
2232                    }
2233                    FrameType::Goodbye if frame.header.channel == 0 => {
2234                        log::info!("subc attach: received channel-0 Goodbye");
2235                        break Ok(ModuleLoopExit::Graceful);
2236                    }
2237                    FrameType::Goodbye => {
2238                        let channel = route_key(frame.header.channel, frame.header.epoch);
2239                        if let Err(error) = teardown_installed_route(
2240                            &writer_tx,
2241                            &dispatch_path_metrics,
2242                            &executor,
2243                            channel,
2244                            "Goodbye",
2245                            None,
2246                            &mut installed_route_epochs,
2247                            &mut routes,
2248                            &mut root_channels,
2249                            &mut bg_subs,
2250                            &mut bg_sub_by_session,
2251                            &mut bg_wake_pending,
2252                            &mut pending_bash_asks,
2253                            &mut live_roots,
2254                            &mut route_bash_cancels,
2255                            &mut pending_binds,
2256                            &mut retry_buffer,
2257                            &mut push_buffer,
2258                            &shutdown,
2259                        )
2260                        .await
2261                        {
2262                            break Err(error);
2263                        }
2264                    }
2265                    FrameType::Response | FrameType::Error if frame.header.channel != 0 => {
2266                        if let Err(error) = handle_bash_elicitation_reply(
2267                            &writer_tx,
2268                            &frame,
2269                            &mut pending_bash_asks,
2270                            &routes,
2271                            &mut live_roots,
2272                            &executor,
2273                            &shutdown,
2274                            &bash_deferred_tx,
2275                            &bash_poll_touch_tx,
2276                            &dispatch_path_metrics,
2277                            &mut route_bash_cancels,
2278                            dispatch,
2279                        )
2280                        .await
2281                        {
2282                            break Err(error);
2283                        }
2284                    }
2285                    FrameType::Request if frame.header.channel == 0 => {
2286                        if let Err(error) = handle_control_request(
2287                            &writer_tx,
2288                            &frame,
2289                            &shared_app,
2290                            &executor,
2291                            &mut live_roots,
2292                            &mut pending_binds,
2293                            &mut installed_route_epochs,
2294                            &mut routes,
2295                            &mut root_channels,
2296                            &mut bg_subs,
2297                            &mut bg_sub_by_session,
2298                            &mut bg_wake_pending,
2299                            &mut pending_bash_asks,
2300                            &mut route_bash_cancels,
2301                            &mut retry_buffer,
2302                            &mut push_buffer,
2303                            &shutdown,
2304                            &control_completion_tx,
2305                            &dispatch_path_metrics,
2306                            &push_senders,
2307                            dispatch,
2308                            user_config_path.as_deref(),
2309                        )
2310                        .await
2311                        {
2312                            break Err(error);
2313                        }
2314                    }
2315                    FrameType::Request => {
2316                        if let Err(error) = handle_tool_call(
2317                            &writer_tx,
2318                            &frame,
2319                            phase_trace,
2320                            &routes,
2321                            &pending_binds,
2322                            &mut live_roots,
2323                            &executor,
2324                            &shutdown,
2325                            &connection_cancel,
2326                            &bash_deferred_tx,
2327                            &bash_poll_touch_tx,
2328                            &dispatch_path_metrics,
2329                            &mut route_bash_cancels,
2330                            &mut pending_bash_asks,
2331                            &mut next_bash_ask_corr,
2332                            &mut bg_subs,
2333                            &mut bg_sub_by_session,
2334                            &mut bg_wake_pending,
2335                            &mut bg_wake_epoch,
2336                            dispatch,
2337                            allow_native_passthrough,
2338                        )
2339                        .await
2340                        {
2341                            break Err(error);
2342                        }
2343                    }
2344                    FrameType::Cancel => {
2345                        let channel = route_key(frame.header.channel, frame.header.epoch);
2346                        if bg_subs.contains_key(&channel) {
2347                            if let Err(error) = end_bg_subscription(
2348                                &writer_tx,
2349                                &dispatch_path_metrics,
2350                                &mut bg_subs,
2351                                &mut bg_sub_by_session,
2352                                &mut bg_wake_pending,
2353                                channel,
2354                                routes.get(&channel),
2355                            )
2356                            .await
2357                            {
2358                                break Err(error);
2359                            }
2360                        }
2361                        if let Err(error) = cancel_pending_bash_ask_for_tool_call(
2362                            &writer_tx,
2363                            &mut pending_bash_asks,
2364                            channel,
2365                            frame.header.corr,
2366                            &routes,
2367                            &mut live_roots,
2368                            &mut route_bash_cancels,
2369                            &shutdown,
2370                            &dispatch_path_metrics,
2371                        )
2372                        .await
2373                        {
2374                            break Err(error);
2375                        }
2376                    }
2377                    // Incoming push messages are ignored here. Cancel frames only
2378                    // stop pending bash elicitation requests; executor-level
2379                    // cancellation for tool calls that are already running is not
2380                    // implemented.
2381                    _ => {}
2382                }
2383            }
2384            Some((root_id, frame)) = reliable_rx.recv() => {
2385                // Reliable Push frames are FIFO and must-deliver, but draining an
2386                // unbounded burst in one current-thread turn can starve RouteBind
2387                // completions. The budget defers excess frames, never drops them.
2388                let (_, deferred) = push::drain_reliable_push_turn(
2389                    &writer_tx,
2390                    &dispatch_path_metrics,
2391                    &routes,
2392                    &root_channels,
2393                    &session_identity,
2394                    &mut retry_buffer,
2395                    &mut push_buffer,
2396                    &mut completed_tasks,
2397                    &bg_sub_by_session,
2398                    &mut bg_wake_pending,
2399                    &mut bg_wake_epoch,
2400                    &mut reliable_rx,
2401                    Some((root_id, frame)),
2402                );
2403                if deferred {
2404                    tokio::task::yield_now().await;
2405                }
2406            }
2407            Some((order, root_id, frame)) = lossy_rx.recv() => {
2408                // When both push lanes have work, handle a small reliable slice before lossy work.
2409                // That ordering lets completed task ids suppress stale BashLongRunning frames.
2410                // The slice stays bounded so reliable bursts cannot monopolize this loop turn.
2411                let (_, deferred) = push::drain_reliable_push_turn(
2412                    &writer_tx,
2413                    &dispatch_path_metrics,
2414                    &routes,
2415                    &root_channels,
2416                    &session_identity,
2417                    &mut retry_buffer,
2418                    &mut push_buffer,
2419                    &mut completed_tasks,
2420                    &bg_sub_by_session,
2421                    &mut bg_wake_pending,
2422                    &mut bg_wake_epoch,
2423                    &mut reliable_rx,
2424                    None,
2425                );
2426                if deferred {
2427                    tokio::task::yield_now().await;
2428                }
2429
2430                // Drain the currently queued burst in one loop turn so lossy
2431                // status/progress updates can be merged before reaching subc's
2432                // shared egress queue. Each lossy frame gets a sequence number
2433                // before it goes to the channel or overflow buffer, so the
2434                // combined batch is sorted back into producer order before
2435                // coalescing drops stale updates for the same key.
2436                let mut batch = vec![(order, root_id, frame)];
2437                while let Ok(item) = lossy_rx.try_recv() {
2438                    batch.push(item);
2439                }
2440                batch.extend(lossy_overflow.drain());
2441                push::process_lossy_push_envelope_batch(
2442                    &writer_tx,
2443                    &dispatch_path_metrics,
2444                    &routes,
2445                    &root_channels,
2446                    &completed_tasks,
2447                    batch,
2448                );
2449            }
2450            Some(done) = bash_deferred_rx.recv() => {
2451                decrement_counted_channel(&dispatch_path_metrics.bash_deferred_queued);
2452                if let Err(error) = bash::handle_bash_deferred_completion(
2453                    &writer_tx,
2454                    done,
2455                    &routes,
2456                    &mut live_roots,
2457                    &mut route_bash_cancels,
2458                    &shutdown,
2459                    &dispatch_path_metrics,
2460                )
2461                .await
2462                {
2463                    break Err(error);
2464                }
2465            }
2466            Some(root_id) = bash_poll_touch_rx.recv() => {
2467                decrement_counted_channel(&dispatch_path_metrics.bash_poll_touch_queued);
2468                if let Some(meta) = live_roots.get_mut(&root_id) {
2469                    meta.note_activity();
2470                }
2471            }
2472            Some(completion) = maintenance_rx.recv() => {
2473                decrement_counted_channel(&dispatch_path_metrics.maintenance_queued);
2474                let root_id = completion.root_id.clone();
2475                let response = completion.response;
2476                let response_is_fatal = response_is_fatal_panic(&response);
2477                let bind_pending = pending_binds
2478                    .values()
2479                    .any(|pending| pending.bind_root_id == root_id);
2480                let requiesce = if let Some(meta) = live_roots.get_mut(&root_id) {
2481                    let defer_requeue = meta.unbound_quiesced || bind_pending;
2482                    note_maintenance_completion(
2483                        meta,
2484                        completion.requeue_kind,
2485                        response_is_fatal,
2486                        defer_requeue,
2487                    );
2488                    should_requiesce_after_maintenance(meta, completion.kind, bind_pending)
2489                } else {
2490                    false
2491                };
2492                if requiesce {
2493                    quiesce_unbound_root(&root_id, &mut live_roots, &executor);
2494                }
2495                push::clear_stale_bg_wakes_for_empty_sessions(
2496                    &root_id,
2497                    &completion.empty_bg_sessions,
2498                    &bg_sub_by_session,
2499                    &mut bg_wake_pending,
2500                    &bg_wake_epoch,
2501                );
2502                if response_is_fatal {
2503                    if let Some(meta) = live_roots.get_mut(&root_id) {
2504                        meta.maintenance_poisoned = true;
2505                    }
2506                    log::warn!(
2507                        "subc attach: maintenance drain observed a fatal actor; deferring teardown until a route request can receive actor_fatal"
2508                    );
2509                }
2510            }
2511            _ = tokio::time::sleep_until(next_drain_at) => {
2512                // Wakes an otherwise-idle loop so the pre-turn drain check
2513                // above runs on schedule; the drain work itself lives there.
2514            }
2515            _ = tokio::time::sleep_until(next_maintenance_at) => {
2516                // Delay cache-draining maintenance until any already-ready
2517                // inbound route/control messages and push completions have run,
2518                // so maintenance does not block the actor from handling the
2519                // first request that arrives after a route bind is acknowledged.
2520                let reaped = reap_idle_roots(
2521                    Instant::now(),
2522                    &mut live_roots,
2523                    &pending_binds,
2524                    &executor,
2525                );
2526                if reaped > 0 {
2527                    log::debug!("subc attach: reaped {reaped} idle root(s)");
2528                }
2529                submit_due_maintenance_jobs(
2530                    &executor,
2531                    &mut live_roots,
2532                    &pending_binds,
2533                    &bg_sub_by_session,
2534                    &bg_wake_pending,
2535                    &bg_wake_epoch,
2536                    &maintenance_tx,
2537                    &dispatch_path_metrics,
2538                );
2539                next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2540            }
2541        }
2542    };
2543
2544    // Sweep pending binds on EVERY loop exit (channel-0 Goodbye, EOF, fatal,
2545    // error): ExecutorInner::drop joins worker threads, so a configure still
2546    // parked at a cancellation checkpoint would wedge module shutdown.
2547    for pending in pending_binds.values() {
2548        let _ = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
2549    }
2550
2551    let mut loop_result = loop_result;
2552    if !pending_bash_asks.is_empty() {
2553        let no_routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
2554        if let Err(error) = settle_all_pending_bash_asks(
2555            &writer_tx,
2556            &mut pending_bash_asks,
2557            &no_routes,
2558            &mut live_roots,
2559            &mut route_bash_cancels,
2560            &shutdown,
2561            &dispatch_path_metrics,
2562        )
2563        .await
2564        {
2565            loop_result = loop_result.and(Err(error));
2566        }
2567    }
2568
2569    // The reader task may be parked on `read_frame`; abort it (we are done with
2570    // the connection) and flush the writer.
2571    connection_cancel.cancel();
2572    reader_task.abort();
2573    drop(writer_tx);
2574    let writer_result = finish_writer_task(writer_task).await;
2575    loop_result.and_then(|exit| writer_result.map(|_| exit))
2576}
2577
2578fn spawn_writer_task<W>(
2579    mut write: W,
2580    mut rx: mpsc::Receiver<WriterFrame>,
2581    metrics: Arc<DispatchPathMetrics>,
2582) -> JoinHandle<Result<(), subc_transport::FrameIoError>>
2583where
2584    W: AsyncWrite + Unpin + Send + 'static,
2585{
2586    tokio::spawn(async move {
2587        let mut write_buffer = Vec::new();
2588        while let Some(mut queued) = rx.recv().await {
2589            let measure = queued.tool_response_trace.is_some();
2590            let dequeued = measure.then(Instant::now);
2591            metrics.writer_active.store(true, Ordering::Relaxed);
2592            decrement_counted_channel(&metrics.writer_queued);
2593            let write_timing =
2594                write_frame_contiguous(&mut write, &queued.frame, &mut write_buffer, measure).await;
2595            metrics.writer_active.store(false, Ordering::Relaxed);
2596            let write_timing = write_timing?;
2597
2598            if let (Some(trace), Some(dequeued), Some(write_timing)) =
2599                (queued.tool_response_trace.take(), dequeued, write_timing)
2600            {
2601                if let Some(completed) = trace.finish(
2602                    dequeued,
2603                    write_timing.write_started,
2604                    write_timing.write_finished,
2605                    write_timing.frame_bytes,
2606                ) {
2607                    log_ctx::with_session(Some(completed.session), || {
2608                        crate::logging::note_tool_call_trace(
2609                            &completed.name,
2610                            &completed.root,
2611                            completed.channel,
2612                            completed.corr,
2613                            completed.phases,
2614                        );
2615                    });
2616                }
2617            }
2618        }
2619        Ok(())
2620    })
2621}
2622
2623struct FrameWriteTiming {
2624    write_started: Instant,
2625    write_finished: Instant,
2626    frame_bytes: usize,
2627}
2628
2629/// Encode one complete frame into the existing reusable buffer and write it
2630/// without interleaving bytes from another channel. Timing is collected only
2631/// for tool responses, so Push and control frames add no clock reads.
2632async fn write_frame_contiguous<W>(
2633    writer: &mut W,
2634    frame: &Frame,
2635    buffer: &mut Vec<u8>,
2636    measure: bool,
2637) -> Result<Option<FrameWriteTiming>, subc_transport::FrameIoError>
2638where
2639    W: AsyncWrite + Unpin,
2640{
2641    if frame.header.len as usize != frame.body.len() {
2642        return Err(subc_transport::FrameIoError::BodyLengthMismatch {
2643            header_len: frame.header.len,
2644            body_len: frame.body.len(),
2645        });
2646    }
2647
2648    let header = frame.header.encode();
2649    buffer.clear();
2650    buffer.reserve(header.len() + frame.body.len());
2651    buffer.extend_from_slice(&header);
2652    buffer.extend_from_slice(&frame.body);
2653    let write_started = measure.then(Instant::now);
2654    writer
2655        .write_all(buffer)
2656        .await
2657        .map_err(subc_transport::FrameIoError::Io)?;
2658    Ok(write_started.map(|write_started| FrameWriteTiming {
2659        write_started,
2660        write_finished: Instant::now(),
2661        frame_bytes: buffer.len(),
2662    }))
2663}
2664
2665fn spawn_reader_task<R>(
2666    mut read: R,
2667    tx: mpsc::Sender<Result<DecodedFrame, SubcError>>,
2668) -> JoinHandle<()>
2669where
2670    R: AsyncRead + Unpin + Send + 'static,
2671{
2672    tokio::spawn(async move {
2673        loop {
2674            match read_frame(&mut read).await {
2675                Ok(Some(frame)) => {
2676                    let decoded = DecodedFrame {
2677                        frame,
2678                        phase_trace: PhaseTrace::new(Instant::now()),
2679                    };
2680                    if tx.send(Ok(decoded)).await.is_err() {
2681                        return;
2682                    }
2683                }
2684                Ok(None) => {
2685                    // EOF: let the loop observe channel close as "daemon closed".
2686                    return;
2687                }
2688                Err(error) => {
2689                    // A killed daemon surfaces as ConnectionReset (RST) on
2690                    // Windows where Unix delivers a clean EOF (FIN); a
2691                    // mid-teardown daemon can also abort the socket. Both mean
2692                    // "daemon went away", not a wire fault — normalize them to
2693                    // the clean-close path so module exit behavior matches
2694                    // across platforms (same class subc-core fixed in d33d9a71).
2695                    if let subc_transport::FrameIoError::Io(io_error) = &error {
2696                        if matches!(
2697                            io_error.kind(),
2698                            std::io::ErrorKind::ConnectionReset
2699                                | std::io::ErrorKind::ConnectionAborted
2700                        ) {
2701                            log::info!(
2702                                "subc attach: connection reset by daemon; treating as close"
2703                            );
2704                            return;
2705                        }
2706                    }
2707                    let _ = tx.send(Err(SubcError::FrameIo(error))).await;
2708                    return;
2709                }
2710            }
2711        }
2712    })
2713}
2714
2715async fn finish_writer_task(
2716    mut writer_task: JoinHandle<Result<(), subc_transport::FrameIoError>>,
2717) -> Result<(), SubcError> {
2718    match tokio::time::timeout(Duration::from_millis(100), &mut writer_task).await {
2719        Ok(Ok(Ok(()))) => Ok(()),
2720        Ok(Ok(Err(error))) => Err(SubcError::FrameIo(error)),
2721        Ok(Err(error)) => Err(SubcError::WriterJoin(error)),
2722        Err(_) => {
2723            writer_task.abort();
2724            Ok(())
2725        }
2726    }
2727}
2728
2729fn register_actor_for_bind(
2730    shared_app: &Arc<App>,
2731    executor: &Arc<Executor>,
2732    push_senders: &PushSenders,
2733    bind_root_id: &ProjectRootId,
2734    route_channel: u16,
2735    root_was_live: bool,
2736) -> bool {
2737    if executor.actor_registered(bind_root_id) {
2738        log::debug!(
2739            "subc attach: reusing actor for route {} root {}",
2740            route_channel,
2741            bind_root_id.as_path().display()
2742        );
2743        return false;
2744    }
2745
2746    if root_was_live {
2747        log::warn!(
2748            "subc attach: recreating missing actor for live root {} on route {}",
2749            bind_root_id.as_path().display(),
2750            route_channel
2751        );
2752    }
2753
2754    let actor_ctx = Arc::new(AppContext::from_app(
2755        Arc::clone(shared_app),
2756        Config::default(),
2757    ));
2758    install_bash_compressor(&actor_ctx);
2759    actor_ctx.set_progress_sender(Some(push::progress_sender_for_root(
2760        push_senders.clone(),
2761        bind_root_id.clone(),
2762    )));
2763    let inserted = executor.register_actor(bind_root_id.clone(), Arc::clone(&actor_ctx));
2764    drop(actor_ctx);
2765    if inserted {
2766        // Do not insert into live_roots until configure succeeds: live_roots
2767        // drives maintenance, and a half-configured new actor must not be
2768        // maintenance-eligible before its route/session identity exists.
2769        log::debug!(
2770            "subc attach: registered actor for route {} root {}",
2771            route_channel,
2772            bind_root_id.as_path().display()
2773        );
2774    } else {
2775        log::debug!(
2776            "subc attach: actor appeared while binding route {} root {}; reusing it",
2777            route_channel,
2778            bind_root_id.as_path().display()
2779        );
2780    }
2781    inserted
2782}
2783
2784fn rollback_pending_bind_actor(
2785    executor: &Arc<Executor>,
2786    live_roots: &HashMap<ProjectRootId, RootMeta>,
2787    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2788    root_id: &ProjectRootId,
2789    inserted_new_actor: bool,
2790) {
2791    if !inserted_new_actor || live_roots.contains_key(root_id) {
2792        return;
2793    }
2794
2795    if let Some((route, pending)) = pending_binds
2796        .iter_mut()
2797        .find(|(_, pending)| &pending.bind_root_id == root_id)
2798    {
2799        pending.inserted_new_actor = true;
2800        log::debug!(
2801            "subc attach: transferred rollback ownership for root {} to pending route {}",
2802            root_id.as_path().display(),
2803            route
2804        );
2805        return;
2806    }
2807
2808    executor.remove_actor(root_id);
2809}
2810
2811fn route_bind_error_code_for_configure_response(response: &Response) -> &'static str {
2812    match response.data.get("code").and_then(|code| code.as_str()) {
2813        // Preserve typed configure rejections across the bind boundary: a
2814        // malformed fed fingerprint means a federation-module bug or
2815        // fingerprint-format drift, and the fed side matches on the code rather
2816        // than parsing prose.
2817        Some("bad_harness_fingerprint") => "bad_harness_fingerprint",
2818        // Cache-key probe failures are transient (fd pressure, git spawn
2819        // contention); the client retries the bind rather than treating the
2820        // root as permanently divergent.
2821        Some("cache_key_probe_failed") => "cache_key_probe_failed",
2822        // Actor lifecycle gaps are transient from the daemon/client viewpoint:
2823        // a fresh bind can create or join a healthy actor, so do not classify
2824        // them as permanent config divergence.
2825        Some("actor_not_registered" | "actor_fatal") => "actor_not_ready",
2826        _ => "config_divergence",
2827    }
2828}
2829
2830fn queue_post_bind_configure_and_completion_maintenance(
2831    root_id: &ProjectRootId,
2832    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2833) {
2834    let Some(meta) = live_roots.get_mut(root_id) else {
2835        return;
2836    };
2837    if meta.maintenance_poisoned || meta.maintenance_pending {
2838        return;
2839    }
2840
2841    meta.maintenance_pending = true;
2842    meta.maintenance_queued_kinds
2843        .push_back(MaintenanceDrainKind::ConfigureTail);
2844    meta.maintenance_queued_kinds
2845        .push_back(MaintenanceDrainKind::CompletionDrains);
2846}
2847
2848#[allow(clippy::too_many_arguments)]
2849async fn handle_route_bind_completion(
2850    tx: &WriterSender,
2851    completion: RouteBindCompletion,
2852    routes: &mut HashMap<RouteChannel, RouteIdentity>,
2853    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2854    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2855    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2856    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2857    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2858    installed_route_epochs: &mut HashMap<u16, u32>,
2859    executor: &Arc<Executor>,
2860    shutdown: &Arc<Notify>,
2861    metrics: &Arc<DispatchPathMetrics>,
2862) -> Result<(), SubcError> {
2863    let route_id = completion.route;
2864    let Some(pending) = pending_binds.remove(&route_id) else {
2865        log::warn!(
2866            "subc attach: dropping RouteBind completion for non-pending route {}",
2867            completion.route
2868        );
2869        rollback_pending_bind_actor(
2870            executor,
2871            live_roots,
2872            pending_binds,
2873            &completion.bind_root_id,
2874            completion.inserted_new_actor,
2875        );
2876        let has_pending_bind = pending_binds
2877            .values()
2878            .any(|pending| pending.bind_root_id == completion.bind_root_id);
2879        if !root_channels
2880            .get(&completion.bind_root_id)
2881            .is_some_and(|channels| !channels.is_empty())
2882            && !has_pending_bind
2883        {
2884            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
2885        }
2886        remove_installed_route(installed_route_epochs, route_id);
2887        return Ok(());
2888    };
2889
2890    if pending.bind_root_id != completion.bind_root_id {
2891        log::warn!(
2892            "subc attach: pending RouteBind root mismatch for route {} (pending {} completion {})",
2893            completion.route,
2894            pending.bind_root_id.as_path().display(),
2895            completion.bind_root_id.as_path().display()
2896        );
2897    }
2898
2899    let inserted_new_actor = pending.inserted_new_actor || completion.inserted_new_actor;
2900    if pending.cancelled {
2901        rollback_pending_bind_actor(
2902            executor,
2903            live_roots,
2904            pending_binds,
2905            &completion.bind_root_id,
2906            inserted_new_actor,
2907        );
2908        let has_pending_bind = pending_binds
2909            .values()
2910            .any(|pending| pending.bind_root_id == completion.bind_root_id);
2911        if !root_channels
2912            .get(&completion.bind_root_id)
2913            .is_some_and(|channels| !channels.is_empty())
2914            && !has_pending_bind
2915        {
2916            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
2917        }
2918        log::debug!(
2919            "subc attach: discarded completed RouteBind for cancelled route {} root {}",
2920            completion.route,
2921            completion.bind_root_id.as_path().display()
2922        );
2923        remove_installed_route(installed_route_epochs, route_id);
2924        return Ok(());
2925    }
2926
2927    let failure = if !completion.configure_response.success {
2928        Some((
2929            &completion.configure_response,
2930            "configure failed during route bind",
2931        ))
2932    } else {
2933        None
2934    };
2935
2936    if let Some((response, fallback)) = failure {
2937        rollback_pending_bind_actor(
2938            executor,
2939            live_roots,
2940            pending_binds,
2941            &completion.bind_root_id,
2942            inserted_new_actor,
2943        );
2944        let has_pending_bind = pending_binds
2945            .values()
2946            .any(|pending| pending.bind_root_id == completion.bind_root_id);
2947        if !root_channels
2948            .get(&completion.bind_root_id)
2949            .is_some_and(|channels| !channels.is_empty())
2950            && !has_pending_bind
2951        {
2952            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
2953        }
2954        let message = response_message(response, fallback);
2955        let fatal = response_is_fatal_panic(response);
2956        let error_code = route_bind_error_code_for_configure_response(response);
2957        send_route_bind_error_parts(
2958            tx,
2959            completion.ver,
2960            completion.corr,
2961            completion.flags,
2962            error_code,
2963            &message,
2964            metrics,
2965        )
2966        .await?;
2967        remove_installed_route(installed_route_epochs, route_id);
2968        if fatal {
2969            signal_fatal_teardown(
2970                tx,
2971                Some(completion.route),
2972                completion.ver,
2973                completion.corr,
2974                shutdown,
2975                metrics,
2976            )
2977            .await;
2978        }
2979        return Ok(());
2980    }
2981
2982    remember_session_identity(session_identity, &completion.identity);
2983    let replay_key = push::ReplayKey::from_identity(&completion.identity);
2984    let bind_trust = completion.identity.trust;
2985    insert_route_channel(routes, root_channels, route_id, completion.identity);
2986    let restore_watcher = live_roots
2987        .get(&completion.bind_root_id)
2988        .is_some_and(|meta| meta.idle_artifacts_evicted || meta.unbound_quiesced);
2989    live_roots
2990        .entry(completion.bind_root_id.clone())
2991        .and_modify(|meta| {
2992            meta.reactivate_bound();
2993            meta.diagnostics_on_edit = completion.diagnostics_on_edit;
2994            meta.maintenance_poisoned = false;
2995        })
2996        .or_insert_with(|| RootMeta::new(Instant::now()));
2997    if let Some(meta) = live_roots.get_mut(&completion.bind_root_id) {
2998        meta.diagnostics_on_edit = completion.diagnostics_on_edit;
2999        meta.maintenance_poisoned = false;
3000    }
3001    if let Some(ctx) = executor.actor_context(&completion.bind_root_id) {
3002        ctx.mark_subc_bound();
3003        if restore_watcher {
3004            crate::commands::configure::ensure_project_watcher(&ctx);
3005        }
3006    }
3007
3008    let ack =
3009        serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).map_err(SubcError::Json)?;
3010    let response = Frame::build_with_version(
3011        completion.ver,
3012        FrameType::Response,
3013        control_flags(),
3014        0,
3015        0,
3016        completion.corr,
3017        ack,
3018    )
3019    .map_err(SubcError::FrameBuild)?;
3020    send_reliable_writer_frame(tx, metrics, response, "RouteBindAck").await?;
3021    queue_post_bind_configure_and_completion_maintenance(&completion.bind_root_id, live_roots);
3022    let replayed = push::replay_buffered_push_frames(
3023        tx,
3024        metrics,
3025        route_id,
3026        push_buffer,
3027        &replay_key,
3028        bind_trust,
3029    );
3030    if replayed > 0 {
3031        log::debug!(
3032            "subc attach: replayed {} buffered Push frame(s) to route {} root {} harness {} session {}",
3033            replayed,
3034            completion.route,
3035            replay_key.root.as_path().display(),
3036            replay_key.harness,
3037            replay_key.session
3038        );
3039    }
3040    log::info!(
3041        "subc attach: route {} bound to root {}",
3042        completion.route,
3043        completion.bind_root_id.as_path().display()
3044    );
3045    Ok(())
3046}
3047
3048async fn expire_overdue_route_binds(
3049    tx: &WriterSender,
3050    executor: &Arc<Executor>,
3051    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3052    installed_route_epochs: &mut HashMap<u16, u32>,
3053    metrics: &DispatchPathMetrics,
3054) -> Result<(), SubcError> {
3055    let now = Instant::now();
3056    let expired: Vec<_> = pending_binds
3057        .iter()
3058        .filter_map(|(route, pending)| {
3059            let age = now.saturating_duration_since(pending.started_at);
3060            (!pending.deadline_reported && age >= ROUTE_BIND_DEADLINE).then(|| {
3061                (
3062                    *route,
3063                    pending.corr,
3064                    pending.ver,
3065                    pending.flags,
3066                    pending.bind_root_id.clone(),
3067                    pending.configure_request_id.clone(),
3068                    age,
3069                )
3070            })
3071        })
3072        .collect();
3073
3074    for (route, corr, ver, flags, root_id, configure_request_id, age) in expired {
3075        if let Some(pending) = pending_binds.get_mut(&route) {
3076            pending.cancelled = true;
3077            pending.deadline_reported = true;
3078            let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
3079            log::debug!(
3080                "subc attach: cancelled overdue RouteBind configure for route {route} ({outcome:?})"
3081            );
3082        }
3083        remove_installed_route(installed_route_epochs, route);
3084        let age_ms = age.as_millis().min(u128::from(u64::MAX)) as u64;
3085        let deadline_ms = ROUTE_BIND_DEADLINE.as_millis();
3086        send_route_bind_error_parts(
3087            tx,
3088            ver,
3089            corr,
3090            flags,
3091            "actor_not_ready",
3092            &format!("route bind deadline exceeded after {age_ms}ms (deadline {deadline_ms}ms)"),
3093            metrics,
3094        )
3095        .await?;
3096        log::warn!(
3097            "subc attach: route {} bind for root {} exceeded {}ms deadline (configure_request_id={})",
3098            route,
3099            root_id.as_path().display(),
3100            deadline_ms,
3101            configure_request_id
3102        );
3103    }
3104
3105    Ok(())
3106}
3107
3108/// channel-0 control requests: RouteBind plus the cached health probe. RouteBind
3109/// still reconciles the route's RootConfig through the executor's Mutating lane
3110/// and resolves completion on a loop-owned control-completion channel so slow
3111/// configure jobs do not block the transport loop.
3112#[allow(clippy::too_many_arguments)]
3113async fn handle_control_request(
3114    tx: &WriterSender,
3115    frame: &Frame,
3116    shared_app: &Arc<App>,
3117    executor: &Arc<Executor>,
3118    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3119    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3120    installed_route_epochs: &mut HashMap<u16, u32>,
3121    routes: &mut HashMap<RouteChannel, RouteIdentity>,
3122    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
3123    bg_subs: &mut HashMap<RouteChannel, BgSub>,
3124    bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
3125    bg_wake_pending: &mut HashSet<RouteChannel>,
3126    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
3127    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
3128    retry_buffer: &mut RetryBuffer,
3129    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
3130    shutdown: &Arc<Notify>,
3131    control_completion_tx: &mpsc::Sender<RouteBindCompletion>,
3132    metrics: &Arc<DispatchPathMetrics>,
3133    push_senders: &PushSenders,
3134    dispatch: DispatchFn,
3135    user_config_path: Option<&Path>,
3136) -> Result<(), SubcError> {
3137    let request =
3138        serde_json::from_slice::<ModuleControlRequest>(&frame.body).map_err(SubcError::Json)?;
3139    match request {
3140        ModuleControlRequest::RouteBind {
3141            route_channel,
3142            epoch,
3143            target: _,
3144            identity,
3145            principal,
3146            consumer_capabilities,
3147        } => {
3148            let route_id = route_key(route_channel, epoch);
3149            if epoch == 0 {
3150                return send_route_bind_error(
3151                    tx,
3152                    frame,
3153                    "config_divergence",
3154                    "route bind uses an invalid channel generation",
3155                    metrics,
3156                )
3157                .await;
3158            }
3159            let mut bind_root_id = None;
3160            if let Some(installed_epoch) = installed_route_epochs.get(&route_channel).copied() {
3161                if installed_epoch >= epoch {
3162                    return send_route_bind_error(
3163                        tx,
3164                        frame,
3165                        "config_divergence",
3166                        "route bind generation is not newer than the installed generation",
3167                        metrics,
3168                    )
3169                    .await;
3170                }
3171
3172                let replacement_root = match ProjectRootId::from_path(&identity.project_root) {
3173                    Ok(root_id) => root_id,
3174                    Err(error) => {
3175                        return send_route_bind_error(
3176                            tx,
3177                            frame,
3178                            "config_divergence",
3179                            &format!("invalid route project root: {error}"),
3180                            metrics,
3181                        )
3182                        .await;
3183                    }
3184                };
3185                teardown_installed_route(
3186                    tx,
3187                    metrics,
3188                    executor,
3189                    route_key(route_channel, installed_epoch),
3190                    "higher-epoch RouteBind",
3191                    Some(&replacement_root),
3192                    installed_route_epochs,
3193                    routes,
3194                    root_channels,
3195                    bg_subs,
3196                    bg_sub_by_session,
3197                    bg_wake_pending,
3198                    pending_bash_asks,
3199                    live_roots,
3200                    route_bash_cancels,
3201                    pending_binds,
3202                    retry_buffer,
3203                    push_buffer,
3204                    shutdown,
3205                )
3206                .await?;
3207                bind_root_id = Some(replacement_root);
3208            }
3209            if pending_binds.contains_key(&route_id) {
3210                return send_route_bind_error(
3211                    tx,
3212                    frame,
3213                    "config_divergence",
3214                    "route bind is already pending for channel",
3215                    metrics,
3216                )
3217                .await;
3218            }
3219            let bind_root_id = match bind_root_id {
3220                Some(root_id) => root_id,
3221                None => match ProjectRootId::from_path(&identity.project_root) {
3222                    Ok(root_id) => root_id,
3223                    Err(error) => {
3224                        return send_route_bind_error(
3225                            tx,
3226                            frame,
3227                            "config_divergence",
3228                            &format!("invalid route project root: {error}"),
3229                            metrics,
3230                        )
3231                        .await;
3232                    }
3233                },
3234            };
3235
3236            // Reconcile RootConfig: build a configure request from the bind
3237            // identity + forwarded config tiers and run it through the executor.
3238            let request_id = format!("subc-bind-{route_channel}");
3239            let bind_project_root = identity.project_root.clone();
3240            let bind_harness = identity.harness.clone();
3241            let bind_session = identity.session.clone();
3242            let bind_trust = trust_for_bind(&bind_harness, &principal);
3243            // Typed capability declaration from the consumer: the facade stamps it
3244            // from the MCP host's initialize-advertised capabilities. Absent
3245            // means no reverse-request capability — flat deny, fail-closed. A
3246            // consumer over-declaring only earns asks that TTL-deny.
3247            let consumer_elicitation_capable = consumer_capabilities
3248                .as_ref()
3249                .is_some_and(|capabilities| capabilities.iter().any(|c| c == "elicitation"));
3250            log::info!(
3251                "subc attach: route {} harness={} principal={} trust={} elicitation={}",
3252                route_channel,
3253                bind_harness,
3254                principal_label(&principal),
3255                bind_trust.label(),
3256                consumer_elicitation_capable
3257            );
3258
3259            // Config is single-per-project, read by AFT directly from the
3260            // CortexKit config files (user: ~/.config/cortexkit/aft.jsonc,
3261            // project: <root>/.cortexkit/aft.jsonc). Wire-relayed config tiers are
3262            // IGNORED entirely: a front (runner, mcp:*, or fed:*) cannot push config over
3263            // the wire. This is what makes config harness-INDEPENDENT — every
3264            // harness binding a project gets the identical on-disk config, so two
3265            // trust domains sharing the per-root actor can never diverge or
3266            // inherit each other's capabilities (the cross-bind escalation class).
3267            // Wire-relayed config tiers (if the protocol still carries them) are
3268            // ignored entirely; the per-tier trust boundary (user trusted, project
3269            // privileged-dropped) is applied to the FILE tiers in handle_configure.
3270            let local_tiers = crate::subc_config::read_local_cortexkit_config_tiers(
3271                user_config_path,
3272                Path::new(&bind_project_root),
3273            );
3274            let config_tiers: Vec<Value> = local_tiers
3275                .iter()
3276                .map(|t| json!({ "tier": t.tier, "source": t.source, "doc": t.doc }))
3277                .collect();
3278            let diagnostics_on_edit = diagnostics_on_edit_from_tiers(&local_tiers);
3279            let configure_json = json!({
3280                "id": request_id,
3281                "command": "configure",
3282                "project_root": bind_project_root,
3283                "harness": bind_harness,
3284                "session_id": bind_session.clone(),
3285                "config": config_tiers,
3286            });
3287            let configure_req = match serde_json::from_value::<RawRequest>(configure_json) {
3288                Ok(req) => req,
3289                Err(error) => {
3290                    return send_route_bind_error(
3291                        tx,
3292                        frame,
3293                        "config_divergence",
3294                        &format!("failed to build configure request: {error}"),
3295                        metrics,
3296                    )
3297                    .await;
3298                }
3299            };
3300
3301            let route_identity = RouteIdentity(Arc::new(RouteIdentityData {
3302                root: bind_root_id.clone(),
3303                project_root: PathBuf::from(&bind_project_root),
3304                harness: bind_harness.clone(),
3305                session: bind_session.clone(),
3306                trust: bind_trust,
3307                consumer_elicitation_capable,
3308            }));
3309            let configure_session = route_identity.session.clone();
3310            let root_was_live = live_roots.contains_key(&bind_root_id);
3311            let inserted_new_actor = register_actor_for_bind(
3312                shared_app,
3313                executor,
3314                push_senders,
3315                &bind_root_id,
3316                route_channel,
3317                root_was_live,
3318            );
3319
3320            let configure_request_id = configure_req.id.clone();
3321            installed_route_epochs.insert(route_channel, epoch);
3322            if let Some(meta) = live_roots.get_mut(&bind_root_id) {
3323                meta.maintenance_queued_kinds.clear();
3324                meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
3325            }
3326            let (configure_rx, configure_cancellation) = executor.submit_cancellable_async(
3327                bind_root_id.clone(),
3328                Lane::Mutating,
3329                configure_request_id.clone(),
3330                Box::new(move |ctx| {
3331                    log_ctx::with_session(Some(configure_session.clone()), || {
3332                        dispatch(configure_req, ctx)
3333                    })
3334                }),
3335            );
3336            pending_binds.insert(
3337                route_id,
3338                PendingBind {
3339                    bind_root_id: bind_root_id.clone(),
3340                    inserted_new_actor,
3341                    cancelled: false,
3342                    configure_request_id: configure_request_id.clone(),
3343                    started_at: Instant::now(),
3344                    warned_half_deadline: false,
3345                    deadline_reported: false,
3346                    corr: frame.header.corr,
3347                    ver: frame.header.ver,
3348                    flags: frame.header.flags,
3349                    cancellation: configure_cancellation,
3350                },
3351            );
3352
3353            let completion_tx = control_completion_tx.clone();
3354            let completion_identity = route_identity;
3355            let completion_root = bind_root_id.clone();
3356            let completion_route_channel = route_channel;
3357            let completion_ver = frame.header.ver;
3358            let completion_corr = frame.header.corr;
3359            let completion_flags = frame.header.flags;
3360            let completion_metrics = Arc::clone(metrics);
3361            tokio::spawn(async move {
3362                let _response_task = ResponseTaskGuard::new(&completion_metrics);
3363                let configure_response =
3364                    await_executor_response(configure_rx, configure_request_id.clone()).await;
3365                // Send the route-bind acknowledgment as soon as configure succeeds.
3366                // Installing completed search or callgraph builds only refreshes cached
3367                // read data, so a later maintenance pass can do it without delaying the
3368                // daemon's confirmation that the route is usable.
3369                let completion = RouteBindCompletion {
3370                    route: route_key(completion_route_channel, epoch),
3371                    identity: completion_identity,
3372                    bind_root_id: completion_root,
3373                    inserted_new_actor,
3374                    configure_response,
3375                    diagnostics_on_edit,
3376                    ver: completion_ver,
3377                    corr: completion_corr,
3378                    flags: completion_flags,
3379                };
3380                if send_counted_channel(
3381                    &completion_tx,
3382                    &completion_metrics.control_completion_queued,
3383                    completion,
3384                )
3385                .await
3386                .is_err()
3387                {
3388                    log::debug!(
3389                        "subc attach: dropped RouteBind completion for route {} after loop exit",
3390                        completion_route_channel
3391                    );
3392                }
3393            });
3394
3395            Ok(())
3396        }
3397        ModuleControlRequest::HealthCheck {} => {
3398            let report = build_health_report(executor, pending_binds, metrics);
3399            let body = serde_json::to_vec(&ModuleControlResponse::from(report))
3400                .map_err(SubcError::Json)?;
3401            let response = Frame::build_with_version(
3402                frame.header.ver,
3403                FrameType::Response,
3404                frame.header.flags,
3405                0,
3406                0,
3407                frame.header.corr,
3408                body,
3409            )
3410            .map_err(SubcError::FrameBuild)?;
3411            send_frame(tx, metrics, response).await
3412        }
3413    }
3414}
3415
3416fn install_bash_compressor(ctx: &AppContext) {
3417    // Mirrors main.rs per-actor compressor installation for subc-created actors.
3418    let filter_registry_handle = ctx.shared_filter_registry();
3419    let compress_flag = ctx.bash_compress_flag();
3420    ctx.bash_background().set_compressor_with_exit_code(
3421        move |command: &str, output: String, exit_code: Option<i32>| {
3422            if !compress_flag.load(std::sync::atomic::Ordering::Relaxed) {
3423                return crate::compress::CompressionResult::new(output);
3424            }
3425            let registry_guard = match filter_registry_handle.read() {
3426                Ok(g) => g,
3427                Err(poisoned) => poisoned.into_inner(),
3428            };
3429            crate::compress::compress_with_registry_exit_code(
3430                command,
3431                &output,
3432                exit_code,
3433                &registry_guard,
3434            )
3435        },
3436    );
3437}
3438
3439fn diagnostics_on_edit_from_tiers(tiers: &[ConfigTier]) -> bool {
3440    let mut diagnostics_on_edit = false;
3441    for tier in tiers {
3442        if let Some(value) = diagnostics_on_edit_from_doc(&tier.doc) {
3443            diagnostics_on_edit = value;
3444        }
3445    }
3446    diagnostics_on_edit
3447}
3448
3449fn diagnostics_on_edit_from_doc(doc: &str) -> Option<bool> {
3450    let stripped = strip_jsonc(doc);
3451    let value = serde_json::from_str::<Value>(&stripped).ok()?;
3452    value
3453        .get("lsp")
3454        .and_then(Value::as_object)?
3455        .get("diagnostics_on_edit")
3456        .and_then(Value::as_bool)
3457}
3458
3459async fn send_route_bind_error(
3460    tx: &WriterSender,
3461    frame: &Frame,
3462    code: &str,
3463    message: &str,
3464    metrics: &DispatchPathMetrics,
3465) -> Result<(), SubcError> {
3466    send_route_bind_error_parts(
3467        tx,
3468        frame.header.ver,
3469        frame.header.corr,
3470        frame.header.flags,
3471        code,
3472        message,
3473        metrics,
3474    )
3475    .await
3476}
3477
3478async fn send_route_bind_error_parts(
3479    tx: &WriterSender,
3480    ver: u8,
3481    corr: u64,
3482    flags: Flags,
3483    code: &str,
3484    message: &str,
3485    metrics: &DispatchPathMetrics,
3486) -> Result<(), SubcError> {
3487    let response = build_error_frame(ver, 0, 0, corr, flags, code, message)?;
3488    send_reliable_writer_frame(tx, metrics, response, "RouteBind error").await?;
3489    log::warn!("subc attach: route bind rejected ({code}): {message}");
3490    Ok(())
3491}
3492
3493/// Route-channel tool call: `{name, arguments}` → executor lane → dispatch to
3494/// the sync command core → wrap the structured Response in a CallToolResult
3495/// `{content, isError}`. Tool-result mapping: the whole `{success, ...}` Response
3496/// serialized into ONE text block; `isError` carries `success == false`.
3497async fn handle_tool_call(
3498    tx: &WriterSender,
3499    frame: &Frame,
3500    mut phase_trace: PhaseTrace,
3501    routes: &HashMap<RouteChannel, RouteIdentity>,
3502    pending_binds: &HashMap<RouteChannel, PendingBind>,
3503    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3504    executor: &Arc<Executor>,
3505    shutdown: &Arc<Notify>,
3506    connection_cancel: &PersistentCancelSignal,
3507    bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
3508    bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
3509    metrics: &Arc<DispatchPathMetrics>,
3510    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
3511    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
3512    next_bash_ask_corr: &mut u64,
3513    bg_subs: &mut HashMap<RouteChannel, BgSub>,
3514    bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
3515    bg_wake_pending: &mut HashSet<RouteChannel>,
3516    bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
3517    dispatch: DispatchFn,
3518    allow_native_passthrough: bool,
3519) -> Result<(), SubcError> {
3520    let route_id = route_key(frame.header.channel, frame.header.epoch);
3521    if pending_binds.contains_key(&route_id) {
3522        let error = build_error_frame(
3523            frame.header.ver,
3524            frame.header.channel,
3525            frame.header.epoch,
3526            frame.header.corr,
3527            frame.header.flags,
3528            "route_not_bound",
3529            "route is not bound before tool call",
3530        )?;
3531        return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
3532    }
3533
3534    let Some(identity) = routes.get(&route_id).cloned() else {
3535        let error = build_error_frame(
3536            frame.header.ver,
3537            frame.header.channel,
3538            frame.header.epoch,
3539            frame.header.corr,
3540            frame.header.flags,
3541            "route_not_bound",
3542            "route is not bound before tool call",
3543        )?;
3544        return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
3545    };
3546    let restore_watcher = live_roots
3547        .get(&identity.root)
3548        .is_some_and(|meta| meta.idle_artifacts_evicted);
3549    if let Some(meta) = live_roots.get_mut(&identity.root) {
3550        meta.reactivate_bound();
3551    }
3552    if restore_watcher {
3553        if let Some(ctx) = executor.actor_context(&identity.root) {
3554            crate::commands::configure::ensure_project_watcher(&ctx);
3555        }
3556    }
3557
3558    let route_request =
3559        serde_json::from_slice::<RouteRequest>(&frame.body).map_err(SubcError::Json)?;
3560    if matches!(
3561        route_request,
3562        RouteRequest::BgEvents(BgEventsRequest {
3563            op: BgEventsOp::BgEvents
3564        })
3565    ) {
3566        if let Some(old_sub) = bg_subs.get(&route_id).copied() {
3567            push::send_reliable_bg_stream_end(tx, metrics, route_id, &old_sub).await?;
3568        }
3569        if !identity.trust.allows_bash_observation() {
3570            bg_subs.remove(&route_id);
3571            bg_wake_pending.remove(&route_id);
3572            remove_bg_subscription_index(bg_sub_by_session, route_id, Some(&identity));
3573            let denied_sub = BgSub {
3574                corr: frame.header.corr,
3575                ver: frame.header.ver,
3576                flags: frame.header.flags,
3577            };
3578            push::send_reliable_bg_stream_end(tx, metrics, route_id, &denied_sub).await?;
3579            return Ok(());
3580        }
3581        bg_subs.insert(
3582            route_id,
3583            BgSub {
3584                corr: frame.header.corr,
3585                ver: frame.header.ver,
3586                flags: frame.header.flags,
3587            },
3588        );
3589        bg_sub_by_session.insert((identity.root.clone(), identity.session.clone()), route_id);
3590        push::arm_bg_wake(
3591            identity.root.clone(),
3592            identity.session.clone(),
3593            route_id,
3594            bg_wake_pending,
3595            bg_wake_epoch,
3596        );
3597        return Ok(());
3598    }
3599
3600    let RouteRequest::ToolCall(call) = route_request else {
3601        unreachable!("background event subscription returned above")
3602    };
3603    let bare_name = call.name;
3604    let arguments = strip_agent_preview_arg_owned(call.arguments);
3605    let format_context = crate::subc_format::FormatContext::from_tool_call(
3606        &bare_name,
3607        &arguments,
3608        identity.project_root.as_path(),
3609    );
3610
3611    let request_id = format!("subc-{}-{}", frame.header.channel, frame.header.corr);
3612    let bind_trust = identity.trust;
3613    let diagnostics_on_edit = live_roots
3614        .get(&identity.root)
3615        .map(|meta| meta.diagnostics_on_edit)
3616        .unwrap_or(false);
3617
3618    if matches!(bind_trust, BindTrust::Untrusted)
3619        && is_bash_family_tool(&bare_name)
3620        && (bare_name != "bash" || !identity.consumer_elicitation_capable)
3621    {
3622        let response = bash::bash_denied_untrusted_response(request_id.clone());
3623        let text = crate::subc_format::format_response_with_context(
3624            &bare_name,
3625            &response,
3626            &format_context,
3627        );
3628        let result = ToolCallResult { text, response };
3629        let response_frame = build_tool_response_frame(
3630            frame.header.ver,
3631            route_id,
3632            frame.header.corr,
3633            frame.header.flags,
3634            &result,
3635        )?;
3636        return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
3637    }
3638
3639    // A non-core name is NOT in the tool manifest. AFT fails closed and
3640    // does not trust subc to enforce the manifest: rejecting here is the
3641    // defense-in-depth backstop that prevents a forwarded native command
3642    // (e.g. `configure`, which would reach handle_configure and bypass
3643    // the RouteBind config-trust cap) from ever reaching dispatch. Only
3644    // the integration-test harness (run_subc_mode_for_test) opens this to
3645    // drive synthetic native commands through the executor.
3646    if !is_subc_agent_core_tool(&bare_name)
3647        && !is_subc_native_plumbing_tool(&bare_name)
3648        && !allow_native_passthrough
3649    {
3650        log::warn!(
3651            "subc tool call: rejecting non-manifest tool name {:?} on route {} (fail-closed)",
3652            bare_name,
3653            frame.header.channel
3654        );
3655        let response = Response::error(
3656            request_id.clone(),
3657            "unknown_tool",
3658            format!("tool {:?} is not in the AFT tool manifest", bare_name),
3659        );
3660        let text = crate::subc_format::format_response_with_context(
3661            &bare_name,
3662            &response,
3663            &format_context,
3664        );
3665        let result = ToolCallResult { text, response };
3666        let response_frame = build_tool_response_frame(
3667            frame.header.ver,
3668            route_id,
3669            frame.header.corr,
3670            frame.header.flags,
3671            &result,
3672        )?;
3673        return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
3674    }
3675
3676    if bare_name == "bash" {
3677        if matches!(bind_trust, BindTrust::Untrusted) {
3678            let plan = match bash::prepare_bash_elicitation_plan(
3679                &arguments,
3680                identity.project_root.as_path(),
3681            ) {
3682                Ok(plan) => plan,
3683                Err(error) => {
3684                    let response = Response::error(request_id.clone(), error.code, error.message);
3685                    let text = crate::subc_format::format_response_with_context(
3686                        &bare_name,
3687                        &response,
3688                        &format_context,
3689                    );
3690                    let result = ToolCallResult { text, response };
3691                    let response_frame = build_tool_response_frame(
3692                        frame.header.ver,
3693                        route_id,
3694                        frame.header.corr,
3695                        frame.header.flags,
3696                        &result,
3697                    )?;
3698                    return send_reliable_writer_frame(
3699                        tx,
3700                        metrics,
3701                        response_frame,
3702                        "tool response",
3703                    )
3704                    .await;
3705                }
3706            };
3707
3708            let meta = live_roots
3709                .entry(identity.root.clone())
3710                .or_insert_with(|| RootMeta::new(Instant::now()));
3711            meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
3712            meta.reactivate_bound();
3713
3714            let route_cancel =
3715                route_bash_cancels
3716                    .entry(route_id)
3717                    .or_insert_with(|| bash::RouteBashCancel {
3718                        token: PersistentCancelSignal::new(),
3719                        active_waits: 0,
3720                    });
3721            route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
3722            let cancel = bash::BashWaitCancel {
3723                connection: connection_cancel.clone(),
3724                route: route_cancel.token.clone(),
3725            };
3726
3727            let reverse_corr =
3728                allocate_reverse_corr(pending_bash_asks, route_id, next_bash_ask_corr);
3729            let ask_frame = build_bash_elicitation_request_frame(
3730                frame.header.ver,
3731                route_id,
3732                reverse_corr,
3733                frame.header.flags,
3734                &plan.command,
3735                &plan.asks,
3736            )?;
3737            pending_bash_asks.insert(
3738                ReverseCorrKey {
3739                    route: route_id,
3740                    corr: reverse_corr,
3741                },
3742                PendingBashAsk {
3743                    route: route_id,
3744                    tool_corr: frame.header.corr,
3745                    tool_flags: frame.header.flags,
3746                    tool_ver: frame.header.ver,
3747                    root: identity.root.clone(),
3748                    project_root: identity.project_root.clone(),
3749                    session_id: identity.session.clone(),
3750                    request_id,
3751                    arguments,
3752                    format_context,
3753                    cancel,
3754                    grants: plan.grants,
3755                    expires_at: Instant::now() + bash_elicitation_timeout(),
3756                },
3757            );
3758            return send_reliable_writer_frame(tx, metrics, ask_frame, "bash elicitation request")
3759                .await;
3760        }
3761
3762        let meta = live_roots
3763            .entry(identity.root.clone())
3764            .or_insert_with(|| RootMeta::new(Instant::now()));
3765        meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
3766        meta.reactivate_bound();
3767
3768        let route_cancel =
3769            route_bash_cancels
3770                .entry(route_id)
3771                .or_insert_with(|| bash::RouteBashCancel {
3772                    token: PersistentCancelSignal::new(),
3773                    active_waits: 0,
3774                });
3775        route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
3776        let cancel = bash::BashWaitCancel {
3777            connection: connection_cancel.clone(),
3778            route: route_cancel.token.clone(),
3779        };
3780
3781        bash::submit_deferred_bash(
3782            executor,
3783            bash_deferred_tx,
3784            bash_poll_touch_tx,
3785            metrics,
3786            dispatch,
3787            identity.root.clone(),
3788            identity.project_root.clone(),
3789            identity.session.clone(),
3790            request_id,
3791            route_id,
3792            frame.header.corr,
3793            frame.header.flags,
3794            frame.header.ver,
3795            arguments,
3796            format_context,
3797            cancel,
3798            bind_trust,
3799            None,
3800        );
3801        return Ok(());
3802    }
3803
3804    let lane = command_lane(&bare_name);
3805    let tool_call_context = ToolCallContext {
3806        project_root: identity.project_root.clone(),
3807        session_id: Some(identity.session.clone()),
3808        request_id: request_id.clone(),
3809        diagnostics_on_edit,
3810        preview: call.preview,
3811    };
3812    let bare_name_for_frame = bare_name.clone();
3813    let identity_for_run = identity.clone();
3814    let completion_session = identity.session.clone();
3815    let completion_root = identity.project_root.clone();
3816    let request_id_for_force = request_id.clone();
3817    let format_context_for_frame = format_context.clone();
3818    let (tool_call_tx, tool_call_rx) = oneshot::channel::<ToolCallCompletion>();
3819    phase_trace.mark_executor_submitted();
3820    let rx = executor.submit_async(
3821        identity.root.clone(),
3822        lane,
3823        request_id.clone(),
3824        Box::new(move |ctx| {
3825            phase_trace.mark_job_admitted();
3826            log_ctx::with_session(Some(identity_for_run.session.clone()), || {
3827                let run = || {
3828                    let finalizer = |response: &mut Response| {
3829                        crate::response_finalize::finalize_response_with_bg_completions(
3830                            response,
3831                            ctx,
3832                            &identity_for_run.session,
3833                            &bare_name,
3834                            bind_trust.allows_bash_observation(),
3835                        );
3836                    };
3837                    match run_tool_call(
3838                        &bare_name,
3839                        &arguments,
3840                        &format_context,
3841                        &tool_call_context,
3842                        ctx,
3843                        &dispatch,
3844                        Some(&finalizer),
3845                        Some(&mut phase_trace),
3846                    ) {
3847                        ToolCallOutcome::Unary(result) => {
3848                            let response = result.response;
3849                            let _ = tool_call_tx.send(ToolCallCompletion {
3850                                text: result.text,
3851                                phase_trace,
3852                            });
3853                            response
3854                        }
3855                    }
3856                };
3857                if matches!(bind_trust, BindTrust::Untrusted) {
3858                    ctx.with_force_restrict(&request_id_for_force, run)
3859                } else {
3860                    run()
3861                }
3862            })
3863        }),
3864    );
3865    let completion_tx = tx.clone();
3866    let completion_shutdown = Arc::clone(shutdown);
3867    let route = route_id;
3868    let corr = frame.header.corr;
3869    let flags = frame.header.flags;
3870    let ver = frame.header.ver;
3871    let completion_metrics = Arc::clone(metrics);
3872    tokio::spawn(async move {
3873        let _response_task = ResponseTaskGuard::new(&completion_metrics);
3874        let response = await_executor_response(rx, request_id.clone()).await;
3875        let (text, phase_trace) = match tool_call_rx.await {
3876            Ok(completion) => (completion.text, Some(completion.phase_trace)),
3877            Err(_) => (
3878                crate::subc_format::format_response_with_context(
3879                    &bare_name_for_frame,
3880                    &response,
3881                    &format_context_for_frame,
3882                ),
3883                None,
3884            ),
3885        };
3886        let result = ToolCallResult { text, response };
3887        let fatal = response_is_fatal_panic(&result.response);
3888        match build_tool_response_frame(ver, route, corr, flags, &result) {
3889            Ok(response_frame) => {
3890                let send_result = if let Some(phase_trace) = phase_trace {
3891                    let trace = ToolResponseWriteTrace::new(
3892                        phase_trace,
3893                        bare_name_for_frame,
3894                        completion_root,
3895                        completion_session,
3896                        route.channel,
3897                        corr,
3898                    );
3899                    send_traced_tool_response_frame(
3900                        &completion_tx,
3901                        &completion_metrics,
3902                        response_frame,
3903                        trace,
3904                    )
3905                    .await
3906                } else {
3907                    send_reliable_writer_frame(
3908                        &completion_tx,
3909                        &completion_metrics,
3910                        response_frame,
3911                        "tool response",
3912                    )
3913                    .await
3914                };
3915                if let Err(error) = send_result {
3916                    log::warn!("subc attach: failed to queue tool response frame: {error}");
3917                }
3918            }
3919            Err(error) => {
3920                log::error!("subc attach: failed to build tool response frame: {error}");
3921            }
3922        }
3923        if fatal {
3924            signal_fatal_teardown(
3925                &completion_tx,
3926                Some(route),
3927                ver,
3928                corr,
3929                &completion_shutdown,
3930                &completion_metrics,
3931            )
3932            .await;
3933        }
3934    });
3935    Ok(())
3936}
3937
3938fn submit_maintenance_job(
3939    executor: &Arc<Executor>,
3940    root_id: ProjectRootId,
3941    kind: MaintenanceDrainKind,
3942    bg_sessions_to_check: Vec<(String, u64)>,
3943    completion_tx: &mpsc::Sender<MaintenanceCompletion>,
3944    metrics: &Arc<DispatchPathMetrics>,
3945) {
3946    let request_id = format!(
3947        "subc-maintenance-drain-{}-{}",
3948        kind.label(),
3949        root_id.as_path().to_string_lossy()
3950    );
3951    let response_id = request_id.clone();
3952    let completion_root_id = root_id.clone();
3953    let maintenance_generation = executor
3954        .actor_context(&root_id)
3955        .map(|ctx| ctx.configure_generation())
3956        .unwrap_or(0);
3957    let (outcome_tx, outcome_rx) = oneshot::channel::<MaintenanceJobOutcome>();
3958    // ConfigureTail runs deferred configure mutations and needs the actor
3959    // epoch write gate. The other drain kinds only mutate subsystem state
3960    // behind that subsystem's own lock, so they run on MaintenanceCommit and
3961    // overlap interactive reads instead of excluding them.
3962    let lane = match kind {
3963        MaintenanceDrainKind::ConfigureTail => Lane::Mutating,
3964        MaintenanceDrainKind::Watcher
3965        | MaintenanceDrainKind::Lsp
3966        | MaintenanceDrainKind::CompletionDrains => Lane::MaintenanceCommit,
3967    };
3968    let rx = executor.submit_maintenance_async(
3969        root_id,
3970        lane,
3971        request_id.clone(),
3972        Box::new(move |ctx| {
3973            let outcome = match kind {
3974                MaintenanceDrainKind::Watcher => {
3975                    let drained = runtime_drain::drain_watcher_events_bounded(
3976                        ctx,
3977                        runtime_drain::WATCHER_PATH_DRAIN_BATCH_CAP,
3978                    );
3979                    MaintenanceJobOutcome {
3980                        empty_bg_sessions: Vec::new(),
3981                        requeue_kind: drained.has_more.then_some(kind),
3982                    }
3983                }
3984                MaintenanceDrainKind::Lsp => {
3985                    let drained = runtime_drain::drain_lsp_events_bounded(
3986                        ctx,
3987                        runtime_drain::LSP_EVENT_DRAIN_BATCH_CAP,
3988                    );
3989                    MaintenanceJobOutcome {
3990                        empty_bg_sessions: Vec::new(),
3991                        requeue_kind: drained.has_more.then_some(kind),
3992                    }
3993                }
3994                MaintenanceDrainKind::ConfigureTail => {
3995                    runtime_drain::drain_deferred_configure_maintenance(ctx);
3996                    runtime_drain::drain_configure_warning_events(ctx);
3997                    MaintenanceJobOutcome::default()
3998                }
3999                MaintenanceDrainKind::CompletionDrains => {
4000                    runtime_drain::drain_search_index_events(ctx);
4001                    runtime_drain::drain_callgraph_store_events(ctx);
4002                    runtime_drain::drain_semantic_index_events(ctx);
4003                    runtime_drain::drain_semantic_refresh_events(ctx);
4004                    runtime_drain::drain_inspect_events_for_generation(ctx, maintenance_generation);
4005                    let empty_bg_sessions = bg_sessions_to_check
4006                        .into_iter()
4007                        .filter(|(session, _)| {
4008                            !ctx.bash_background()
4009                                .has_completions_for_session(Some(session.as_str()))
4010                        })
4011                        .collect();
4012                    MaintenanceJobOutcome {
4013                        empty_bg_sessions,
4014                        requeue_kind: None,
4015                    }
4016                }
4017            };
4018            let requeued = outcome.requeue_kind.is_some();
4019            let _ = outcome_tx.send(outcome);
4020            Response::success(
4021                response_id,
4022                json!({ "drained": true, "kind": kind.label(), "requeued": requeued }),
4023            )
4024        }),
4025    );
4026    let completion_tx = completion_tx.clone();
4027    let completion_metrics = Arc::clone(metrics);
4028    tokio::spawn(async move {
4029        let _response_task = ResponseTaskGuard::new(&completion_metrics);
4030        let response = await_executor_response(rx, request_id).await;
4031        let outcome = outcome_rx.await.unwrap_or_default();
4032        let _ = send_counted_channel(
4033            &completion_tx,
4034            &completion_metrics.maintenance_queued,
4035            MaintenanceCompletion {
4036                root_id: completion_root_id,
4037                kind,
4038                response,
4039                empty_bg_sessions: outcome.empty_bg_sessions,
4040                requeue_kind: outcome.requeue_kind,
4041            },
4042        )
4043        .await;
4044    });
4045}
4046
4047async fn await_executor_response(rx: oneshot::Receiver<Response>, request_id: String) -> Response {
4048    rx.await
4049        .unwrap_or_else(|_| Response::error(request_id, "internal_error", "executor dropped"))
4050}
4051async fn signal_fatal_teardown(
4052    tx: &WriterSender,
4053    route: Option<RouteChannel>,
4054    ver: u8,
4055    corr: u64,
4056    shutdown: &Arc<Notify>,
4057    metrics: &DispatchPathMetrics,
4058) {
4059    if let Some(route) = route {
4060        if let Ok(frame) = build_goodbye_frame(ver, route.channel, route.epoch, corr) {
4061            if let Err(error) = send_frame(tx, metrics, frame).await {
4062                log::warn!(
4063                    "subc attach: failed to queue fatal route Goodbye for route {route}: {error}"
4064                );
4065            }
4066        }
4067    }
4068    if let Ok(frame) = build_goodbye_frame(ver, 0, 0, 0) {
4069        if let Err(error) = send_frame(tx, metrics, frame).await {
4070            log::warn!("subc attach: failed to queue fatal channel-0 Goodbye: {error}");
4071        }
4072    }
4073    shutdown.notify_one();
4074}
4075#[derive(Debug, Deserialize)]
4076#[serde(untagged)]
4077enum RouteRequest {
4078    BgEvents(BgEventsRequest),
4079    ToolCall(ToolCallRequest),
4080}
4081
4082#[derive(Debug, Deserialize)]
4083struct BgEventsRequest {
4084    op: BgEventsOp,
4085}
4086
4087#[derive(Debug, Deserialize)]
4088#[serde(rename_all = "snake_case")]
4089enum BgEventsOp {
4090    BgEvents,
4091}
4092
4093#[derive(Debug, Deserialize)]
4094struct ToolCallRequest {
4095    name: String,
4096    #[serde(default)]
4097    arguments: Value,
4098    /// Server-owned preview control (B1c-0): the plugin's mutation flow is
4099    /// preview -> permission ask -> apply. Dropping this field made "preview"
4100    /// calls mutate disk before the permission prompt and the subsequent
4101    /// apply fail with not-found.
4102    #[serde(default)]
4103    preview: bool,
4104}
4105
4106#[cfg(test)]
4107pub(crate) mod test_support {
4108    use super::*;
4109    use crate::bash_background::BgTaskStatus;
4110    use crate::protocol::{
4111        BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, ConfigureWarningsFrame,
4112        ProgressFrame, StatusChangedFrame,
4113    };
4114    use serde_json::json;
4115
4116    pub(super) fn test_root(name: &str) -> (tempfile::TempDir, ProjectRootId) {
4117        let dir = tempfile::Builder::new()
4118            .prefix(name)
4119            .tempdir()
4120            .expect("temp root");
4121        let root = ProjectRootId::from_path(dir.path()).expect("project root id");
4122        (dir, root)
4123    }
4124
4125    pub(super) fn test_ctx() -> Arc<AppContext> {
4126        Arc::new(AppContext::new(
4127            Box::new(crate::parser::TreeSitterProvider::new()),
4128            crate::config::Config::default(),
4129        ))
4130    }
4131
4132    pub(super) fn status_frame(seq: u64) -> PushFrame {
4133        status_frame_with_session(seq, None)
4134    }
4135
4136    pub(super) fn status_frame_with_session(seq: u64, session_id: Option<&str>) -> PushFrame {
4137        PushFrame::StatusChanged(StatusChangedFrame {
4138            frame_type: "status_changed",
4139            session_id: session_id.map(str::to_string),
4140            snapshot: json!({ "seq": seq }),
4141        })
4142    }
4143
4144    pub(super) fn completion_frame(task_id: &str) -> PushFrame {
4145        completion_frame_with_session(task_id, "session-1")
4146    }
4147
4148    pub(super) fn completion_frame_with_session(task_id: &str, session_id: &str) -> PushFrame {
4149        PushFrame::BashCompleted(BashCompletedFrame {
4150            frame_type: "bash_completed",
4151            task_id: task_id.to_string(),
4152            session_id: session_id.to_string(),
4153            status: BgTaskStatus::Completed,
4154            exit_code: Some(0),
4155            command: format!("echo {task_id}"),
4156            output_preview: String::new(),
4157            output_truncated: false,
4158            original_tokens: None,
4159            compressed_tokens: None,
4160            tokens_skipped: false,
4161        })
4162    }
4163
4164    pub(super) fn long_running_frame(task_id: &str, elapsed_ms: u64) -> PushFrame {
4165        long_running_frame_with_session(task_id, "session-1", elapsed_ms)
4166    }
4167
4168    pub(super) fn long_running_frame_with_session(
4169        task_id: &str,
4170        session_id: &str,
4171        elapsed_ms: u64,
4172    ) -> PushFrame {
4173        PushFrame::BashLongRunning(BashLongRunningFrame {
4174            frame_type: "bash_long_running",
4175            task_id: task_id.to_string(),
4176            session_id: session_id.to_string(),
4177            command: format!("sleep {elapsed_ms}"),
4178            elapsed_ms,
4179        })
4180    }
4181
4182    pub(super) fn pattern_match_frame(session_id: &str) -> PushFrame {
4183        PushFrame::BashPatternMatch(BashPatternMatchFrame {
4184            frame_type: "bash_pattern_match",
4185            task_id: "task-pattern".to_string(),
4186            session_id: session_id.to_string(),
4187            watch_id: "watch-1".to_string(),
4188            match_text: "needle".to_string(),
4189            match_offset: 7,
4190            context: "haystack needle".to_string(),
4191            once: true,
4192            reason: "pattern_match",
4193        })
4194    }
4195
4196    pub(super) fn configure_warnings_frame(session_id: Option<&str>) -> PushFrame {
4197        PushFrame::ConfigureWarnings(ConfigureWarningsFrame {
4198            frame_type: "configure_warnings",
4199            session_id: session_id.map(str::to_string),
4200            project_root: "/tmp/subc-test".to_string(),
4201            warnings: Vec::new(),
4202        })
4203    }
4204
4205    pub(super) fn route_identity(root: &ProjectRootId, session_id: &str) -> RouteIdentity {
4206        route_identity_with_trust(root, session_id, BindTrust::FirstParty)
4207    }
4208
4209    pub(super) fn route_identity_with_trust(
4210        root: &ProjectRootId,
4211        session_id: &str,
4212        trust: BindTrust,
4213    ) -> RouteIdentity {
4214        RouteIdentity(Arc::new(RouteIdentityData {
4215            root: root.clone(),
4216            project_root: root.as_path().to_path_buf(),
4217            harness: "opencode".to_string(),
4218            session: session_id.to_string(),
4219            trust,
4220            consumer_elicitation_capable: false,
4221        }))
4222    }
4223
4224    pub(super) fn progress_frame(request_id: &str, kind: ProgressKind, chunk: &str) -> PushFrame {
4225        PushFrame::Progress(ProgressFrame::new(request_id, kind, chunk))
4226    }
4227
4228    pub(super) fn status_seq(frame: &PushFrame) -> Option<u64> {
4229        match frame {
4230            PushFrame::StatusChanged(status) => status.snapshot.get("seq").and_then(|v| v.as_u64()),
4231            _ => None,
4232        }
4233    }
4234
4235    pub(super) fn completion_task(frame: &PushFrame) -> Option<&str> {
4236        match frame {
4237            PushFrame::BashCompleted(completion) => Some(completion.task_id.as_str()),
4238            _ => None,
4239        }
4240    }
4241
4242    pub(super) fn push_frame_task_id(frame: &Frame) -> Option<String> {
4243        let body: serde_json::Value = serde_json::from_slice(&frame.body).expect("push body");
4244        body.get("task_id")
4245            .and_then(serde_json::Value::as_str)
4246            .map(str::to_string)
4247    }
4248}
4249
4250#[cfg(test)]
4251mod tests {
4252    use super::test_support::{completion_frame, test_ctx, test_root};
4253    use super::*;
4254
4255    fn attach_error(kind: io::ErrorKind) -> SubcError {
4256        SubcError::Connect {
4257            endpoint: "127.0.0.1:1".to_string(),
4258            source: io::Error::new(kind, "constructed attach failure"),
4259        }
4260    }
4261
4262    fn auth_io_error(kind: io::ErrorKind) -> SubcError {
4263        SubcError::Auth {
4264            endpoint: "127.0.0.1:1".to_string(),
4265            source: subc_transport::AuthError::Io {
4266                stage: subc_transport::AuthStage::ServerProof,
4267                source: io::Error::new(kind, "constructed auth failure"),
4268            },
4269        }
4270    }
4271
4272    #[test]
4273    fn initial_attach_error_classifier_distinguishes_transient_and_permanent_failures() {
4274        let transient_errors = vec![
4275            attach_error(io::ErrorKind::ConnectionRefused),
4276            attach_error(io::ErrorKind::TimedOut),
4277            attach_error(io::ErrorKind::ConnectionReset),
4278            auth_io_error(io::ErrorKind::ConnectionAborted),
4279            auth_io_error(io::ErrorKind::BrokenPipe),
4280            SubcError::Auth {
4281                endpoint: "127.0.0.1:1".to_string(),
4282                source: subc_transport::AuthError::UnexpectedEof {
4283                    stage: subc_transport::AuthStage::ServerProof,
4284                    expected: 4,
4285                    actual: 0,
4286                },
4287            },
4288            SubcError::Auth {
4289                endpoint: "127.0.0.1:1".to_string(),
4290                source: subc_transport::AuthError::Timeout {
4291                    stage: subc_transport::AuthStage::ServerProof,
4292                    deadline: AUTH_DEADLINE,
4293                },
4294            },
4295        ];
4296        for error in &transient_errors {
4297            assert_eq!(
4298                classify_attach_error(error),
4299                AttachErrorClass::Transient,
4300                "expected transient: {error}"
4301            );
4302        }
4303
4304        let permanent_errors = vec![
4305            attach_error(io::ErrorKind::PermissionDenied),
4306            auth_io_error(io::ErrorKind::InvalidData),
4307            SubcError::Auth {
4308                endpoint: "127.0.0.1:1".to_string(),
4309                source: subc_transport::AuthError::InvalidServerProof,
4310            },
4311            SubcError::Auth {
4312                endpoint: "127.0.0.1:1".to_string(),
4313                source: subc_transport::AuthError::DaemonIdMismatch,
4314            },
4315            SubcError::ConnectionFile {
4316                path: PathBuf::from("subc-connection.json"),
4317                source: subc_transport::ConnectionFileError::Invalid {
4318                    reason: "constructed invalid file".to_string(),
4319                },
4320            },
4321            SubcError::NoEndpoint {
4322                path: PathBuf::from("subc-connection.json"),
4323            },
4324            SubcError::InvalidEndpoint {
4325                path: PathBuf::from("subc-connection.json"),
4326                endpoint: "not-an-ip:1234".to_string(),
4327            },
4328        ];
4329        for error in &permanent_errors {
4330            assert_eq!(
4331                classify_attach_error(error),
4332                AttachErrorClass::Permanent,
4333                "expected permanent: {error}"
4334            );
4335        }
4336    }
4337
4338    #[test]
4339    fn incompatible_wire_version_is_rejected_before_tcp_connect() {
4340        let conn_dir = tempfile::tempdir().expect("connection tempdir");
4341        let conn_path = conn_dir.path().join("subc-connection.json");
4342        let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind listener");
4343        listener
4344            .set_nonblocking(true)
4345            .expect("set listener nonblocking");
4346        let port = listener.local_addr().expect("listener addr").port();
4347        connection_file::write_atomic(
4348            &conn_path,
4349            &connection_file::ConnectionInfo {
4350                schema: connection_file::SCHEMA_VERSION,
4351                wire_version: Some(PROTOCOL_VERSION.wrapping_add(1)),
4352                endpoints: vec![connection_file::Endpoint {
4353                    host: "127.0.0.1".to_string(),
4354                    port,
4355                }],
4356                key: vec![0x42; subc_transport::KEY_LEN],
4357                daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
4358                pid: std::process::id(),
4359                daemon_ver: "subc-test".to_string(),
4360            },
4361        )
4362        .expect("write connection file");
4363
4364        let runtime = tokio::runtime::Builder::new_current_thread()
4365            .enable_all()
4366            .build()
4367            .expect("test runtime");
4368        let result = runtime.block_on(connect_and_authenticate_with_policy(
4369            &conn_path,
4370            AttachRetryPolicy {
4371                budget: Duration::from_secs(1),
4372                initial_backoff: Duration::from_millis(5),
4373                max_backoff: Duration::from_millis(10),
4374                jitter_percent: 0,
4375            },
4376        ));
4377        assert!(matches!(
4378            result,
4379            Err(SubcError::ConnectionFile {
4380                source: connection_file::ConnectionFileError::WireVersionMismatch { .. },
4381                ..
4382            })
4383        ));
4384        assert!(matches!(
4385            listener.accept(),
4386            Err(error) if error.kind() == std::io::ErrorKind::WouldBlock
4387        ));
4388    }
4389
4390    #[test]
4391    fn initial_attach_unreachable_endpoint_retries_until_budget_then_fails_loud() {
4392        let conn_dir = tempfile::tempdir().expect("connection tempdir");
4393        let conn_path = conn_dir.path().join("subc-connection.json");
4394        let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve port");
4395        let port = listener.local_addr().expect("reserved addr").port();
4396        drop(listener);
4397        connection_file::write_atomic(
4398            &conn_path,
4399            &connection_file::ConnectionInfo {
4400                schema: connection_file::SCHEMA_VERSION,
4401                wire_version: Some(PROTOCOL_VERSION),
4402                endpoints: vec![connection_file::Endpoint {
4403                    host: "127.0.0.1".to_string(),
4404                    port,
4405                }],
4406                key: vec![0x42; subc_transport::KEY_LEN],
4407                daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
4408                pid: std::process::id(),
4409                daemon_ver: "subc-test".to_string(),
4410            },
4411        )
4412        .expect("write connection file");
4413
4414        let policy = AttachRetryPolicy {
4415            budget: Duration::from_millis(40),
4416            initial_backoff: Duration::from_millis(5),
4417            max_backoff: Duration::from_millis(10),
4418            jitter_percent: 0,
4419        };
4420        let runtime = tokio::runtime::Builder::new_current_thread()
4421            .enable_all()
4422            .build()
4423            .expect("test runtime");
4424        let started_at = Instant::now();
4425        let result = runtime.block_on(connect_and_authenticate_with_policy(&conn_path, policy));
4426        let elapsed = started_at.elapsed();
4427        let error = match result {
4428            Ok(_) => panic!("unreachable endpoint unexpectedly attached"),
4429            Err(error) => error,
4430        };
4431
4432        assert!(matches!(error, SubcError::Connect { .. }), "{error}");
4433        assert!(
4434            elapsed >= Duration::from_millis(35),
4435            "retry budget ended too early: {elapsed:?}"
4436        );
4437        assert!(
4438            elapsed < Duration::from_secs(1),
4439            "retry budget was not bounded: {elapsed:?}"
4440        );
4441    }
4442
4443    fn due_maintenance_jobs_without_actor_context(
4444        live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4445        budget: usize,
4446        pending_bind_roots: &HashSet<ProjectRootId>,
4447    ) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
4448        due_maintenance_jobs(
4449            live_roots,
4450            None,
4451            &HashMap::new(),
4452            &HashSet::new(),
4453            budget,
4454            pending_bind_roots,
4455        )
4456    }
4457
4458    fn actor_ctx_with_dirty_search_index(
4459        root: &Path,
4460        storage: &Path,
4461        file_name: &str,
4462        old_contents: &str,
4463        new_contents: &str,
4464    ) -> (Arc<AppContext>, PathBuf, PathBuf) {
4465        let file = root.join(file_name);
4466        std::fs::write(&file, old_contents).expect("write source");
4467        let canonical_root = std::fs::canonicalize(root).expect("canonical root");
4468        let ctx = Arc::new(AppContext::new(
4469            Box::new(crate::parser::TreeSitterProvider::new()),
4470            Config {
4471                project_root: Some(root.to_path_buf()),
4472                storage_dir: Some(storage.to_path_buf()),
4473                ..Config::default()
4474            },
4475        ));
4476        ctx.set_canonical_cache_root(canonical_root.clone());
4477
4478        let cache_dir = crate::search_index::resolve_cache_dir(&canonical_root, Some(storage));
4479        let mut index = crate::search_index::SearchIndex::build(&canonical_root);
4480        let git_head = index.stored_git_head().map(str::to_owned);
4481        index.write_to_disk(&cache_dir, git_head.as_deref());
4482
4483        std::fs::write(&file, new_contents).expect("edit source");
4484        index.update_file(&file);
4485        *ctx.search_index()
4486            .write()
4487            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
4488        (ctx, canonical_root, cache_dir)
4489    }
4490
4491    #[test]
4492    fn graceful_shutdown_flushes_every_actor_search_index() {
4493        let storage = tempfile::tempdir().expect("storage tempdir");
4494        let (root1_dir, root1) = test_root("shutdown-flush-root-1");
4495        let (root2_dir, root2) = test_root("shutdown-flush-root-2");
4496        let (ctx1, canonical_root1, cache_dir1) = actor_ctx_with_dirty_search_index(
4497            root1_dir.path(),
4498            storage.path(),
4499            "alpha.txt",
4500            "old actor one token\n",
4501            "new actor one token\n",
4502        );
4503        let (ctx2, canonical_root2, cache_dir2) = actor_ctx_with_dirty_search_index(
4504            root2_dir.path(),
4505            storage.path(),
4506            "beta.txt",
4507            "old actor two token\n",
4508            "new actor two token\n",
4509        );
4510
4511        let executor = Executor::new();
4512        assert!(executor.register_actor(root1.clone(), Arc::clone(&ctx1)));
4513        assert!(executor.register_actor(root2.clone(), Arc::clone(&ctx2)));
4514
4515        flush_actor_indexes_on_graceful_shutdown(&executor.actor_contexts());
4516
4517        let mut restored1 =
4518            crate::search_index::SearchIndex::read_from_disk(&cache_dir1, &canonical_root1)
4519                .expect("load flushed root one index");
4520        restored1.ready = true;
4521        assert_eq!(
4522            restored1
4523                .grep("new actor one token", true, &[], &[], &canonical_root1, 10)
4524                .matches
4525                .len(),
4526            1,
4527            "graceful subc shutdown should flush the first root's trigram delta"
4528        );
4529
4530        let mut restored2 =
4531            crate::search_index::SearchIndex::read_from_disk(&cache_dir2, &canonical_root2)
4532                .expect("load flushed root two index");
4533        restored2.ready = true;
4534        assert_eq!(
4535            restored2
4536                .grep("new actor two token", true, &[], &[], &canonical_root2, 10)
4537                .matches
4538                .len(),
4539            1,
4540            "graceful subc shutdown should flush every registered root"
4541        );
4542    }
4543
4544    #[test]
4545    fn idle_root_reaper_closes_artifacts_and_stops_watcher() {
4546        let _ = env_logger::builder().is_test(true).try_init();
4547        let (root_dir, root) = test_root("idle-root-reaper");
4548        let storage = tempfile::tempdir().expect("storage tempdir");
4549        std::fs::write(
4550            root_dir.path().join("main.rs"),
4551            "fn entry() { leaf(); }\nfn leaf() {}\n",
4552        )
4553        .expect("source file");
4554        let canonical_root = std::fs::canonicalize(root_dir.path()).expect("canonical root");
4555        let app = App::default_shared();
4556        let ctx = Arc::new(AppContext::from_app(
4557            Arc::clone(&app),
4558            Config {
4559                project_root: Some(canonical_root.clone()),
4560                storage_dir: Some(storage.path().to_path_buf()),
4561                callgraph_store: true,
4562                search_index: true,
4563                ..Config::default()
4564            },
4565        ));
4566        ctx.set_canonical_cache_root(canonical_root.clone());
4567        assert!(ctx
4568            .ensure_callgraph_store()
4569            .expect("build callgraph store")
4570            .is_some());
4571
4572        let cache_dir =
4573            crate::search_index::resolve_cache_dir(&canonical_root, Some(storage.path()));
4574        let mut index = crate::search_index::SearchIndex::build(&canonical_root);
4575        let git_head = index.stored_git_head().map(str::to_owned);
4576        index.write_to_disk(&cache_dir, git_head.as_deref());
4577        *ctx.search_index()
4578            .write()
4579            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
4580        // Seed a completed warm verification so the test can prove eviction
4581        // downgrades it to Strict rather than passing vacuously.
4582        let seeded_generation =
4583            crate::cache_freshness::artifact_generation(&cache_dir.join("cache.bin"))
4584                .expect("seeded artifact generation");
4585        crate::cache_freshness::record_verify_completed(
4586            &canonical_root,
4587            crate::cache_freshness::VerifyArtifact::Search,
4588            Some(seeded_generation),
4589        );
4590        assert!(
4591            matches!(
4592                crate::cache_freshness::warm_verify_plan(
4593                    canonical_root.as_path(),
4594                    crate::cache_freshness::VerifyArtifact::Search,
4595                    Some(seeded_generation),
4596                ),
4597                crate::cache_freshness::WarmVerifyPlan::Skip
4598            ),
4599            "memo must be warm before eviction for the downgrade assertion to bite"
4600        );
4601
4602        let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
4603        let _dispatch_tx = dispatch_tx;
4604        let shutdown = Arc::new(AtomicBool::new(false));
4605        let thread_shutdown = Arc::clone(&shutdown);
4606        let join = std::thread::spawn(move || {
4607            while !thread_shutdown.load(Ordering::SeqCst) {
4608                std::thread::yield_now();
4609            }
4610        });
4611        ctx.install_watcher_runtime(
4612            dispatch_rx,
4613            crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
4614        );
4615        assert_eq!(ctx.watcher_registry_count(), 1);
4616
4617        let executor = Arc::new(Executor::new());
4618        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
4619        let mut live_roots = HashMap::new();
4620        let mut meta = RootMeta::new(Instant::now());
4621        meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
4622        live_roots.insert(root.clone(), meta);
4623
4624        let message = idle_root_eviction_message(&root, &ctx.memory_root_snapshot(), None);
4625        assert!(message.contains("evicted idle root"));
4626        assert!(message.contains("freed ~"));
4627        assert!(message.contains("semantic"));
4628        assert!(!message.contains("semantic not estimated retained"));
4629        assert!(message.contains("trigram"));
4630        assert!(message.contains("retained: bash"));
4631        assert!(message.contains("parser_pool"));
4632
4633        assert_eq!(
4634            reap_idle_roots(Instant::now(), &mut live_roots, &HashMap::new(), &executor),
4635            1
4636        );
4637        assert!(ctx.search_index().read().unwrap().is_none());
4638        assert_eq!(ctx.watcher_registry_count(), 0);
4639        // The watcher stopped with the eviction, so the idle interval is
4640        // unobserved: the pre-seeded warm-verify memo (Skip) must fall back to
4641        // strict content verification (stat-first would miss same-size,
4642        // preserved-mtime edits made while nobody was watching).
4643        assert!(
4644            matches!(
4645                crate::cache_freshness::warm_verify_plan(
4646                    canonical_root.as_path(),
4647                    crate::cache_freshness::VerifyArtifact::Search,
4648                    Some(seeded_generation),
4649                ),
4650                crate::cache_freshness::WarmVerifyPlan::Strict
4651            ),
4652            "idle eviction must force strict re-verification"
4653        );
4654        assert!(
4655            crate::search_index::SearchIndex::read_from_disk(&cache_dir, &canonical_root).is_some()
4656        );
4657        assert!(ctx
4658            .ensure_callgraph_store()
4659            .expect("reopen callgraph store")
4660            .is_some());
4661        assert!(live_roots[&root].idle_artifacts_evicted);
4662    }
4663
4664    #[test]
4665    fn idle_root_reaper_applies_ttl_to_unbound_roots() {
4666        let (_root_dir, root) = test_root("idle-root-ttl-gate");
4667        let ctx = test_ctx();
4668        let executor = Arc::new(Executor::new());
4669        assert!(executor.register_actor(root.clone(), ctx));
4670        let now = Instant::now();
4671        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(now))]);
4672
4673        // A recently-unbound root stays warm: a transient unbind (host
4674        // restart) must not pay the strict-verify + forced-rebuild teardown
4675        // on the next maintenance sweep.
4676        assert_eq!(
4677            reap_idle_roots(now, &mut live_roots, &HashMap::new(), &executor),
4678            0
4679        );
4680        assert!(!live_roots[&root].idle_artifacts_evicted);
4681
4682        // Past the TTL the same unbound root pays the full teardown. The
4683        // pending reconciliation paths retained across the transient-unbind
4684        // window would block eviction forever through
4685        // `artifact_eviction_blocked`; the reaper disposes them because the
4686        // strict gap invalidation subsumes their purpose.
4687        let ctx = executor.actor_context(&root).expect("actor context");
4688        ctx.mark_subc_unbound();
4689        ctx.add_pending_search_index_paths([root.as_path().join("retained.rs")]);
4690        assert_eq!(
4691            reap_idle_roots(
4692                now + IDLE_ROOT_TTL,
4693                &mut live_roots,
4694                &HashMap::new(),
4695                &executor,
4696            ),
4697            1
4698        );
4699        assert!(live_roots[&root].idle_artifacts_evicted);
4700        assert!(
4701            ctx.take_pending_search_index_paths().is_empty(),
4702            "TTL eviction must dispose retained pending reconciliation paths"
4703        );
4704    }
4705
4706    #[test]
4707    fn blocked_ttl_eviction_restores_taken_pending_reconciliation_state() {
4708        let (_root_dir, root) = test_root("ttl-eviction-blocked-restore");
4709        let ctx = test_ctx();
4710        let executor = Arc::new(Executor::new());
4711        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
4712        ctx.mark_subc_unbound();
4713
4714        // Retained pending path from the transient-unbind window plus a
4715        // SECONDARY eviction blocker (a non-ready resident search index, the
4716        // dirty-index blocker in artifact_eviction_blocked). Disposal must be
4717        // transactional: the blocked eviction may be followed by a rebind, and
4718        // the path is the only repair record for its consumed watcher event.
4719        let pending = root.as_path().join("edited-while-unbound.rs");
4720        ctx.add_pending_search_index_paths([pending.clone()]);
4721        let dirty_source = root.as_path().join("dirty.rs");
4722        std::fs::write(&dirty_source, "fn dirty() {}\n").expect("dirty source");
4723        let mut dirty = crate::search_index::SearchIndex::new();
4724        dirty.ready = true;
4725        dirty.update_file(&dirty_source);
4726        *ctx.search_index()
4727            .write()
4728            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(dirty);
4729        assert!(ctx.artifact_eviction_blocked());
4730
4731        let mut live_roots = HashMap::new();
4732        let mut meta = RootMeta::new(Instant::now());
4733        meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
4734        live_roots.insert(root.clone(), meta);
4735
4736        assert_eq!(
4737            reap_idle_roots(Instant::now(), &mut live_roots, &HashMap::new(), &executor),
4738            0,
4739            "the dirty index must still block this eviction"
4740        );
4741        assert_eq!(
4742            ctx.take_pending_search_index_paths(),
4743            vec![pending],
4744            "a blocked eviction must restore the taken pending paths"
4745        );
4746    }
4747
4748    #[test]
4749    fn unbound_root_quiesces_maintenance_without_removing_actor() {
4750        let (_root_dir, root) = test_root("unbound-root-quiesce");
4751        let ctx = test_ctx();
4752        let executor = Arc::new(Executor::new());
4753        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
4754        let mut meta = RootMeta::new(Instant::now());
4755        meta.maintenance_pending = true;
4756        meta.maintenance_jobs_in_flight = 1;
4757        meta.maintenance_queued_kinds
4758            .push_back(MaintenanceDrainKind::ConfigureTail);
4759        let mut live_roots = HashMap::from([(root.clone(), meta)]);
4760        // Warm state planted before the unbind: quiesce must keep it.
4761        *ctx.search_index()
4762            .write()
4763            .unwrap_or_else(std::sync::PoisonError::into_inner) =
4764            Some(crate::search_index::SearchIndex::new());
4765        ctx.set_cache_writer_capabilities(true, true);
4766        let pending = root.as_path().join("pending.rs");
4767        ctx.add_pending_search_index_paths([pending.clone()]);
4768        // A warm verify memo must survive the transient unbind: the watcher
4769        // keeps running, so no unobserved window exists and the next warm
4770        // reload must not pay a strict full-corpus re-hash.
4771        let canonical_root = root.as_path().to_path_buf();
4772        let artifact = canonical_root.join("cache.bin");
4773        std::fs::write(&artifact, b"warm-artifact").expect("write artifact");
4774        let seeded_generation = crate::cache_freshness::artifact_generation(&artifact);
4775        crate::cache_freshness::record_verify_completed(
4776            &canonical_root,
4777            crate::cache_freshness::VerifyArtifact::Search,
4778            seeded_generation,
4779        );
4780        assert!(matches!(
4781            crate::cache_freshness::warm_verify_plan(
4782                &canonical_root,
4783                crate::cache_freshness::VerifyArtifact::Search,
4784                seeded_generation,
4785            ),
4786            crate::cache_freshness::WarmVerifyPlan::Skip
4787        ));
4788        // A live watcher runtime must survive quiesce (its events accumulate
4789        // for the rebind replay).
4790        let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
4791        let _dispatch_tx = dispatch_tx;
4792        let shutdown = Arc::new(AtomicBool::new(false));
4793        let thread_shutdown = Arc::clone(&shutdown);
4794        let join = std::thread::spawn(move || {
4795            while !thread_shutdown.load(Ordering::SeqCst) {
4796                std::thread::yield_now();
4797            }
4798        });
4799        ctx.install_watcher_runtime(
4800            dispatch_rx,
4801            crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
4802        );
4803        assert!(ctx.watcher_runtime_active());
4804
4805        quiesce_unbound_root(&root, &mut live_roots, &executor);
4806        let meta = &live_roots[&root];
4807        assert!(meta.unbound_quiesced);
4808        assert!(ctx.subc_unbound_quiesced());
4809        assert!(meta.maintenance_pending);
4810        assert!(meta.maintenance_queued_kinds.is_empty());
4811        assert!(executor.actor_registered(&root));
4812        // Transient unbind keeps the root warm: resident artifacts stay
4813        // resident, no forced callgraph rebuild is planted, and pending
4814        // reconciliation paths survive for the rebind replay.
4815        assert!(
4816            ctx.search_index()
4817                .read()
4818                .unwrap_or_else(std::sync::PoisonError::into_inner)
4819                .is_some(),
4820            "quiesce must not evict resident artifacts"
4821        );
4822        assert_eq!(
4823            ctx.pending_callgraph_store_force_token(),
4824            None,
4825            "quiesce must not force a callgraph rebuild"
4826        );
4827        assert_eq!(
4828            ctx.take_pending_search_index_paths(),
4829            vec![pending],
4830            "quiesce must retain pending watcher-derived paths"
4831        );
4832        assert!(
4833            matches!(
4834                crate::cache_freshness::warm_verify_plan(
4835                    &canonical_root,
4836                    crate::cache_freshness::VerifyArtifact::Search,
4837                    seeded_generation,
4838                ),
4839                crate::cache_freshness::WarmVerifyPlan::Skip
4840            ),
4841            "quiesce must not invalidate the warm verify memo"
4842        );
4843        assert!(
4844            ctx.watcher_runtime_active(),
4845            "quiesce must not stop a running watcher"
4846        );
4847        ctx.stop_watcher_runtime();
4848
4849        let meta = live_roots.get_mut(&root).expect("root metadata");
4850        note_maintenance_completion(
4851            meta,
4852            Some(MaintenanceDrainKind::ConfigureTail),
4853            false,
4854            meta.unbound_quiesced,
4855        );
4856        assert!(!meta.maintenance_pending);
4857        assert!(meta.maintenance_queued_kinds.is_empty());
4858    }
4859
4860    #[test]
4861    fn same_root_higher_epoch_replacement_does_not_quiesce_between_generations() {
4862        let (_dir, root) = test_root("same-root-replacement");
4863        let route = route_key(7, 1);
4864        let installed_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
4865        let root_channels = HashMap::new();
4866
4867        assert!(!route_removal_will_quiesce_root(
4868            &root,
4869            route,
4870            &installed_channels,
4871            false,
4872            Some(&root),
4873        ));
4874        assert!(route_removal_will_quiesce_root(
4875            &root,
4876            route,
4877            &installed_channels,
4878            false,
4879            None,
4880        ));
4881        assert!(!should_quiesce_removed_root(
4882            &root,
4883            &root_channels,
4884            false,
4885            Some(&root),
4886        ));
4887        assert!(should_quiesce_removed_root(
4888            &root,
4889            &root_channels,
4890            false,
4891            None,
4892        ));
4893        assert!(!should_quiesce_removed_root(
4894            &root,
4895            &root_channels,
4896            true,
4897            None,
4898        ));
4899    }
4900
4901    #[test]
4902    fn root_quiesces_only_after_its_last_route_is_removed_and_reactivates_on_bind() {
4903        let (_root_dir, root) = test_root("unbound-root-route-count");
4904        let executor = Arc::new(Executor::new());
4905        assert!(executor.register_actor(root.clone(), test_ctx()));
4906        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
4907        let mut root_channels = HashMap::from([(
4908            root.clone(),
4909            HashSet::from([route_key(7, 1), route_key(8, 1)]),
4910        )]);
4911
4912        remove_root_channel(&mut root_channels, &root, route_key(7, 1));
4913        if !root_channels.contains_key(&root) {
4914            quiesce_unbound_root(&root, &mut live_roots, &executor);
4915        }
4916        assert!(!live_roots[&root].unbound_quiesced);
4917
4918        remove_root_channel(&mut root_channels, &root, route_key(8, 1));
4919        if !root_channels.contains_key(&root) {
4920            quiesce_unbound_root(&root, &mut live_roots, &executor);
4921        }
4922        assert!(live_roots[&root].unbound_quiesced);
4923
4924        live_roots
4925            .get_mut(&root)
4926            .expect("root metadata")
4927            .note_activity();
4928        assert!(
4929            live_roots[&root].unbound_quiesced,
4930            "late asynchronous activity must not reactivate an unbound root"
4931        );
4932
4933        live_roots
4934            .get_mut(&root)
4935            .expect("root metadata")
4936            .reactivate_bound();
4937        assert!(!live_roots[&root].unbound_quiesced);
4938    }
4939
4940    #[test]
4941    fn allocator_pressure_relief_requires_every_root_to_be_idle() {
4942        let (_idle_dir, idle_root) = test_root("allocator-relief-idle");
4943        let (_active_dir, active_root) = test_root("allocator-relief-active");
4944        let now = Instant::now();
4945        let mut live_roots = HashMap::new();
4946        let mut idle = RootMeta::new(now);
4947        idle.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
4948        live_roots.insert(idle_root, idle);
4949        assert!(process_has_been_idle(now, &live_roots));
4950
4951        live_roots.insert(active_root.clone(), RootMeta::new(now));
4952        assert!(!process_has_been_idle(now, &live_roots));
4953
4954        let active = live_roots
4955            .get_mut(&active_root)
4956            .expect("active root metadata");
4957        active.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
4958        active.active_bash_waits = 1;
4959        assert!(!process_has_been_idle(now, &live_roots));
4960    }
4961
4962    #[test]
4963    fn pressure_relief_log_reports_before_and_after_measurements() {
4964        let allocator = crate::memory::AllocatorMemorySnapshot {
4965            status: "measured",
4966            bytes_in_use: Some(8 * 1024 * 1024),
4967            size_allocated: Some(12 * 1024 * 1024),
4968            retained_slack_bytes: Some(4 * 1024 * 1024),
4969            not_estimated: None,
4970        };
4971        let relief = crate::memory::AllocatorPressureRelief {
4972            bytes_released: 3 * 1024 * 1024,
4973            rss_before_bytes: Some(20 * 1024 * 1024),
4974            rss_after_bytes: Some(17 * 1024 * 1024),
4975            allocator_before: allocator.clone(),
4976            allocator_after: crate::memory::AllocatorMemorySnapshot {
4977                size_allocated: Some(9 * 1024 * 1024),
4978                retained_slack_bytes: Some(1024 * 1024),
4979                ..allocator
4980            },
4981        };
4982        let message = pressure_relief_label(&relief);
4983        assert!(message.contains("RSS 20.0 MB -> 17.0 MB"));
4984        assert!(message.contains("allocated 12.0 MB -> 9.0 MB"));
4985        assert!(message.contains("slack 4.0 MB -> 1.0 MB"));
4986        assert!(message.contains("reported 3.0 MB released"));
4987    }
4988
4989    #[test]
4990    fn due_maintenance_jobs_skip_poisoned_roots() {
4991        let (_healthy_dir, healthy_root) = test_root("maintenance-healthy");
4992        let (_poisoned_dir, poisoned_root) = test_root("maintenance-poisoned");
4993        let mut live_roots = HashMap::new();
4994        live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
4995        let mut poisoned_meta = RootMeta::new(Instant::now());
4996        poisoned_meta.maintenance_poisoned = true;
4997        live_roots.insert(poisoned_root.clone(), poisoned_meta);
4998
4999        let (due, deferred) = due_maintenance_jobs_without_actor_context(
5000            &mut live_roots,
5001            MAINTENANCE_SUBMIT_BUDGET,
5002            &HashSet::new(),
5003        );
5004
5005        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
5006        assert!(due.iter().all(|(root, _)| root == &healthy_root));
5007        assert!(!deferred);
5008        assert!(live_roots[&healthy_root].maintenance_pending);
5009        assert_eq!(
5010            live_roots[&healthy_root].maintenance_jobs_in_flight,
5011            INITIAL_MAINTENANCE_JOB_COUNT
5012        );
5013        assert!(!live_roots[&poisoned_root].maintenance_pending);
5014    }
5015
5016    #[test]
5017    fn due_maintenance_jobs_do_not_restart_quiesced_root_work() {
5018        let (_dir, root) = test_root("maintenance-unbound");
5019        let mut meta = RootMeta::new(Instant::now());
5020        meta.unbound_quiesced = true;
5021        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5022
5023        let (due, deferred) = due_maintenance_jobs_without_actor_context(
5024            &mut live_roots,
5025            MAINTENANCE_SUBMIT_BUDGET,
5026            &HashSet::new(),
5027        );
5028
5029        assert!(due.is_empty());
5030        assert!(!deferred);
5031        assert!(!live_roots[&root].maintenance_pending);
5032    }
5033
5034    #[test]
5035    fn idle_bg_subscription_queues_no_jobs_until_a_wake_arrives() {
5036        let (_dir, root) = test_root("maintenance-idle-bg-subscription");
5037        let ctx = test_ctx();
5038        assert!(!ctx.completion_drains_have_work());
5039
5040        let executor = Executor::new();
5041        assert!(executor.register_actor(root.clone(), ctx));
5042        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
5043        let session = "idle-session".to_string();
5044        let channel = route_key(17, 1);
5045        let bg_sub_by_session = HashMap::from([((root.clone(), session.clone()), channel)]);
5046        let mut bg_wake_pending = HashSet::new();
5047
5048        let (idle_tick_jobs, deferred) = due_maintenance_jobs(
5049            &mut live_roots,
5050            Some(&executor),
5051            &bg_sub_by_session,
5052            &bg_wake_pending,
5053            MAINTENANCE_SUBMIT_BUDGET,
5054            &HashSet::new(),
5055        );
5056        assert!(idle_tick_jobs.is_empty());
5057        assert!(!deferred);
5058        assert!(!live_roots[&root].maintenance_pending);
5059
5060        // A completion can arm its wake after the idle tick's probes. The wake
5061        // remains loop-owned state, so the following tick must observe it.
5062        let mut bg_wake_epoch = HashMap::new();
5063        push::arm_bg_wake(
5064            root.clone(),
5065            session,
5066            channel,
5067            &mut bg_wake_pending,
5068            &mut bg_wake_epoch,
5069        );
5070        let (next_tick_jobs, deferred) = due_maintenance_jobs(
5071            &mut live_roots,
5072            Some(&executor),
5073            &bg_sub_by_session,
5074            &bg_wake_pending,
5075            MAINTENANCE_SUBMIT_BUDGET,
5076            &HashSet::new(),
5077        );
5078        assert_eq!(
5079            next_tick_jobs,
5080            vec![(root, MaintenanceDrainKind::CompletionDrains)]
5081        );
5082        assert!(!deferred);
5083    }
5084
5085    #[tokio::test]
5086    async fn subc_configure_tail_precedes_completed_search_install() {
5087        let root_dir = tempfile::tempdir().unwrap();
5088        let storage = tempfile::tempdir().unwrap();
5089        let root = ProjectRootId::from_path(root_dir.path()).unwrap();
5090        let (ctx, ignored_path) =
5091            runtime_drain::configure_search_order_context_for_test(root_dir.path(), storage.path());
5092        let ctx = Arc::new(ctx);
5093        assert!(!runtime_drain::watcher_path_is_ignored_by_current_matcher(
5094            &ctx,
5095            &ignored_path
5096        ));
5097
5098        let executor = Arc::new(Executor::new());
5099        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5100        let metrics = Arc::new(DispatchPathMetrics::new());
5101        let (completion_tx, mut completion_rx) = mpsc::channel(4);
5102        submit_maintenance_job(
5103            &executor,
5104            root.clone(),
5105            MaintenanceDrainKind::ConfigureTail,
5106            Vec::new(),
5107            &completion_tx,
5108            &metrics,
5109        );
5110        submit_maintenance_job(
5111            &executor,
5112            root,
5113            MaintenanceDrainKind::CompletionDrains,
5114            Vec::new(),
5115            &completion_tx,
5116            &metrics,
5117        );
5118
5119        let first = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
5120            .await
5121            .expect("configure-tail completion timed out")
5122            .expect("configure-tail completion channel closed");
5123        let second = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
5124            .await
5125            .expect("completion-drains completion timed out")
5126            .expect("completion-drains completion channel closed");
5127        assert!(first.response.id.contains("configure-tail"));
5128        assert!(second.response.id.contains("completion-drains"));
5129        assert!(runtime_drain::watcher_path_is_ignored_by_current_matcher(
5130            &ctx,
5131            &ignored_path
5132        ));
5133        assert_eq!(
5134            ctx.search_index()
5135                .read()
5136                .unwrap_or_else(std::sync::PoisonError::into_inner)
5137                .as_ref()
5138                .expect("completed search index installed")
5139                .file_count(),
5140            0,
5141            "configure must install the ignore matcher before pending paths replay"
5142        );
5143        ctx.stop_watcher_runtime();
5144    }
5145
5146    #[test]
5147    fn post_bind_configure_and_completion_jobs_are_queued_in_order() {
5148        let (_dir, root) = test_root("maintenance-post-bind");
5149        let mut live_roots = HashMap::new();
5150        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
5151
5152        queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
5153        queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
5154
5155        let meta = live_roots.get(&root).expect("root metadata");
5156        assert!(meta.maintenance_pending);
5157        assert_eq!(meta.maintenance_jobs_in_flight, 0);
5158        assert_eq!(
5159            meta.maintenance_queued_kinds
5160                .iter()
5161                .copied()
5162                .collect::<Vec<_>>(),
5163            vec![
5164                MaintenanceDrainKind::ConfigureTail,
5165                MaintenanceDrainKind::CompletionDrains,
5166            ]
5167        );
5168
5169        let (due, deferred) = due_maintenance_jobs_without_actor_context(
5170            &mut live_roots,
5171            MAINTENANCE_SUBMIT_BUDGET,
5172            &HashSet::new(),
5173        );
5174
5175        assert_eq!(
5176            due,
5177            vec![
5178                (root.clone(), MaintenanceDrainKind::ConfigureTail),
5179                (root.clone(), MaintenanceDrainKind::CompletionDrains),
5180            ]
5181        );
5182        assert!(!deferred);
5183        assert_eq!(live_roots[&root].maintenance_jobs_in_flight, 2);
5184        assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
5185    }
5186
5187    #[test]
5188    fn due_maintenance_jobs_defers_unsubmitted_roots_without_marking_pending() {
5189        let mut live_roots = HashMap::new();
5190        let mut root_ids = Vec::new();
5191        let mut _dirs = Vec::new();
5192        for index in 0..4 {
5193            let (dir, root_id) = test_root(&format!("maintenance-budget-{index}"));
5194            live_roots.insert(root_id.clone(), RootMeta::new(Instant::now()));
5195            root_ids.push(root_id);
5196            _dirs.push(dir);
5197        }
5198
5199        let small_budget = INITIAL_MAINTENANCE_JOB_COUNT + 1;
5200        let (first_due, first_deferred) = due_maintenance_jobs_without_actor_context(
5201            &mut live_roots,
5202            small_budget,
5203            &HashSet::new(),
5204        );
5205
5206        assert_eq!(first_due.len(), small_budget);
5207        assert!(first_deferred);
5208        let first_due_set: HashSet<_> = first_due.into_iter().map(|(root, _)| root).collect();
5209        assert!(first_due_set
5210            .iter()
5211            .all(|root| live_roots[root].maintenance_pending));
5212        assert!(first_due_set
5213            .iter()
5214            .any(|root| !live_roots[root].maintenance_queued_kinds.is_empty()));
5215
5216        let all_roots: HashSet<_> = root_ids.into_iter().collect();
5217        let deferred_roots: HashSet<_> = all_roots.difference(&first_due_set).cloned().collect();
5218        assert!(deferred_roots
5219            .iter()
5220            .all(|root| !live_roots[root].maintenance_pending));
5221    }
5222
5223    #[test]
5224    fn due_maintenance_jobs_defers_pending_bind_roots() {
5225        let (_bind_dir, bind_root) = test_root("maintenance-pending-bind");
5226        let (_healthy_dir, healthy_root) = test_root("maintenance-no-bind");
5227        let mut live_roots = HashMap::new();
5228        live_roots.insert(bind_root.clone(), RootMeta::new(Instant::now()));
5229        live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
5230        let pending_bind_roots = HashSet::from([bind_root.clone()]);
5231
5232        let (due, deferred) = due_maintenance_jobs_without_actor_context(
5233            &mut live_roots,
5234            usize::MAX,
5235            &pending_bind_roots,
5236        );
5237
5238        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
5239        assert!(due.iter().all(|(root, _)| root == &healthy_root));
5240        assert!(!deferred);
5241        assert!(!live_roots[&bind_root].maintenance_pending);
5242        assert!(live_roots[&bind_root].maintenance_queued_kinds.is_empty());
5243    }
5244
5245    #[test]
5246    fn maintenance_pending_survives_requeue_and_clears_after_final_batch() {
5247        let (_dir, root) = test_root("maintenance-requeue");
5248        let mut live_roots = HashMap::new();
5249        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
5250        let (due, deferred) = due_maintenance_jobs_without_actor_context(
5251            &mut live_roots,
5252            usize::MAX,
5253            &HashSet::new(),
5254        );
5255        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
5256        assert!(due.iter().all(|(due_root, _)| due_root == &root));
5257        assert!(!deferred);
5258
5259        let meta = live_roots.get_mut(&root).unwrap();
5260        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, false);
5261        assert!(meta.maintenance_pending);
5262        assert_eq!(
5263            meta.maintenance_jobs_in_flight,
5264            INITIAL_MAINTENANCE_JOB_COUNT - 1
5265        );
5266        assert_eq!(meta.maintenance_queued_kinds.len(), 1);
5267
5268        let (requeued, deferred) =
5269            due_maintenance_jobs_without_actor_context(&mut live_roots, 1, &HashSet::new());
5270        assert_eq!(
5271            requeued,
5272            vec![(root.clone(), MaintenanceDrainKind::Watcher)]
5273        );
5274        assert!(!deferred);
5275        let meta = live_roots.get_mut(&root).unwrap();
5276        assert_eq!(
5277            meta.maintenance_jobs_in_flight,
5278            INITIAL_MAINTENANCE_JOB_COUNT
5279        );
5280        assert!(meta.maintenance_queued_kinds.is_empty());
5281
5282        for _ in 0..INITIAL_MAINTENANCE_JOB_COUNT {
5283            note_maintenance_completion(meta, None, false, false);
5284        }
5285        assert!(!meta.maintenance_pending);
5286        assert_eq!(meta.maintenance_jobs_in_flight, 0);
5287    }
5288
5289    #[test]
5290    fn maintenance_requeue_drops_while_bind_is_pending() {
5291        let (_dir, root) = test_root("maintenance-bind-requeue");
5292        let mut live_roots = HashMap::new();
5293        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
5294        let (due, _) = due_maintenance_jobs_without_actor_context(
5295            &mut live_roots,
5296            usize::MAX,
5297            &HashSet::new(),
5298        );
5299        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
5300
5301        let meta = live_roots.get_mut(&root).unwrap();
5302        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, true);
5303
5304        assert_eq!(
5305            meta.maintenance_jobs_in_flight,
5306            INITIAL_MAINTENANCE_JOB_COUNT - 1
5307        );
5308        assert!(meta.maintenance_queued_kinds.is_empty());
5309        assert!(meta.maintenance_pending);
5310    }
5311
5312    #[test]
5313    fn parked_lsp_completion_never_requiesces_or_cancels_a_pending_bind() {
5314        let mut meta = RootMeta::new(Instant::now());
5315        meta.unbound_quiesced = true;
5316
5317        assert!(!should_requiesce_after_maintenance(
5318            &meta,
5319            MaintenanceDrainKind::Lsp,
5320            false,
5321        ));
5322        assert!(!should_requiesce_after_maintenance(
5323            &meta,
5324            MaintenanceDrainKind::ConfigureTail,
5325            true,
5326        ));
5327        assert!(should_requiesce_after_maintenance(
5328            &meta,
5329            MaintenanceDrainKind::ConfigureTail,
5330            false,
5331        ));
5332    }
5333
5334    #[test]
5335    fn maintenance_pending_clears_and_poison_stops_requeue_after_fatal() {
5336        let (_dir, root) = test_root("maintenance-fatal");
5337        let mut live_roots = HashMap::new();
5338        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
5339        let (due, _) = due_maintenance_jobs_without_actor_context(
5340            &mut live_roots,
5341            usize::MAX,
5342            &HashSet::new(),
5343        );
5344        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
5345
5346        let meta = live_roots.get_mut(&root).unwrap();
5347        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), true, false);
5348        assert!(meta.maintenance_poisoned);
5349        assert!(meta.maintenance_queued_kinds.is_empty());
5350
5351        for _ in 1..INITIAL_MAINTENANCE_JOB_COUNT {
5352            note_maintenance_completion(meta, None, false, false);
5353        }
5354        assert!(!meta.maintenance_pending);
5355        assert_eq!(meta.maintenance_jobs_in_flight, 0);
5356    }
5357
5358    #[test]
5359    fn trust_for_principal_matrix() {
5360        assert_eq!(
5361            trust_for_principal(&Some(Principal::Direct)),
5362            BindTrust::FirstParty
5363        );
5364        assert_eq!(
5365            trust_for_principal(&Some(Principal::Reserved {
5366                module_id: "llm-runner".to_string(),
5367            })),
5368            BindTrust::FirstParty
5369        );
5370        assert_eq!(
5371            trust_for_principal(&Some(Principal::Reserved {
5372                module_id: "aft".to_string(),
5373            })),
5374            BindTrust::FirstParty
5375        );
5376        assert_eq!(
5377            trust_for_principal(&Some(Principal::Reserved {
5378                module_id: "subc-mcp".to_string(),
5379            })),
5380            BindTrust::Untrusted
5381        );
5382        assert_eq!(
5383            trust_for_principal(&Some(Principal::Reserved {
5384                module_id: "anything-unknown".to_string(),
5385            })),
5386            BindTrust::Untrusted
5387        );
5388        assert_eq!(
5389            trust_for_principal(&Some(Principal::Unverified)),
5390            BindTrust::Untrusted
5391        );
5392        assert_eq!(trust_for_principal(&None), BindTrust::Untrusted);
5393    }
5394
5395    #[test]
5396    fn fed_harness_class_maps_to_untrusted_regardless_of_fingerprint_value() {
5397        let principal = Some(Principal::Direct);
5398        let fingerprint_a = "fed:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
5399        let fingerprint_b = "fed:0123456789abcdef111111111111111111111111111111111111111111111111";
5400
5401        assert_eq!(
5402            trust_for_bind(fingerprint_a, &principal),
5403            BindTrust::Untrusted
5404        );
5405        assert_eq!(
5406            trust_for_bind(fingerprint_b, &principal),
5407            BindTrust::Untrusted
5408        );
5409    }
5410
5411    #[tokio::test]
5412    async fn persistent_cancel_resolves_when_fired_before_await() {
5413        // The lost-wakeup guard: cancel() fires exactly once via notify_waiters()
5414        // (no stored permit). A waiter that registers AFTER the cancel must still
5415        // observe it via the flag; a waiter racing the cancel must still be woken.
5416        let signal = PersistentCancelSignal::new();
5417        signal.cancel();
5418        // Fired before we ever call cancelled() — must return immediately, not park.
5419        tokio::time::timeout(Duration::from_secs(1), signal.cancelled())
5420            .await
5421            .expect("cancelled() must resolve when cancel fired beforehand");
5422
5423        // A fresh signal cancelled concurrently with an in-flight cancelled().
5424        let racing = PersistentCancelSignal::new();
5425        let racing_for_task = racing.clone();
5426        let waiter = tokio::spawn(async move { racing_for_task.cancelled().await });
5427        racing.cancel();
5428        tokio::time::timeout(Duration::from_secs(1), waiter)
5429            .await
5430            .expect("cancelled() must resolve when cancel races the await")
5431            .expect("waiter task panicked");
5432    }
5433
5434    #[test]
5435    fn ingress_epoch_validation_silently_drops_every_stale_route_frame() {
5436        let installed = HashMap::from([(7, 9)]);
5437        for ty in [
5438            FrameType::Request,
5439            FrameType::Response,
5440            FrameType::Error,
5441            FrameType::Push,
5442            FrameType::Cancel,
5443            FrameType::Goodbye,
5444        ] {
5445            let body = if ty.is_pure_header() {
5446                Vec::new()
5447            } else {
5448                br#"{}"#.to_vec()
5449            };
5450            let stale = Frame::build(ty, control_flags(), 7, 8, 41, body).unwrap();
5451            assert!(!ingress_route_is_current(&installed, &stale), "{ty:?}");
5452        }
5453
5454        let uninstalled = Frame::build(
5455            FrameType::Request,
5456            control_flags(),
5457            8,
5458            1,
5459            42,
5460            br#"{}"#.to_vec(),
5461        )
5462        .unwrap();
5463        assert!(!ingress_route_is_current(&installed, &uninstalled));
5464
5465        let current = Frame::build(
5466            FrameType::Request,
5467            control_flags(),
5468            7,
5469            9,
5470            43,
5471            br#"{}"#.to_vec(),
5472        )
5473        .unwrap();
5474        let control = Frame::build(FrameType::Ping, control_flags(), 0, 0, 44, Vec::new()).unwrap();
5475        assert!(ingress_route_is_current(&installed, &current));
5476        assert!(ingress_route_is_current(&installed, &control));
5477        assert_eq!(installed, HashMap::from([(7, 9)]));
5478    }
5479
5480    #[tokio::test]
5481    async fn route_bind_ack_precedes_route_egress_in_writer_queue() {
5482        let (_dir, root) = test_root("route-bind-b2-ordering");
5483        let route = route_key(7, 3);
5484        let identity = RouteIdentity(Arc::new(RouteIdentityData {
5485            root: root.clone(),
5486            project_root: root.as_path().to_path_buf(),
5487            harness: "opencode".to_string(),
5488            session: "b2-session".to_string(),
5489            trust: BindTrust::FirstParty,
5490            consumer_elicitation_capable: false,
5491        }));
5492        let replay_key = push::ReplayKey::from_identity(&identity);
5493        let completion = RouteBindCompletion {
5494            route,
5495            identity,
5496            bind_root_id: root.clone(),
5497            inserted_new_actor: false,
5498            configure_response: Response::success("subc-bind-7", json!({})),
5499            diagnostics_on_edit: false,
5500            ver: PROTOCOL_VERSION,
5501            corr: 91,
5502            flags: control_flags(),
5503        };
5504        let mut pending_binds = HashMap::from([(
5505            route,
5506            PendingBind {
5507                bind_root_id: root,
5508                inserted_new_actor: false,
5509                cancelled: false,
5510                configure_request_id: "subc-bind-7".to_string(),
5511                started_at: Instant::now(),
5512                warned_half_deadline: false,
5513                deadline_reported: false,
5514                corr: 91,
5515                ver: PROTOCOL_VERSION,
5516                flags: control_flags(),
5517                cancellation: crate::executor::JobCancellation::new(),
5518            },
5519        )]);
5520        let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
5521        let mut push_buffer =
5522            HashMap::from([(replay_key, VecDeque::from([completion_frame("b2-replay")]))]);
5523        let (writer_tx, mut writer_rx) = mpsc::channel(8);
5524        let metrics = Arc::new(DispatchPathMetrics::new());
5525
5526        handle_route_bind_completion(
5527            &writer_tx,
5528            completion,
5529            &mut HashMap::new(),
5530            &mut HashMap::new(),
5531            &mut HashMap::new(),
5532            &mut push_buffer,
5533            &mut HashMap::new(),
5534            &mut pending_binds,
5535            &mut installed_route_epochs,
5536            &Arc::new(Executor::new()),
5537            &Arc::new(Notify::new()),
5538            &metrics,
5539        )
5540        .await
5541        .unwrap();
5542
5543        let ack = writer_rx.try_recv().expect("RouteBindAck");
5544        assert_eq!(ack.header.ty, FrameType::Response);
5545        assert_eq!((ack.header.channel, ack.header.epoch), (0, 0));
5546        let route_frame = writer_rx.try_recv().expect("post-ack route frame");
5547        assert_eq!(route_frame.header.ty, FrameType::Push);
5548        assert_eq!(
5549            (route_frame.header.channel, route_frame.header.epoch),
5550            (route.channel, route.epoch)
5551        );
5552    }
5553}