Skip to main content

aft/subc/
mod.rs

1//! subc daemon attach — transport edge.
2//!
3//! When AFT is launched as `aft --subc <connection-file>`, it does NOT run the
4//! standalone NDJSON-over-stdin loop. Instead it connects to a running subc
5//! daemon over loopback TCP, authenticates with the pre-envelope HMAC handshake
6//! (`subc-transport`), then speaks the subc frame protocol (`subc-protocol`):
7//! ModuleHello → HelloAck (register as a tool provider), then a channel-0
8//! control loop (Ping/Pong, RouteBind) plus route-channel tool calls.
9//!
10//! Concurrency: subc routes tool calls through the executor. The tokio
11//! edge never dispatches against `AppContext` inline; per-actor executor lanes
12//! own the reader/mutator epoch, while a writer task serializes outbound frames.
13
14use std::collections::{HashMap, HashSet, VecDeque};
15use std::fmt;
16use std::io;
17use std::net::{IpAddr, SocketAddr};
18use std::ops::Deref;
19use std::path::{Path, PathBuf};
20use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
21use std::sync::{Arc, LazyLock};
22use std::time::{Duration, Instant};
23
24use serde::Deserialize;
25use serde_json::{json, Value};
26
27use crate::config::Config;
28use crate::config_resolve::ConfigTier;
29use crate::context::{App, AppContext, ProgressSender, RootHealthSnapshot};
30use crate::executor::{Executor, Lane};
31use crate::jsonc::strip_jsonc;
32use crate::log_ctx;
33use crate::path_identity::ProjectRootId;
34use crate::protocol::{ProgressKind, PushFrame, RawRequest, Response};
35use crate::run_tool_call::{
36    run_tool_call, strip_agent_preview_arg_owned, PhaseTrace, ToolCallContext, ToolCallOutcome,
37    ToolCallResult,
38};
39use crate::runtime_drain;
40use crate::sandbox_spawn::{AuthenticatedPrincipal, PrincipalTrust};
41
42use subc_protocol::manifest::{
43    Bindings, Concurrency, ExecutionMode, IdentityBinding, IdentityScope, ModuleManifest,
44    ProviderRole, StorageBinding, StorageKind, StorageScope, Tool, TrustTier,
45};
46use subc_protocol::session::{
47    HealthReport, HealthStatus, ModuleControlRequest, ModuleControlResponse,
48    MODULE_CONTROL_OP_HEALTH_CHECK,
49};
50use subc_protocol::{
51    ErrorBody, Flags, Frame, FrameType, ModuleHelloBody, Principal, Priority, PROTOCOL_VERSION,
52};
53use subc_transport::{authenticate_client, connection_file, read_frame, write_frame};
54use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
55use tokio::net::TcpStream;
56use tokio::sync::{mpsc, oneshot, Notify};
57use tokio::task::JoinHandle;
58
59/// Per-attempt handshake deadline. The initial attach loop has a separate total
60/// budget so a stalled peer cannot consume an unbounded supervisor launch window.
61const AUTH_DEADLINE: Duration = Duration::from_secs(5);
62const ATTACH_RETRY_BUDGET: Duration = Duration::from_secs(60);
63const ATTACH_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(250);
64const ATTACH_RETRY_MAX_BACKOFF: Duration = Duration::from_secs(5);
65const ATTACH_RETRY_JITTER_PERCENT: u64 = 20;
66
67/// Correlation id for the initial ModuleHello (channel 0).
68const HELLO_CORR: u64 = 1;
69
70/// Per-session in-memory replay cap for must-deliver Push frames. This covers
71/// detach/re-attach while AFT stays alive; cross-restart replay is phased later.
72const PUSH_BUFFER_MAX_PER_KEY: usize = 256;
73
74/// Bounded guard for control-frame sends. If the daemon stops reading and the
75/// writer queue stays full, tear the subc edge down instead of stalling the
76/// route loop indefinitely.
77const CONTROL_SEND_TIMEOUT: Duration = Duration::from_millis(250);
78
79/// Cadence for the loop's deadline-driven drain work (retry-buffer flush,
80/// bg-wake emission, maintenance submission). Checked at the top of every
81/// loop turn so busy select arms cannot starve it.
82const DRAIN_TICK_PERIOD: Duration = Duration::from_millis(250);
83
84/// Root-scoped stores and watcher runtimes are reopened lazily after this
85/// period without tool traffic. Keeping the value fixed avoids per-client
86/// eviction policies competing inside the module loop.
87const IDLE_ROOT_TTL: Duration = Duration::from_secs(30 * 60);
88
89const WRITER_QUEUE_CAPACITY: usize = 256;
90
91/// Keep reliable Push bursts from monopolizing the current-thread subc loop;
92/// any remaining must-deliver frames stay queued for the next loop turn.
93const RELIABLE_PUSH_DRAIN_BUDGET: usize = 32;
94
95/// Limit maintenance submissions per tick so background drains cannot delay
96/// control-plane work such as completed RouteBind acknowledgements.
97///
98/// The decomposed maintenance pass charges this budget by Mutating job, not by
99/// root. Size the default burst for one maintenance pass over eight live roots,
100/// while follow-up batches still re-enter the capped queue instead of bypassing
101/// the budget.
102const MAINTENANCE_SUBMIT_BUDGET: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len() * 8;
103const INITIAL_MAINTENANCE_DRAIN_KINDS: [MaintenanceDrainKind; 4] = [
104    MaintenanceDrainKind::Watcher,
105    MaintenanceDrainKind::Lsp,
106    MaintenanceDrainKind::ConfigureTail,
107    MaintenanceDrainKind::CompletionDrains,
108];
109#[cfg(test)]
110const INITIAL_MAINTENANCE_JOB_COUNT: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len();
111
112const RELIABLE_WRITER_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(10);
113const RELIABLE_WRITER_RETRY_MAX_BACKOFF: Duration = Duration::from_millis(250);
114
115const DISPATCH_PATH_BIND_WARN_AFTER: Duration = Duration::from_secs(6);
116const ROUTE_BIND_DEADLINE: Duration = Duration::from_secs(12);
117
118/// Small bounded memory of completed task ids used to suppress stale lossy
119/// long-running reminders that arrive after their reliable completion event.
120const COMPLETED_TASK_SUPPRESSION_MAX: usize = 4096;
121
122/// Bash foreground orchestration polls detached tasks with short read-lane jobs.
123/// The sleep between polls is outside the executor so no read or write worker is
124/// pinned while a foreground command is still running.
125const PENDING_POLL_INTERVAL: Duration = Duration::from_millis(100);
126
127/// Host elicitation asks fail closed if the MCP facade does not answer promptly.
128const BASH_ELICITATION_TIMEOUT: Duration = Duration::from_secs(60);
129const BASH_ELICITATION_CREATE_METHOD: &str = "elicitation/create";
130
131#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
132struct RouteChannel {
133    channel: u16,
134    epoch: u32,
135}
136
137impl fmt::Display for RouteChannel {
138    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
139        write!(f, "{}@{}", self.channel, self.epoch)
140    }
141}
142
143type PushEnvelope = (ProjectRootId, PushFrame);
144type LossyPushEnvelope = (u64, ProjectRootId, PushFrame);
145type RetryBuffer = HashMap<RouteChannel, VecDeque<(push::ReplayKey, PushFrame)>>;
146mod bash;
147mod health;
148mod manifest;
149mod push;
150mod wire;
151
152use self::health::{
153    build_health_report, warn_slow_pending_binds, DispatchPathMetrics, ReapBlockerCensus,
154    ResponseTaskGuard,
155};
156use self::manifest::{
157    build_manifest, command_lane, control_flags, control_ops, is_bash_family_tool,
158    is_subc_agent_core_tool, is_subc_native_plumbing_tool,
159};
160pub use self::wire::SubcError;
161
162/// Test-only view of the fail-closed tool-call gate: would `name` be admitted
163/// on a bound route (as an agent tool or native plumbing)? Used by the
164/// plugin-send drift guard in `subc_plumbing_drift_test.rs`.
165pub fn is_tool_call_admitted_for_test(name: &str) -> bool {
166    manifest::is_subc_agent_core_tool(name) || manifest::is_subc_native_plumbing_tool(name)
167}
168use self::wire::{
169    build_error_frame, build_goodbye_frame, build_tool_response_frame, decrement_counted_channel,
170    response_is_fatal_panic, response_message, send_counted_channel, send_frame,
171    send_reliable_writer_frame, send_traced_tool_response_frame, ToolResponseWriteTrace,
172    WriterFrame, WriterSender,
173};
174
175struct DecodedFrame {
176    frame: Frame,
177    phase_trace: PhaseTrace,
178}
179
180struct ToolCallCompletion {
181    text: String,
182    phase_trace: PhaseTrace,
183}
184
185#[derive(Clone)]
186struct PushSenders {
187    lossy_tx: mpsc::Sender<LossyPushEnvelope>,
188    reliable_tx: mpsc::UnboundedSender<PushEnvelope>,
189    lossy_overflow: Arc<push::LossyOverflow>,
190    lossy_seq: Arc<AtomicU64>,
191}
192
193#[derive(Clone)]
194struct PersistentCancelSignal {
195    inner: Arc<PersistentCancelInner>,
196}
197
198struct PersistentCancelInner {
199    cancelled: AtomicBool,
200    notify: Notify,
201}
202
203impl PersistentCancelSignal {
204    fn new() -> Self {
205        Self {
206            inner: Arc::new(PersistentCancelInner {
207                cancelled: AtomicBool::new(false),
208                notify: Notify::new(),
209            }),
210        }
211    }
212
213    fn cancel(&self) {
214        if !self.inner.cancelled.swap(true, Ordering::SeqCst) {
215            self.inner.notify.notify_waiters();
216        }
217    }
218
219    fn is_cancelled(&self) -> bool {
220        self.inner.cancelled.load(Ordering::SeqCst)
221    }
222
223    async fn cancelled(&self) {
224        // `enable()` REGISTERS this waiter before we read the flag, closing the
225        // lost-wakeup window: `notify_waiters()` only wakes already-registered
226        // waiters and stores no permit, so without enable() a `cancel()` firing
227        // between the flag read and `.await` would be missed and the future
228        // would park forever (cancel() fires only once). With enable(), a cancel
229        // racing the flag read still wakes the registered waiter. The loop is a
230        // belt-and-suspenders re-check on spurious wakeups.
231        loop {
232            let notified = self.inner.notify.notified();
233            tokio::pin!(notified);
234            notified.as_mut().enable();
235            if self.is_cancelled() {
236                return;
237            }
238            notified.await;
239        }
240    }
241}
242
243#[derive(Debug, Clone, Copy, PartialEq, Eq)]
244pub(crate) enum BindTrust {
245    FirstParty,
246    Untrusted,
247}
248
249impl BindTrust {
250    fn allows_bash_observation(self) -> bool {
251        matches!(self, Self::FirstParty)
252    }
253
254    fn label(self) -> &'static str {
255        match self {
256            Self::FirstParty => "first_party",
257            Self::Untrusted => "untrusted",
258        }
259    }
260
261    fn sandbox_trust(self) -> PrincipalTrust {
262        match self {
263            Self::FirstParty => PrincipalTrust::FirstParty,
264            Self::Untrusted => PrincipalTrust::Untrusted,
265        }
266    }
267}
268
269pub(super) fn trust_for_principal(principal: &Option<Principal>) -> BindTrust {
270    match principal {
271        Some(Principal::Direct) => BindTrust::FirstParty,
272        // Module renames are flag-days: the daemon registry refuses duplicate
273        // active ids, so a renaming module cannot advertise both names during
274        // its transition. This allowlist is DIALLED, not dialling — it must
275        // accept a module's NEW name in a released binary before the module
276        // starts using it, and the old name stays until the flip has settled.
277        // That is why transitional pairs appear here: llm-runner/broca was the
278        // previous rename, alfonso-core/prefrontal is the current one. When
279        // retiring an old name, confirm the fleet no longer spawns it — a
280        // stale entry here is inert, but a missing one silently downgrades a
281        // first-party module to Untrusted and revokes its bash access.
282        Some(Principal::Reserved { module_id })
283            if module_id == "llm-runner"
284                || module_id == "aft"
285                || module_id == "broca"
286                || module_id == "alfonso-core"
287                || module_id == "prefrontal"
288                || module_id == "prefrontal-core" =>
289        {
290            BindTrust::FirstParty
291        }
292        Some(Principal::Reserved { .. }) | Some(Principal::Unverified) | None => {
293            BindTrust::Untrusted
294        }
295    }
296}
297
298fn harness_forces_untrusted(harness: &str) -> bool {
299    harness.starts_with("fed:")
300}
301
302pub(super) fn trust_for_bind(harness: &str, principal: &Option<Principal>) -> BindTrust {
303    if harness_forces_untrusted(harness) {
304        BindTrust::Untrusted
305    } else {
306        trust_for_principal(principal)
307    }
308}
309
310fn principal_id(principal: &Option<Principal>) -> Option<String> {
311    match principal {
312        Some(Principal::Direct) => Some("direct".to_string()),
313        Some(Principal::Reserved { module_id }) => Some(format!("reserved:{module_id}")),
314        Some(Principal::Unverified) => Some("unverified".to_string()),
315        None => None,
316    }
317}
318
319fn principal_label(principal: &Option<Principal>) -> String {
320    principal_id(principal).unwrap_or_else(|| "absent".to_string())
321}
322
323#[derive(Debug)]
324/// Per-root route metadata owned by the subc loop. The `active_bash_waits` field
325/// counts detached bash processes that are still being observed for this root.
326/// Any future logic that evicts roots based on idle time must not evict a root
327/// while this count is greater than zero, because a foreground bash response may
328/// still arrive later.
329struct RootMeta {
330    maintenance_pending: bool,
331    maintenance_jobs_in_flight: usize,
332    maintenance_queued_kinds: VecDeque<MaintenanceDrainKind>,
333    maintenance_last_submitted: Option<Instant>,
334    maintenance_poisoned: bool,
335    last_touched: Instant,
336    diagnostics_on_edit: bool,
337    active_bash_waits: usize,
338    idle_artifacts_evicted: bool,
339    unbound_quiesced: bool,
340    consecutive_missing_sweeps: u8,
341}
342
343#[derive(Debug)]
344struct PendingBind {
345    bind_root_id: ProjectRootId,
346    inserted_new_actor: bool,
347    cancelled: bool,
348    configure_request_id: String,
349    started_at: Instant,
350    warned_half_deadline: bool,
351    deadline_reported: bool,
352    corr: u64,
353    ver: u8,
354    flags: Flags,
355    /// Exact-job cancellation for the submitted configure: Goodbye and
356    /// deadline expiry cancel the executor job operationally (queued jobs are
357    /// removed, running configures return at their next checkpoint) instead of
358    /// only marking bookkeeping.
359    cancellation: crate::executor::JobCancellation,
360}
361
362struct RouteBindCompletion {
363    route: RouteChannel,
364    identity: RouteIdentity,
365    bind_root_id: ProjectRootId,
366    inserted_new_actor: bool,
367    configure_response: Response,
368    diagnostics_on_edit: bool,
369    ver: u8,
370    corr: u64,
371    flags: Flags,
372}
373
374#[derive(Debug, Clone)]
375struct RouteIdentity(Arc<RouteIdentityData>);
376
377#[derive(Debug)]
378struct RouteIdentityData {
379    root: ProjectRootId,
380    project_root: PathBuf,
381    harness: String,
382    session: String,
383    trust: BindTrust,
384    spawn_principal: AuthenticatedPrincipal,
385    consumer_elicitation_capable: bool,
386}
387
388impl Deref for RouteIdentity {
389    type Target = RouteIdentityData;
390
391    fn deref(&self) -> &Self::Target {
392        &self.0
393    }
394}
395
396#[derive(Debug, Clone)]
397struct RetainedSessionIdentity {
398    harness: String,
399    trust: BindTrust,
400}
401
402#[derive(Clone, Copy)]
403struct BgSub {
404    corr: u64,
405    ver: u8,
406    flags: Flags,
407}
408
409struct MaintenanceCompletion {
410    root_id: ProjectRootId,
411    kind: MaintenanceDrainKind,
412    response: Response,
413    empty_bg_sessions: Vec<(String, u64)>,
414    requeue_kind: Option<MaintenanceDrainKind>,
415}
416
417#[derive(Clone, Copy, Debug, PartialEq, Eq)]
418enum MaintenanceDrainKind {
419    Watcher,
420    Lsp,
421    ConfigureTail,
422    CompletionDrains,
423}
424
425impl MaintenanceDrainKind {
426    fn label(self) -> &'static str {
427        match self {
428            Self::Watcher => "watcher",
429            Self::Lsp => "lsp",
430            Self::ConfigureTail => "configure-tail",
431            Self::CompletionDrains => "completion-drains",
432        }
433    }
434}
435
436#[derive(Debug, Default)]
437struct MaintenanceJobOutcome {
438    empty_bg_sessions: Vec<(String, u64)>,
439    requeue_kind: Option<MaintenanceDrainKind>,
440}
441
442#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
443struct ReverseCorrKey {
444    route: RouteChannel,
445    corr: u64,
446}
447
448struct PendingBashAsk {
449    route: RouteChannel,
450    tool_corr: u64,
451    tool_flags: Flags,
452    tool_ver: u8,
453    root: ProjectRootId,
454    project_root: PathBuf,
455    session_id: String,
456    spawn_principal: AuthenticatedPrincipal,
457    request_id: String,
458    arguments: Value,
459    format_context: crate::subc_format::FormatContext,
460    cancel: bash::BashWaitCancel,
461    grants: Vec<String>,
462    expires_at: Instant,
463}
464
465impl RootMeta {
466    fn new(now: Instant) -> Self {
467        Self {
468            maintenance_pending: false,
469            maintenance_jobs_in_flight: 0,
470            maintenance_queued_kinds: VecDeque::new(),
471            maintenance_last_submitted: None,
472            maintenance_poisoned: false,
473            last_touched: now,
474            diagnostics_on_edit: false,
475            active_bash_waits: 0,
476            idle_artifacts_evicted: false,
477            unbound_quiesced: false,
478            consecutive_missing_sweeps: 0,
479        }
480    }
481
482    fn note_activity(&mut self) {
483        self.last_touched = Instant::now();
484    }
485
486    fn reactivate_bound(&mut self) {
487        self.note_activity();
488        self.idle_artifacts_evicted = false;
489        self.unbound_quiesced = false;
490    }
491}
492
493fn due_maintenance_jobs(
494    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
495    executor: Option<&Executor>,
496    bg_sub_by_session: &HashMap<(ProjectRootId, String), RouteChannel>,
497    bg_wake_pending: &HashSet<RouteChannel>,
498    budget: usize,
499    pending_bind_roots: &HashSet<ProjectRootId>,
500) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
501    let mut jobs = Vec::new();
502    let mut deferred = false;
503    let mut roots = live_roots.keys().cloned().collect::<Vec<_>>();
504    roots.sort_by(|left, right| {
505        let left_last = live_roots
506            .get(left)
507            .and_then(|meta| meta.maintenance_last_submitted);
508        let right_last = live_roots
509            .get(right)
510            .and_then(|meta| meta.maintenance_last_submitted);
511        left_last
512            .cmp(&right_last)
513            .then_with(|| left.as_path().cmp(right.as_path()))
514    });
515
516    for root_id in roots {
517        let Some(meta) = live_roots.get_mut(&root_id) else {
518            continue;
519        };
520        if meta.maintenance_poisoned {
521            continue;
522        }
523
524        if pending_bind_roots.contains(&root_id) {
525            if meta.maintenance_pending || !meta.maintenance_queued_kinds.is_empty() {
526                deferred = true;
527            }
528            continue;
529        }
530
531        if !meta.maintenance_pending {
532            if jobs.len() >= budget {
533                deferred = true;
534                continue;
535            }
536            // Only enqueue kinds with pending work. Probes are cheap and
537            // fail-open (contended sources count as pending), so an idle root
538            // costs four probes per tick instead of four dispatched jobs.
539            let executor_actor_context =
540                executor.and_then(|executor| executor.actor_context(&root_id));
541            let root_has_pending_bg_wake =
542                bg_sub_by_session.iter().any(|((sub_root, _), channel)| {
543                    sub_root == &root_id && bg_wake_pending.contains(channel)
544                });
545            let kinds_with_work: Vec<MaintenanceDrainKind> = match executor_actor_context {
546                Some(ctx) => INITIAL_MAINTENANCE_DRAIN_KINDS
547                    .into_iter()
548                    .filter(|kind| {
549                        if meta.unbound_quiesced && !matches!(kind, MaintenanceDrainKind::Lsp) {
550                            return false;
551                        }
552                        match kind {
553                            MaintenanceDrainKind::Watcher => ctx.watcher_drain_has_work(),
554                            MaintenanceDrainKind::Lsp => ctx.lsp_drain_has_work(),
555                            MaintenanceDrainKind::ConfigureTail => ctx.configure_tail_has_work(),
556                            // Every CompletionDrains source is visible at this enqueue site:
557                            // AppContext probes completion queues, this loop owns bg wakes,
558                            // and queued continuations bypass probing via maintenance_pending.
559                            // New drain sources must expose a probe here rather than making
560                            // every subscribed root fail open again.
561                            MaintenanceDrainKind::CompletionDrains => {
562                                root_has_pending_bg_wake || ctx.completion_drains_have_work()
563                            }
564                        }
565                    })
566                    .collect(),
567                None if meta.unbound_quiesced => Vec::new(),
568                // No context handle (actor gone mid-tick): enqueue everything.
569                None => INITIAL_MAINTENANCE_DRAIN_KINDS.to_vec(),
570            };
571            if kinds_with_work.is_empty() {
572                continue;
573            }
574            meta.maintenance_pending = true;
575            meta.maintenance_queued_kinds.extend(kinds_with_work);
576        }
577
578        while let Some(kind) = meta.maintenance_queued_kinds.pop_front() {
579            if jobs.len() >= budget {
580                meta.maintenance_queued_kinds.push_front(kind);
581                deferred = true;
582                break;
583            }
584            meta.maintenance_jobs_in_flight += 1;
585            meta.maintenance_last_submitted = Some(Instant::now());
586            jobs.push((root_id.clone(), kind));
587        }
588
589        meta.maintenance_pending =
590            meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
591    }
592
593    (jobs, deferred)
594}
595
596fn eviction_estimate_label(estimate: &crate::memory::MemoryEstimate) -> String {
597    match estimate.estimated_bytes {
598        Some(bytes) => format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
599        None if estimate.status == "busy" => "busy".to_string(),
600        None => "not estimated".to_string(),
601    }
602}
603
604fn optional_memory_label(bytes: Option<u64>) -> String {
605    bytes.map_or_else(
606        || "not estimated".to_string(),
607        |bytes| format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
608    )
609}
610
611fn pressure_relief_label(relief: &crate::memory::AllocatorPressureRelief) -> String {
612    format!(
613        "; allocator pressure relief: RSS {} -> {}, in-use {} -> {}, allocated {} -> {}, slack {} -> {}, allocator reported {:.1} MB released",
614        optional_memory_label(relief.rss_before_bytes),
615        optional_memory_label(relief.rss_after_bytes),
616        optional_memory_label(relief.allocator_before.bytes_in_use),
617        optional_memory_label(relief.allocator_after.bytes_in_use),
618        optional_memory_label(relief.allocator_before.size_allocated),
619        optional_memory_label(relief.allocator_after.size_allocated),
620        optional_memory_label(relief.allocator_before.retained_slack_bytes),
621        optional_memory_label(relief.allocator_after.retained_slack_bytes),
622        relief.bytes_released as f64 / (1024.0 * 1024.0),
623    )
624}
625
626fn idle_root_eviction_message(
627    root_id: &ProjectRootId,
628    memory: &crate::memory::RootMemorySnapshot,
629    pressure_relief: Option<&crate::memory::AllocatorPressureRelief>,
630) -> String {
631    // Bash, LSP, and parser state remain resident. The freed total is deliberately
632    // only the known-byte portion of handles eviction actually drops.
633    let freed_bytes = [
634        &memory.semantic,
635        &memory.trigram,
636        &memory.symbols,
637        &memory.callgraph,
638        &memory.inspect,
639    ]
640    .iter()
641    .filter_map(|estimate| estimate.estimated_bytes)
642    .fold(0u64, u64::saturating_add);
643    let mut message = format!(
644        "evicted idle root {}: freed ~{:.1} MB (semantic {}, trigram {}, symbols {}, callgraph {}, inspect {}; retained: bash {}, lsp {}, parser_pool {})",
645        root_id.as_path().display(),
646        freed_bytes as f64 / (1024.0 * 1024.0),
647        eviction_estimate_label(&memory.semantic),
648        eviction_estimate_label(&memory.trigram),
649        eviction_estimate_label(&memory.symbols),
650        eviction_estimate_label(&memory.callgraph),
651        eviction_estimate_label(&memory.inspect),
652        eviction_estimate_label(&memory.bash),
653        eviction_estimate_label(&memory.lsp),
654        eviction_estimate_label(&memory.parser_pool),
655    );
656    if let Some(pressure_relief) = pressure_relief {
657        message.push_str(&pressure_relief_label(pressure_relief));
658    }
659    message
660}
661
662fn process_has_been_idle(now: Instant, live_roots: &HashMap<ProjectRootId, RootMeta>) -> bool {
663    !live_roots.is_empty()
664        && live_roots.values().all(|meta| {
665            now.saturating_duration_since(meta.last_touched) >= IDLE_ROOT_TTL
666                && meta.active_bash_waits == 0
667                && !meta.maintenance_pending
668                && meta.maintenance_queued_kinds.is_empty()
669        })
670}
671
672fn allocator_pressure_relief_after_idle_sweep(
673    now: Instant,
674    live_roots: &HashMap<ProjectRootId, RootMeta>,
675    executor: &Executor,
676) -> Option<crate::memory::AllocatorPressureRelief> {
677    if !process_has_been_idle(now, live_roots)
678        || live_roots.keys().any(|root_id| {
679            executor
680                .actor_context(root_id)
681                .is_some_and(|ctx| ctx.artifact_eviction_blocked())
682        })
683    {
684        return None;
685    }
686
687    #[cfg(target_os = "macos")]
688    {
689        Some(crate::memory::relieve_allocator_pressure())
690    }
691    #[cfg(not(target_os = "macos"))]
692    {
693        None
694    }
695}
696
697fn quiesce_unbound_root(
698    root_id: &ProjectRootId,
699    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
700    executor: &Arc<Executor>,
701) {
702    let Some(meta) = live_roots.get_mut(root_id) else {
703        return;
704    };
705
706    let ctx = executor.actor_context(root_id);
707    if let Some(ctx) = ctx.as_ref() {
708        // Close lifecycle admission before touching scheduler queues. A running
709        // ConfigureTail cannot release gates, install a watcher, or reserve a
710        // callgraph build after this transition becomes visible.
711        ctx.mark_subc_unbound();
712    }
713    let cancelled = executor.cancel_queued_maintenance(root_id);
714    // Transient unbind keeps the root WARM: the watcher stays running (its
715    // events accumulate and replay on rebind, so no unobserved gap exists) and
716    // resident artifacts stay resident. Host restarts unbind every root and
717    // rebind seconds later; stopping the watcher here would force strict
718    // re-verification plus a full callgraph rebuild on every restart. The
719    // expensive teardown (watcher stop + gap invalidation) belongs to the
720    // idle-TTL reaper and the root-deleted path.
721    let discarded = ctx
722        .map(|ctx| crate::commands::configure::cancel_deferred_configure_maintenance(&ctx))
723        .unwrap_or(0);
724    meta.unbound_quiesced = true;
725    meta.maintenance_queued_kinds.clear();
726    meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
727    log::debug!(
728        "subc attach: quiesced unbound root {} (cancelled {} queued maintenance job(s), cancelled {} configure maintenance job(s))",
729        root_id.as_path().display(),
730        cancelled,
731        discarded
732    );
733}
734
735#[allow(clippy::too_many_arguments)]
736fn quiesce_connection_roots(
737    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
738    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
739    routes: &mut HashMap<RouteChannel, RouteIdentity>,
740    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
741    installed_route_epochs: &mut HashMap<u16, u32>,
742    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
743    executor: &Arc<Executor>,
744) {
745    for cancel in route_bash_cancels.values() {
746        cancel.token.cancel();
747    }
748    route_bash_cancels.clear();
749
750    let mut roots = live_roots.keys().cloned().collect::<HashSet<_>>();
751    for pending in pending_binds.values_mut() {
752        pending.cancelled = true;
753        roots.insert(pending.bind_root_id.clone());
754        let _ = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
755    }
756
757    // A connection exit abandons every installed route at once. Close lifecycle
758    // admission before cancelling maintenance so no deferred worker can restore
759    // root activity after the loop-owned route tables disappear.
760    for root_id in roots {
761        if live_roots.contains_key(&root_id) {
762            quiesce_unbound_root(&root_id, live_roots, executor);
763        } else if let Some(ctx) = executor.actor_context(&root_id) {
764            ctx.mark_subc_unbound();
765            executor.cancel_queued_maintenance(&root_id);
766            crate::commands::configure::cancel_deferred_configure_maintenance(&ctx);
767        }
768    }
769
770    routes.clear();
771    root_channels.clear();
772    installed_route_epochs.clear();
773}
774
775/// Per-channel epoch watermarks for roots reclaimed without a client Goodbye.
776/// The 16-bit channel space bounds this map, and no root identity or resource
777/// handle is retained. It exists only so late requests receive a typed error.
778#[derive(Debug, Default)]
779struct ReclaimedRoutes {
780    highest_epoch_by_channel: HashMap<u16, u32>,
781}
782
783impl ReclaimedRoutes {
784    fn insert(&mut self, route: RouteChannel) {
785        self.highest_epoch_by_channel
786            .entry(route.channel)
787            .and_modify(|epoch| *epoch = (*epoch).max(route.epoch))
788            .or_insert(route.epoch);
789    }
790
791    fn contains(&self, route: RouteChannel) -> bool {
792        self.highest_epoch_by_channel
793            .get(&route.channel)
794            .is_some_and(|epoch| route.epoch <= *epoch)
795    }
796}
797
798#[derive(Debug, Default)]
799struct IdleReapOutcome {
800    evicted: usize,
801    forgotten_deleted_roots: Vec<ProjectRootId>,
802}
803
804fn reap_idle_roots(
805    now: Instant,
806    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
807    pending_binds: &HashMap<RouteChannel, PendingBind>,
808    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
809    executor: &Arc<Executor>,
810    metrics: &DispatchPathMetrics,
811) -> IdleReapOutcome {
812    let pending_bind_roots = pending_binds
813        .values()
814        .map(|pending| pending.bind_root_id.clone())
815        .collect::<HashSet<_>>();
816    let mut census = ReapBlockerCensus::default();
817    let mut candidates = Vec::new();
818
819    for (root_id, meta) in live_roots.iter_mut() {
820        let deleted = !root_id.as_path().exists();
821        if deleted {
822            // A missing directory makes a bound route obsolete, but one failed
823            // lookup is not enough evidence to tear down a client-visible actor.
824            // Requiring two maintenance sweeps protects atomic replacement and
825            // transient filesystem failures; observing the path resets the proof.
826            meta.consecutive_missing_sweeps = meta.consecutive_missing_sweeps.saturating_add(1);
827        } else {
828            meta.consecutive_missing_sweeps = 0;
829        }
830        let deletion_confirmed = meta.consecutive_missing_sweeps >= 2;
831        let has_bound_route = root_channels
832            .get(root_id)
833            .is_some_and(|channels| !channels.is_empty());
834        let has_pending_bind = pending_bind_roots.contains(root_id);
835
836        if deleted {
837            let mut retained = false;
838            if !deletion_confirmed {
839                census.absence_unconfirmed += 1;
840                retained = true;
841            }
842            // Once absence is confirmed, the directory cannot serve this route
843            // again. Neither a stale route nor the lack of normal unbind cleanup
844            // justifies retaining the root; purge removes the route after retirement.
845            if meta.active_bash_waits > 0 {
846                census.bash_waits += 1;
847                retained = true;
848            }
849            if meta.maintenance_pending {
850                census.maintenance_pending += 1;
851                retained = true;
852            }
853            if !meta.maintenance_queued_kinds.is_empty() {
854                census.maintenance_queued += 1;
855                retained = true;
856            }
857            if has_pending_bind {
858                census.pending_binds += 1;
859                retained = true;
860            }
861            match executor.try_actor_is_idle(root_id) {
862                Some(true) => {}
863                Some(false) => {
864                    census.actor_busy += 1;
865                    retained = true;
866                }
867                None => {
868                    census.actor_state_busy += 1;
869                    retained = true;
870                }
871            }
872            if retained {
873                census.deleted_retained += 1;
874                continue;
875            }
876        } else {
877            // Route teardown marks the lifecycle admission gate before the last
878            // channel disappears. Requiring zero bound channels and a quiesced
879            // lifecycle prevents a still-bound root from losing its watcher.
880            if has_bound_route
881                || !meta.unbound_quiesced
882                || meta.idle_artifacts_evicted
883                || now.saturating_duration_since(meta.last_touched) < IDLE_ROOT_TTL
884                || meta.active_bash_waits > 0
885                || meta.maintenance_pending
886                || !meta.maintenance_queued_kinds.is_empty()
887                || has_pending_bind
888                || !executor.actor_is_idle(root_id)
889            {
890                continue;
891            }
892        }
893        candidates.push((root_id.clone(), deleted));
894    }
895
896    let mut reaped = Vec::new();
897    let mut forgotten_deleted_roots = Vec::new();
898    for (root_id, deleted) in candidates {
899        let Some(ctx) = executor.actor_context(&root_id) else {
900            if deleted {
901                census.deleted_retained += 1;
902                census.actor_busy += 1;
903            }
904            continue;
905        };
906        // A TTL-aged unbound root retained its watcher-derived pending paths
907        // across the transient-unbind window. Strict gap invalidation subsumes
908        // them, but every abort path must restore them because a rebind can
909        // still happen until eviction commits.
910        let taken_pending = Some(ctx.take_pending_reconciliation_state());
911        if ctx.artifact_eviction_blocked() {
912            if let Some(pending) = taken_pending {
913                ctx.restore_pending_reconciliation_state(pending);
914            }
915            if deleted {
916                census.deleted_retained += 1;
917                census.artifact_eviction_blocked += 1;
918            }
919            continue;
920        }
921        let memory_before = ctx.memory_root_snapshot();
922        if !ctx.evict_idle_artifacts() {
923            if let Some(pending) = taken_pending {
924                ctx.restore_pending_reconciliation_state(pending);
925            }
926            if deleted {
927                census.deleted_retained += 1;
928                census.artifact_eviction_failed += 1;
929            }
930            continue;
931        }
932        drop(taken_pending);
933        ctx.stop_watcher_runtime_in_background();
934        // Edits during watcher downtime are unobserved. Advance publication
935        // epochs and force strict verification before any later warm reload.
936        ctx.invalidate_artifacts_after_watcher_gap();
937
938        if deleted {
939            if executor.retire_idle_actor_in_background(&root_id) {
940                live_roots.remove(&root_id);
941                forgotten_deleted_roots.push(root_id.clone());
942            } else {
943                census.deleted_retained += 1;
944                census.actor_busy += 1;
945            }
946        } else {
947            if let Some(meta) = live_roots.get_mut(&root_id) {
948                meta.idle_artifacts_evicted = true;
949            }
950            ctx.release_idle_reopenable_resources_in_background();
951        }
952        reaped.push((root_id, memory_before));
953    }
954
955    metrics.record_reap(census);
956    if census.deleted_retained > 0 {
957        log::info!(
958            "subc attach: retained {} deleted root(s) during idle reap; blockers={}",
959            census.deleted_retained,
960            census.blocker_histogram()
961        );
962    }
963
964    let pressure_relief = (!reaped.is_empty())
965        .then(|| allocator_pressure_relief_after_idle_sweep(now, live_roots, executor))
966        .flatten();
967    for (root_id, memory_before) in &reaped {
968        log::info!(
969            "{}",
970            idle_root_eviction_message(root_id, memory_before, pressure_relief.as_ref())
971        );
972    }
973    IdleReapOutcome {
974        evicted: reaped.len(),
975        forgotten_deleted_roots,
976    }
977}
978
979#[allow(clippy::too_many_arguments)]
980fn purge_deleted_root_residents(
981    root_id: &ProjectRootId,
982    routes: &mut HashMap<RouteChannel, RouteIdentity>,
983    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
984    installed_route_epochs: &mut HashMap<u16, u32>,
985    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
986    retry_buffer: &mut RetryBuffer,
987    reclaimed_routes: &mut ReclaimedRoutes,
988    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
989    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
990    bg_subs: &mut HashMap<RouteChannel, BgSub>,
991    bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
992    bg_wake_pending: &mut HashSet<RouteChannel>,
993    bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
994    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
995) {
996    let mut stale_routes = root_channels.get(root_id).cloned().unwrap_or_default();
997    stale_routes.extend(
998        routes
999            .iter()
1000            .filter_map(|(route, identity)| (&identity.root == root_id).then_some(*route)),
1001    );
1002    stale_routes.extend(
1003        bg_sub_by_session
1004            .iter()
1005            .filter_map(|((root, _), route)| (root == root_id).then_some(*route)),
1006    );
1007    stale_routes.extend(
1008        pending_bash_asks
1009            .values()
1010            .filter_map(|ask| (&ask.root == root_id).then_some(ask.route)),
1011    );
1012
1013    for route in stale_routes {
1014        reclaimed_routes.insert(route);
1015        remove_installed_route(installed_route_epochs, route);
1016        remove_route_channel(routes, root_channels, route);
1017        if let Some(cancel) = route_bash_cancels.remove(&route) {
1018            cancel.token.cancel();
1019        }
1020        retry_buffer.remove(&route);
1021        bg_subs.remove(&route);
1022        bg_wake_pending.remove(&route);
1023    }
1024    root_channels.remove(root_id);
1025    session_identity.retain(|(root, _), _| root != root_id);
1026    push_buffer.retain(|key, _| &key.root != root_id);
1027    bg_wake_epoch.retain(|(root, _), _| root != root_id);
1028    pending_bash_asks.retain(|_, ask| &ask.root != root_id);
1029    bg_sub_by_session.retain(|(root, _), _| root != root_id);
1030
1031    log::info!(
1032        "subc attach: fully forgot deleted root {}",
1033        root_id.as_path().display()
1034    );
1035}
1036
1037#[allow(clippy::too_many_arguments)]
1038fn submit_due_maintenance_jobs(
1039    executor: &Arc<Executor>,
1040    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1041    pending_binds: &HashMap<RouteChannel, PendingBind>,
1042    bg_sub_by_session: &HashMap<(ProjectRootId, String), RouteChannel>,
1043    bg_wake_pending: &HashSet<RouteChannel>,
1044    bg_wake_epoch: &HashMap<(ProjectRootId, String), u64>,
1045    maintenance_tx: &mpsc::Sender<MaintenanceCompletion>,
1046    metrics: &Arc<DispatchPathMetrics>,
1047) {
1048    let pending_bind_roots = pending_binds
1049        .values()
1050        .map(|pending| pending.bind_root_id.clone())
1051        .collect::<HashSet<_>>();
1052    let (due_jobs, deferred_jobs) = due_maintenance_jobs(
1053        live_roots,
1054        Some(executor),
1055        bg_sub_by_session,
1056        bg_wake_pending,
1057        MAINTENANCE_SUBMIT_BUDGET,
1058        &pending_bind_roots,
1059    );
1060    if deferred_jobs {
1061        metrics
1062            .maintenance_budget_deferrals
1063            .fetch_add(1, Ordering::Relaxed);
1064    }
1065    for (root_id, kind) in due_jobs {
1066        let bg_sessions_to_check = if kind == MaintenanceDrainKind::CompletionDrains {
1067            bg_sub_by_session
1068                .iter()
1069                .filter_map(|((root, session), _)| {
1070                    if root == &root_id {
1071                        Some((
1072                            session.clone(),
1073                            bg_wake_epoch
1074                                .get(&(root_id.clone(), session.clone()))
1075                                .copied()
1076                                .unwrap_or(0),
1077                        ))
1078                    } else {
1079                        None
1080                    }
1081                })
1082                .collect()
1083        } else {
1084            Vec::new()
1085        };
1086        submit_maintenance_job(
1087            executor,
1088            root_id,
1089            kind,
1090            bg_sessions_to_check,
1091            maintenance_tx,
1092            metrics,
1093        );
1094    }
1095}
1096
1097fn should_requiesce_after_maintenance(
1098    meta: &RootMeta,
1099    completed_kind: MaintenanceDrainKind,
1100    bind_pending: bool,
1101) -> bool {
1102    meta.unbound_quiesced && completed_kind != MaintenanceDrainKind::Lsp && !bind_pending
1103}
1104
1105fn note_maintenance_completion(
1106    meta: &mut RootMeta,
1107    requeue_kind: Option<MaintenanceDrainKind>,
1108    fatal: bool,
1109    defer_requeue: bool,
1110) {
1111    if fatal {
1112        meta.maintenance_poisoned = true;
1113    }
1114
1115    if let Some(kind) = requeue_kind.filter(|_| !meta.maintenance_poisoned && !defer_requeue) {
1116        meta.maintenance_queued_kinds.push_back(kind);
1117    }
1118
1119    meta.maintenance_jobs_in_flight = meta.maintenance_jobs_in_flight.saturating_sub(1);
1120    meta.maintenance_pending =
1121        meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
1122}
1123
1124fn route_key(channel: u16, epoch: u32) -> RouteChannel {
1125    RouteChannel { channel, epoch }
1126}
1127
1128fn remove_installed_route(installed_epochs: &mut HashMap<u16, u32>, route: RouteChannel) {
1129    if installed_epochs.get(&route.channel).copied() == Some(route.epoch) {
1130        installed_epochs.remove(&route.channel);
1131    }
1132}
1133
1134fn ingress_route_should_be_processed(
1135    installed_epochs: &HashMap<u16, u32>,
1136    reclaimed_routes: &ReclaimedRoutes,
1137    frame: &Frame,
1138) -> bool {
1139    if frame.header.channel == 0
1140        || installed_epochs.get(&frame.header.channel).copied() == Some(frame.header.epoch)
1141    {
1142        return true;
1143    }
1144
1145    // A late request for a reclaimed root reaches the normal unknown-route
1146    // handler, which returns the typed `route_not_bound` error. Other stale or
1147    // never-installed generations remain silent so they cannot affect a newer
1148    // route or change the protocol's rejected-bind behavior.
1149    frame.header.ty == FrameType::Request
1150        && reclaimed_routes.contains(route_key(frame.header.channel, frame.header.epoch))
1151}
1152
1153fn bash_elicitation_timeout() -> Duration {
1154    if cfg!(debug_assertions) {
1155        if let Ok(raw) = std::env::var("AFT_TEST_SUBC_BASH_ELICITATION_TTL_MS") {
1156            if let Ok(ms) = raw.parse::<u64>() {
1157                if ms > 0 {
1158                    return Duration::from_millis(ms);
1159                }
1160            }
1161        }
1162    }
1163    BASH_ELICITATION_TIMEOUT
1164}
1165
1166fn allocate_reverse_corr(
1167    pending_bash_asks: &HashMap<ReverseCorrKey, PendingBashAsk>,
1168    route: RouteChannel,
1169    next_corr: &mut u64,
1170) -> u64 {
1171    loop {
1172        let corr = *next_corr;
1173        *next_corr = (*next_corr).wrapping_add(1).max(1);
1174        if !pending_bash_asks.contains_key(&ReverseCorrKey { route, corr }) {
1175            return corr;
1176        }
1177    }
1178}
1179
1180fn bash_permission_kind_label(kind: &crate::bash_permissions::PermissionKind) -> &'static str {
1181    match kind {
1182        crate::bash_permissions::PermissionKind::ExternalDirectory => "external directory",
1183        crate::bash_permissions::PermissionKind::Bash => "bash",
1184    }
1185}
1186
1187fn bash_elicitation_patterns(asks: &[crate::bash_permissions::PermissionAsk]) -> Vec<String> {
1188    let mut patterns = Vec::new();
1189    let mut seen = HashSet::new();
1190    for ask in asks {
1191        for pattern in ask.patterns.iter().chain(ask.always.iter()) {
1192            if seen.insert(pattern.clone()) {
1193                patterns.push(pattern.clone());
1194            }
1195        }
1196    }
1197    patterns
1198}
1199
1200fn bash_elicitation_message(
1201    command: &str,
1202    asks: &[crate::bash_permissions::PermissionAsk],
1203) -> String {
1204    let command = command.split_whitespace().collect::<Vec<_>>().join(" ");
1205    let patterns = bash_elicitation_patterns(asks);
1206    let pattern_text = if patterns.is_empty() {
1207        "no matched permission patterns".to_string()
1208    } else {
1209        patterns.join(", ")
1210    };
1211    let ask_kinds = asks
1212        .iter()
1213        .map(|ask| bash_permission_kind_label(&ask.kind))
1214        .collect::<HashSet<_>>()
1215        .into_iter()
1216        .collect::<Vec<_>>()
1217        .join(", ");
1218    if ask_kinds.is_empty() {
1219        format!("Allow bash command `{command}`? Matched patterns: {pattern_text}")
1220    } else {
1221        format!("Allow bash command `{command}`? Matched {ask_kinds} patterns: {pattern_text}")
1222    }
1223}
1224
1225fn bash_elicitation_request_body(
1226    command: &str,
1227    asks: &[crate::bash_permissions::PermissionAsk],
1228) -> Value {
1229    json!({
1230        "method": BASH_ELICITATION_CREATE_METHOD,
1231        "params": {
1232            "mode": "form",
1233            "message": bash_elicitation_message(command, asks),
1234            "requestedSchema": {
1235                "type": "object",
1236                "properties": {
1237                    "decision": {
1238                        "type": "string",
1239                        "enum": ["allow", "deny"],
1240                        "description": "Choose allow to run this bash command once, or deny to block it."
1241                    }
1242                },
1243                "required": ["decision"],
1244                "additionalProperties": false
1245            },
1246            "_meta": {
1247                "aft": {
1248                    "tool": "bash",
1249                    "command": command,
1250                    "asks": asks
1251                }
1252            }
1253        }
1254    })
1255}
1256
1257fn build_bash_elicitation_request_frame(
1258    ver: u8,
1259    route: RouteChannel,
1260    corr: u64,
1261    flags: Flags,
1262    command: &str,
1263    asks: &[crate::bash_permissions::PermissionAsk],
1264) -> Result<Frame, SubcError> {
1265    let body = bash_elicitation_request_body(command, asks);
1266    Frame::build_with_version(
1267        ver,
1268        FrameType::Request,
1269        flags,
1270        route.channel,
1271        route.epoch,
1272        corr,
1273        serde_json::to_vec(&body).map_err(SubcError::Json)?,
1274    )
1275    .map_err(SubcError::FrameBuild)
1276}
1277
1278fn bash_elicitation_reply_is_allow(body: &[u8]) -> bool {
1279    let Ok(value) = serde_json::from_slice::<Value>(body) else {
1280        return false;
1281    };
1282    flat_bash_elicitation_reply_is_allow(&value) || mcp_bash_elicitation_reply_is_allow(&value)
1283}
1284
1285fn flat_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1286    let Some(object) = value.as_object() else {
1287        return false;
1288    };
1289    object.len() == 1 && object.get("decision").and_then(Value::as_str) == Some("allow")
1290}
1291
1292fn mcp_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1293    let Some(object) = value.as_object() else {
1294        return false;
1295    };
1296    if object.len() != 2 || object.get("action").and_then(Value::as_str) != Some("accept") {
1297        return false;
1298    }
1299    let Some(content) = object.get("content").and_then(Value::as_object) else {
1300        return false;
1301    };
1302    content.len() == 1 && content.get("decision").and_then(Value::as_str) == Some("allow")
1303}
1304
1305#[allow(clippy::too_many_arguments)]
1306async fn settle_pending_bash_ask_denied(
1307    tx: &WriterSender,
1308    pending: PendingBashAsk,
1309    routes: &HashMap<RouteChannel, RouteIdentity>,
1310    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1311    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1312    shutdown: &Arc<Notify>,
1313    metrics: &DispatchPathMetrics,
1314) -> Result<(), SubcError> {
1315    let completion = bash::bash_denied_untrusted_completion(
1316        pending.route,
1317        pending.tool_corr,
1318        pending.tool_flags,
1319        pending.tool_ver,
1320        pending.root,
1321        pending.request_id,
1322        pending.format_context,
1323    );
1324    bash::handle_bash_deferred_completion(
1325        tx,
1326        completion,
1327        routes,
1328        live_roots,
1329        route_bash_cancels,
1330        shutdown,
1331        metrics,
1332    )
1333    .await
1334}
1335
1336fn take_pending_bash_asks_for_route(
1337    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1338    route: RouteChannel,
1339) -> Vec<PendingBashAsk> {
1340    let keys = pending_bash_asks
1341        .keys()
1342        .copied()
1343        .filter(|key| key.route == route)
1344        .collect::<Vec<_>>();
1345    keys.into_iter()
1346        .filter_map(|key| pending_bash_asks.remove(&key))
1347        .collect()
1348}
1349
1350#[allow(clippy::too_many_arguments)]
1351async fn settle_pending_bash_asks_for_route(
1352    tx: &WriterSender,
1353    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1354    route: RouteChannel,
1355    routes: &HashMap<RouteChannel, RouteIdentity>,
1356    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1357    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1358    shutdown: &Arc<Notify>,
1359    metrics: &DispatchPathMetrics,
1360) -> Result<(), SubcError> {
1361    for pending in take_pending_bash_asks_for_route(pending_bash_asks, route) {
1362        settle_pending_bash_ask_denied(
1363            tx,
1364            pending,
1365            routes,
1366            live_roots,
1367            route_bash_cancels,
1368            shutdown,
1369            metrics,
1370        )
1371        .await?;
1372    }
1373    Ok(())
1374}
1375
1376#[allow(clippy::too_many_arguments)]
1377async fn settle_all_pending_bash_asks(
1378    tx: &WriterSender,
1379    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1380    routes: &HashMap<RouteChannel, RouteIdentity>,
1381    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1382    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1383    shutdown: &Arc<Notify>,
1384    metrics: &DispatchPathMetrics,
1385) -> Result<(), SubcError> {
1386    let pending = pending_bash_asks
1387        .drain()
1388        .map(|(_, pending)| pending)
1389        .collect::<Vec<_>>();
1390    for pending in pending {
1391        settle_pending_bash_ask_denied(
1392            tx,
1393            pending,
1394            routes,
1395            live_roots,
1396            route_bash_cancels,
1397            shutdown,
1398            metrics,
1399        )
1400        .await?;
1401    }
1402    Ok(())
1403}
1404
1405#[allow(clippy::too_many_arguments)]
1406async fn expire_pending_bash_asks(
1407    tx: &WriterSender,
1408    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1409    routes: &HashMap<RouteChannel, RouteIdentity>,
1410    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1411    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1412    shutdown: &Arc<Notify>,
1413    metrics: &DispatchPathMetrics,
1414) -> Result<(), SubcError> {
1415    let now = Instant::now();
1416    let expired = pending_bash_asks
1417        .iter()
1418        .filter_map(|(key, pending)| (pending.expires_at <= now).then_some(*key))
1419        .collect::<Vec<_>>();
1420    for key in expired {
1421        if let Some(pending) = pending_bash_asks.remove(&key) {
1422            log::debug!(
1423                "subc attach: bash elicitation request {} on route {} expired fail-closed",
1424                key.corr,
1425                pending.route
1426            );
1427            settle_pending_bash_ask_denied(
1428                tx,
1429                pending,
1430                routes,
1431                live_roots,
1432                route_bash_cancels,
1433                shutdown,
1434                metrics,
1435            )
1436            .await?;
1437        }
1438    }
1439    Ok(())
1440}
1441
1442#[allow(clippy::too_many_arguments)]
1443async fn handle_bash_elicitation_reply(
1444    tx: &WriterSender,
1445    frame: &Frame,
1446    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1447    routes: &HashMap<RouteChannel, RouteIdentity>,
1448    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1449    executor: &Arc<Executor>,
1450    shutdown: &Arc<Notify>,
1451    bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
1452    bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
1453    metrics: &Arc<DispatchPathMetrics>,
1454    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1455    dispatch: DispatchFn,
1456) -> Result<(), SubcError> {
1457    let key = ReverseCorrKey {
1458        route: route_key(frame.header.channel, frame.header.epoch),
1459        corr: frame.header.corr,
1460    };
1461    let Some(pending) = pending_bash_asks.remove(&key) else {
1462        return Ok(());
1463    };
1464
1465    if frame.header.ty == FrameType::Response && bash_elicitation_reply_is_allow(&frame.body) {
1466        if routes.contains_key(&key.route) {
1467            bash::submit_deferred_bash(
1468                executor,
1469                bash_deferred_tx,
1470                bash_poll_touch_tx,
1471                metrics,
1472                dispatch,
1473                pending.root,
1474                pending.project_root,
1475                pending.session_id,
1476                pending.request_id,
1477                pending.route,
1478                pending.tool_corr,
1479                pending.tool_flags,
1480                pending.tool_ver,
1481                pending.arguments,
1482                pending.format_context,
1483                pending.cancel,
1484                BindTrust::Untrusted,
1485                pending.spawn_principal,
1486                Some(pending.grants),
1487            );
1488            return Ok(());
1489        }
1490        log::debug!(
1491            "subc attach: dropping allowed bash elicitation reply {} for unbound route {}",
1492            key.corr,
1493            pending.route
1494        );
1495    }
1496
1497    settle_pending_bash_ask_denied(
1498        tx,
1499        pending,
1500        routes,
1501        live_roots,
1502        route_bash_cancels,
1503        shutdown,
1504        metrics,
1505    )
1506    .await
1507}
1508
1509#[allow(clippy::too_many_arguments)]
1510async fn cancel_pending_bash_ask_for_tool_call(
1511    tx: &WriterSender,
1512    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1513    route: RouteChannel,
1514    tool_corr: u64,
1515    routes: &HashMap<RouteChannel, RouteIdentity>,
1516    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1517    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1518    shutdown: &Arc<Notify>,
1519    metrics: &DispatchPathMetrics,
1520) -> Result<(), SubcError> {
1521    let keys = pending_bash_asks
1522        .iter()
1523        .filter_map(|(key, pending)| {
1524            (key.route == route && pending.tool_corr == tool_corr).then_some(*key)
1525        })
1526        .collect::<Vec<_>>();
1527    for key in keys {
1528        if let Some(pending) = pending_bash_asks.remove(&key) {
1529            settle_pending_bash_ask_denied(
1530                tx,
1531                pending,
1532                routes,
1533                live_roots,
1534                route_bash_cancels,
1535                shutdown,
1536                metrics,
1537            )
1538            .await?;
1539        }
1540    }
1541    Ok(())
1542}
1543
1544fn remove_root_channel(
1545    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1546    root: &ProjectRootId,
1547    channel: RouteChannel,
1548) {
1549    let remove_root = if let Some(channels) = root_channels.get_mut(root) {
1550        channels.remove(&channel);
1551        channels.is_empty()
1552    } else {
1553        false
1554    };
1555    if remove_root {
1556        root_channels.remove(root);
1557    }
1558}
1559
1560fn remove_route_channel(
1561    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1562    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1563    channel: RouteChannel,
1564) -> Option<RouteIdentity> {
1565    let removed = routes.remove(&channel);
1566    if let Some(identity) = &removed {
1567        remove_root_channel(root_channels, &identity.root, channel);
1568    }
1569    removed
1570}
1571
1572fn insert_route_channel(
1573    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1574    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1575    channel: RouteChannel,
1576    identity: RouteIdentity,
1577) {
1578    if let Some(previous) = routes.insert(channel, identity.clone()) {
1579        remove_root_channel(root_channels, &previous.root, channel);
1580    }
1581    root_channels
1582        .entry(identity.root.clone())
1583        .or_default()
1584        .insert(channel);
1585}
1586
1587fn remove_bg_subscription_index(
1588    bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1589    channel: RouteChannel,
1590    identity: Option<&RouteIdentity>,
1591) {
1592    if let Some(identity) = identity {
1593        let key = (identity.root.clone(), identity.session.clone());
1594        if bg_sub_by_session.get(&key).copied() == Some(channel) {
1595            bg_sub_by_session.remove(&key);
1596        }
1597    } else {
1598        bg_sub_by_session.retain(|_, mapped_channel| *mapped_channel != channel);
1599    }
1600}
1601
1602fn route_removal_will_quiesce_root(
1603    root: &ProjectRootId,
1604    route: RouteChannel,
1605    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1606    has_pending_bind: bool,
1607    replacement_root: Option<&ProjectRootId>,
1608) -> bool {
1609    let removes_last_route = root_channels
1610        .get(root)
1611        .is_some_and(|channels| channels.len() == 1 && channels.contains(&route));
1612    removes_last_route && !has_pending_bind && replacement_root != Some(root)
1613}
1614
1615fn should_quiesce_removed_root(
1616    root: &ProjectRootId,
1617    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1618    has_pending_bind: bool,
1619    replacement_root: Option<&ProjectRootId>,
1620) -> bool {
1621    !root_channels.contains_key(root) && !has_pending_bind && replacement_root != Some(root)
1622}
1623
1624async fn end_bg_subscription(
1625    writer_tx: &WriterSender,
1626    metrics: &DispatchPathMetrics,
1627    bg_subs: &mut HashMap<RouteChannel, BgSub>,
1628    bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1629    bg_wake_pending: &mut HashSet<RouteChannel>,
1630    channel: RouteChannel,
1631    identity: Option<&RouteIdentity>,
1632) -> Result<(), SubcError> {
1633    if let Some(sub) = bg_subs.remove(&channel) {
1634        bg_wake_pending.remove(&channel);
1635        remove_bg_subscription_index(bg_sub_by_session, channel, identity);
1636        push::send_reliable_bg_stream_end(writer_tx, metrics, channel, &sub).await?;
1637    }
1638    Ok(())
1639}
1640
1641#[allow(clippy::too_many_arguments)]
1642async fn teardown_installed_route(
1643    tx: &WriterSender,
1644    metrics: &DispatchPathMetrics,
1645    executor: &Arc<Executor>,
1646    channel: RouteChannel,
1647    cancellation_reason: &str,
1648    replacement_root: Option<&ProjectRootId>,
1649    installed_route_epochs: &mut HashMap<u16, u32>,
1650    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1651    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1652    bg_subs: &mut HashMap<RouteChannel, BgSub>,
1653    bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1654    bg_wake_pending: &mut HashSet<RouteChannel>,
1655    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1656    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1657    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1658    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
1659    retry_buffer: &mut RetryBuffer,
1660    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1661    shutdown: &Arc<Notify>,
1662) -> Result<(), SubcError> {
1663    remove_installed_route(installed_route_epochs, channel);
1664    end_bg_subscription(
1665        tx,
1666        metrics,
1667        bg_subs,
1668        bg_sub_by_session,
1669        bg_wake_pending,
1670        channel,
1671        routes.get(&channel),
1672    )
1673    .await?;
1674    settle_pending_bash_asks_for_route(
1675        tx,
1676        pending_bash_asks,
1677        channel,
1678        routes,
1679        live_roots,
1680        route_bash_cancels,
1681        shutdown,
1682        metrics,
1683    )
1684    .await?;
1685    if let Some(cancel) = route_bash_cancels.remove(&channel) {
1686        cancel.token.cancel();
1687    }
1688    if let Some(pending) = pending_binds.get_mut(&channel) {
1689        pending.cancelled = true;
1690        let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
1691        log::debug!(
1692            "subc attach: cancelled pending RouteBind for route {} on {cancellation_reason} (configure job: {outcome:?})",
1693            channel.channel
1694        );
1695    }
1696    let migrated = push::migrate_retry_buffer_to_push_buffer(retry_buffer, channel, push_buffer);
1697    if let Some(identity) = routes.get(&channel) {
1698        let has_pending_bind = pending_binds
1699            .values()
1700            .any(|pending| pending.bind_root_id == identity.root);
1701        if route_removal_will_quiesce_root(
1702            &identity.root,
1703            channel,
1704            root_channels,
1705            has_pending_bind,
1706            replacement_root,
1707        ) {
1708            if let Some(ctx) = executor.actor_context(&identity.root) {
1709                // Fence deferred admissions before the final route disappears
1710                // from the loop-owned routing tables.
1711                ctx.mark_subc_unbound();
1712            }
1713        }
1714    }
1715    if let Some(identity) = remove_route_channel(routes, root_channels, channel) {
1716        if migrated > 0 {
1717            log::debug!(
1718                "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from route {} into detach replay",
1719                channel.channel
1720            );
1721        }
1722        if let Some(meta) = live_roots.get_mut(&identity.root) {
1723            let idle_for = meta.last_touched.elapsed();
1724            meta.note_activity();
1725            log::debug!(
1726                "subc attach: route {} torn down for root {} harness {} session {} (last touched {:?} ago)",
1727                channel.channel,
1728                identity.root.as_path().display(),
1729                identity.harness,
1730                identity.session,
1731                idle_for
1732            );
1733        } else {
1734            log::debug!(
1735                "subc attach: route {} torn down for root {} harness {} session {}",
1736                channel.channel,
1737                identity.root.as_path().display(),
1738                identity.harness,
1739                identity.session
1740            );
1741        }
1742        let has_pending_bind = pending_binds
1743            .values()
1744            .any(|pending| pending.bind_root_id == identity.root);
1745        if should_quiesce_removed_root(
1746            &identity.root,
1747            root_channels,
1748            has_pending_bind,
1749            replacement_root,
1750        ) {
1751            quiesce_unbound_root(&identity.root, live_roots, executor);
1752        }
1753    } else {
1754        if migrated > 0 {
1755            log::debug!(
1756                "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from unbound route {} into detach replay",
1757                channel.channel
1758            );
1759        }
1760        log::debug!("subc attach: unbound route {} torn down", channel.channel);
1761    }
1762    Ok(())
1763}
1764
1765fn remember_session_identity(
1766    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1767    identity: &RouteIdentity,
1768) {
1769    let key = (identity.root.clone(), identity.session.clone());
1770    if matches!(identity.trust, BindTrust::Untrusted)
1771        && session_identity
1772            .get(&key)
1773            .is_some_and(|retained| matches!(retained.trust, BindTrust::FirstParty))
1774    {
1775        return;
1776    }
1777
1778    // Retained after route Goodbye so reliable session-scoped frames emitted while
1779    // the session is detached can still be keyed by the full (root,harness,session)
1780    // replay triple. Untrusted binds never overwrite a retained first-party
1781    // session identity, because bash completion replay is an observation channel.
1782    session_identity.insert(
1783        key,
1784        RetainedSessionIdentity {
1785            harness: identity.harness.clone(),
1786            trust: identity.trust,
1787        },
1788    );
1789}
1790
1791fn replay_key_for_session(
1792    session_identity: &HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1793    root: &ProjectRootId,
1794    session: &str,
1795) -> Option<(push::ReplayKey, BindTrust)> {
1796    let retained = session_identity.get(&(root.clone(), session.to_string()))?;
1797    Some((
1798        push::ReplayKey {
1799            root: root.clone(),
1800            harness: retained.harness.clone(),
1801            session: session.to_string(),
1802        },
1803        retained.trust,
1804    ))
1805}
1806/// Sync command dispatch, passed in from `main` (the binary owns the command
1807/// table). Invoked only inside executor jobs in subc mode.
1808pub type DispatchFn = fn(RawRequest, &AppContext) -> Response;
1809
1810#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1811enum ModuleLoopExit {
1812    Graceful,
1813    SkipSearchFlush,
1814}
1815
1816/// Entry point for `aft --subc <connection-file>`. Synchronous on the outside;
1817/// owns an isolated current-thread tokio runtime for the async transport.
1818/// Returns `Err` (fail-loud) on any connect/auth/protocol failure — we never
1819/// fall back to the standalone loop, to avoid split-brain index state.
1820pub fn run_subc_mode(
1821    connection_file_path: &Path,
1822    ctx: Arc<AppContext>,
1823    executor: Arc<Executor>,
1824    dispatch: DispatchFn,
1825    user_config_path: Option<PathBuf>,
1826) -> Result<(), SubcError> {
1827    // Production NEVER allows non-manifest tool names on route channels: AFT
1828    // fails closed and does not trust subc to enforce the manifest. The
1829    // test-only harness sets this through `run_subc_mode_for_test`.
1830    run_subc_mode_inner(
1831        connection_file_path,
1832        ctx,
1833        executor,
1834        dispatch,
1835        user_config_path,
1836        false,
1837    )
1838}
1839
1840fn run_subc_mode_inner(
1841    connection_file_path: &Path,
1842    ctx: Arc<AppContext>,
1843    executor: Arc<Executor>,
1844    dispatch: DispatchFn,
1845    user_config_path: Option<PathBuf>,
1846    allow_native_passthrough: bool,
1847) -> Result<(), SubcError> {
1848    let runtime = tokio::runtime::Builder::new_current_thread()
1849        .enable_all()
1850        .build()
1851        .map_err(SubcError::Runtime)?;
1852
1853    let executor_for_loop = Arc::clone(&executor);
1854    let loop_result = runtime.block_on(async move {
1855        let shared_app = ctx.app();
1856        drop(ctx);
1857        let stream = connect_and_authenticate(connection_file_path).await?;
1858        log::info!(
1859            "subc attach: authenticated to daemon via {}",
1860            connection_file_path.display()
1861        );
1862        let (read_half, write_half) = tokio::io::split(stream);
1863        run_module_loop(
1864            read_half,
1865            write_half,
1866            shared_app,
1867            executor_for_loop,
1868            dispatch,
1869            user_config_path,
1870            allow_native_passthrough,
1871        )
1872        .await
1873    });
1874
1875    let actor_contexts = executor.actor_contexts();
1876    if matches!(loop_result, Ok(ModuleLoopExit::Graceful)) {
1877        // EOF/Goodbye teardown flushes each root's index deltas and queued
1878        // callgraph refreshes. Fatal/panic teardown skips this best-effort work.
1879        flush_actor_indexes_on_graceful_shutdown(&actor_contexts);
1880    }
1881    for actor_ctx in &actor_contexts {
1882        actor_ctx.lsp().shutdown_all();
1883        actor_ctx.bash_background().detach();
1884    }
1885
1886    loop_result.map(|_| ())
1887}
1888
1889fn flush_actor_indexes_on_graceful_shutdown(actor_contexts: &[Arc<AppContext>]) {
1890    for actor_ctx in actor_contexts {
1891        let _ = actor_ctx.flush_search_index_on_graceful_shutdown();
1892    }
1893    let _ = crate::callgraph_store::flush_callgraph_store_refreshes_on_graceful_shutdown();
1894}
1895
1896/// Test-only entry that enables the non-manifest native-command passthrough on
1897/// route channels. Integration tests drive synthetic native commands (`glob`,
1898/// `callers`, `subc_test_echo_session`, …) through the executor to exercise
1899/// mechanics; production callers use [`run_subc_mode`], which fails closed.
1900#[doc(hidden)]
1901pub fn run_subc_mode_for_test(
1902    connection_file_path: &Path,
1903    ctx: Arc<AppContext>,
1904    executor: Arc<Executor>,
1905    dispatch: DispatchFn,
1906    user_config_path: Option<PathBuf>,
1907) -> Result<(), SubcError> {
1908    run_subc_mode_inner(
1909        connection_file_path,
1910        ctx,
1911        executor,
1912        dispatch,
1913        user_config_path,
1914        true,
1915    )
1916}
1917
1918#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1919enum AttachErrorClass {
1920    Transient,
1921    Permanent,
1922}
1923
1924impl fmt::Display for AttachErrorClass {
1925    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1926        match self {
1927            Self::Transient => f.write_str("transient"),
1928            Self::Permanent => f.write_str("permanent"),
1929        }
1930    }
1931}
1932
1933#[derive(Clone, Copy)]
1934struct AttachRetryPolicy {
1935    budget: Duration,
1936    initial_backoff: Duration,
1937    max_backoff: Duration,
1938    jitter_percent: u64,
1939}
1940
1941const ATTACH_RETRY_POLICY: AttachRetryPolicy = AttachRetryPolicy {
1942    budget: ATTACH_RETRY_BUDGET,
1943    initial_backoff: ATTACH_RETRY_INITIAL_BACKOFF,
1944    max_backoff: ATTACH_RETRY_MAX_BACKOFF,
1945    jitter_percent: ATTACH_RETRY_JITTER_PERCENT,
1946};
1947
1948/// Retry only failures that can be caused by a daemon bounce or an interrupted
1949/// handshake. Protocol and credential failures are permanent for this process.
1950fn classify_attach_error(error: &SubcError) -> AttachErrorClass {
1951    let transient = match error {
1952        SubcError::Connect { source, .. } => is_transient_attach_io(source.kind()),
1953        SubcError::Auth { source, .. } => match source {
1954            subc_transport::AuthError::Timeout { .. }
1955            | subc_transport::AuthError::UnexpectedEof { .. } => true,
1956            subc_transport::AuthError::Io { source, .. } => is_transient_attach_io(source.kind()),
1957            _ => false,
1958        },
1959        _ => false,
1960    };
1961    if transient {
1962        AttachErrorClass::Transient
1963    } else {
1964        AttachErrorClass::Permanent
1965    }
1966}
1967
1968fn is_transient_attach_io(kind: io::ErrorKind) -> bool {
1969    matches!(
1970        kind,
1971        io::ErrorKind::ConnectionRefused
1972            | io::ErrorKind::TimedOut
1973            | io::ErrorKind::ConnectionReset
1974            | io::ErrorKind::ConnectionAborted
1975            | io::ErrorKind::BrokenPipe
1976            | io::ErrorKind::UnexpectedEof
1977    )
1978}
1979
1980/// Read the connection file → resolve the first endpoint → TCP connect → HMAC
1981/// handshake. Transient initial-attach failures retry on fresh sockets and reread
1982/// the file so a daemon bounce can publish a new endpoint or authentication key.
1983async fn connect_and_authenticate(connection_file_path: &Path) -> Result<TcpStream, SubcError> {
1984    connect_and_authenticate_with_policy(connection_file_path, ATTACH_RETRY_POLICY).await
1985}
1986
1987async fn connect_and_authenticate_with_policy(
1988    connection_file_path: &Path,
1989    policy: AttachRetryPolicy,
1990) -> Result<TcpStream, SubcError> {
1991    let started_at = Instant::now();
1992    let deadline = started_at + policy.budget;
1993    let mut attempt = 0_u32;
1994    let mut backoff = policy.initial_backoff;
1995    let mut history = Vec::new();
1996
1997    loop {
1998        attempt = attempt.saturating_add(1);
1999        let error = match connect_and_authenticate_once(connection_file_path, deadline).await {
2000            Ok(stream) => return Ok(stream),
2001            Err(error) => error,
2002        };
2003        let class = classify_attach_error(&error);
2004        let error_text = error.to_string().lines().collect::<Vec<_>>().join(" ");
2005        history.push(format!("attempt {attempt} [{class}]: {error_text}"));
2006
2007        if class == AttachErrorClass::Permanent {
2008            log_attach_final_failure(started_at.elapsed(), &history);
2009            return Err(error);
2010        }
2011
2012        let remaining = deadline.saturating_duration_since(Instant::now());
2013        if remaining.is_zero() {
2014            log_attach_final_failure(started_at.elapsed(), &history);
2015            return Err(error);
2016        }
2017
2018        let delay = jittered_attach_delay(backoff, policy.jitter_percent, attempt).min(remaining);
2019        log::info!(
2020            "subc attach retry: attempt {attempt} failed; error_class={class}; error={error_text}; next_delay={delay:?}"
2021        );
2022        tokio::time::sleep(delay).await;
2023
2024        if Instant::now() >= deadline {
2025            log_attach_final_failure(started_at.elapsed(), &history);
2026            return Err(error);
2027        }
2028        backoff = backoff.saturating_mul(2).min(policy.max_backoff);
2029    }
2030}
2031
2032fn jittered_attach_delay(base: Duration, jitter_percent: u64, attempt: u32) -> Duration {
2033    let jitter_percent = jitter_percent.min(100);
2034    if jitter_percent == 0 {
2035        return base;
2036    }
2037
2038    let mut random_bytes = [0_u8; 8];
2039    let random = if getrandom::fill(&mut random_bytes).is_ok() {
2040        u64::from_le_bytes(random_bytes)
2041    } else {
2042        let timestamp = std::time::SystemTime::now()
2043            .duration_since(std::time::UNIX_EPOCH)
2044            .unwrap_or_default()
2045            .subsec_nanos();
2046        u64::from(timestamp) ^ u64::from(attempt)
2047    };
2048    let span = jitter_percent.saturating_mul(2).saturating_add(1);
2049    let multiplier_percent = 100 - jitter_percent + random % span;
2050    let base_millis = u64::try_from(base.as_millis()).unwrap_or(u64::MAX);
2051    Duration::from_millis(base_millis.saturating_mul(multiplier_percent) / 100)
2052}
2053
2054fn log_attach_final_failure(elapsed: Duration, history: &[String]) {
2055    log::error!(
2056        "subc initial attach failed after {} attempt(s) in {elapsed:?}; attempt history: {}",
2057        history.len(),
2058        history.join(" | ")
2059    );
2060}
2061
2062async fn connect_and_authenticate_once(
2063    connection_file_path: &Path,
2064    deadline: Instant,
2065) -> Result<TcpStream, SubcError> {
2066    // This read intentionally lives inside the per-attempt function. The daemon
2067    // publishes connection files atomically and may change both port and key.
2068    let conn = connection_file::read_for_client(connection_file_path).map_err(|source| {
2069        SubcError::ConnectionFile {
2070            path: connection_file_path.to_path_buf(),
2071            source,
2072        }
2073    })?;
2074
2075    let endpoint = conn
2076        .endpoints
2077        .first()
2078        .ok_or_else(|| SubcError::NoEndpoint {
2079            path: connection_file_path.to_path_buf(),
2080        })?;
2081    let endpoint_label = format!("{}:{}", endpoint.host, endpoint.port);
2082    let ip = endpoint
2083        .host
2084        .parse::<IpAddr>()
2085        .map_err(|_| SubcError::InvalidEndpoint {
2086            path: connection_file_path.to_path_buf(),
2087            endpoint: endpoint_label.clone(),
2088        })?;
2089    let addr = SocketAddr::new(ip, endpoint.port);
2090
2091    let connect_budget = deadline.saturating_duration_since(Instant::now());
2092    let mut stream = tokio::time::timeout(connect_budget, TcpStream::connect(addr))
2093        .await
2094        .map_err(|_| SubcError::Connect {
2095            endpoint: endpoint_label.clone(),
2096            source: io::Error::new(
2097                io::ErrorKind::TimedOut,
2098                "initial subc attach retry budget elapsed during TCP connect",
2099            ),
2100        })?
2101        .map_err(|source| SubcError::Connect {
2102            endpoint: endpoint_label.clone(),
2103            source,
2104        })?;
2105    stream
2106        .set_nodelay(true)
2107        .map_err(|source| SubcError::Connect {
2108            endpoint: endpoint_label.clone(),
2109            source,
2110        })?;
2111
2112    let auth_budget = AUTH_DEADLINE.min(deadline.saturating_duration_since(Instant::now()));
2113    authenticate_client(&mut stream, &conn, auth_budget)
2114        .await
2115        .map_err(|source| SubcError::Auth {
2116            endpoint: endpoint_label,
2117            source,
2118        })?;
2119
2120    Ok(stream)
2121}
2122
2123#[allow(clippy::too_many_arguments)]
2124async fn process_route_bind_completion(
2125    writer_tx: &WriterSender,
2126    completion: RouteBindCompletion,
2127    routes: &mut HashMap<RouteChannel, RouteIdentity>,
2128    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2129    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2130    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2131    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2132    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2133    installed_route_epochs: &mut HashMap<u16, u32>,
2134    executor: &Arc<Executor>,
2135    shutdown: &Arc<Notify>,
2136    metrics: &Arc<DispatchPathMetrics>,
2137) -> Result<(), SubcError> {
2138    decrement_counted_channel(&metrics.control_completion_queued);
2139    handle_route_bind_completion(
2140        writer_tx,
2141        completion,
2142        routes,
2143        root_channels,
2144        session_identity,
2145        push_buffer,
2146        live_roots,
2147        pending_binds,
2148        installed_route_epochs,
2149        executor,
2150        shutdown,
2151        metrics,
2152    )
2153    .await
2154}
2155
2156#[allow(clippy::too_many_arguments)]
2157async fn drain_pending_route_bind_completions(
2158    control_completion_rx: &mut mpsc::Receiver<RouteBindCompletion>,
2159    writer_tx: &WriterSender,
2160    routes: &mut HashMap<RouteChannel, RouteIdentity>,
2161    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2162    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2163    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2164    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2165    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2166    installed_route_epochs: &mut HashMap<u16, u32>,
2167    executor: &Arc<Executor>,
2168    shutdown: &Arc<Notify>,
2169    metrics: &Arc<DispatchPathMetrics>,
2170) -> Result<usize, SubcError> {
2171    let mut drained = 0;
2172    while let Ok(completion) = control_completion_rx.try_recv() {
2173        process_route_bind_completion(
2174            writer_tx,
2175            completion,
2176            routes,
2177            root_channels,
2178            session_identity,
2179            push_buffer,
2180            live_roots,
2181            pending_binds,
2182            installed_route_epochs,
2183            executor,
2184            shutdown,
2185            metrics,
2186        )
2187        .await?;
2188        drained += 1;
2189    }
2190    Ok(drained)
2191}
2192
2193/// ModuleHello → HelloAck → control/route loop. Runs until the daemon closes
2194/// the connection (EOF), sends channel-0 Goodbye, or a fatal mutating executor
2195/// response requests whole-connection teardown.
2196async fn run_module_loop<R, W>(
2197    mut read: R,
2198    mut write: W,
2199    shared_app: Arc<App>,
2200    executor: Arc<Executor>,
2201    dispatch: DispatchFn,
2202    user_config_path: Option<PathBuf>,
2203    allow_native_passthrough: bool,
2204) -> Result<ModuleLoopExit, SubcError>
2205where
2206    R: AsyncRead + Unpin + Send + 'static,
2207    W: AsyncWrite + Unpin + Send + 'static,
2208{
2209    // ModuleHello: register as a tool provider and advertise the supported control-plane operations.
2210    // Echo the one-time launch nonce the daemon injected via SUBC_LAUNCH_NONCE so a
2211    // reserved module_id's HELLO is accepted; absent for non-reserved/self-connect.
2212    let hello = ModuleHelloBody {
2213        manifest: build_manifest(),
2214        protocol_ver: PROTOCOL_VERSION,
2215        control_ops: control_ops(),
2216        launch_nonce: std::env::var("SUBC_LAUNCH_NONCE").ok(),
2217    };
2218    let hello_frame = Frame::build(
2219        FrameType::Hello,
2220        control_flags(),
2221        0,
2222        0,
2223        HELLO_CORR,
2224        serde_json::to_vec(&hello).map_err(SubcError::Json)?,
2225    )
2226    .map_err(SubcError::FrameBuild)?;
2227    write_frame(&mut write, &hello_frame)
2228        .await
2229        .map_err(SubcError::FrameIo)?;
2230
2231    // Expect HelloAck (registered) or a channel-0 Error (manifest/version reject).
2232    match read_frame(&mut read).await.map_err(SubcError::FrameIo)? {
2233        None => return Err(SubcError::ClosedBeforeHelloAck),
2234        Some(frame) => match frame.header.ty {
2235            FrameType::HelloAck => {
2236                log::info!("subc attach: registered (HelloAck received)");
2237            }
2238            FrameType::Error => {
2239                let body = serde_json::from_slice::<ErrorBody>(&frame.body).ok();
2240                return Err(SubcError::HelloRejected { body });
2241            }
2242            other => return Err(SubcError::UnexpectedFrame { ty: other }),
2243        },
2244    }
2245
2246    let dispatch_path_metrics = Arc::new(DispatchPathMetrics::new());
2247    let (writer_tx, writer_rx) = mpsc::channel::<WriterFrame>(WRITER_QUEUE_CAPACITY);
2248    let writer_task = spawn_writer_task(write, writer_rx, Arc::clone(&dispatch_path_metrics));
2249    // `read_frame` is NOT cancellation-safe, so it must never sit directly inside
2250    // the `select!` below: a drain-interval tick (or shutdown) firing while a
2251    // frame is mid-transit would drop the partially-consumed bytes and desync the
2252    // stream (the next read would parse a body byte as a frame header). A
2253    // dedicated reader task owns the socket, reads whole frames sequentially, and
2254    // forwards them over a channel; the loop selects on the cancel-safe `recv()`.
2255    let (reader_tx, mut reader_rx) = mpsc::channel::<Result<DecodedFrame, SubcError>>(256);
2256    let reader_task = spawn_reader_task(read, reader_tx);
2257    let shutdown = Arc::new(Notify::new());
2258    // Drain-tick deadline is tracked manually and checked at the TOP of every
2259    // loop turn rather than as an Interval select arm: the select below is
2260    // `biased` (bind completions first), and biased polling means a saturated
2261    // higher arm (sustained lossy push traffic keeps lossy_rx always-ready)
2262    // would starve every arm below it, including a timer arm — leaving
2263    // backpressured reliable frames parked in the retry buffer past their
2264    // delivery deadline. The pre-turn check cannot be starved by arm order;
2265    // the sleep_until arm below only exists to wake an otherwise-idle loop.
2266    let mut next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2267    let mut next_maintenance_at = next_drain_at;
2268    // Rate-limit stamp for opportunistic macOS allocator slack relief (checked
2269    // on the maintenance tick; policy shared with standalone via memory.rs).
2270    #[cfg(target_os = "macos")]
2271    let mut last_slack_relief: Option<std::time::Instant> = None;
2272    let (maintenance_tx, mut maintenance_rx) = mpsc::channel::<MaintenanceCompletion>(256);
2273    let (bash_deferred_tx, mut bash_deferred_rx) =
2274        mpsc::channel::<bash::BashDeferredCompletion>(256);
2275    let (bash_poll_touch_tx, mut bash_poll_touch_rx) = mpsc::channel::<ProjectRootId>(256);
2276    let (control_completion_tx, mut control_completion_rx) =
2277        mpsc::channel::<RouteBindCompletion>(256);
2278    let (lossy_tx, mut lossy_rx) = mpsc::channel::<LossyPushEnvelope>(1024);
2279    let lossy_overflow = Arc::new(push::LossyOverflow::default());
2280    let lossy_seq = Arc::new(AtomicU64::new(0));
2281    let (reliable_tx, mut reliable_rx) = mpsc::unbounded_channel::<PushEnvelope>();
2282    let push_senders = PushSenders {
2283        lossy_tx,
2284        reliable_tx,
2285        lossy_overflow: Arc::clone(&lossy_overflow),
2286        lossy_seq,
2287    };
2288    let connection_cancel = PersistentCancelSignal::new();
2289    let mut installed_route_epochs: HashMap<u16, u32> = HashMap::new();
2290    let mut routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
2291    let mut bg_subs: HashMap<RouteChannel, BgSub> = HashMap::new();
2292    let mut bg_sub_by_session: HashMap<(ProjectRootId, String), RouteChannel> = HashMap::new();
2293    let mut bg_wake_pending: HashSet<RouteChannel> = HashSet::new();
2294    let mut bg_wake_epoch: HashMap<(ProjectRootId, String), u64> = HashMap::new();
2295    let mut root_channels: HashMap<ProjectRootId, HashSet<RouteChannel>> = HashMap::new();
2296    let mut session_identity: HashMap<(ProjectRootId, String), RetainedSessionIdentity> =
2297        HashMap::new();
2298    let mut push_buffer: HashMap<push::ReplayKey, VecDeque<PushFrame>> = HashMap::new();
2299    let mut retry_buffer: RetryBuffer = HashMap::new();
2300    let mut reclaimed_routes = ReclaimedRoutes::default();
2301    let mut completed_tasks = push::CompletedTaskIds::default();
2302    let mut live_roots: HashMap<ProjectRootId, RootMeta> = HashMap::new();
2303    let mut pending_binds: HashMap<RouteChannel, PendingBind> = HashMap::new();
2304    let mut pending_bash_asks: HashMap<ReverseCorrKey, PendingBashAsk> = HashMap::new();
2305    let mut next_bash_ask_corr: u64 = 1;
2306    let mut route_bash_cancels: HashMap<RouteChannel, bash::RouteBashCancel> = HashMap::new();
2307
2308    let loop_result: Result<ModuleLoopExit, SubcError> = loop {
2309        shared_app.set_open_route_count(routes.len());
2310        crate::logging::perf_tick(Some(&executor));
2311        dispatch_path_metrics.mark_frame_loop_tick();
2312        if let Err(error) = expire_pending_bash_asks(
2313            &writer_tx,
2314            &mut pending_bash_asks,
2315            &routes,
2316            &mut live_roots,
2317            &mut route_bash_cancels,
2318            &shutdown,
2319            &dispatch_path_metrics,
2320        )
2321        .await
2322        {
2323            break Err(error);
2324        }
2325
2326        // RouteBind completions are control-plane unblockers. Drain any completed
2327        // binds before entering other branch work so Push and maintenance bursts
2328        // can only add one loop-turn of latency.
2329        match drain_pending_route_bind_completions(
2330            &mut control_completion_rx,
2331            &writer_tx,
2332            &mut routes,
2333            &mut root_channels,
2334            &mut session_identity,
2335            &mut push_buffer,
2336            &mut live_roots,
2337            &mut pending_binds,
2338            &mut installed_route_epochs,
2339            &executor,
2340            &shutdown,
2341            &dispatch_path_metrics,
2342        )
2343        .await
2344        {
2345            Ok(drained) => {
2346                if drained > 0 {
2347                    next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2348                }
2349            }
2350            Err(error) => break Err(error),
2351        }
2352
2353        if tokio::time::Instant::now() >= next_drain_at {
2354            push::emit_bg_event_wakes(
2355                &writer_tx,
2356                &dispatch_path_metrics,
2357                &bg_subs,
2358                &mut bg_wake_pending,
2359            );
2360            warn_slow_pending_binds(&mut pending_binds, &executor);
2361            if let Err(error) = expire_overdue_route_binds(
2362                &writer_tx,
2363                &executor,
2364                &mut pending_binds,
2365                &mut installed_route_epochs,
2366                &dispatch_path_metrics,
2367            )
2368            .await
2369            {
2370                break Err(error);
2371            }
2372
2373            let retried = push::drain_retry_buffers_for_bound_routes(
2374                &writer_tx,
2375                &dispatch_path_metrics,
2376                &routes,
2377                &mut retry_buffer,
2378            );
2379            if retried > 0 {
2380                log::debug!(
2381                    "subc attach: retried {retried} reliable Push frame(s) after writer backpressure"
2382                );
2383            }
2384
2385            next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2386        }
2387
2388        // A lossy emitter may place its newest update in the overflow buffer
2389        // when the bounded channel is full, while this receive loop is draining
2390        // the channel. Drain overflow before selecting again so that raced
2391        // update is delivered on the next timer tick instead of waiting for
2392        // another lossy enqueue.
2393        let overflow_batch = lossy_overflow.drain();
2394        if !overflow_batch.is_empty() {
2395            let (_, deferred) = push::drain_reliable_push_turn(
2396                &writer_tx,
2397                &dispatch_path_metrics,
2398                &routes,
2399                &root_channels,
2400                &session_identity,
2401                &mut retry_buffer,
2402                &mut push_buffer,
2403                &mut completed_tasks,
2404                &bg_sub_by_session,
2405                &mut bg_wake_pending,
2406                &mut bg_wake_epoch,
2407                &mut reliable_rx,
2408                None,
2409            );
2410            if deferred {
2411                tokio::task::yield_now().await;
2412            }
2413
2414            let mut batch = Vec::new();
2415            while let Ok(item) = lossy_rx.try_recv() {
2416                batch.push(item);
2417            }
2418            batch.extend(overflow_batch);
2419            push::process_lossy_push_envelope_batch(
2420                &writer_tx,
2421                &dispatch_path_metrics,
2422                &routes,
2423                &root_channels,
2424                &completed_tasks,
2425                batch,
2426            );
2427        }
2428
2429        tokio::select! {
2430            biased;
2431            Some(completion) = control_completion_rx.recv() => {
2432                if let Err(error) = process_route_bind_completion(
2433                    &writer_tx,
2434                    completion,
2435                    &mut routes,
2436                    &mut root_channels,
2437                    &mut session_identity,
2438                    &mut push_buffer,
2439                    &mut live_roots,
2440                    &mut pending_binds,
2441                    &mut installed_route_epochs,
2442                    &executor,
2443                    &shutdown,
2444                    &dispatch_path_metrics,
2445                )
2446                .await
2447                {
2448                    break Err(error);
2449                }
2450                next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2451            }
2452            _ = shutdown.notified() => {
2453                log::warn!("subc attach: fatal executor response requested teardown");
2454                break Ok(ModuleLoopExit::SkipSearchFlush);
2455            }
2456            maybe_frame = reader_rx.recv() => {
2457                let frame = match maybe_frame {
2458                    None => {
2459                        log::info!("subc attach: daemon closed connection");
2460                        break Ok(ModuleLoopExit::Graceful);
2461                    }
2462                    Some(Err(error)) => break Err(error),
2463                    Some(Ok(frame)) => frame,
2464                };
2465                let phase_trace = frame.phase_trace;
2466                let frame = frame.frame;
2467
2468                if !ingress_route_should_be_processed(
2469                    &installed_route_epochs,
2470                    &reclaimed_routes,
2471                    &frame,
2472                ) {
2473                    log::debug!(
2474                        "subc attach: silently dropping {:?} for uninstalled route {}@{}",
2475                        frame.header.ty,
2476                        frame.header.channel,
2477                        frame.header.epoch
2478                    );
2479                    continue;
2480                }
2481
2482                match frame.header.ty {
2483                    FrameType::Ping if frame.header.channel == 0 => {
2484                        let pong = match Frame::build_with_version(
2485                            frame.header.ver,
2486                            FrameType::Pong,
2487                            frame.header.flags,
2488                            0,
2489                            0,
2490                            frame.header.corr,
2491                            Vec::new(),
2492                        ) {
2493                            Ok(pong) => pong,
2494                            Err(error) => break Err(SubcError::FrameBuild(error)),
2495                        };
2496                        if let Err(error) = send_frame(&writer_tx, &dispatch_path_metrics, pong).await {
2497                            break Err(error);
2498                        }
2499                    }
2500                    FrameType::Goodbye if frame.header.channel == 0 => {
2501                        log::info!("subc attach: received channel-0 Goodbye");
2502                        break Ok(ModuleLoopExit::Graceful);
2503                    }
2504                    FrameType::Goodbye => {
2505                        let channel = route_key(frame.header.channel, frame.header.epoch);
2506                        if let Err(error) = teardown_installed_route(
2507                            &writer_tx,
2508                            &dispatch_path_metrics,
2509                            &executor,
2510                            channel,
2511                            "Goodbye",
2512                            None,
2513                            &mut installed_route_epochs,
2514                            &mut routes,
2515                            &mut root_channels,
2516                            &mut bg_subs,
2517                            &mut bg_sub_by_session,
2518                            &mut bg_wake_pending,
2519                            &mut pending_bash_asks,
2520                            &mut live_roots,
2521                            &mut route_bash_cancels,
2522                            &mut pending_binds,
2523                            &mut retry_buffer,
2524                            &mut push_buffer,
2525                            &shutdown,
2526                        )
2527                        .await
2528                        {
2529                            break Err(error);
2530                        }
2531                    }
2532                    FrameType::Response | FrameType::Error if frame.header.channel != 0 => {
2533                        if let Err(error) = handle_bash_elicitation_reply(
2534                            &writer_tx,
2535                            &frame,
2536                            &mut pending_bash_asks,
2537                            &routes,
2538                            &mut live_roots,
2539                            &executor,
2540                            &shutdown,
2541                            &bash_deferred_tx,
2542                            &bash_poll_touch_tx,
2543                            &dispatch_path_metrics,
2544                            &mut route_bash_cancels,
2545                            dispatch,
2546                        )
2547                        .await
2548                        {
2549                            break Err(error);
2550                        }
2551                    }
2552                    FrameType::Request if frame.header.channel == 0 => {
2553                        if let Err(error) = handle_control_request(
2554                            &writer_tx,
2555                            &frame,
2556                            &shared_app,
2557                            &executor,
2558                            &mut live_roots,
2559                            &mut pending_binds,
2560                            &mut installed_route_epochs,
2561                            &mut routes,
2562                            &mut root_channels,
2563                            &mut bg_subs,
2564                            &mut bg_sub_by_session,
2565                            &mut bg_wake_pending,
2566                            &mut pending_bash_asks,
2567                            &mut route_bash_cancels,
2568                            &mut retry_buffer,
2569                            &mut push_buffer,
2570                            &shutdown,
2571                            &control_completion_tx,
2572                            &dispatch_path_metrics,
2573                            &push_senders,
2574                            dispatch,
2575                            user_config_path.as_deref(),
2576                        )
2577                        .await
2578                        {
2579                            break Err(error);
2580                        }
2581                    }
2582                    FrameType::Request => {
2583                        if let Err(error) = handle_tool_call(
2584                            &writer_tx,
2585                            &frame,
2586                            phase_trace,
2587                            &routes,
2588                            &pending_binds,
2589                            &mut live_roots,
2590                            &executor,
2591                            &shutdown,
2592                            &connection_cancel,
2593                            &bash_deferred_tx,
2594                            &bash_poll_touch_tx,
2595                            &dispatch_path_metrics,
2596                            &mut route_bash_cancels,
2597                            &mut pending_bash_asks,
2598                            &mut next_bash_ask_corr,
2599                            &mut bg_subs,
2600                            &mut bg_sub_by_session,
2601                            &mut bg_wake_pending,
2602                            &mut bg_wake_epoch,
2603                            dispatch,
2604                            allow_native_passthrough,
2605                        )
2606                        .await
2607                        {
2608                            break Err(error);
2609                        }
2610                    }
2611                    FrameType::Cancel => {
2612                        let channel = route_key(frame.header.channel, frame.header.epoch);
2613                        if bg_subs.contains_key(&channel) {
2614                            if let Err(error) = end_bg_subscription(
2615                                &writer_tx,
2616                                &dispatch_path_metrics,
2617                                &mut bg_subs,
2618                                &mut bg_sub_by_session,
2619                                &mut bg_wake_pending,
2620                                channel,
2621                                routes.get(&channel),
2622                            )
2623                            .await
2624                            {
2625                                break Err(error);
2626                            }
2627                        }
2628                        if let Err(error) = cancel_pending_bash_ask_for_tool_call(
2629                            &writer_tx,
2630                            &mut pending_bash_asks,
2631                            channel,
2632                            frame.header.corr,
2633                            &routes,
2634                            &mut live_roots,
2635                            &mut route_bash_cancels,
2636                            &shutdown,
2637                            &dispatch_path_metrics,
2638                        )
2639                        .await
2640                        {
2641                            break Err(error);
2642                        }
2643                    }
2644                    // Incoming push messages are ignored here. Cancel frames only
2645                    // stop pending bash elicitation requests; executor-level
2646                    // cancellation for tool calls that are already running is not
2647                    // implemented.
2648                    _ => {}
2649                }
2650            }
2651            Some((root_id, frame)) = reliable_rx.recv() => {
2652                // Reliable Push frames are FIFO and must-deliver, but draining an
2653                // unbounded burst in one current-thread turn can starve RouteBind
2654                // completions. The budget defers excess frames, never drops them.
2655                let (_, deferred) = push::drain_reliable_push_turn(
2656                    &writer_tx,
2657                    &dispatch_path_metrics,
2658                    &routes,
2659                    &root_channels,
2660                    &session_identity,
2661                    &mut retry_buffer,
2662                    &mut push_buffer,
2663                    &mut completed_tasks,
2664                    &bg_sub_by_session,
2665                    &mut bg_wake_pending,
2666                    &mut bg_wake_epoch,
2667                    &mut reliable_rx,
2668                    Some((root_id, frame)),
2669                );
2670                if deferred {
2671                    tokio::task::yield_now().await;
2672                }
2673            }
2674            Some((order, root_id, frame)) = lossy_rx.recv() => {
2675                // When both push lanes have work, handle a small reliable slice before lossy work.
2676                // That ordering lets completed task ids suppress stale BashLongRunning frames.
2677                // The slice stays bounded so reliable bursts cannot monopolize this loop turn.
2678                let (_, deferred) = push::drain_reliable_push_turn(
2679                    &writer_tx,
2680                    &dispatch_path_metrics,
2681                    &routes,
2682                    &root_channels,
2683                    &session_identity,
2684                    &mut retry_buffer,
2685                    &mut push_buffer,
2686                    &mut completed_tasks,
2687                    &bg_sub_by_session,
2688                    &mut bg_wake_pending,
2689                    &mut bg_wake_epoch,
2690                    &mut reliable_rx,
2691                    None,
2692                );
2693                if deferred {
2694                    tokio::task::yield_now().await;
2695                }
2696
2697                // Drain the currently queued burst in one loop turn so lossy
2698                // status/progress updates can be merged before reaching subc's
2699                // shared egress queue. Each lossy frame gets a sequence number
2700                // before it goes to the channel or overflow buffer, so the
2701                // combined batch is sorted back into producer order before
2702                // coalescing drops stale updates for the same key.
2703                let mut batch = vec![(order, root_id, frame)];
2704                while let Ok(item) = lossy_rx.try_recv() {
2705                    batch.push(item);
2706                }
2707                batch.extend(lossy_overflow.drain());
2708                push::process_lossy_push_envelope_batch(
2709                    &writer_tx,
2710                    &dispatch_path_metrics,
2711                    &routes,
2712                    &root_channels,
2713                    &completed_tasks,
2714                    batch,
2715                );
2716            }
2717            Some(done) = bash_deferred_rx.recv() => {
2718                decrement_counted_channel(&dispatch_path_metrics.bash_deferred_queued);
2719                if let Err(error) = bash::handle_bash_deferred_completion(
2720                    &writer_tx,
2721                    done,
2722                    &routes,
2723                    &mut live_roots,
2724                    &mut route_bash_cancels,
2725                    &shutdown,
2726                    &dispatch_path_metrics,
2727                )
2728                .await
2729                {
2730                    break Err(error);
2731                }
2732            }
2733            Some(root_id) = bash_poll_touch_rx.recv() => {
2734                decrement_counted_channel(&dispatch_path_metrics.bash_poll_touch_queued);
2735                if let Some(meta) = live_roots.get_mut(&root_id) {
2736                    meta.note_activity();
2737                }
2738            }
2739            Some(completion) = maintenance_rx.recv() => {
2740                decrement_counted_channel(&dispatch_path_metrics.maintenance_queued);
2741                let root_id = completion.root_id.clone();
2742                let response = completion.response;
2743                let response_is_fatal = response_is_fatal_panic(&response);
2744                let bind_pending = pending_binds
2745                    .values()
2746                    .any(|pending| pending.bind_root_id == root_id);
2747                let requiesce = if let Some(meta) = live_roots.get_mut(&root_id) {
2748                    let defer_requeue = meta.unbound_quiesced || bind_pending;
2749                    note_maintenance_completion(
2750                        meta,
2751                        completion.requeue_kind,
2752                        response_is_fatal,
2753                        defer_requeue,
2754                    );
2755                    should_requiesce_after_maintenance(meta, completion.kind, bind_pending)
2756                } else {
2757                    false
2758                };
2759                if requiesce {
2760                    quiesce_unbound_root(&root_id, &mut live_roots, &executor);
2761                }
2762                push::clear_stale_bg_wakes_for_empty_sessions(
2763                    &root_id,
2764                    &completion.empty_bg_sessions,
2765                    &bg_sub_by_session,
2766                    &mut bg_wake_pending,
2767                    &bg_wake_epoch,
2768                );
2769                if response_is_fatal {
2770                    if let Some(meta) = live_roots.get_mut(&root_id) {
2771                        meta.maintenance_poisoned = true;
2772                    }
2773                    log::warn!(
2774                        "subc attach: maintenance drain observed a fatal actor; deferring teardown until a route request can receive actor_fatal"
2775                    );
2776                }
2777            }
2778            _ = tokio::time::sleep_until(next_drain_at) => {
2779                // Wakes an otherwise-idle loop so the pre-turn drain check
2780                // above runs on schedule; the drain work itself lives there.
2781            }
2782            _ = tokio::time::sleep_until(next_maintenance_at) => {
2783                // Delay cache-draining maintenance until any already-ready
2784                // inbound route/control messages and push completions have run,
2785                // so maintenance does not block the actor from handling the
2786                // first request that arrives after a route bind is acknowledged.
2787                let reaped_lsp_children = shared_app
2788                    .lsp_child_registry()
2789                    .reap_children_with_gone_cwd();
2790                if reaped_lsp_children > 0 {
2791                    log::warn!(
2792                        "subc attach: reaped {reaped_lsp_children} LSP child process group(s) whose cwd no longer exists"
2793                    );
2794                }
2795                let reap = reap_idle_roots(
2796                    Instant::now(),
2797                    &mut live_roots,
2798                    &pending_binds,
2799                    &root_channels,
2800                    &executor,
2801                    &dispatch_path_metrics,
2802                );
2803                for root_id in &reap.forgotten_deleted_roots {
2804                    purge_deleted_root_residents(
2805                        root_id,
2806                        &mut routes,
2807                        &mut root_channels,
2808                        &mut installed_route_epochs,
2809                        &mut route_bash_cancels,
2810                        &mut retry_buffer,
2811                        &mut reclaimed_routes,
2812                        &mut session_identity,
2813                        &mut push_buffer,
2814                        &mut bg_subs,
2815                        &mut bg_sub_by_session,
2816                        &mut bg_wake_pending,
2817                        &mut bg_wake_epoch,
2818                        &mut pending_bash_asks,
2819                    );
2820                }
2821                if reap.evicted > 0 {
2822                    log::debug!("subc attach: reaped {} idle root(s)", reap.evicted);
2823                }
2824                submit_due_maintenance_jobs(
2825                    &executor,
2826                    &mut live_roots,
2827                    &pending_binds,
2828                    &bg_sub_by_session,
2829                    &bg_wake_pending,
2830                    &bg_wake_epoch,
2831                    &maintenance_tx,
2832                    &dispatch_path_metrics,
2833                );
2834                // Opportunistic allocator relief, independent of the idle
2835                // sweep: the sweep's whole-process idle gate never opens while
2836                // any session stays active, which let freed warm-up arenas sit
2837                // resident for the process lifetime (5.1 GB RSS over ~600 MB
2838                // live). Slack threshold + spacing live in memory.rs; the pass
2839                // itself runs on a detached thread.
2840                #[cfg(target_os = "macos")]
2841                {
2842                    let now_std = std::time::Instant::now();
2843                    if crate::memory::spawn_allocator_slack_relief_if_due(
2844                        last_slack_relief,
2845                        now_std,
2846                    ) {
2847                        last_slack_relief = Some(now_std);
2848                    }
2849                }
2850                next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2851            }
2852        }
2853    };
2854
2855    shared_app.set_open_route_count(0);
2856
2857    connection_cancel.cancel();
2858    // Channel-0 Goodbye, EOF, and fatal exits bypass per-route Goodbye. Settle
2859    // their root lifecycle state before loop-owned routing metadata is dropped.
2860    quiesce_connection_roots(
2861        &mut live_roots,
2862        &mut pending_binds,
2863        &mut routes,
2864        &mut root_channels,
2865        &mut installed_route_epochs,
2866        &mut route_bash_cancels,
2867        &executor,
2868    );
2869
2870    let mut loop_result = loop_result;
2871    if !pending_bash_asks.is_empty() {
2872        let no_routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
2873        if let Err(error) = settle_all_pending_bash_asks(
2874            &writer_tx,
2875            &mut pending_bash_asks,
2876            &no_routes,
2877            &mut live_roots,
2878            &mut route_bash_cancels,
2879            &shutdown,
2880            &dispatch_path_metrics,
2881        )
2882        .await
2883        {
2884            loop_result = loop_result.and(Err(error));
2885        }
2886    }
2887
2888    // The reader task may be parked on `read_frame`; abort it (we are done with
2889    // the connection) and flush the writer.
2890    reader_task.abort();
2891    drop(writer_tx);
2892    let writer_result = finish_writer_task(writer_task).await;
2893    loop_result.and_then(|exit| writer_result.map(|_| exit))
2894}
2895
2896fn spawn_writer_task<W>(
2897    mut write: W,
2898    mut rx: mpsc::Receiver<WriterFrame>,
2899    metrics: Arc<DispatchPathMetrics>,
2900) -> JoinHandle<Result<(), subc_transport::FrameIoError>>
2901where
2902    W: AsyncWrite + Unpin + Send + 'static,
2903{
2904    tokio::spawn(async move {
2905        let mut write_buffer = Vec::new();
2906        while let Some(mut queued) = rx.recv().await {
2907            let measure = queued.tool_response_trace.is_some();
2908            let dequeued = measure.then(Instant::now);
2909            metrics.writer_active.store(true, Ordering::Relaxed);
2910            decrement_counted_channel(&metrics.writer_queued);
2911            let write_timing = write_frame_contiguous(
2912                &mut write,
2913                queued.frame(),
2914                queued.body(),
2915                &mut write_buffer,
2916                measure,
2917            )
2918            .await;
2919            metrics.writer_active.store(false, Ordering::Relaxed);
2920            let write_timing = write_timing?;
2921
2922            if let (Some(trace), Some(dequeued), Some(write_timing)) =
2923                (queued.tool_response_trace.take(), dequeued, write_timing)
2924            {
2925                if let Some(completed) = trace.finish(
2926                    dequeued,
2927                    write_timing.write_started,
2928                    write_timing.write_finished,
2929                    write_timing.frame_bytes,
2930                ) {
2931                    log_ctx::with_session(Some(completed.session), || {
2932                        crate::logging::note_tool_call_trace(
2933                            &completed.name,
2934                            &completed.root,
2935                            completed.channel,
2936                            completed.corr,
2937                            completed.phases,
2938                        );
2939                    });
2940                }
2941            }
2942        }
2943        Ok(())
2944    })
2945}
2946
2947struct FrameWriteTiming {
2948    write_started: Instant,
2949    write_finished: Instant,
2950    frame_bytes: usize,
2951}
2952
2953/// Encode one complete frame into the existing reusable buffer and write it
2954/// without interleaving bytes from another channel. Timing is collected only
2955/// for tool responses, so Push and control frames add no clock reads.
2956async fn write_frame_contiguous<W>(
2957    writer: &mut W,
2958    frame: &Frame,
2959    body: &[u8],
2960    buffer: &mut Vec<u8>,
2961    measure: bool,
2962) -> Result<Option<FrameWriteTiming>, subc_transport::FrameIoError>
2963where
2964    W: AsyncWrite + Unpin,
2965{
2966    if frame.header.len as usize != body.len() {
2967        return Err(subc_transport::FrameIoError::BodyLengthMismatch {
2968            header_len: frame.header.len,
2969            body_len: body.len(),
2970        });
2971    }
2972
2973    let header = frame.header.encode();
2974    buffer.clear();
2975    buffer.reserve(header.len() + body.len());
2976    buffer.extend_from_slice(&header);
2977    buffer.extend_from_slice(body);
2978    let write_started = measure.then(Instant::now);
2979    writer
2980        .write_all(buffer)
2981        .await
2982        .map_err(subc_transport::FrameIoError::Io)?;
2983    Ok(write_started.map(|write_started| FrameWriteTiming {
2984        write_started,
2985        write_finished: Instant::now(),
2986        frame_bytes: buffer.len(),
2987    }))
2988}
2989
2990fn spawn_reader_task<R>(
2991    mut read: R,
2992    tx: mpsc::Sender<Result<DecodedFrame, SubcError>>,
2993) -> JoinHandle<()>
2994where
2995    R: AsyncRead + Unpin + Send + 'static,
2996{
2997    tokio::spawn(async move {
2998        loop {
2999            match read_frame(&mut read).await {
3000                Ok(Some(frame)) => {
3001                    let decoded = DecodedFrame {
3002                        frame,
3003                        phase_trace: PhaseTrace::new(Instant::now()),
3004                    };
3005                    if tx.send(Ok(decoded)).await.is_err() {
3006                        return;
3007                    }
3008                }
3009                Ok(None) => {
3010                    // EOF: let the loop observe channel close as "daemon closed".
3011                    return;
3012                }
3013                Err(error) => {
3014                    // A killed daemon surfaces as ConnectionReset (RST) on
3015                    // Windows where Unix delivers a clean EOF (FIN); a
3016                    // mid-teardown daemon can also abort the socket. Both mean
3017                    // "daemon went away", not a wire fault — normalize them to
3018                    // the clean-close path so module exit behavior matches
3019                    // across platforms (same class subc-core fixed in d33d9a71).
3020                    if let subc_transport::FrameIoError::Io(io_error) = &error {
3021                        if matches!(
3022                            io_error.kind(),
3023                            std::io::ErrorKind::ConnectionReset
3024                                | std::io::ErrorKind::ConnectionAborted
3025                        ) {
3026                            log::info!(
3027                                "subc attach: connection reset by daemon; treating as close"
3028                            );
3029                            return;
3030                        }
3031                    }
3032                    let _ = tx.send(Err(SubcError::FrameIo(error))).await;
3033                    return;
3034                }
3035            }
3036        }
3037    })
3038}
3039
3040async fn finish_writer_task(
3041    mut writer_task: JoinHandle<Result<(), subc_transport::FrameIoError>>,
3042) -> Result<(), SubcError> {
3043    match tokio::time::timeout(Duration::from_millis(100), &mut writer_task).await {
3044        Ok(Ok(Ok(()))) => Ok(()),
3045        Ok(Ok(Err(error))) => Err(SubcError::FrameIo(error)),
3046        Ok(Err(error)) => Err(SubcError::WriterJoin(error)),
3047        Err(_) => {
3048            writer_task.abort();
3049            Ok(())
3050        }
3051    }
3052}
3053
3054fn register_actor_for_bind(
3055    shared_app: &Arc<App>,
3056    executor: &Arc<Executor>,
3057    push_senders: &PushSenders,
3058    bind_root_id: &ProjectRootId,
3059    route_channel: u16,
3060    root_was_live: bool,
3061) -> bool {
3062    if executor.actor_registered(bind_root_id) {
3063        log::debug!(
3064            "subc attach: reusing actor for route {} root {}",
3065            route_channel,
3066            bind_root_id.as_path().display()
3067        );
3068        return false;
3069    }
3070
3071    if root_was_live {
3072        log::warn!(
3073            "subc attach: recreating missing actor for live root {} on route {}",
3074            bind_root_id.as_path().display(),
3075            route_channel
3076        );
3077    }
3078
3079    let actor_ctx = Arc::new(AppContext::from_app(
3080        Arc::clone(shared_app),
3081        Config::default(),
3082    ));
3083    install_bash_compressor(&actor_ctx);
3084    actor_ctx.set_progress_sender(Some(push::progress_sender_for_root(
3085        push_senders.clone(),
3086        bind_root_id.clone(),
3087    )));
3088    let inserted = executor.register_actor(bind_root_id.clone(), Arc::clone(&actor_ctx));
3089    drop(actor_ctx);
3090    if inserted {
3091        // Do not insert into live_roots until configure succeeds: live_roots
3092        // drives maintenance, and a half-configured new actor must not be
3093        // maintenance-eligible before its route/session identity exists.
3094        log::debug!(
3095            "subc attach: registered actor for route {} root {}",
3096            route_channel,
3097            bind_root_id.as_path().display()
3098        );
3099    } else {
3100        log::debug!(
3101            "subc attach: actor appeared while binding route {} root {}; reusing it",
3102            route_channel,
3103            bind_root_id.as_path().display()
3104        );
3105    }
3106    inserted
3107}
3108
3109fn rollback_pending_bind_actor(
3110    executor: &Arc<Executor>,
3111    live_roots: &HashMap<ProjectRootId, RootMeta>,
3112    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3113    root_id: &ProjectRootId,
3114    inserted_new_actor: bool,
3115) {
3116    if !inserted_new_actor || live_roots.contains_key(root_id) {
3117        return;
3118    }
3119
3120    if let Some((route, pending)) = pending_binds
3121        .iter_mut()
3122        .find(|(_, pending)| &pending.bind_root_id == root_id)
3123    {
3124        pending.inserted_new_actor = true;
3125        log::debug!(
3126            "subc attach: transferred rollback ownership for root {} to pending route {}",
3127            root_id.as_path().display(),
3128            route
3129        );
3130        return;
3131    }
3132
3133    executor.remove_actor(root_id);
3134}
3135
3136fn route_bind_error_code_for_configure_response(response: &Response) -> &'static str {
3137    match response.data.get("code").and_then(|code| code.as_str()) {
3138        // Preserve typed configure rejections across the bind boundary: a
3139        // malformed fed fingerprint means a federation-module bug or
3140        // fingerprint-format drift, and the fed side matches on the code rather
3141        // than parsing prose.
3142        Some("bad_harness_fingerprint") => "bad_harness_fingerprint",
3143        // Cache-key probe failures are transient (fd pressure, git spawn
3144        // contention); the client retries the bind rather than treating the
3145        // root as permanently divergent.
3146        Some("cache_key_probe_failed") => "cache_key_probe_failed",
3147        // Actor lifecycle gaps are transient from the daemon/client viewpoint:
3148        // a fresh bind can create or join a healthy actor, so do not classify
3149        // them as permanent config divergence.
3150        Some("actor_not_registered" | "actor_fatal") => "actor_not_ready",
3151        _ => "config_divergence",
3152    }
3153}
3154
3155fn queue_post_bind_configure_and_completion_maintenance(
3156    root_id: &ProjectRootId,
3157    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3158) {
3159    let Some(meta) = live_roots.get_mut(root_id) else {
3160        return;
3161    };
3162    if meta.maintenance_poisoned || meta.maintenance_pending {
3163        return;
3164    }
3165
3166    meta.maintenance_pending = true;
3167    meta.maintenance_queued_kinds
3168        .push_back(MaintenanceDrainKind::ConfigureTail);
3169    meta.maintenance_queued_kinds
3170        .push_back(MaintenanceDrainKind::CompletionDrains);
3171}
3172
3173#[allow(clippy::too_many_arguments)]
3174async fn handle_route_bind_completion(
3175    tx: &WriterSender,
3176    completion: RouteBindCompletion,
3177    routes: &mut HashMap<RouteChannel, RouteIdentity>,
3178    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
3179    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
3180    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
3181    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3182    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3183    installed_route_epochs: &mut HashMap<u16, u32>,
3184    executor: &Arc<Executor>,
3185    shutdown: &Arc<Notify>,
3186    metrics: &Arc<DispatchPathMetrics>,
3187) -> Result<(), SubcError> {
3188    let route_id = completion.route;
3189    let Some(pending) = pending_binds.remove(&route_id) else {
3190        log::warn!(
3191            "subc attach: dropping RouteBind completion for non-pending route {}",
3192            completion.route
3193        );
3194        rollback_pending_bind_actor(
3195            executor,
3196            live_roots,
3197            pending_binds,
3198            &completion.bind_root_id,
3199            completion.inserted_new_actor,
3200        );
3201        let has_pending_bind = pending_binds
3202            .values()
3203            .any(|pending| pending.bind_root_id == completion.bind_root_id);
3204        if !root_channels
3205            .get(&completion.bind_root_id)
3206            .is_some_and(|channels| !channels.is_empty())
3207            && !has_pending_bind
3208        {
3209            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3210        }
3211        remove_installed_route(installed_route_epochs, route_id);
3212        return Ok(());
3213    };
3214
3215    if pending.bind_root_id != completion.bind_root_id {
3216        log::warn!(
3217            "subc attach: pending RouteBind root mismatch for route {} (pending {} completion {})",
3218            completion.route,
3219            pending.bind_root_id.as_path().display(),
3220            completion.bind_root_id.as_path().display()
3221        );
3222    }
3223
3224    let inserted_new_actor = pending.inserted_new_actor || completion.inserted_new_actor;
3225    if pending.cancelled {
3226        rollback_pending_bind_actor(
3227            executor,
3228            live_roots,
3229            pending_binds,
3230            &completion.bind_root_id,
3231            inserted_new_actor,
3232        );
3233        let has_pending_bind = pending_binds
3234            .values()
3235            .any(|pending| pending.bind_root_id == completion.bind_root_id);
3236        if !root_channels
3237            .get(&completion.bind_root_id)
3238            .is_some_and(|channels| !channels.is_empty())
3239            && !has_pending_bind
3240        {
3241            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3242        }
3243        log::debug!(
3244            "subc attach: discarded completed RouteBind for cancelled route {} root {}",
3245            completion.route,
3246            completion.bind_root_id.as_path().display()
3247        );
3248        remove_installed_route(installed_route_epochs, route_id);
3249        return Ok(());
3250    }
3251
3252    let failure = if !completion.configure_response.success {
3253        Some((
3254            &completion.configure_response,
3255            "configure failed during route bind",
3256        ))
3257    } else {
3258        None
3259    };
3260
3261    if let Some((response, fallback)) = failure {
3262        rollback_pending_bind_actor(
3263            executor,
3264            live_roots,
3265            pending_binds,
3266            &completion.bind_root_id,
3267            inserted_new_actor,
3268        );
3269        let has_pending_bind = pending_binds
3270            .values()
3271            .any(|pending| pending.bind_root_id == completion.bind_root_id);
3272        if !root_channels
3273            .get(&completion.bind_root_id)
3274            .is_some_and(|channels| !channels.is_empty())
3275            && !has_pending_bind
3276        {
3277            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3278        }
3279        let message = response_message(response, fallback);
3280        let fatal = response_is_fatal_panic(response);
3281        let error_code = route_bind_error_code_for_configure_response(response);
3282        send_route_bind_error_parts(
3283            tx,
3284            completion.ver,
3285            completion.corr,
3286            completion.flags,
3287            error_code,
3288            &message,
3289            metrics,
3290        )
3291        .await?;
3292        remove_installed_route(installed_route_epochs, route_id);
3293        if fatal {
3294            signal_fatal_teardown(
3295                tx,
3296                Some(completion.route),
3297                completion.ver,
3298                completion.corr,
3299                shutdown,
3300                metrics,
3301            )
3302            .await;
3303        }
3304        return Ok(());
3305    }
3306
3307    remember_session_identity(session_identity, &completion.identity);
3308    let replay_key = push::ReplayKey::from_identity(&completion.identity);
3309    let bind_trust = completion.identity.trust;
3310    insert_route_channel(routes, root_channels, route_id, completion.identity);
3311    let restore_watcher = live_roots
3312        .get(&completion.bind_root_id)
3313        .is_some_and(|meta| meta.idle_artifacts_evicted || meta.unbound_quiesced);
3314    live_roots
3315        .entry(completion.bind_root_id.clone())
3316        .and_modify(|meta| {
3317            meta.reactivate_bound();
3318            meta.diagnostics_on_edit = completion.diagnostics_on_edit;
3319            meta.maintenance_poisoned = false;
3320        })
3321        .or_insert_with(|| RootMeta::new(Instant::now()));
3322    if let Some(meta) = live_roots.get_mut(&completion.bind_root_id) {
3323        meta.diagnostics_on_edit = completion.diagnostics_on_edit;
3324        meta.maintenance_poisoned = false;
3325    }
3326    if let Some(ctx) = executor.actor_context(&completion.bind_root_id) {
3327        ctx.mark_subc_bound();
3328        if restore_watcher {
3329            crate::commands::configure::ensure_project_watcher(&ctx);
3330        }
3331    }
3332
3333    let ack =
3334        serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).map_err(SubcError::Json)?;
3335    let response = Frame::build_with_version(
3336        completion.ver,
3337        FrameType::Response,
3338        control_flags(),
3339        0,
3340        0,
3341        completion.corr,
3342        ack,
3343    )
3344    .map_err(SubcError::FrameBuild)?;
3345    send_reliable_writer_frame(tx, metrics, response, "RouteBindAck").await?;
3346    queue_post_bind_configure_and_completion_maintenance(&completion.bind_root_id, live_roots);
3347    let replayed = push::replay_buffered_push_frames(
3348        tx,
3349        metrics,
3350        route_id,
3351        push_buffer,
3352        &replay_key,
3353        bind_trust,
3354    );
3355    if replayed > 0 {
3356        log::debug!(
3357            "subc attach: replayed {} buffered Push frame(s) to route {} root {} harness {} session {}",
3358            replayed,
3359            completion.route,
3360            replay_key.root.as_path().display(),
3361            replay_key.harness,
3362            replay_key.session
3363        );
3364    }
3365    log::info!(
3366        "subc attach: route {} bound to root {}",
3367        completion.route,
3368        completion.bind_root_id.as_path().display()
3369    );
3370    Ok(())
3371}
3372
3373async fn expire_overdue_route_binds(
3374    tx: &WriterSender,
3375    executor: &Arc<Executor>,
3376    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3377    installed_route_epochs: &mut HashMap<u16, u32>,
3378    metrics: &DispatchPathMetrics,
3379) -> Result<(), SubcError> {
3380    let now = Instant::now();
3381    let expired: Vec<_> = pending_binds
3382        .iter()
3383        .filter_map(|(route, pending)| {
3384            let age = now.saturating_duration_since(pending.started_at);
3385            (!pending.deadline_reported && age >= ROUTE_BIND_DEADLINE).then(|| {
3386                (
3387                    *route,
3388                    pending.corr,
3389                    pending.ver,
3390                    pending.flags,
3391                    pending.bind_root_id.clone(),
3392                    pending.configure_request_id.clone(),
3393                    age,
3394                )
3395            })
3396        })
3397        .collect();
3398
3399    for (route, corr, ver, flags, root_id, configure_request_id, age) in expired {
3400        if let Some(pending) = pending_binds.get_mut(&route) {
3401            pending.cancelled = true;
3402            pending.deadline_reported = true;
3403            let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
3404            log::debug!(
3405                "subc attach: cancelled overdue RouteBind configure for route {route} ({outcome:?})"
3406            );
3407        }
3408        remove_installed_route(installed_route_epochs, route);
3409        let age_ms = age.as_millis().min(u128::from(u64::MAX)) as u64;
3410        let deadline_ms = ROUTE_BIND_DEADLINE.as_millis();
3411        send_route_bind_error_parts(
3412            tx,
3413            ver,
3414            corr,
3415            flags,
3416            "actor_not_ready",
3417            &format!("route bind deadline exceeded after {age_ms}ms (deadline {deadline_ms}ms)"),
3418            metrics,
3419        )
3420        .await?;
3421        log::warn!(
3422            "subc attach: route {} bind for root {} exceeded {}ms deadline (configure_request_id={})",
3423            route,
3424            root_id.as_path().display(),
3425            deadline_ms,
3426            configure_request_id
3427        );
3428    }
3429
3430    Ok(())
3431}
3432
3433/// channel-0 control requests: RouteBind plus the cached health probe. RouteBind
3434/// still reconciles the route's RootConfig through the executor's Mutating lane
3435/// and resolves completion on a loop-owned control-completion channel so slow
3436/// configure jobs do not block the transport loop.
3437#[allow(clippy::too_many_arguments)]
3438async fn handle_control_request(
3439    tx: &WriterSender,
3440    frame: &Frame,
3441    shared_app: &Arc<App>,
3442    executor: &Arc<Executor>,
3443    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3444    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3445    installed_route_epochs: &mut HashMap<u16, u32>,
3446    routes: &mut HashMap<RouteChannel, RouteIdentity>,
3447    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
3448    bg_subs: &mut HashMap<RouteChannel, BgSub>,
3449    bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
3450    bg_wake_pending: &mut HashSet<RouteChannel>,
3451    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
3452    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
3453    retry_buffer: &mut RetryBuffer,
3454    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
3455    shutdown: &Arc<Notify>,
3456    control_completion_tx: &mpsc::Sender<RouteBindCompletion>,
3457    metrics: &Arc<DispatchPathMetrics>,
3458    push_senders: &PushSenders,
3459    dispatch: DispatchFn,
3460    user_config_path: Option<&Path>,
3461) -> Result<(), SubcError> {
3462    let request =
3463        serde_json::from_slice::<ModuleControlRequest>(&frame.body).map_err(SubcError::Json)?;
3464    match request {
3465        ModuleControlRequest::RouteBind {
3466            route_channel,
3467            epoch,
3468            target: _,
3469            identity,
3470            principal,
3471            consumer_capabilities,
3472        } => {
3473            let route_id = route_key(route_channel, epoch);
3474            if epoch == 0 {
3475                return send_route_bind_error(
3476                    tx,
3477                    frame,
3478                    "config_divergence",
3479                    "route bind uses an invalid channel generation",
3480                    metrics,
3481                )
3482                .await;
3483            }
3484            let mut bind_root_id = None;
3485            if let Some(installed_epoch) = installed_route_epochs.get(&route_channel).copied() {
3486                if installed_epoch >= epoch {
3487                    return send_route_bind_error(
3488                        tx,
3489                        frame,
3490                        "config_divergence",
3491                        "route bind generation is not newer than the installed generation",
3492                        metrics,
3493                    )
3494                    .await;
3495                }
3496
3497                let replacement_root = match ProjectRootId::from_path(&identity.project_root) {
3498                    Ok(root_id) => root_id,
3499                    Err(error) => {
3500                        return send_route_bind_error(
3501                            tx,
3502                            frame,
3503                            "config_divergence",
3504                            &format!("invalid route project root: {error}"),
3505                            metrics,
3506                        )
3507                        .await;
3508                    }
3509                };
3510                teardown_installed_route(
3511                    tx,
3512                    metrics,
3513                    executor,
3514                    route_key(route_channel, installed_epoch),
3515                    "higher-epoch RouteBind",
3516                    Some(&replacement_root),
3517                    installed_route_epochs,
3518                    routes,
3519                    root_channels,
3520                    bg_subs,
3521                    bg_sub_by_session,
3522                    bg_wake_pending,
3523                    pending_bash_asks,
3524                    live_roots,
3525                    route_bash_cancels,
3526                    pending_binds,
3527                    retry_buffer,
3528                    push_buffer,
3529                    shutdown,
3530                )
3531                .await?;
3532                bind_root_id = Some(replacement_root);
3533            }
3534            if pending_binds.contains_key(&route_id) {
3535                return send_route_bind_error(
3536                    tx,
3537                    frame,
3538                    "config_divergence",
3539                    "route bind is already pending for channel",
3540                    metrics,
3541                )
3542                .await;
3543            }
3544            let bind_root_id = match bind_root_id {
3545                Some(root_id) => root_id,
3546                None => match ProjectRootId::from_path(&identity.project_root) {
3547                    Ok(root_id) => root_id,
3548                    Err(error) => {
3549                        return send_route_bind_error(
3550                            tx,
3551                            frame,
3552                            "config_divergence",
3553                            &format!("invalid route project root: {error}"),
3554                            metrics,
3555                        )
3556                        .await;
3557                    }
3558                },
3559            };
3560
3561            // Reconcile RootConfig: build a configure request from the bind
3562            // identity + forwarded config tiers and run it through the executor.
3563            let request_id = format!("subc-bind-{route_channel}");
3564            let bind_project_root = identity.project_root.clone();
3565            let bind_harness = identity.harness.clone();
3566            let bind_session = identity.session.clone();
3567            let bind_trust = trust_for_bind(&bind_harness, &principal);
3568            let bind_principal_id = principal_id(&principal);
3569            // Typed capability declaration from the consumer: the facade stamps it
3570            // from the MCP host's initialize-advertised capabilities. Absent
3571            // means no reverse-request capability — flat deny, fail-closed. A
3572            // consumer over-declaring only earns asks that TTL-deny.
3573            let consumer_elicitation_capable = consumer_capabilities
3574                .as_ref()
3575                .is_some_and(|capabilities| capabilities.iter().any(|c| c == "elicitation"));
3576            log::info!(
3577                "subc attach: route {} harness={} principal={} trust={} elicitation={}",
3578                route_channel,
3579                bind_harness,
3580                principal_label(&principal),
3581                bind_trust.label(),
3582                consumer_elicitation_capable
3583            );
3584
3585            // Config is single-per-project, read by AFT directly from the
3586            // CortexKit config files (user: ~/.config/cortexkit/aft.jsonc,
3587            // project: <root>/.cortexkit/aft.jsonc). Wire-relayed config tiers are
3588            // IGNORED entirely: a front (runner, mcp:*, or fed:*) cannot push config over
3589            // the wire. This is what makes config harness-INDEPENDENT — every
3590            // harness binding a project gets the identical on-disk config, so two
3591            // trust domains sharing the per-root actor can never diverge or
3592            // inherit each other's capabilities (the cross-bind escalation class).
3593            // Wire-relayed config tiers (if the protocol still carries them) are
3594            // ignored entirely; the per-tier trust boundary (user trusted, project
3595            // privileged-dropped) is applied to the FILE tiers in handle_configure.
3596            let local_tiers = crate::subc_config::read_local_cortexkit_config_tiers(
3597                user_config_path,
3598                Path::new(&bind_project_root),
3599            );
3600            let config_tiers: Vec<Value> = local_tiers
3601                .iter()
3602                .map(|t| json!({ "tier": t.tier, "source": t.source, "doc": t.doc }))
3603                .collect();
3604            let diagnostics_on_edit = diagnostics_on_edit_from_tiers(&local_tiers);
3605            let configure_json = json!({
3606                "id": request_id,
3607                "command": "configure",
3608                "project_root": bind_project_root,
3609                "harness": bind_harness,
3610                "session_id": bind_session.clone(),
3611                "config": config_tiers,
3612            });
3613            let configure_req = match serde_json::from_value::<RawRequest>(configure_json) {
3614                Ok(req) => req,
3615                Err(error) => {
3616                    return send_route_bind_error(
3617                        tx,
3618                        frame,
3619                        "config_divergence",
3620                        &format!("failed to build configure request: {error}"),
3621                        metrics,
3622                    )
3623                    .await;
3624                }
3625            };
3626
3627            let route_identity = RouteIdentity(Arc::new(RouteIdentityData {
3628                root: bind_root_id.clone(),
3629                project_root: PathBuf::from(&bind_project_root),
3630                harness: bind_harness.clone(),
3631                session: bind_session.clone(),
3632                trust: bind_trust,
3633                spawn_principal: AuthenticatedPrincipal::RouteBind {
3634                    trust: bind_trust.sandbox_trust(),
3635                    route_channel,
3636                    route_epoch: epoch,
3637                    project_root: PathBuf::from(&bind_project_root),
3638                    harness: bind_harness.clone(),
3639                    session_id: bind_session.clone(),
3640                    principal_id: bind_principal_id,
3641                },
3642                consumer_elicitation_capable,
3643            }));
3644            let configure_session = route_identity.session.clone();
3645            let root_was_live = live_roots.contains_key(&bind_root_id);
3646            let inserted_new_actor = register_actor_for_bind(
3647                shared_app,
3648                executor,
3649                push_senders,
3650                &bind_root_id,
3651                route_channel,
3652                root_was_live,
3653            );
3654
3655            let configure_request_id = configure_req.id.clone();
3656            installed_route_epochs.insert(route_channel, epoch);
3657            if let Some(meta) = live_roots.get_mut(&bind_root_id) {
3658                meta.maintenance_queued_kinds.clear();
3659                meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
3660            }
3661            let (configure_rx, configure_cancellation) = executor.submit_cancellable_async(
3662                bind_root_id.clone(),
3663                Lane::Mutating,
3664                configure_request_id.clone(),
3665                Box::new(move |ctx| {
3666                    log_ctx::with_session(Some(configure_session.clone()), || {
3667                        dispatch(configure_req, ctx)
3668                    })
3669                }),
3670            );
3671            pending_binds.insert(
3672                route_id,
3673                PendingBind {
3674                    bind_root_id: bind_root_id.clone(),
3675                    inserted_new_actor,
3676                    cancelled: false,
3677                    configure_request_id: configure_request_id.clone(),
3678                    started_at: Instant::now(),
3679                    warned_half_deadline: false,
3680                    deadline_reported: false,
3681                    corr: frame.header.corr,
3682                    ver: frame.header.ver,
3683                    flags: frame.header.flags,
3684                    cancellation: configure_cancellation,
3685                },
3686            );
3687
3688            let completion_tx = control_completion_tx.clone();
3689            let completion_identity = route_identity;
3690            let completion_root = bind_root_id.clone();
3691            let completion_route_channel = route_channel;
3692            let completion_ver = frame.header.ver;
3693            let completion_corr = frame.header.corr;
3694            let completion_flags = frame.header.flags;
3695            let completion_metrics = Arc::clone(metrics);
3696            tokio::spawn(async move {
3697                let _response_task = ResponseTaskGuard::new(&completion_metrics);
3698                let configure_response =
3699                    await_executor_response(configure_rx, configure_request_id.clone()).await;
3700                // Send the route-bind acknowledgment as soon as configure succeeds.
3701                // Installing completed search or callgraph builds only refreshes cached
3702                // read data, so a later maintenance pass can do it without delaying the
3703                // daemon's confirmation that the route is usable.
3704                let completion = RouteBindCompletion {
3705                    route: route_key(completion_route_channel, epoch),
3706                    identity: completion_identity,
3707                    bind_root_id: completion_root,
3708                    inserted_new_actor,
3709                    configure_response,
3710                    diagnostics_on_edit,
3711                    ver: completion_ver,
3712                    corr: completion_corr,
3713                    flags: completion_flags,
3714                };
3715                if send_counted_channel(
3716                    &completion_tx,
3717                    &completion_metrics.control_completion_queued,
3718                    completion,
3719                )
3720                .await
3721                .is_err()
3722                {
3723                    log::debug!(
3724                        "subc attach: dropped RouteBind completion for route {} after loop exit",
3725                        completion_route_channel
3726                    );
3727                }
3728            });
3729
3730            Ok(())
3731        }
3732        ModuleControlRequest::HealthCheck {} => {
3733            let report = build_health_report(executor, pending_binds, metrics, shared_app);
3734            let body = serde_json::to_vec(&ModuleControlResponse::from(report))
3735                .map_err(SubcError::Json)?;
3736            let response = Frame::build_with_version(
3737                frame.header.ver,
3738                FrameType::Response,
3739                frame.header.flags,
3740                0,
3741                0,
3742                frame.header.corr,
3743                body,
3744            )
3745            .map_err(SubcError::FrameBuild)?;
3746            send_frame(tx, metrics, response).await
3747        }
3748    }
3749}
3750
3751fn install_bash_compressor(ctx: &AppContext) {
3752    // Mirrors main.rs per-actor compressor installation for subc-created actors.
3753    let filter_registry_handle = ctx.shared_filter_registry();
3754    let compress_flag = ctx.bash_compress_flag();
3755    ctx.bash_background().set_compressor_with_exit_code(
3756        move |command: &str, output: String, exit_code: Option<i32>| {
3757            if !compress_flag.load(std::sync::atomic::Ordering::Relaxed) {
3758                return crate::compress::CompressionResult::new(output);
3759            }
3760            let registry_guard = match filter_registry_handle.read() {
3761                Ok(g) => g,
3762                Err(poisoned) => poisoned.into_inner(),
3763            };
3764            crate::compress::compress_with_registry_exit_code(
3765                command,
3766                &output,
3767                exit_code,
3768                &registry_guard,
3769            )
3770        },
3771    );
3772}
3773
3774fn diagnostics_on_edit_from_tiers(tiers: &[ConfigTier]) -> bool {
3775    let mut diagnostics_on_edit = false;
3776    for tier in tiers {
3777        if let Some(value) = diagnostics_on_edit_from_doc(&tier.doc) {
3778            diagnostics_on_edit = value;
3779        }
3780    }
3781    diagnostics_on_edit
3782}
3783
3784fn diagnostics_on_edit_from_doc(doc: &str) -> Option<bool> {
3785    let stripped = strip_jsonc(doc);
3786    let value = serde_json::from_str::<Value>(&stripped).ok()?;
3787    value
3788        .get("lsp")
3789        .and_then(Value::as_object)?
3790        .get("diagnostics_on_edit")
3791        .and_then(Value::as_bool)
3792}
3793
3794async fn send_route_bind_error(
3795    tx: &WriterSender,
3796    frame: &Frame,
3797    code: &str,
3798    message: &str,
3799    metrics: &DispatchPathMetrics,
3800) -> Result<(), SubcError> {
3801    send_route_bind_error_parts(
3802        tx,
3803        frame.header.ver,
3804        frame.header.corr,
3805        frame.header.flags,
3806        code,
3807        message,
3808        metrics,
3809    )
3810    .await
3811}
3812
3813async fn send_route_bind_error_parts(
3814    tx: &WriterSender,
3815    ver: u8,
3816    corr: u64,
3817    flags: Flags,
3818    code: &str,
3819    message: &str,
3820    metrics: &DispatchPathMetrics,
3821) -> Result<(), SubcError> {
3822    let response = build_error_frame(ver, 0, 0, corr, flags, code, message)?;
3823    send_reliable_writer_frame(tx, metrics, response, "RouteBind error").await?;
3824    log::warn!("subc attach: route bind rejected ({code}): {message}");
3825    Ok(())
3826}
3827
3828/// Route-channel tool call: `{name, arguments}` → executor lane → dispatch to
3829/// the sync command core → wrap the structured Response in a CallToolResult
3830/// `{content, isError}`. Tool-result mapping: the whole `{success, ...}` Response
3831/// serialized into ONE text block; `isError` carries `success == false`.
3832async fn handle_tool_call(
3833    tx: &WriterSender,
3834    frame: &Frame,
3835    mut phase_trace: PhaseTrace,
3836    routes: &HashMap<RouteChannel, RouteIdentity>,
3837    pending_binds: &HashMap<RouteChannel, PendingBind>,
3838    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3839    executor: &Arc<Executor>,
3840    shutdown: &Arc<Notify>,
3841    connection_cancel: &PersistentCancelSignal,
3842    bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
3843    bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
3844    metrics: &Arc<DispatchPathMetrics>,
3845    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
3846    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
3847    next_bash_ask_corr: &mut u64,
3848    bg_subs: &mut HashMap<RouteChannel, BgSub>,
3849    bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
3850    bg_wake_pending: &mut HashSet<RouteChannel>,
3851    bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
3852    dispatch: DispatchFn,
3853    allow_native_passthrough: bool,
3854) -> Result<(), SubcError> {
3855    let route_id = route_key(frame.header.channel, frame.header.epoch);
3856    if pending_binds.contains_key(&route_id) {
3857        let error = build_error_frame(
3858            frame.header.ver,
3859            frame.header.channel,
3860            frame.header.epoch,
3861            frame.header.corr,
3862            frame.header.flags,
3863            "route_not_bound",
3864            "route is not bound before tool call",
3865        )?;
3866        return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
3867    }
3868
3869    let Some(identity) = routes.get(&route_id).cloned() else {
3870        let error = build_error_frame(
3871            frame.header.ver,
3872            frame.header.channel,
3873            frame.header.epoch,
3874            frame.header.corr,
3875            frame.header.flags,
3876            "route_not_bound",
3877            "route is not bound before tool call",
3878        )?;
3879        return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
3880    };
3881    let restore_watcher = live_roots
3882        .get(&identity.root)
3883        .is_some_and(|meta| meta.idle_artifacts_evicted);
3884    if let Some(meta) = live_roots.get_mut(&identity.root) {
3885        meta.reactivate_bound();
3886    }
3887    if restore_watcher {
3888        if let Some(ctx) = executor.actor_context(&identity.root) {
3889            crate::commands::configure::ensure_project_watcher(&ctx);
3890        }
3891    }
3892
3893    let route_request =
3894        serde_json::from_slice::<RouteRequest>(&frame.body).map_err(SubcError::Json)?;
3895    if matches!(
3896        route_request,
3897        RouteRequest::BgEvents(BgEventsRequest {
3898            op: BgEventsOp::BgEvents
3899        })
3900    ) {
3901        if let Some(old_sub) = bg_subs.get(&route_id).copied() {
3902            push::send_reliable_bg_stream_end(tx, metrics, route_id, &old_sub).await?;
3903        }
3904        if !identity.trust.allows_bash_observation() {
3905            bg_subs.remove(&route_id);
3906            bg_wake_pending.remove(&route_id);
3907            remove_bg_subscription_index(bg_sub_by_session, route_id, Some(&identity));
3908            let denied_sub = BgSub {
3909                corr: frame.header.corr,
3910                ver: frame.header.ver,
3911                flags: frame.header.flags,
3912            };
3913            push::send_reliable_bg_stream_end(tx, metrics, route_id, &denied_sub).await?;
3914            return Ok(());
3915        }
3916        bg_subs.insert(
3917            route_id,
3918            BgSub {
3919                corr: frame.header.corr,
3920                ver: frame.header.ver,
3921                flags: frame.header.flags,
3922            },
3923        );
3924        bg_sub_by_session.insert((identity.root.clone(), identity.session.clone()), route_id);
3925        push::arm_bg_wake(
3926            identity.root.clone(),
3927            identity.session.clone(),
3928            route_id,
3929            bg_wake_pending,
3930            bg_wake_epoch,
3931        );
3932        return Ok(());
3933    }
3934
3935    let RouteRequest::ToolCall(call) = route_request else {
3936        unreachable!("background event subscription returned above")
3937    };
3938    let bare_name = call.name;
3939    let arguments = strip_agent_preview_arg_owned(call.arguments);
3940    let format_context = crate::subc_format::FormatContext::from_tool_call(
3941        &bare_name,
3942        &arguments,
3943        identity.project_root.as_path(),
3944    );
3945
3946    let request_id = format!("subc-{}-{}", frame.header.channel, frame.header.corr);
3947    let bind_trust = identity.trust;
3948    let diagnostics_on_edit = live_roots
3949        .get(&identity.root)
3950        .map(|meta| meta.diagnostics_on_edit)
3951        .unwrap_or(false);
3952
3953    let requests_host = bare_name == "bash"
3954        && arguments
3955            .get("sandbox")
3956            .or_else(|| {
3957                arguments
3958                    .get("params")
3959                    .and_then(|params| params.get("sandbox"))
3960            })
3961            .and_then(Value::as_str)
3962            == Some("host");
3963    if matches!(bind_trust, BindTrust::Untrusted) && requests_host {
3964        let response = Response::error(
3965            request_id.clone(),
3966            "sandbox_escalation_denied",
3967            "sandbox host escalation is unavailable to untrusted principals",
3968        );
3969        let text = crate::subc_format::format_response_with_context(
3970            &bare_name,
3971            &response,
3972            &format_context,
3973        );
3974        let result = ToolCallResult { text, response };
3975        let response_frame = build_tool_response_frame(
3976            frame.header.ver,
3977            route_id,
3978            frame.header.corr,
3979            frame.header.flags,
3980            &result,
3981            bind_trust,
3982        )?;
3983        return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
3984    }
3985
3986    if matches!(bind_trust, BindTrust::Untrusted)
3987        && is_bash_family_tool(&bare_name)
3988        && (bare_name != "bash" || !identity.consumer_elicitation_capable)
3989    {
3990        let response = bash::bash_denied_untrusted_response(request_id.clone());
3991        let text = crate::subc_format::format_response_with_context(
3992            &bare_name,
3993            &response,
3994            &format_context,
3995        );
3996        let result = ToolCallResult { text, response };
3997        let response_frame = build_tool_response_frame(
3998            frame.header.ver,
3999            route_id,
4000            frame.header.corr,
4001            frame.header.flags,
4002            &result,
4003            bind_trust,
4004        )?;
4005        return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4006    }
4007
4008    // A non-core name is NOT in the tool manifest. AFT fails closed and
4009    // does not trust subc to enforce the manifest: rejecting here is the
4010    // defense-in-depth backstop that prevents a forwarded native command
4011    // (e.g. `configure`, which would reach handle_configure and bypass
4012    // the RouteBind config-trust cap) from ever reaching dispatch. Only
4013    // the integration-test harness (run_subc_mode_for_test) opens this to
4014    // drive synthetic native commands through the executor.
4015    if !is_subc_agent_core_tool(&bare_name)
4016        && !is_subc_native_plumbing_tool(&bare_name)
4017        && !allow_native_passthrough
4018    {
4019        log::warn!(
4020            "subc tool call: rejecting non-manifest tool name {:?} on route {} (fail-closed)",
4021            bare_name,
4022            frame.header.channel
4023        );
4024        let response = Response::error(
4025            request_id.clone(),
4026            "unknown_tool",
4027            format!("tool {:?} is not in the AFT tool manifest", bare_name),
4028        );
4029        let text = crate::subc_format::format_response_with_context(
4030            &bare_name,
4031            &response,
4032            &format_context,
4033        );
4034        let result = ToolCallResult { text, response };
4035        let response_frame = build_tool_response_frame(
4036            frame.header.ver,
4037            route_id,
4038            frame.header.corr,
4039            frame.header.flags,
4040            &result,
4041            bind_trust,
4042        )?;
4043        return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4044    }
4045
4046    if bare_name == "bash" {
4047        if matches!(bind_trust, BindTrust::Untrusted) {
4048            let plan = match bash::prepare_bash_elicitation_plan(
4049                &arguments,
4050                identity.project_root.as_path(),
4051            ) {
4052                Ok(plan) => plan,
4053                Err(error) => {
4054                    let response = Response::error(request_id.clone(), error.code, error.message);
4055                    let text = crate::subc_format::format_response_with_context(
4056                        &bare_name,
4057                        &response,
4058                        &format_context,
4059                    );
4060                    let result = ToolCallResult { text, response };
4061                    let response_frame = build_tool_response_frame(
4062                        frame.header.ver,
4063                        route_id,
4064                        frame.header.corr,
4065                        frame.header.flags,
4066                        &result,
4067                        bind_trust,
4068                    )?;
4069                    return send_reliable_writer_frame(
4070                        tx,
4071                        metrics,
4072                        response_frame,
4073                        "tool response",
4074                    )
4075                    .await;
4076                }
4077            };
4078
4079            let reverse_corr =
4080                allocate_reverse_corr(pending_bash_asks, route_id, next_bash_ask_corr);
4081            let ask_frame = build_bash_elicitation_request_frame(
4082                frame.header.ver,
4083                route_id,
4084                reverse_corr,
4085                frame.header.flags,
4086                &plan.command,
4087                &plan.asks,
4088            )?;
4089
4090            let meta = live_roots
4091                .entry(identity.root.clone())
4092                .or_insert_with(|| RootMeta::new(Instant::now()));
4093            meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
4094            meta.reactivate_bound();
4095
4096            let route_cancel =
4097                route_bash_cancels
4098                    .entry(route_id)
4099                    .or_insert_with(|| bash::RouteBashCancel {
4100                        token: PersistentCancelSignal::new(),
4101                        active_waits: 0,
4102                    });
4103            route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
4104            let cancel = bash::BashWaitCancel {
4105                connection: connection_cancel.clone(),
4106                route: route_cancel.token.clone(),
4107            };
4108            pending_bash_asks.insert(
4109                ReverseCorrKey {
4110                    route: route_id,
4111                    corr: reverse_corr,
4112                },
4113                PendingBashAsk {
4114                    route: route_id,
4115                    tool_corr: frame.header.corr,
4116                    tool_flags: frame.header.flags,
4117                    tool_ver: frame.header.ver,
4118                    root: identity.root.clone(),
4119                    project_root: identity.project_root.clone(),
4120                    session_id: identity.session.clone(),
4121                    spawn_principal: identity.spawn_principal.clone(),
4122                    request_id,
4123                    arguments,
4124                    format_context,
4125                    cancel,
4126                    grants: plan.grants,
4127                    expires_at: Instant::now() + bash_elicitation_timeout(),
4128                },
4129            );
4130            return send_reliable_writer_frame(tx, metrics, ask_frame, "bash elicitation request")
4131                .await;
4132        }
4133
4134        let meta = live_roots
4135            .entry(identity.root.clone())
4136            .or_insert_with(|| RootMeta::new(Instant::now()));
4137        meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
4138        meta.reactivate_bound();
4139
4140        let route_cancel =
4141            route_bash_cancels
4142                .entry(route_id)
4143                .or_insert_with(|| bash::RouteBashCancel {
4144                    token: PersistentCancelSignal::new(),
4145                    active_waits: 0,
4146                });
4147        route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
4148        let cancel = bash::BashWaitCancel {
4149            connection: connection_cancel.clone(),
4150            route: route_cancel.token.clone(),
4151        };
4152
4153        bash::submit_deferred_bash(
4154            executor,
4155            bash_deferred_tx,
4156            bash_poll_touch_tx,
4157            metrics,
4158            dispatch,
4159            identity.root.clone(),
4160            identity.project_root.clone(),
4161            identity.session.clone(),
4162            request_id,
4163            route_id,
4164            frame.header.corr,
4165            frame.header.flags,
4166            frame.header.ver,
4167            arguments,
4168            format_context,
4169            cancel,
4170            bind_trust,
4171            identity.spawn_principal.clone(),
4172            None,
4173        );
4174        return Ok(());
4175    }
4176
4177    let lane = command_lane(&bare_name);
4178    let tool_call_context = ToolCallContext {
4179        project_root: identity.project_root.clone(),
4180        session_id: Some(identity.session.clone()),
4181        request_id: request_id.clone(),
4182        diagnostics_on_edit,
4183        preview: call.preview,
4184    };
4185    let bare_name_for_frame = bare_name.clone();
4186    let identity_for_run = identity.clone();
4187    let completion_session = identity.session.clone();
4188    let completion_root = identity.project_root.clone();
4189    let request_id_for_force = request_id.clone();
4190    let format_context_for_frame = format_context.clone();
4191    let (tool_call_tx, tool_call_rx) = oneshot::channel::<ToolCallCompletion>();
4192    phase_trace.mark_executor_submitted();
4193    let rx = executor.submit_async(
4194        identity.root.clone(),
4195        lane,
4196        request_id.clone(),
4197        Box::new(move |ctx| {
4198            phase_trace.mark_job_admitted();
4199            log_ctx::with_session(Some(identity_for_run.session.clone()), || {
4200                let run = || {
4201                    let finalizer = |response: &mut Response| {
4202                        crate::response_finalize::finalize_response_with_bg_completions(
4203                            response,
4204                            ctx,
4205                            &identity_for_run.session,
4206                            &bare_name,
4207                            bind_trust.allows_bash_observation(),
4208                        );
4209                    };
4210                    match run_tool_call(
4211                        &bare_name,
4212                        arguments,
4213                        &format_context,
4214                        &tool_call_context,
4215                        ctx,
4216                        &dispatch,
4217                        Some(&finalizer),
4218                        Some(&mut phase_trace),
4219                    ) {
4220                        ToolCallOutcome::Unary(result) => {
4221                            let response = result.response;
4222                            let _ = tool_call_tx.send(ToolCallCompletion {
4223                                text: result.text,
4224                                phase_trace,
4225                            });
4226                            response
4227                        }
4228                    }
4229                };
4230                if matches!(bind_trust, BindTrust::Untrusted) {
4231                    ctx.with_force_restrict(&request_id_for_force, run)
4232                } else {
4233                    run()
4234                }
4235            })
4236        }),
4237    );
4238    let completion_tx = tx.clone();
4239    let completion_shutdown = Arc::clone(shutdown);
4240    let route = route_id;
4241    let corr = frame.header.corr;
4242    let flags = frame.header.flags;
4243    let ver = frame.header.ver;
4244    let completion_metrics = Arc::clone(metrics);
4245    tokio::spawn(async move {
4246        let _response_task = ResponseTaskGuard::new(&completion_metrics);
4247        let response = await_executor_response(rx, request_id.clone()).await;
4248        let (text, phase_trace) = match tool_call_rx.await {
4249            Ok(completion) => (completion.text, Some(completion.phase_trace)),
4250            Err(_) => (
4251                crate::subc_format::format_response_with_context(
4252                    &bare_name_for_frame,
4253                    &response,
4254                    &format_context_for_frame,
4255                ),
4256                None,
4257            ),
4258        };
4259        let result = ToolCallResult { text, response };
4260        let fatal = response_is_fatal_panic(&result.response);
4261        match build_tool_response_frame(ver, route, corr, flags, &result, bind_trust) {
4262            Ok(response_frame) => {
4263                let send_result = if let Some(phase_trace) = phase_trace {
4264                    let trace = ToolResponseWriteTrace::new(
4265                        phase_trace,
4266                        bare_name_for_frame,
4267                        completion_root,
4268                        completion_session,
4269                        route.channel,
4270                        corr,
4271                    );
4272                    send_traced_tool_response_frame(
4273                        &completion_tx,
4274                        &completion_metrics,
4275                        response_frame,
4276                        trace,
4277                    )
4278                    .await
4279                } else {
4280                    send_reliable_writer_frame(
4281                        &completion_tx,
4282                        &completion_metrics,
4283                        response_frame,
4284                        "tool response",
4285                    )
4286                    .await
4287                };
4288                if let Err(error) = send_result {
4289                    log::warn!("subc attach: failed to queue tool response frame: {error}");
4290                }
4291            }
4292            Err(error) => {
4293                log::error!("subc attach: failed to build tool response frame: {error}");
4294            }
4295        }
4296        if fatal {
4297            signal_fatal_teardown(
4298                &completion_tx,
4299                Some(route),
4300                ver,
4301                corr,
4302                &completion_shutdown,
4303                &completion_metrics,
4304            )
4305            .await;
4306        }
4307    });
4308    Ok(())
4309}
4310
4311fn submit_maintenance_job(
4312    executor: &Arc<Executor>,
4313    root_id: ProjectRootId,
4314    kind: MaintenanceDrainKind,
4315    bg_sessions_to_check: Vec<(String, u64)>,
4316    completion_tx: &mpsc::Sender<MaintenanceCompletion>,
4317    metrics: &Arc<DispatchPathMetrics>,
4318) {
4319    let request_id = format!(
4320        "subc-maintenance-drain-{}-{}",
4321        kind.label(),
4322        root_id.as_path().to_string_lossy()
4323    );
4324    let response_id = request_id.clone();
4325    let completion_root_id = root_id.clone();
4326    let maintenance_generation = executor
4327        .actor_context(&root_id)
4328        .map(|ctx| ctx.configure_generation())
4329        .unwrap_or(0);
4330    let (outcome_tx, outcome_rx) = oneshot::channel::<MaintenanceJobOutcome>();
4331    // ConfigureTail runs deferred configure mutations and needs the actor
4332    // epoch write gate. The other drain kinds only mutate subsystem state
4333    // behind that subsystem's own lock, so they run on MaintenanceCommit and
4334    // overlap interactive reads instead of excluding them.
4335    let lane = match kind {
4336        MaintenanceDrainKind::ConfigureTail => Lane::Mutating,
4337        MaintenanceDrainKind::Watcher
4338        | MaintenanceDrainKind::Lsp
4339        | MaintenanceDrainKind::CompletionDrains => Lane::MaintenanceCommit,
4340    };
4341    let rx = executor.submit_maintenance_async(
4342        root_id,
4343        lane,
4344        request_id.clone(),
4345        Box::new(move |ctx| {
4346            let outcome = match kind {
4347                MaintenanceDrainKind::Watcher => {
4348                    let drained = runtime_drain::drain_watcher_events_bounded(
4349                        ctx,
4350                        runtime_drain::WATCHER_PATH_DRAIN_BATCH_CAP,
4351                    );
4352                    MaintenanceJobOutcome {
4353                        empty_bg_sessions: Vec::new(),
4354                        requeue_kind: drained.has_more.then_some(kind),
4355                    }
4356                }
4357                MaintenanceDrainKind::Lsp => {
4358                    let drained = runtime_drain::drain_lsp_events_bounded(
4359                        ctx,
4360                        runtime_drain::LSP_EVENT_DRAIN_BATCH_CAP,
4361                    );
4362                    MaintenanceJobOutcome {
4363                        empty_bg_sessions: Vec::new(),
4364                        requeue_kind: drained.has_more.then_some(kind),
4365                    }
4366                }
4367                MaintenanceDrainKind::ConfigureTail => {
4368                    runtime_drain::drain_deferred_configure_maintenance(ctx);
4369                    runtime_drain::drain_configure_warning_events(ctx);
4370                    MaintenanceJobOutcome::default()
4371                }
4372                MaintenanceDrainKind::CompletionDrains => {
4373                    runtime_drain::drain_search_index_events(ctx);
4374                    runtime_drain::drain_callgraph_store_events(ctx);
4375                    runtime_drain::drain_semantic_index_events(ctx);
4376                    runtime_drain::drain_semantic_refresh_events(ctx);
4377                    runtime_drain::drain_inspect_events_for_generation(ctx, maintenance_generation);
4378                    let empty_bg_sessions = bg_sessions_to_check
4379                        .into_iter()
4380                        .filter(|(session, _)| {
4381                            !ctx.bash_background()
4382                                .has_completions_for_session(Some(session.as_str()))
4383                        })
4384                        .collect();
4385                    MaintenanceJobOutcome {
4386                        empty_bg_sessions,
4387                        requeue_kind: None,
4388                    }
4389                }
4390            };
4391            let requeued = outcome.requeue_kind.is_some();
4392            let _ = outcome_tx.send(outcome);
4393            Response::success(
4394                response_id,
4395                json!({ "drained": true, "kind": kind.label(), "requeued": requeued }),
4396            )
4397        }),
4398    );
4399    let completion_tx = completion_tx.clone();
4400    let completion_metrics = Arc::clone(metrics);
4401    tokio::spawn(async move {
4402        let _response_task = ResponseTaskGuard::new(&completion_metrics);
4403        let response = await_executor_response(rx, request_id).await;
4404        let outcome = outcome_rx.await.unwrap_or_default();
4405        let _ = send_counted_channel(
4406            &completion_tx,
4407            &completion_metrics.maintenance_queued,
4408            MaintenanceCompletion {
4409                root_id: completion_root_id,
4410                kind,
4411                response,
4412                empty_bg_sessions: outcome.empty_bg_sessions,
4413                requeue_kind: outcome.requeue_kind,
4414            },
4415        )
4416        .await;
4417    });
4418}
4419
4420async fn await_executor_response(rx: oneshot::Receiver<Response>, request_id: String) -> Response {
4421    rx.await
4422        .unwrap_or_else(|_| Response::error(request_id, "internal_error", "executor dropped"))
4423}
4424async fn signal_fatal_teardown(
4425    tx: &WriterSender,
4426    route: Option<RouteChannel>,
4427    ver: u8,
4428    corr: u64,
4429    shutdown: &Arc<Notify>,
4430    metrics: &DispatchPathMetrics,
4431) {
4432    if let Some(route) = route {
4433        if let Ok(frame) = build_goodbye_frame(ver, route.channel, route.epoch, corr) {
4434            if let Err(error) = send_frame(tx, metrics, frame).await {
4435                log::warn!(
4436                    "subc attach: failed to queue fatal route Goodbye for route {route}: {error}"
4437                );
4438            }
4439        }
4440    }
4441    if let Ok(frame) = build_goodbye_frame(ver, 0, 0, 0) {
4442        if let Err(error) = send_frame(tx, metrics, frame).await {
4443            log::warn!("subc attach: failed to queue fatal channel-0 Goodbye: {error}");
4444        }
4445    }
4446    shutdown.notify_one();
4447}
4448#[derive(Debug, Deserialize)]
4449#[serde(untagged)]
4450enum RouteRequest {
4451    BgEvents(BgEventsRequest),
4452    ToolCall(ToolCallRequest),
4453}
4454
4455#[derive(Debug, Deserialize)]
4456struct BgEventsRequest {
4457    op: BgEventsOp,
4458}
4459
4460#[derive(Debug, Deserialize)]
4461#[serde(rename_all = "snake_case")]
4462enum BgEventsOp {
4463    BgEvents,
4464}
4465
4466#[derive(Debug, Deserialize)]
4467struct ToolCallRequest {
4468    name: String,
4469    #[serde(default)]
4470    arguments: Value,
4471    /// Server-owned preview control (B1c-0): the plugin's mutation flow is
4472    /// preview -> permission ask -> apply. Dropping this field made "preview"
4473    /// calls mutate disk before the permission prompt and the subsequent
4474    /// apply fail with not-found.
4475    #[serde(default)]
4476    preview: bool,
4477}
4478
4479#[cfg(test)]
4480pub(crate) mod test_support {
4481    use super::*;
4482    use crate::bash_background::BgTaskStatus;
4483    use crate::protocol::{
4484        BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, ConfigureWarningsFrame,
4485        ProgressFrame, StatusChangedFrame,
4486    };
4487    use serde_json::json;
4488
4489    pub(super) fn test_root(name: &str) -> (tempfile::TempDir, ProjectRootId) {
4490        let dir = tempfile::Builder::new()
4491            .prefix(name)
4492            .tempdir()
4493            .expect("temp root");
4494        let root = ProjectRootId::from_path(dir.path()).expect("project root id");
4495        (dir, root)
4496    }
4497
4498    pub(super) fn test_ctx() -> Arc<AppContext> {
4499        Arc::new(AppContext::new(
4500            Box::new(crate::parser::TreeSitterProvider::new()),
4501            crate::config::Config::default(),
4502        ))
4503    }
4504
4505    pub(super) fn wait_for_watcher_count(ctx: &AppContext, expected: usize) {
4506        let deadline = Instant::now() + Duration::from_secs(30);
4507        loop {
4508            let observed = ctx.watcher_registry_count();
4509            if observed == expected {
4510                return;
4511            }
4512            assert!(
4513                Instant::now() < deadline,
4514                "watcher count did not settle before deadline: expected={expected}, observed={observed}"
4515            );
4516            std::thread::sleep(Duration::from_millis(50));
4517        }
4518    }
4519
4520    /// Sweep until `root` is forgotten, mirroring how production reaps.
4521    ///
4522    /// `reap_idle_roots` probes actor idleness with a try-lock and retains the
4523    /// root when the scheduler holds that lock — correct behavior, since the
4524    /// real caller sweeps on a timer and simply catches the root next tick. A
4525    /// test that asserts a single sweep succeeds is therefore asserting it wins
4526    /// a lock race that nothing in production depends on: `register_actor` wakes
4527    /// the scheduler, which grabs the same lock, and on a loaded runner that
4528    /// window is wide enough to lose. Sweep to the outcome instead.
4529    pub(super) fn reap_until_forgotten(
4530        root: &ProjectRootId,
4531        live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4532        pending_binds: &HashMap<RouteChannel, PendingBind>,
4533        root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
4534        executor: &Arc<Executor>,
4535        metrics: &DispatchPathMetrics,
4536    ) -> IdleReapOutcome {
4537        let deadline = Instant::now() + Duration::from_secs(30);
4538        loop {
4539            let outcome = reap_idle_roots(
4540                Instant::now(),
4541                live_roots,
4542                pending_binds,
4543                root_channels,
4544                executor,
4545                metrics,
4546            );
4547            if outcome.forgotten_deleted_roots.contains(root) {
4548                return outcome;
4549            }
4550            assert!(
4551                Instant::now() < deadline,
4552                "deleted root was never forgotten: {root:?}"
4553            );
4554            std::thread::sleep(Duration::from_millis(10));
4555        }
4556    }
4557
4558    pub(super) fn wait_for_actor_root_count(app: &App, expected: usize) {
4559        let deadline = Instant::now() + Duration::from_secs(30);
4560        loop {
4561            let observed = app.actor_root_count();
4562            if observed == expected {
4563                return;
4564            }
4565            assert!(
4566                Instant::now() < deadline,
4567                "actor root count did not settle before deadline: expected={expected}, observed={observed}"
4568            );
4569            std::thread::sleep(Duration::from_millis(50));
4570        }
4571    }
4572
4573    pub(super) fn status_frame(seq: u64) -> PushFrame {
4574        status_frame_with_session(seq, None)
4575    }
4576
4577    pub(super) fn status_frame_with_session(seq: u64, session_id: Option<&str>) -> PushFrame {
4578        PushFrame::StatusChanged(StatusChangedFrame {
4579            frame_type: "status_changed",
4580            session_id: session_id.map(str::to_string),
4581            snapshot: json!({ "seq": seq }),
4582        })
4583    }
4584
4585    pub(super) fn completion_frame(task_id: &str) -> PushFrame {
4586        completion_frame_with_session(task_id, "session-1")
4587    }
4588
4589    pub(super) fn completion_frame_with_session(task_id: &str, session_id: &str) -> PushFrame {
4590        PushFrame::BashCompleted(BashCompletedFrame {
4591            frame_type: "bash_completed",
4592            task_id: task_id.to_string(),
4593            session_id: session_id.to_string(),
4594            status: BgTaskStatus::Completed,
4595            exit_code: Some(0),
4596            command: format!("echo {task_id}"),
4597            output_preview: String::new(),
4598            output_truncated: false,
4599            original_tokens: None,
4600            compressed_tokens: None,
4601            tokens_skipped: false,
4602        })
4603    }
4604
4605    pub(super) fn long_running_frame(task_id: &str, elapsed_ms: u64) -> PushFrame {
4606        long_running_frame_with_session(task_id, "session-1", elapsed_ms)
4607    }
4608
4609    pub(super) fn long_running_frame_with_session(
4610        task_id: &str,
4611        session_id: &str,
4612        elapsed_ms: u64,
4613    ) -> PushFrame {
4614        PushFrame::BashLongRunning(BashLongRunningFrame {
4615            frame_type: "bash_long_running",
4616            task_id: task_id.to_string(),
4617            session_id: session_id.to_string(),
4618            command: format!("sleep {elapsed_ms}"),
4619            elapsed_ms,
4620        })
4621    }
4622
4623    pub(super) fn pattern_match_frame(session_id: &str) -> PushFrame {
4624        PushFrame::BashPatternMatch(BashPatternMatchFrame {
4625            frame_type: "bash_pattern_match",
4626            task_id: "task-pattern".to_string(),
4627            session_id: session_id.to_string(),
4628            watch_id: "watch-1".to_string(),
4629            match_text: "needle".to_string(),
4630            match_offset: 7,
4631            context: "haystack needle".to_string(),
4632            once: true,
4633            reason: "pattern_match",
4634        })
4635    }
4636
4637    pub(super) fn configure_warnings_frame(session_id: Option<&str>) -> PushFrame {
4638        PushFrame::ConfigureWarnings(ConfigureWarningsFrame {
4639            frame_type: "configure_warnings",
4640            session_id: session_id.map(str::to_string),
4641            project_root: "/tmp/subc-test".to_string(),
4642            warnings: Vec::new(),
4643        })
4644    }
4645
4646    pub(super) fn route_identity(root: &ProjectRootId, session_id: &str) -> RouteIdentity {
4647        route_identity_with_trust(root, session_id, BindTrust::FirstParty)
4648    }
4649
4650    pub(super) fn route_identity_with_trust(
4651        root: &ProjectRootId,
4652        session_id: &str,
4653        trust: BindTrust,
4654    ) -> RouteIdentity {
4655        RouteIdentity(Arc::new(RouteIdentityData {
4656            root: root.clone(),
4657            project_root: root.as_path().to_path_buf(),
4658            harness: "opencode".to_string(),
4659            session: session_id.to_string(),
4660            trust,
4661            spawn_principal: AuthenticatedPrincipal::RouteBind {
4662                trust: trust.sandbox_trust(),
4663                route_channel: 0,
4664                route_epoch: 0,
4665                project_root: root.as_path().to_path_buf(),
4666                harness: "opencode".to_string(),
4667                session_id: session_id.to_string(),
4668                principal_id: Some(match trust {
4669                    BindTrust::FirstParty => "direct".to_string(),
4670                    BindTrust::Untrusted => "unverified".to_string(),
4671                }),
4672            },
4673            consumer_elicitation_capable: false,
4674        }))
4675    }
4676
4677    pub(super) fn progress_frame(request_id: &str, kind: ProgressKind, chunk: &str) -> PushFrame {
4678        PushFrame::Progress(ProgressFrame::new(request_id, kind, chunk))
4679    }
4680
4681    pub(super) fn status_seq(frame: &PushFrame) -> Option<u64> {
4682        match frame {
4683            PushFrame::StatusChanged(status) => status.snapshot.get("seq").and_then(|v| v.as_u64()),
4684            _ => None,
4685        }
4686    }
4687
4688    pub(super) fn completion_task(frame: &PushFrame) -> Option<&str> {
4689        match frame {
4690            PushFrame::BashCompleted(completion) => Some(completion.task_id.as_str()),
4691            _ => None,
4692        }
4693    }
4694
4695    pub(super) fn push_frame_task_id(frame: &Frame) -> Option<String> {
4696        let body: serde_json::Value = serde_json::from_slice(&frame.body).expect("push body");
4697        body.get("task_id")
4698            .and_then(serde_json::Value::as_str)
4699            .map(str::to_string)
4700    }
4701}
4702
4703#[cfg(test)]
4704mod tests {
4705    use super::test_support::{
4706        completion_frame, reap_until_forgotten, route_identity, test_ctx, test_root,
4707        wait_for_actor_root_count, wait_for_watcher_count,
4708    };
4709    use super::*;
4710
4711    fn attach_error(kind: io::ErrorKind) -> SubcError {
4712        SubcError::Connect {
4713            endpoint: "127.0.0.1:1".to_string(),
4714            source: io::Error::new(kind, "constructed attach failure"),
4715        }
4716    }
4717
4718    fn auth_io_error(kind: io::ErrorKind) -> SubcError {
4719        SubcError::Auth {
4720            endpoint: "127.0.0.1:1".to_string(),
4721            source: subc_transport::AuthError::Io {
4722                stage: subc_transport::AuthStage::ServerProof,
4723                source: io::Error::new(kind, "constructed auth failure"),
4724            },
4725        }
4726    }
4727
4728    #[test]
4729    fn initial_attach_error_classifier_distinguishes_transient_and_permanent_failures() {
4730        let transient_errors = vec![
4731            attach_error(io::ErrorKind::ConnectionRefused),
4732            attach_error(io::ErrorKind::TimedOut),
4733            attach_error(io::ErrorKind::ConnectionReset),
4734            auth_io_error(io::ErrorKind::ConnectionAborted),
4735            auth_io_error(io::ErrorKind::BrokenPipe),
4736            SubcError::Auth {
4737                endpoint: "127.0.0.1:1".to_string(),
4738                source: subc_transport::AuthError::UnexpectedEof {
4739                    stage: subc_transport::AuthStage::ServerProof,
4740                    expected: 4,
4741                    actual: 0,
4742                },
4743            },
4744            SubcError::Auth {
4745                endpoint: "127.0.0.1:1".to_string(),
4746                source: subc_transport::AuthError::Timeout {
4747                    stage: subc_transport::AuthStage::ServerProof,
4748                    deadline: AUTH_DEADLINE,
4749                },
4750            },
4751        ];
4752        for error in &transient_errors {
4753            assert_eq!(
4754                classify_attach_error(error),
4755                AttachErrorClass::Transient,
4756                "expected transient: {error}"
4757            );
4758        }
4759
4760        let permanent_errors = vec![
4761            attach_error(io::ErrorKind::PermissionDenied),
4762            auth_io_error(io::ErrorKind::InvalidData),
4763            SubcError::Auth {
4764                endpoint: "127.0.0.1:1".to_string(),
4765                source: subc_transport::AuthError::InvalidServerProof,
4766            },
4767            SubcError::Auth {
4768                endpoint: "127.0.0.1:1".to_string(),
4769                source: subc_transport::AuthError::DaemonIdMismatch,
4770            },
4771            SubcError::ConnectionFile {
4772                path: PathBuf::from("subc-connection.json"),
4773                source: subc_transport::ConnectionFileError::Invalid {
4774                    reason: "constructed invalid file".to_string(),
4775                },
4776            },
4777            SubcError::NoEndpoint {
4778                path: PathBuf::from("subc-connection.json"),
4779            },
4780            SubcError::InvalidEndpoint {
4781                path: PathBuf::from("subc-connection.json"),
4782                endpoint: "not-an-ip:1234".to_string(),
4783            },
4784        ];
4785        for error in &permanent_errors {
4786            assert_eq!(
4787                classify_attach_error(error),
4788                AttachErrorClass::Permanent,
4789                "expected permanent: {error}"
4790            );
4791        }
4792    }
4793
4794    #[test]
4795    fn incompatible_wire_version_is_rejected_before_tcp_connect() {
4796        let conn_dir = tempfile::tempdir().expect("connection tempdir");
4797        let conn_path = conn_dir.path().join("subc-connection.json");
4798        let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind listener");
4799        listener
4800            .set_nonblocking(true)
4801            .expect("set listener nonblocking");
4802        let port = listener.local_addr().expect("listener addr").port();
4803        connection_file::write_atomic(
4804            &conn_path,
4805            &connection_file::ConnectionInfo {
4806                schema: connection_file::SCHEMA_VERSION,
4807                wire_version: Some(PROTOCOL_VERSION.wrapping_add(1)),
4808                endpoints: vec![connection_file::Endpoint {
4809                    host: "127.0.0.1".to_string(),
4810                    port,
4811                }],
4812                key: vec![0x42; subc_transport::KEY_LEN],
4813                daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
4814                pid: std::process::id(),
4815                daemon_ver: "subc-test".to_string(),
4816            },
4817        )
4818        .expect("write connection file");
4819
4820        let runtime = tokio::runtime::Builder::new_current_thread()
4821            .enable_all()
4822            .build()
4823            .expect("test runtime");
4824        let result = runtime.block_on(connect_and_authenticate_with_policy(
4825            &conn_path,
4826            AttachRetryPolicy {
4827                budget: Duration::from_secs(1),
4828                initial_backoff: Duration::from_millis(5),
4829                max_backoff: Duration::from_millis(10),
4830                jitter_percent: 0,
4831            },
4832        ));
4833        assert!(matches!(
4834            result,
4835            Err(SubcError::ConnectionFile {
4836                source: connection_file::ConnectionFileError::WireVersionMismatch { .. },
4837                ..
4838            })
4839        ));
4840        assert!(matches!(
4841            listener.accept(),
4842            Err(error) if error.kind() == std::io::ErrorKind::WouldBlock
4843        ));
4844    }
4845
4846    #[test]
4847    fn initial_attach_unreachable_endpoint_retries_until_budget_then_fails_loud() {
4848        let conn_dir = tempfile::tempdir().expect("connection tempdir");
4849        let conn_path = conn_dir.path().join("subc-connection.json");
4850        let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve port");
4851        let port = listener.local_addr().expect("reserved addr").port();
4852        drop(listener);
4853        connection_file::write_atomic(
4854            &conn_path,
4855            &connection_file::ConnectionInfo {
4856                schema: connection_file::SCHEMA_VERSION,
4857                wire_version: Some(PROTOCOL_VERSION),
4858                endpoints: vec![connection_file::Endpoint {
4859                    host: "127.0.0.1".to_string(),
4860                    port,
4861                }],
4862                key: vec![0x42; subc_transport::KEY_LEN],
4863                daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
4864                pid: std::process::id(),
4865                daemon_ver: "subc-test".to_string(),
4866            },
4867        )
4868        .expect("write connection file");
4869
4870        let policy = AttachRetryPolicy {
4871            budget: Duration::from_millis(40),
4872            initial_backoff: Duration::from_millis(5),
4873            max_backoff: Duration::from_millis(10),
4874            jitter_percent: 0,
4875        };
4876        let runtime = tokio::runtime::Builder::new_current_thread()
4877            .enable_all()
4878            .build()
4879            .expect("test runtime");
4880        let started_at = Instant::now();
4881        let result = runtime.block_on(connect_and_authenticate_with_policy(&conn_path, policy));
4882        let elapsed = started_at.elapsed();
4883        let error = match result {
4884            Ok(_) => panic!("unreachable endpoint unexpectedly attached"),
4885            Err(error) => error,
4886        };
4887
4888        assert!(matches!(error, SubcError::Connect { .. }), "{error}");
4889        assert!(
4890            elapsed >= Duration::from_millis(35),
4891            "retry budget ended too early: {elapsed:?}"
4892        );
4893        assert!(
4894            elapsed < Duration::from_secs(1),
4895            "retry budget was not bounded: {elapsed:?}"
4896        );
4897    }
4898
4899    fn due_maintenance_jobs_without_actor_context(
4900        live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4901        budget: usize,
4902        pending_bind_roots: &HashSet<ProjectRootId>,
4903    ) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
4904        due_maintenance_jobs(
4905            live_roots,
4906            None,
4907            &HashMap::new(),
4908            &HashSet::new(),
4909            budget,
4910            pending_bind_roots,
4911        )
4912    }
4913
4914    fn actor_ctx_with_dirty_search_index(
4915        root: &Path,
4916        storage: &Path,
4917        file_name: &str,
4918        old_contents: &str,
4919        new_contents: &str,
4920    ) -> (Arc<AppContext>, PathBuf, PathBuf) {
4921        let file = root.join(file_name);
4922        std::fs::write(&file, old_contents).expect("write source");
4923        let canonical_root = std::fs::canonicalize(root).expect("canonical root");
4924        let ctx = Arc::new(AppContext::new(
4925            Box::new(crate::parser::TreeSitterProvider::new()),
4926            Config {
4927                project_root: Some(root.to_path_buf()),
4928                storage_dir: Some(storage.to_path_buf()),
4929                ..Config::default()
4930            },
4931        ));
4932        ctx.set_canonical_cache_root(canonical_root.clone());
4933
4934        let cache_dir = crate::search_index::resolve_cache_dir(&canonical_root, Some(storage));
4935        let mut index = crate::search_index::SearchIndex::build(&canonical_root);
4936        let git_head = index.stored_git_head().map(str::to_owned);
4937        index.write_to_disk(&cache_dir, git_head.as_deref());
4938
4939        std::fs::write(&file, new_contents).expect("edit source");
4940        index.update_file(&file);
4941        *ctx.search_index()
4942            .write()
4943            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
4944        (ctx, canonical_root, cache_dir)
4945    }
4946
4947    #[test]
4948    fn graceful_shutdown_flushes_every_actor_search_index() {
4949        let storage = tempfile::tempdir().expect("storage tempdir");
4950        let (root1_dir, root1) = test_root("shutdown-flush-root-1");
4951        let (root2_dir, root2) = test_root("shutdown-flush-root-2");
4952        let (ctx1, canonical_root1, cache_dir1) = actor_ctx_with_dirty_search_index(
4953            root1_dir.path(),
4954            storage.path(),
4955            "alpha.txt",
4956            "old actor one token\n",
4957            "new actor one token\n",
4958        );
4959        let (ctx2, canonical_root2, cache_dir2) = actor_ctx_with_dirty_search_index(
4960            root2_dir.path(),
4961            storage.path(),
4962            "beta.txt",
4963            "old actor two token\n",
4964            "new actor two token\n",
4965        );
4966
4967        let executor = Executor::new();
4968        assert!(executor.register_actor(root1.clone(), Arc::clone(&ctx1)));
4969        assert!(executor.register_actor(root2.clone(), Arc::clone(&ctx2)));
4970
4971        flush_actor_indexes_on_graceful_shutdown(&executor.actor_contexts());
4972
4973        let mut restored1 =
4974            crate::search_index::SearchIndex::read_from_disk(&cache_dir1, &canonical_root1)
4975                .expect("load flushed root one index");
4976        restored1.ready = true;
4977        assert_eq!(
4978            restored1
4979                .grep("new actor one token", true, &[], &[], &canonical_root1, 10)
4980                .matches
4981                .len(),
4982            1,
4983            "graceful subc shutdown should flush the first root's trigram delta"
4984        );
4985
4986        let mut restored2 =
4987            crate::search_index::SearchIndex::read_from_disk(&cache_dir2, &canonical_root2)
4988                .expect("load flushed root two index");
4989        restored2.ready = true;
4990        assert_eq!(
4991            restored2
4992                .grep("new actor two token", true, &[], &[], &canonical_root2, 10)
4993                .matches
4994                .len(),
4995            1,
4996            "graceful subc shutdown should flush every registered root"
4997        );
4998    }
4999
5000    #[test]
5001    fn idle_root_reaper_closes_artifacts_and_stops_watcher() {
5002        let _ = env_logger::builder().is_test(true).try_init();
5003        let (root_dir, root) = test_root("idle-root-reaper");
5004        let storage = tempfile::tempdir().expect("storage tempdir");
5005        std::fs::write(
5006            root_dir.path().join("main.rs"),
5007            "fn entry() { leaf(); }\nfn leaf() {}\n",
5008        )
5009        .expect("source file");
5010        let canonical_root = std::fs::canonicalize(root_dir.path()).expect("canonical root");
5011        let app = App::default_shared();
5012        let ctx = Arc::new(AppContext::from_app(
5013            Arc::clone(&app),
5014            Config {
5015                project_root: Some(canonical_root.clone()),
5016                storage_dir: Some(storage.path().to_path_buf()),
5017                callgraph_store: true,
5018                search_index: true,
5019                ..Config::default()
5020            },
5021        ));
5022        ctx.set_canonical_cache_root(canonical_root.clone());
5023        assert!(ctx
5024            .ensure_callgraph_store()
5025            .expect("build callgraph store")
5026            .is_some());
5027
5028        let cache_dir =
5029            crate::search_index::resolve_cache_dir(&canonical_root, Some(storage.path()));
5030        let mut index = crate::search_index::SearchIndex::build(&canonical_root);
5031        let git_head = index.stored_git_head().map(str::to_owned);
5032        index.write_to_disk(&cache_dir, git_head.as_deref());
5033        *ctx.search_index()
5034            .write()
5035            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
5036        // Seed a completed warm verification so the test can prove eviction
5037        // downgrades it to Strict rather than passing vacuously.
5038        let seeded_generation =
5039            crate::cache_freshness::artifact_generation(&cache_dir.join("cache.bin"))
5040                .expect("seeded artifact generation");
5041        crate::cache_freshness::record_verify_completed(
5042            &canonical_root,
5043            crate::cache_freshness::VerifyArtifact::Search,
5044            Some(seeded_generation),
5045        );
5046        assert!(
5047            matches!(
5048                crate::cache_freshness::warm_verify_plan(
5049                    canonical_root.as_path(),
5050                    crate::cache_freshness::VerifyArtifact::Search,
5051                    Some(seeded_generation),
5052                ),
5053                crate::cache_freshness::WarmVerifyPlan::Skip
5054            ),
5055            "memo must be warm before eviction for the downgrade assertion to bite"
5056        );
5057
5058        let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
5059        let _dispatch_tx = dispatch_tx;
5060        let shutdown = Arc::new(AtomicBool::new(false));
5061        let thread_shutdown = Arc::clone(&shutdown);
5062        let join = std::thread::spawn(move || {
5063            while !thread_shutdown.load(Ordering::SeqCst) {
5064                std::thread::yield_now();
5065            }
5066        });
5067        ctx.install_watcher_runtime(
5068            dispatch_rx,
5069            crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
5070        );
5071        wait_for_watcher_count(&ctx, 1);
5072
5073        let executor = Arc::new(Executor::new());
5074        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5075        ctx.mark_subc_unbound();
5076        let mut live_roots = HashMap::new();
5077        let mut meta = RootMeta::new(Instant::now());
5078        meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
5079        meta.unbound_quiesced = true;
5080        live_roots.insert(root.clone(), meta);
5081
5082        let message = idle_root_eviction_message(&root, &ctx.memory_root_snapshot(), None);
5083        assert!(message.contains("evicted idle root"));
5084        assert!(message.contains("freed ~"));
5085        assert!(message.contains("semantic"));
5086        assert!(!message.contains("semantic not estimated retained"));
5087        assert!(message.contains("trigram"));
5088        assert!(message.contains("retained: bash"));
5089        assert!(message.contains("parser_pool"));
5090
5091        assert_eq!(
5092            reap_idle_roots(
5093                Instant::now(),
5094                &mut live_roots,
5095                &HashMap::new(),
5096                &HashMap::new(),
5097                &executor,
5098                &DispatchPathMetrics::new(),
5099            )
5100            .evicted,
5101            1
5102        );
5103        assert!(ctx.search_index().read().unwrap().is_none());
5104        wait_for_watcher_count(&ctx, 0);
5105        // The watcher stopped with the eviction, so the idle interval is
5106        // unobserved: the pre-seeded warm-verify memo (Skip) must fall back to
5107        // strict content verification (stat-first would miss same-size,
5108        // preserved-mtime edits made while nobody was watching).
5109        assert!(
5110            matches!(
5111                crate::cache_freshness::warm_verify_plan(
5112                    canonical_root.as_path(),
5113                    crate::cache_freshness::VerifyArtifact::Search,
5114                    Some(seeded_generation),
5115                ),
5116                crate::cache_freshness::WarmVerifyPlan::Strict
5117            ),
5118            "idle eviction must force strict re-verification"
5119        );
5120        assert!(
5121            crate::search_index::SearchIndex::read_from_disk(&cache_dir, &canonical_root).is_some()
5122        );
5123        ctx.mark_subc_bound();
5124        assert!(ctx
5125            .ensure_callgraph_store()
5126            .expect("reopen callgraph store")
5127            .is_some());
5128        assert!(live_roots[&root].idle_artifacts_evicted);
5129    }
5130
5131    #[test]
5132    fn idle_root_reaper_applies_ttl_to_unbound_roots() {
5133        let (_root_dir, root) = test_root("idle-root-ttl-gate");
5134        let ctx = test_ctx();
5135        let executor = Arc::new(Executor::new());
5136        assert!(executor.register_actor(root.clone(), ctx));
5137        let ctx = executor.actor_context(&root).expect("actor context");
5138        ctx.mark_subc_unbound();
5139        let now = Instant::now();
5140        let mut meta = RootMeta::new(now);
5141        meta.unbound_quiesced = true;
5142        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5143
5144        // A recently-unbound root stays warm: a transient unbind (host
5145        // restart) must not pay the strict-verify + forced-rebuild teardown
5146        // on the next maintenance sweep.
5147        assert_eq!(
5148            reap_idle_roots(
5149                now,
5150                &mut live_roots,
5151                &HashMap::new(),
5152                &HashMap::new(),
5153                &executor,
5154                &DispatchPathMetrics::new(),
5155            )
5156            .evicted,
5157            0
5158        );
5159        assert!(!live_roots[&root].idle_artifacts_evicted);
5160
5161        // Past the TTL the same unbound root pays the full teardown. The
5162        // pending reconciliation paths retained across the transient-unbind
5163        // window would block eviction forever through
5164        // `artifact_eviction_blocked`; the reaper disposes them because the
5165        // strict gap invalidation subsumes their purpose.
5166        ctx.add_pending_search_index_paths([root.as_path().join("retained.rs")]);
5167        assert_eq!(
5168            reap_idle_roots(
5169                now + IDLE_ROOT_TTL,
5170                &mut live_roots,
5171                &HashMap::new(),
5172                &HashMap::new(),
5173                &executor,
5174                &DispatchPathMetrics::new(),
5175            )
5176            .evicted,
5177            1
5178        );
5179        assert!(live_roots[&root].idle_artifacts_evicted);
5180        assert!(
5181            ctx.take_pending_search_index_paths().is_empty(),
5182            "TTL eviction must dispose retained pending reconciliation paths"
5183        );
5184    }
5185
5186    #[test]
5187    fn blocked_ttl_eviction_restores_taken_pending_reconciliation_state() {
5188        let (_root_dir, root) = test_root("ttl-eviction-blocked-restore");
5189        let ctx = test_ctx();
5190        let executor = Arc::new(Executor::new());
5191        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5192        ctx.mark_subc_unbound();
5193
5194        // Retained pending path from the transient-unbind window plus a
5195        // SECONDARY eviction blocker (a non-ready resident search index, the
5196        // dirty-index blocker in artifact_eviction_blocked). Disposal must be
5197        // transactional: the blocked eviction may be followed by a rebind, and
5198        // the path is the only repair record for its consumed watcher event.
5199        let pending = root.as_path().join("edited-while-unbound.rs");
5200        ctx.add_pending_search_index_paths([pending.clone()]);
5201        let dirty_source = root.as_path().join("dirty.rs");
5202        std::fs::write(&dirty_source, "fn dirty() {}\n").expect("dirty source");
5203        let mut dirty = crate::search_index::SearchIndex::new();
5204        dirty.ready = true;
5205        dirty.update_file(&dirty_source);
5206        *ctx.search_index()
5207            .write()
5208            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(dirty);
5209        assert!(ctx.artifact_eviction_blocked());
5210
5211        let mut live_roots = HashMap::new();
5212        let mut meta = RootMeta::new(Instant::now());
5213        meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
5214        meta.unbound_quiesced = true;
5215        live_roots.insert(root.clone(), meta);
5216
5217        assert_eq!(
5218            reap_idle_roots(
5219                Instant::now(),
5220                &mut live_roots,
5221                &HashMap::new(),
5222                &HashMap::new(),
5223                &executor,
5224                &DispatchPathMetrics::new(),
5225            )
5226            .evicted,
5227            0,
5228            "the dirty index must still block this eviction"
5229        );
5230        assert_eq!(
5231            ctx.take_pending_search_index_paths(),
5232            vec![pending],
5233            "a blocked eviction must restore the taken pending paths"
5234        );
5235    }
5236
5237    #[test]
5238    fn idle_reap_with_bound_route_keeps_watcher_running() {
5239        let (_root_dir, root) = test_root("bound-root-reap-gate");
5240        let ctx = test_ctx();
5241        let executor = Arc::new(Executor::new());
5242        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5243
5244        let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
5245        let _dispatch_tx = dispatch_tx;
5246        let shutdown = Arc::new(AtomicBool::new(false));
5247        let thread_shutdown = Arc::clone(&shutdown);
5248        let join = std::thread::spawn(move || {
5249            while !thread_shutdown.load(Ordering::SeqCst) {
5250                std::thread::yield_now();
5251            }
5252        });
5253        ctx.install_watcher_runtime(
5254            dispatch_rx,
5255            crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
5256        );
5257
5258        let mut meta = RootMeta::new(Instant::now());
5259        meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
5260        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5261        let bound = HashMap::from([(root, HashSet::from([route_key(7, 1)]))]);
5262        assert_eq!(
5263            reap_idle_roots(
5264                Instant::now(),
5265                &mut live_roots,
5266                &HashMap::new(),
5267                &bound,
5268                &executor,
5269                &DispatchPathMetrics::new(),
5270            )
5271            .evicted,
5272            0
5273        );
5274        wait_for_watcher_count(&ctx, 1);
5275        ctx.stop_watcher_runtime_in_background();
5276        wait_for_watcher_count(&ctx, 0);
5277    }
5278
5279    #[test]
5280    fn deleted_root_with_bound_route_is_reclaimed_after_confirmation_and_routes_are_purged() {
5281        let (root_dir, root) = test_root("deleted-bound-root-reap");
5282        let executor = Arc::new(Executor::new());
5283        assert!(executor.register_actor(root.clone(), test_ctx()));
5284        root_dir.close().expect("delete project root");
5285
5286        let route = route_key(19, 3);
5287        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
5288        let cancel_signal = PersistentCancelSignal::new();
5289        let mut routes = HashMap::from([(route, route_identity(&root, "deleted-route"))]);
5290        let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
5291        let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
5292        let mut route_bash_cancels = HashMap::from([(
5293            route,
5294            bash::RouteBashCancel {
5295                token: cancel_signal.clone(),
5296                active_waits: 0,
5297            },
5298        )]);
5299        let metrics = DispatchPathMetrics::new();
5300
5301        let first = reap_idle_roots(
5302            Instant::now(),
5303            &mut live_roots,
5304            &HashMap::new(),
5305            &root_channels,
5306            &executor,
5307            &metrics,
5308        );
5309        assert!(first.forgotten_deleted_roots.is_empty());
5310        assert!(executor.actor_registered(&root));
5311
5312        let mut forgotten = Vec::new();
5313        for _ in 0..100 {
5314            let outcome = reap_idle_roots(
5315                Instant::now(),
5316                &mut live_roots,
5317                &HashMap::new(),
5318                &root_channels,
5319                &executor,
5320                &metrics,
5321            );
5322            if !outcome.forgotten_deleted_roots.is_empty() {
5323                forgotten = outcome.forgotten_deleted_roots;
5324                break;
5325            }
5326            std::thread::sleep(Duration::from_millis(10));
5327        }
5328        assert_eq!(forgotten, vec![root.clone()]);
5329        assert!(!executor.actor_registered(&root));
5330
5331        let mut retry_buffer = HashMap::new();
5332        let mut reclaimed_routes = ReclaimedRoutes::default();
5333        let mut session_identity = HashMap::new();
5334        let mut push_buffer = HashMap::new();
5335        let mut bg_subs = HashMap::new();
5336        let mut bg_sub_by_session = HashMap::new();
5337        let mut bg_wake_pending = HashSet::new();
5338        let mut bg_wake_epoch = HashMap::new();
5339        let mut pending_bash_asks = HashMap::new();
5340        purge_deleted_root_residents(
5341            &root,
5342            &mut routes,
5343            &mut root_channels,
5344            &mut installed_route_epochs,
5345            &mut route_bash_cancels,
5346            &mut retry_buffer,
5347            &mut reclaimed_routes,
5348            &mut session_identity,
5349            &mut push_buffer,
5350            &mut bg_subs,
5351            &mut bg_sub_by_session,
5352            &mut bg_wake_pending,
5353            &mut bg_wake_epoch,
5354            &mut pending_bash_asks,
5355        );
5356
5357        assert!(routes.is_empty());
5358        assert!(root_channels.is_empty());
5359        assert!(installed_route_epochs.is_empty());
5360        assert!(route_bash_cancels.is_empty());
5361        assert!(reclaimed_routes.contains(route));
5362        assert!(cancel_signal.is_cancelled());
5363    }
5364
5365    /// The control for the deleted-root reclamation above: a root whose
5366    /// directory still EXISTS must stay retained while it holds a bound route,
5367    /// even with every other reap precondition satisfied. Without this, the
5368    /// suite cannot tell "reclaim roots that are provably gone" apart from
5369    /// "reap any root that looks idle" — the second would tear down live
5370    /// sessions, and both satisfy the deleted-root tests.
5371    #[test]
5372    fn live_root_with_bound_route_is_never_reclaimed() {
5373        let (_root_dir, root) = test_root("live-bound-root-retained");
5374        let ctx = test_ctx();
5375        ctx.mark_subc_unbound();
5376        let executor = Arc::new(Executor::new());
5377        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5378
5379        let route = RouteChannel {
5380            channel: 7,
5381            epoch: 1,
5382        };
5383        let mut meta = RootMeta::new(Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1));
5384        meta.unbound_quiesced = true;
5385        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5386        let root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
5387
5388        // Sweep three times because reclamation requires the root path to be
5389        // observed missing on two CONSECUTIVE sweeps. A single sweep would pass
5390        // here even if reclamation were wrongly unconditional, since the first
5391        // absence never reclaims on its own.
5392        for _ in 0..3 {
5393            let outcome = reap_idle_roots(
5394                Instant::now(),
5395                &mut live_roots,
5396                &HashMap::new(),
5397                &root_channels,
5398                &executor,
5399                &DispatchPathMetrics::new(),
5400            );
5401            assert!(
5402                outcome.forgotten_deleted_roots.is_empty(),
5403                "a root whose directory exists must never be forgotten"
5404            );
5405        }
5406
5407        assert!(live_roots.contains_key(&root), "live root must be retained");
5408        assert!(
5409            executor.actor_registered(&root),
5410            "live root's actor must survive"
5411        );
5412        assert!(
5413            root.as_path().exists(),
5414            "test vehicle must keep the directory alive; otherwise this control proves nothing"
5415        );
5416    }
5417
5418    #[test]
5419    fn deleted_root_is_not_reclaimed_on_first_absence_observation() {
5420        let (root_dir, root) = test_root("deleted-root-first-observation");
5421        let ctx = test_ctx();
5422        ctx.mark_subc_unbound();
5423        let executor = Arc::new(Executor::new());
5424        assert!(executor.register_actor(root.clone(), ctx));
5425        root_dir.close().expect("delete project root");
5426
5427        let mut meta = RootMeta::new(Instant::now());
5428        meta.unbound_quiesced = true;
5429        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5430        let outcome = reap_idle_roots(
5431            Instant::now(),
5432            &mut live_roots,
5433            &HashMap::new(),
5434            &HashMap::new(),
5435            &executor,
5436            &DispatchPathMetrics::new(),
5437        );
5438
5439        assert!(outcome.forgotten_deleted_roots.is_empty());
5440        assert!(live_roots.contains_key(&root));
5441        assert!(executor.actor_registered(&root));
5442    }
5443
5444    #[test]
5445    fn observing_root_again_resets_deleted_sweep_confirmation() {
5446        let (root_dir, root) = test_root("deleted-root-observation-reset");
5447        let ctx = test_ctx();
5448        ctx.mark_subc_unbound();
5449        let executor = Arc::new(Executor::new());
5450        assert!(executor.register_actor(root.clone(), ctx));
5451        root_dir.close().expect("delete project root");
5452
5453        let mut meta = RootMeta::new(Instant::now());
5454        meta.unbound_quiesced = true;
5455        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5456        let pending_binds = HashMap::new();
5457        let root_channels = HashMap::new();
5458        let metrics = DispatchPathMetrics::new();
5459
5460        let first = reap_idle_roots(
5461            Instant::now(),
5462            &mut live_roots,
5463            &pending_binds,
5464            &root_channels,
5465            &executor,
5466            &metrics,
5467        );
5468        assert!(first.forgotten_deleted_roots.is_empty());
5469
5470        std::fs::create_dir_all(root.as_path()).expect("restore project root");
5471        reap_idle_roots(
5472            Instant::now(),
5473            &mut live_roots,
5474            &pending_binds,
5475            &root_channels,
5476            &executor,
5477            &metrics,
5478        );
5479        std::fs::remove_dir_all(root.as_path()).expect("delete project root again");
5480
5481        let after_reset = reap_idle_roots(
5482            Instant::now(),
5483            &mut live_roots,
5484            &pending_binds,
5485            &root_channels,
5486            &executor,
5487            &metrics,
5488        );
5489        assert!(after_reset.forgotten_deleted_roots.is_empty());
5490        assert!(live_roots.contains_key(&root));
5491        assert!(executor.actor_registered(&root));
5492    }
5493
5494    #[test]
5495    fn deleted_idle_root_is_fully_forgotten_and_status_counts_drop() {
5496        let (root_dir, root) = test_root("deleted-root-reap");
5497        let app = App::default_shared();
5498        let ctx = Arc::new(AppContext::from_app(
5499            Arc::clone(&app),
5500            Config {
5501                project_root: Some(root.as_path().to_path_buf()),
5502                ..Config::default()
5503            },
5504        ));
5505        ctx.set_canonical_cache_root(root.as_path().to_path_buf());
5506        ctx.mark_subc_unbound();
5507        let executor = Arc::new(Executor::new());
5508        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5509        assert_eq!(app.actor_root_count(), 1);
5510        drop(ctx);
5511        root_dir.close().expect("delete project root");
5512
5513        let mut meta = RootMeta::new(Instant::now());
5514        meta.unbound_quiesced = true;
5515        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5516        let outcome = reap_until_forgotten(
5517            &root,
5518            &mut live_roots,
5519            &HashMap::new(),
5520            &HashMap::new(),
5521            &executor,
5522            &DispatchPathMetrics::new(),
5523        );
5524        assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
5525        assert!(!executor.actor_registered(&root));
5526        assert!(!live_roots.contains_key(&root));
5527        wait_for_actor_root_count(&app, 0);
5528
5529        let status_ctx = AppContext::from_app(app, Config::default());
5530        let status = status_ctx.build_status_snapshot();
5531        assert_eq!(status["runtime"]["live_actor_roots"], 0);
5532        assert_eq!(status["runtime"]["open_routes"], 0);
5533    }
5534
5535    #[test]
5536    fn deleted_root_reap_blocker_census_is_exposed_in_health_metrics() {
5537        let (root_dir, root) = test_root("deleted-root-reap-census");
5538        let executor = Arc::new(Executor::new());
5539        assert!(executor.register_actor(root.clone(), test_ctx()));
5540        root_dir.close().expect("delete project root");
5541
5542        let mut live_roots = HashMap::from([(root, RootMeta::new(Instant::now()))]);
5543        let metrics = DispatchPathMetrics::new();
5544        let outcome = reap_idle_roots(
5545            Instant::now(),
5546            &mut live_roots,
5547            &HashMap::new(),
5548            &HashMap::new(),
5549            &executor,
5550            &metrics,
5551        );
5552        assert_eq!(outcome.evicted, 0);
5553
5554        let report = build_health_report(
5555            &executor,
5556            &HashMap::new(),
5557            &metrics,
5558            &crate::context::App::default_shared(),
5559        );
5560        let reap = report
5561            .metrics
5562            .as_ref()
5563            .and_then(|metrics| metrics.get("reap"))
5564            .expect("reap health metrics");
5565        assert_eq!(reap["deleted_retained"].as_u64(), Some(1));
5566        assert_eq!(reap["blockers"]["absence_unconfirmed"].as_u64(), Some(1));
5567        assert_eq!(reap["blockers"]["unbound_quiesced"].as_u64(), Some(0));
5568        assert_eq!(reap["blockers"]["actor_busy"].as_u64(), Some(0));
5569    }
5570
5571    #[test]
5572    fn connection_exit_quiesces_queued_maintenance_and_deleted_root_is_purged() {
5573        let (root_dir, root) = test_root("connection-exit-deleted-root");
5574        let executor = Arc::new(Executor::new());
5575        assert!(executor.register_actor(root.clone(), test_ctx()));
5576
5577        let route = route_key(11, 1);
5578        let mut meta = RootMeta::new(Instant::now());
5579        meta.maintenance_pending = true;
5580        meta.maintenance_queued_kinds
5581            .push_back(MaintenanceDrainKind::CompletionDrains);
5582        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5583        let mut pending_binds = HashMap::new();
5584        let mut routes = HashMap::from([(route, route_identity(&root, "abandoned"))]);
5585        let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
5586        let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
5587        let mut route_bash_cancels = HashMap::new();
5588
5589        quiesce_connection_roots(
5590            &mut live_roots,
5591            &mut pending_binds,
5592            &mut routes,
5593            &mut root_channels,
5594            &mut installed_route_epochs,
5595            &mut route_bash_cancels,
5596            &executor,
5597        );
5598        assert!(live_roots[&root].unbound_quiesced);
5599        assert!(!live_roots[&root].maintenance_pending);
5600        assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
5601        assert!(routes.is_empty());
5602        assert!(root_channels.is_empty());
5603
5604        root_dir.close().expect("delete project root");
5605        let metrics = DispatchPathMetrics::new();
5606        let outcome = reap_until_forgotten(
5607            &root,
5608            &mut live_roots,
5609            &pending_binds,
5610            &root_channels,
5611            &executor,
5612            &metrics,
5613        );
5614        let mut session_identity = HashMap::new();
5615        let mut push_buffer = HashMap::new();
5616        let mut bg_subs = HashMap::new();
5617        let mut bg_sub_by_session = HashMap::new();
5618        let mut bg_wake_pending = HashSet::new();
5619        let mut bg_wake_epoch = HashMap::new();
5620        let mut pending_bash_asks = HashMap::new();
5621        let mut retry_buffer = HashMap::new();
5622        let mut reclaimed_routes = ReclaimedRoutes::default();
5623        for forgotten in &outcome.forgotten_deleted_roots {
5624            purge_deleted_root_residents(
5625                forgotten,
5626                &mut routes,
5627                &mut root_channels,
5628                &mut installed_route_epochs,
5629                &mut route_bash_cancels,
5630                &mut retry_buffer,
5631                &mut reclaimed_routes,
5632                &mut session_identity,
5633                &mut push_buffer,
5634                &mut bg_subs,
5635                &mut bg_sub_by_session,
5636                &mut bg_wake_pending,
5637                &mut bg_wake_epoch,
5638                &mut pending_bash_asks,
5639            );
5640        }
5641
5642        assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
5643        assert!(!executor.actor_registered(&root));
5644        assert!(!live_roots.contains_key(&root));
5645    }
5646
5647    #[test]
5648    fn unbound_root_quiesces_maintenance_without_removing_actor() {
5649        let (_root_dir, root) = test_root("unbound-root-quiesce");
5650        let ctx = test_ctx();
5651        let executor = Arc::new(Executor::new());
5652        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5653        let mut meta = RootMeta::new(Instant::now());
5654        meta.maintenance_pending = true;
5655        meta.maintenance_jobs_in_flight = 1;
5656        meta.maintenance_queued_kinds
5657            .push_back(MaintenanceDrainKind::ConfigureTail);
5658        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5659        // Warm state planted before the unbind: quiesce must keep it.
5660        *ctx.search_index()
5661            .write()
5662            .unwrap_or_else(std::sync::PoisonError::into_inner) =
5663            Some(crate::search_index::SearchIndex::new());
5664        ctx.set_cache_writer_capabilities(true, true);
5665        let pending = root.as_path().join("pending.rs");
5666        ctx.add_pending_search_index_paths([pending.clone()]);
5667        // A warm verify memo must survive the transient unbind: the watcher
5668        // keeps running, so no unobserved window exists and the next warm
5669        // reload must not pay a strict full-corpus re-hash.
5670        let canonical_root = root.as_path().to_path_buf();
5671        let artifact = canonical_root.join("cache.bin");
5672        std::fs::write(&artifact, b"warm-artifact").expect("write artifact");
5673        let seeded_generation = crate::cache_freshness::artifact_generation(&artifact);
5674        crate::cache_freshness::record_verify_completed(
5675            &canonical_root,
5676            crate::cache_freshness::VerifyArtifact::Search,
5677            seeded_generation,
5678        );
5679        assert!(matches!(
5680            crate::cache_freshness::warm_verify_plan(
5681                &canonical_root,
5682                crate::cache_freshness::VerifyArtifact::Search,
5683                seeded_generation,
5684            ),
5685            crate::cache_freshness::WarmVerifyPlan::Skip
5686        ));
5687        // A live watcher runtime must survive quiesce (its events accumulate
5688        // for the rebind replay).
5689        let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
5690        let _dispatch_tx = dispatch_tx;
5691        let shutdown = Arc::new(AtomicBool::new(false));
5692        let thread_shutdown = Arc::clone(&shutdown);
5693        let join = std::thread::spawn(move || {
5694            while !thread_shutdown.load(Ordering::SeqCst) {
5695                std::thread::yield_now();
5696            }
5697        });
5698        ctx.install_watcher_runtime(
5699            dispatch_rx,
5700            crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
5701        );
5702        assert!(ctx.watcher_runtime_active());
5703
5704        quiesce_unbound_root(&root, &mut live_roots, &executor);
5705        let meta = &live_roots[&root];
5706        assert!(meta.unbound_quiesced);
5707        assert!(ctx.subc_unbound_quiesced());
5708        assert!(meta.maintenance_pending);
5709        assert!(meta.maintenance_queued_kinds.is_empty());
5710        assert!(executor.actor_registered(&root));
5711        // Transient unbind keeps the root warm: resident artifacts stay
5712        // resident, no forced callgraph rebuild is planted, and pending
5713        // reconciliation paths survive for the rebind replay.
5714        assert!(
5715            ctx.search_index()
5716                .read()
5717                .unwrap_or_else(std::sync::PoisonError::into_inner)
5718                .is_some(),
5719            "quiesce must not evict resident artifacts"
5720        );
5721        assert_eq!(
5722            ctx.pending_callgraph_store_force_token(),
5723            None,
5724            "quiesce must not force a callgraph rebuild"
5725        );
5726        assert_eq!(
5727            ctx.take_pending_search_index_paths(),
5728            vec![pending],
5729            "quiesce must retain pending watcher-derived paths"
5730        );
5731        assert!(
5732            matches!(
5733                crate::cache_freshness::warm_verify_plan(
5734                    &canonical_root,
5735                    crate::cache_freshness::VerifyArtifact::Search,
5736                    seeded_generation,
5737                ),
5738                crate::cache_freshness::WarmVerifyPlan::Skip
5739            ),
5740            "quiesce must not invalidate the warm verify memo"
5741        );
5742        assert!(
5743            ctx.watcher_runtime_active(),
5744            "quiesce must not stop a running watcher"
5745        );
5746        ctx.stop_watcher_runtime();
5747
5748        let meta = live_roots.get_mut(&root).expect("root metadata");
5749        note_maintenance_completion(
5750            meta,
5751            Some(MaintenanceDrainKind::ConfigureTail),
5752            false,
5753            meta.unbound_quiesced,
5754        );
5755        assert!(!meta.maintenance_pending);
5756        assert!(meta.maintenance_queued_kinds.is_empty());
5757    }
5758
5759    #[test]
5760    fn same_root_higher_epoch_replacement_does_not_quiesce_between_generations() {
5761        let (_dir, root) = test_root("same-root-replacement");
5762        let route = route_key(7, 1);
5763        let installed_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
5764        let root_channels = HashMap::new();
5765
5766        assert!(!route_removal_will_quiesce_root(
5767            &root,
5768            route,
5769            &installed_channels,
5770            false,
5771            Some(&root),
5772        ));
5773        assert!(route_removal_will_quiesce_root(
5774            &root,
5775            route,
5776            &installed_channels,
5777            false,
5778            None,
5779        ));
5780        assert!(!should_quiesce_removed_root(
5781            &root,
5782            &root_channels,
5783            false,
5784            Some(&root),
5785        ));
5786        assert!(should_quiesce_removed_root(
5787            &root,
5788            &root_channels,
5789            false,
5790            None,
5791        ));
5792        assert!(!should_quiesce_removed_root(
5793            &root,
5794            &root_channels,
5795            true,
5796            None,
5797        ));
5798    }
5799
5800    #[test]
5801    fn root_quiesces_only_after_its_last_route_is_removed_and_reactivates_on_bind() {
5802        let (_root_dir, root) = test_root("unbound-root-route-count");
5803        let executor = Arc::new(Executor::new());
5804        assert!(executor.register_actor(root.clone(), test_ctx()));
5805        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
5806        let mut root_channels = HashMap::from([(
5807            root.clone(),
5808            HashSet::from([route_key(7, 1), route_key(8, 1)]),
5809        )]);
5810
5811        remove_root_channel(&mut root_channels, &root, route_key(7, 1));
5812        if !root_channels.contains_key(&root) {
5813            quiesce_unbound_root(&root, &mut live_roots, &executor);
5814        }
5815        assert!(!live_roots[&root].unbound_quiesced);
5816
5817        remove_root_channel(&mut root_channels, &root, route_key(8, 1));
5818        if !root_channels.contains_key(&root) {
5819            quiesce_unbound_root(&root, &mut live_roots, &executor);
5820        }
5821        assert!(live_roots[&root].unbound_quiesced);
5822
5823        live_roots
5824            .get_mut(&root)
5825            .expect("root metadata")
5826            .note_activity();
5827        assert!(
5828            live_roots[&root].unbound_quiesced,
5829            "late asynchronous activity must not reactivate an unbound root"
5830        );
5831
5832        live_roots
5833            .get_mut(&root)
5834            .expect("root metadata")
5835            .reactivate_bound();
5836        assert!(!live_roots[&root].unbound_quiesced);
5837    }
5838
5839    #[test]
5840    fn allocator_pressure_relief_requires_every_root_to_be_idle() {
5841        let (_idle_dir, idle_root) = test_root("allocator-relief-idle");
5842        let (_active_dir, active_root) = test_root("allocator-relief-active");
5843        let now = Instant::now();
5844        let mut live_roots = HashMap::new();
5845        let mut idle = RootMeta::new(now);
5846        idle.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
5847        live_roots.insert(idle_root, idle);
5848        assert!(process_has_been_idle(now, &live_roots));
5849
5850        live_roots.insert(active_root.clone(), RootMeta::new(now));
5851        assert!(!process_has_been_idle(now, &live_roots));
5852
5853        let active = live_roots
5854            .get_mut(&active_root)
5855            .expect("active root metadata");
5856        active.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
5857        active.active_bash_waits = 1;
5858        assert!(!process_has_been_idle(now, &live_roots));
5859    }
5860
5861    #[test]
5862    fn pressure_relief_log_reports_before_and_after_measurements() {
5863        let allocator = crate::memory::AllocatorMemorySnapshot {
5864            status: "measured",
5865            bytes_in_use: Some(8 * 1024 * 1024),
5866            size_allocated: Some(12 * 1024 * 1024),
5867            retained_slack_bytes: Some(4 * 1024 * 1024),
5868            not_estimated: None,
5869        };
5870        let relief = crate::memory::AllocatorPressureRelief {
5871            bytes_released: 3 * 1024 * 1024,
5872            rss_before_bytes: Some(20 * 1024 * 1024),
5873            rss_after_bytes: Some(17 * 1024 * 1024),
5874            allocator_before: allocator.clone(),
5875            allocator_after: crate::memory::AllocatorMemorySnapshot {
5876                size_allocated: Some(9 * 1024 * 1024),
5877                retained_slack_bytes: Some(1024 * 1024),
5878                ..allocator
5879            },
5880        };
5881        let message = pressure_relief_label(&relief);
5882        assert!(message.contains("RSS 20.0 MB -> 17.0 MB"));
5883        assert!(message.contains("allocated 12.0 MB -> 9.0 MB"));
5884        assert!(message.contains("slack 4.0 MB -> 1.0 MB"));
5885        assert!(message.contains("reported 3.0 MB released"));
5886    }
5887
5888    #[test]
5889    fn due_maintenance_jobs_skip_poisoned_roots() {
5890        let (_healthy_dir, healthy_root) = test_root("maintenance-healthy");
5891        let (_poisoned_dir, poisoned_root) = test_root("maintenance-poisoned");
5892        let mut live_roots = HashMap::new();
5893        live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
5894        let mut poisoned_meta = RootMeta::new(Instant::now());
5895        poisoned_meta.maintenance_poisoned = true;
5896        live_roots.insert(poisoned_root.clone(), poisoned_meta);
5897
5898        let (due, deferred) = due_maintenance_jobs_without_actor_context(
5899            &mut live_roots,
5900            MAINTENANCE_SUBMIT_BUDGET,
5901            &HashSet::new(),
5902        );
5903
5904        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
5905        assert!(due.iter().all(|(root, _)| root == &healthy_root));
5906        assert!(!deferred);
5907        assert!(live_roots[&healthy_root].maintenance_pending);
5908        assert_eq!(
5909            live_roots[&healthy_root].maintenance_jobs_in_flight,
5910            INITIAL_MAINTENANCE_JOB_COUNT
5911        );
5912        assert!(!live_roots[&poisoned_root].maintenance_pending);
5913    }
5914
5915    #[test]
5916    fn due_maintenance_jobs_do_not_restart_quiesced_root_work() {
5917        let (_dir, root) = test_root("maintenance-unbound");
5918        let mut meta = RootMeta::new(Instant::now());
5919        meta.unbound_quiesced = true;
5920        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5921
5922        let (due, deferred) = due_maintenance_jobs_without_actor_context(
5923            &mut live_roots,
5924            MAINTENANCE_SUBMIT_BUDGET,
5925            &HashSet::new(),
5926        );
5927
5928        assert!(due.is_empty());
5929        assert!(!deferred);
5930        assert!(!live_roots[&root].maintenance_pending);
5931    }
5932
5933    #[test]
5934    fn idle_bg_subscription_queues_no_jobs_until_a_wake_arrives() {
5935        let (_dir, root) = test_root("maintenance-idle-bg-subscription");
5936        let ctx = test_ctx();
5937        assert!(!ctx.completion_drains_have_work());
5938
5939        let executor = Executor::new();
5940        assert!(executor.register_actor(root.clone(), ctx));
5941        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
5942        let session = "idle-session".to_string();
5943        let channel = route_key(17, 1);
5944        let bg_sub_by_session = HashMap::from([((root.clone(), session.clone()), channel)]);
5945        let mut bg_wake_pending = HashSet::new();
5946
5947        let (idle_tick_jobs, deferred) = due_maintenance_jobs(
5948            &mut live_roots,
5949            Some(&executor),
5950            &bg_sub_by_session,
5951            &bg_wake_pending,
5952            MAINTENANCE_SUBMIT_BUDGET,
5953            &HashSet::new(),
5954        );
5955        assert!(idle_tick_jobs.is_empty());
5956        assert!(!deferred);
5957        assert!(!live_roots[&root].maintenance_pending);
5958
5959        // A completion can arm its wake after the idle tick's probes. The wake
5960        // remains loop-owned state, so the following tick must observe it.
5961        let mut bg_wake_epoch = HashMap::new();
5962        push::arm_bg_wake(
5963            root.clone(),
5964            session,
5965            channel,
5966            &mut bg_wake_pending,
5967            &mut bg_wake_epoch,
5968        );
5969        let (next_tick_jobs, deferred) = due_maintenance_jobs(
5970            &mut live_roots,
5971            Some(&executor),
5972            &bg_sub_by_session,
5973            &bg_wake_pending,
5974            MAINTENANCE_SUBMIT_BUDGET,
5975            &HashSet::new(),
5976        );
5977        assert_eq!(
5978            next_tick_jobs,
5979            vec![(root, MaintenanceDrainKind::CompletionDrains)]
5980        );
5981        assert!(!deferred);
5982    }
5983
5984    #[tokio::test]
5985    async fn subc_configure_tail_precedes_completed_search_install() {
5986        let root_dir = tempfile::tempdir().unwrap();
5987        let storage = tempfile::tempdir().unwrap();
5988        let root = ProjectRootId::from_path(root_dir.path()).unwrap();
5989        let (ctx, ignored_path) =
5990            runtime_drain::configure_search_order_context_for_test(root_dir.path(), storage.path());
5991        let ctx = Arc::new(ctx);
5992        assert!(!runtime_drain::watcher_path_is_ignored_by_current_matcher(
5993            &ctx,
5994            &ignored_path
5995        ));
5996
5997        let executor = Arc::new(Executor::new());
5998        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5999        let metrics = Arc::new(DispatchPathMetrics::new());
6000        let (completion_tx, mut completion_rx) = mpsc::channel(4);
6001        submit_maintenance_job(
6002            &executor,
6003            root.clone(),
6004            MaintenanceDrainKind::ConfigureTail,
6005            Vec::new(),
6006            &completion_tx,
6007            &metrics,
6008        );
6009        submit_maintenance_job(
6010            &executor,
6011            root,
6012            MaintenanceDrainKind::CompletionDrains,
6013            Vec::new(),
6014            &completion_tx,
6015            &metrics,
6016        );
6017
6018        let first = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
6019            .await
6020            .expect("configure-tail completion timed out")
6021            .expect("configure-tail completion channel closed");
6022        let second = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
6023            .await
6024            .expect("completion-drains completion timed out")
6025            .expect("completion-drains completion channel closed");
6026        assert!(first.response.id.contains("configure-tail"));
6027        assert!(second.response.id.contains("completion-drains"));
6028        assert!(runtime_drain::watcher_path_is_ignored_by_current_matcher(
6029            &ctx,
6030            &ignored_path
6031        ));
6032        assert_eq!(
6033            ctx.search_index()
6034                .read()
6035                .unwrap_or_else(std::sync::PoisonError::into_inner)
6036                .as_ref()
6037                .expect("completed search index installed")
6038                .file_count(),
6039            0,
6040            "configure must install the ignore matcher before pending paths replay"
6041        );
6042        ctx.stop_watcher_runtime();
6043    }
6044
6045    #[test]
6046    fn post_bind_configure_and_completion_jobs_are_queued_in_order() {
6047        let (_dir, root) = test_root("maintenance-post-bind");
6048        let mut live_roots = HashMap::new();
6049        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6050
6051        queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
6052        queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
6053
6054        let meta = live_roots.get(&root).expect("root metadata");
6055        assert!(meta.maintenance_pending);
6056        assert_eq!(meta.maintenance_jobs_in_flight, 0);
6057        assert_eq!(
6058            meta.maintenance_queued_kinds
6059                .iter()
6060                .copied()
6061                .collect::<Vec<_>>(),
6062            vec![
6063                MaintenanceDrainKind::ConfigureTail,
6064                MaintenanceDrainKind::CompletionDrains,
6065            ]
6066        );
6067
6068        let (due, deferred) = due_maintenance_jobs_without_actor_context(
6069            &mut live_roots,
6070            MAINTENANCE_SUBMIT_BUDGET,
6071            &HashSet::new(),
6072        );
6073
6074        assert_eq!(
6075            due,
6076            vec![
6077                (root.clone(), MaintenanceDrainKind::ConfigureTail),
6078                (root.clone(), MaintenanceDrainKind::CompletionDrains),
6079            ]
6080        );
6081        assert!(!deferred);
6082        assert_eq!(live_roots[&root].maintenance_jobs_in_flight, 2);
6083        assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
6084    }
6085
6086    #[test]
6087    fn due_maintenance_jobs_defers_unsubmitted_roots_without_marking_pending() {
6088        let mut live_roots = HashMap::new();
6089        let mut root_ids = Vec::new();
6090        let mut _dirs = Vec::new();
6091        for index in 0..4 {
6092            let (dir, root_id) = test_root(&format!("maintenance-budget-{index}"));
6093            live_roots.insert(root_id.clone(), RootMeta::new(Instant::now()));
6094            root_ids.push(root_id);
6095            _dirs.push(dir);
6096        }
6097
6098        let small_budget = INITIAL_MAINTENANCE_JOB_COUNT + 1;
6099        let (first_due, first_deferred) = due_maintenance_jobs_without_actor_context(
6100            &mut live_roots,
6101            small_budget,
6102            &HashSet::new(),
6103        );
6104
6105        assert_eq!(first_due.len(), small_budget);
6106        assert!(first_deferred);
6107        let first_due_set: HashSet<_> = first_due.into_iter().map(|(root, _)| root).collect();
6108        assert!(first_due_set
6109            .iter()
6110            .all(|root| live_roots[root].maintenance_pending));
6111        assert!(first_due_set
6112            .iter()
6113            .any(|root| !live_roots[root].maintenance_queued_kinds.is_empty()));
6114
6115        let all_roots: HashSet<_> = root_ids.into_iter().collect();
6116        let deferred_roots: HashSet<_> = all_roots.difference(&first_due_set).cloned().collect();
6117        assert!(deferred_roots
6118            .iter()
6119            .all(|root| !live_roots[root].maintenance_pending));
6120    }
6121
6122    #[test]
6123    fn due_maintenance_jobs_defers_pending_bind_roots() {
6124        let (_bind_dir, bind_root) = test_root("maintenance-pending-bind");
6125        let (_healthy_dir, healthy_root) = test_root("maintenance-no-bind");
6126        let mut live_roots = HashMap::new();
6127        live_roots.insert(bind_root.clone(), RootMeta::new(Instant::now()));
6128        live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
6129        let pending_bind_roots = HashSet::from([bind_root.clone()]);
6130
6131        let (due, deferred) = due_maintenance_jobs_without_actor_context(
6132            &mut live_roots,
6133            usize::MAX,
6134            &pending_bind_roots,
6135        );
6136
6137        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6138        assert!(due.iter().all(|(root, _)| root == &healthy_root));
6139        assert!(!deferred);
6140        assert!(!live_roots[&bind_root].maintenance_pending);
6141        assert!(live_roots[&bind_root].maintenance_queued_kinds.is_empty());
6142    }
6143
6144    #[test]
6145    fn maintenance_pending_survives_requeue_and_clears_after_final_batch() {
6146        let (_dir, root) = test_root("maintenance-requeue");
6147        let mut live_roots = HashMap::new();
6148        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6149        let (due, deferred) = due_maintenance_jobs_without_actor_context(
6150            &mut live_roots,
6151            usize::MAX,
6152            &HashSet::new(),
6153        );
6154        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6155        assert!(due.iter().all(|(due_root, _)| due_root == &root));
6156        assert!(!deferred);
6157
6158        let meta = live_roots.get_mut(&root).unwrap();
6159        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, false);
6160        assert!(meta.maintenance_pending);
6161        assert_eq!(
6162            meta.maintenance_jobs_in_flight,
6163            INITIAL_MAINTENANCE_JOB_COUNT - 1
6164        );
6165        assert_eq!(meta.maintenance_queued_kinds.len(), 1);
6166
6167        let (requeued, deferred) =
6168            due_maintenance_jobs_without_actor_context(&mut live_roots, 1, &HashSet::new());
6169        assert_eq!(
6170            requeued,
6171            vec![(root.clone(), MaintenanceDrainKind::Watcher)]
6172        );
6173        assert!(!deferred);
6174        let meta = live_roots.get_mut(&root).unwrap();
6175        assert_eq!(
6176            meta.maintenance_jobs_in_flight,
6177            INITIAL_MAINTENANCE_JOB_COUNT
6178        );
6179        assert!(meta.maintenance_queued_kinds.is_empty());
6180
6181        for _ in 0..INITIAL_MAINTENANCE_JOB_COUNT {
6182            note_maintenance_completion(meta, None, false, false);
6183        }
6184        assert!(!meta.maintenance_pending);
6185        assert_eq!(meta.maintenance_jobs_in_flight, 0);
6186    }
6187
6188    #[test]
6189    fn maintenance_requeue_drops_while_bind_is_pending() {
6190        let (_dir, root) = test_root("maintenance-bind-requeue");
6191        let mut live_roots = HashMap::new();
6192        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6193        let (due, _) = due_maintenance_jobs_without_actor_context(
6194            &mut live_roots,
6195            usize::MAX,
6196            &HashSet::new(),
6197        );
6198        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6199
6200        let meta = live_roots.get_mut(&root).unwrap();
6201        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, true);
6202
6203        assert_eq!(
6204            meta.maintenance_jobs_in_flight,
6205            INITIAL_MAINTENANCE_JOB_COUNT - 1
6206        );
6207        assert!(meta.maintenance_queued_kinds.is_empty());
6208        assert!(meta.maintenance_pending);
6209    }
6210
6211    #[test]
6212    fn parked_lsp_completion_never_requiesces_or_cancels_a_pending_bind() {
6213        let mut meta = RootMeta::new(Instant::now());
6214        meta.unbound_quiesced = true;
6215
6216        assert!(!should_requiesce_after_maintenance(
6217            &meta,
6218            MaintenanceDrainKind::Lsp,
6219            false,
6220        ));
6221        assert!(!should_requiesce_after_maintenance(
6222            &meta,
6223            MaintenanceDrainKind::ConfigureTail,
6224            true,
6225        ));
6226        assert!(should_requiesce_after_maintenance(
6227            &meta,
6228            MaintenanceDrainKind::ConfigureTail,
6229            false,
6230        ));
6231    }
6232
6233    #[test]
6234    fn maintenance_pending_clears_and_poison_stops_requeue_after_fatal() {
6235        let (_dir, root) = test_root("maintenance-fatal");
6236        let mut live_roots = HashMap::new();
6237        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6238        let (due, _) = due_maintenance_jobs_without_actor_context(
6239            &mut live_roots,
6240            usize::MAX,
6241            &HashSet::new(),
6242        );
6243        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6244
6245        let meta = live_roots.get_mut(&root).unwrap();
6246        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), true, false);
6247        assert!(meta.maintenance_poisoned);
6248        assert!(meta.maintenance_queued_kinds.is_empty());
6249
6250        for _ in 1..INITIAL_MAINTENANCE_JOB_COUNT {
6251            note_maintenance_completion(meta, None, false, false);
6252        }
6253        assert!(!meta.maintenance_pending);
6254        assert_eq!(meta.maintenance_jobs_in_flight, 0);
6255    }
6256
6257    #[test]
6258    fn trust_for_principal_matrix() {
6259        assert_eq!(
6260            trust_for_principal(&Some(Principal::Direct)),
6261            BindTrust::FirstParty
6262        );
6263        // Every first-party reserved id is asserted BY NAME, in one loop over
6264        // the full set, so two failure classes stay distinguishable: an empty
6265        // or broken allowlist reddens every name at once, while a dropped
6266        // single entry (the rename hazard) reddens exactly the missing name.
6267        // Both halves of each transitional rename pair stay listed until the
6268        // flip settles (see the allowlist comment).
6269        for module_id in [
6270            "llm-runner",
6271            "aft",
6272            "broca",
6273            "alfonso-core",
6274            "prefrontal",
6275            "prefrontal-core",
6276        ] {
6277            assert_eq!(
6278                trust_for_principal(&Some(Principal::Reserved {
6279                    module_id: module_id.to_string(),
6280                })),
6281                BindTrust::FirstParty,
6282                "reserved module id '{module_id}' must resolve to first-party trust"
6283            );
6284        }
6285        assert_eq!(
6286            trust_for_principal(&Some(Principal::Reserved {
6287                module_id: "subc-mcp".to_string(),
6288            })),
6289            BindTrust::Untrusted
6290        );
6291        assert_eq!(
6292            trust_for_principal(&Some(Principal::Reserved {
6293                module_id: "anything-unknown".to_string(),
6294            })),
6295            BindTrust::Untrusted
6296        );
6297        assert_eq!(
6298            trust_for_principal(&Some(Principal::Unverified)),
6299            BindTrust::Untrusted
6300        );
6301        assert_eq!(trust_for_principal(&None), BindTrust::Untrusted);
6302    }
6303
6304    #[test]
6305    fn fed_harness_class_maps_to_untrusted_regardless_of_fingerprint_value() {
6306        let principal = Some(Principal::Direct);
6307        let fingerprint_a = "fed:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
6308        let fingerprint_b = "fed:0123456789abcdef111111111111111111111111111111111111111111111111";
6309
6310        assert_eq!(
6311            trust_for_bind(fingerprint_a, &principal),
6312            BindTrust::Untrusted
6313        );
6314        assert_eq!(
6315            trust_for_bind(fingerprint_b, &principal),
6316            BindTrust::Untrusted
6317        );
6318    }
6319
6320    /// The table above proves `trust_for_principal` maps correctly, and the
6321    /// test above proves the `fed:` harness override wins — but neither
6322    /// exercises the ordinary path, so an implementation that ignored the
6323    /// principal entirely for non-fed harnesses would satisfy both. Pin the
6324    /// delegation itself: on a normal harness the verdict must still come from
6325    /// the principal, in both directions.
6326    #[test]
6327    fn trust_for_bind_delegates_to_the_principal_on_ordinary_harnesses() {
6328        for harness in ["opencode", "pi", "runner", "mcp:claude"] {
6329            assert_eq!(
6330                trust_for_bind(harness, &Some(Principal::Direct)),
6331                BindTrust::FirstParty,
6332                "a direct principal must stay first-party on {harness}"
6333            );
6334            assert_eq!(
6335                trust_for_bind(harness, &Some(Principal::Unverified)),
6336                BindTrust::Untrusted,
6337                "an unverified principal must stay untrusted on {harness}"
6338            );
6339            assert_eq!(
6340                trust_for_bind(harness, &None),
6341                BindTrust::Untrusted,
6342                "an absent principal must fail closed on {harness}"
6343            );
6344            assert_eq!(
6345                trust_for_bind(
6346                    harness,
6347                    &Some(Principal::Reserved {
6348                        module_id: "subc-mcp".to_string(),
6349                    })
6350                ),
6351                BindTrust::Untrusted,
6352                "a non-allowlisted reserved module must stay untrusted on {harness}"
6353            );
6354        }
6355    }
6356
6357    #[tokio::test]
6358    async fn persistent_cancel_resolves_when_fired_before_await() {
6359        // The lost-wakeup guard: cancel() fires exactly once via notify_waiters()
6360        // (no stored permit). A waiter that registers AFTER the cancel must still
6361        // observe it via the flag; a waiter racing the cancel must still be woken.
6362        let signal = PersistentCancelSignal::new();
6363        signal.cancel();
6364        // Fired before we ever call cancelled() — must return immediately, not park.
6365        tokio::time::timeout(Duration::from_secs(1), signal.cancelled())
6366            .await
6367            .expect("cancelled() must resolve when cancel fired beforehand");
6368
6369        // A fresh signal cancelled concurrently with an in-flight cancelled().
6370        let racing = PersistentCancelSignal::new();
6371        let racing_for_task = racing.clone();
6372        let waiter = tokio::spawn(async move { racing_for_task.cancelled().await });
6373        racing.cancel();
6374        tokio::time::timeout(Duration::from_secs(1), waiter)
6375            .await
6376            .expect("cancelled() must resolve when cancel races the await")
6377            .expect("waiter task panicked");
6378    }
6379
6380    #[test]
6381    fn ingress_epoch_validation_rejects_reclaimed_requests_and_drops_other_stale_epochs() {
6382        let installed = HashMap::from([(7, 9)]);
6383        let mut reclaimed = ReclaimedRoutes::default();
6384        reclaimed.insert(route_key(8, 1));
6385        for ty in [
6386            FrameType::Request,
6387            FrameType::Response,
6388            FrameType::Error,
6389            FrameType::Push,
6390            FrameType::Cancel,
6391            FrameType::Goodbye,
6392        ] {
6393            let body = if ty.is_pure_header() {
6394                Vec::new()
6395            } else {
6396                br#"{}"#.to_vec()
6397            };
6398            let stale = Frame::build(ty, control_flags(), 7, 8, 41, body).unwrap();
6399            assert!(
6400                !ingress_route_should_be_processed(&installed, &reclaimed, &stale),
6401                "{ty:?}"
6402            );
6403        }
6404
6405        let reclaimed_request = Frame::build(
6406            FrameType::Request,
6407            control_flags(),
6408            8,
6409            1,
6410            42,
6411            br#"{}"#.to_vec(),
6412        )
6413        .unwrap();
6414        assert!(ingress_route_should_be_processed(
6415            &installed,
6416            &reclaimed,
6417            &reclaimed_request
6418        ));
6419
6420        let never_installed = Frame::build(
6421            FrameType::Request,
6422            control_flags(),
6423            9,
6424            1,
6425            43,
6426            br#"{}"#.to_vec(),
6427        )
6428        .unwrap();
6429        assert!(!ingress_route_should_be_processed(
6430            &installed,
6431            &reclaimed,
6432            &never_installed
6433        ));
6434
6435        let current = Frame::build(
6436            FrameType::Request,
6437            control_flags(),
6438            7,
6439            9,
6440            43,
6441            br#"{}"#.to_vec(),
6442        )
6443        .unwrap();
6444        let control = Frame::build(FrameType::Ping, control_flags(), 0, 0, 44, Vec::new()).unwrap();
6445        assert!(ingress_route_should_be_processed(
6446            &installed, &reclaimed, &current
6447        ));
6448        assert!(ingress_route_should_be_processed(
6449            &installed, &reclaimed, &control
6450        ));
6451        assert_eq!(installed, HashMap::from([(7, 9)]));
6452    }
6453
6454    #[tokio::test]
6455    async fn route_bind_ack_precedes_route_egress_in_writer_queue() {
6456        let (_dir, root) = test_root("route-bind-b2-ordering");
6457        let route = route_key(7, 3);
6458        let identity = RouteIdentity(Arc::new(RouteIdentityData {
6459            root: root.clone(),
6460            project_root: root.as_path().to_path_buf(),
6461            harness: "opencode".to_string(),
6462            session: "b2-session".to_string(),
6463            trust: BindTrust::FirstParty,
6464            spawn_principal: AuthenticatedPrincipal::FirstParty,
6465            consumer_elicitation_capable: false,
6466        }));
6467        let replay_key = push::ReplayKey::from_identity(&identity);
6468        let completion = RouteBindCompletion {
6469            route,
6470            identity,
6471            bind_root_id: root.clone(),
6472            inserted_new_actor: false,
6473            configure_response: Response::success("subc-bind-7", json!({})),
6474            diagnostics_on_edit: false,
6475            ver: PROTOCOL_VERSION,
6476            corr: 91,
6477            flags: control_flags(),
6478        };
6479        let mut pending_binds = HashMap::from([(
6480            route,
6481            PendingBind {
6482                bind_root_id: root,
6483                inserted_new_actor: false,
6484                cancelled: false,
6485                configure_request_id: "subc-bind-7".to_string(),
6486                started_at: Instant::now(),
6487                warned_half_deadline: false,
6488                deadline_reported: false,
6489                corr: 91,
6490                ver: PROTOCOL_VERSION,
6491                flags: control_flags(),
6492                cancellation: crate::executor::JobCancellation::new(),
6493            },
6494        )]);
6495        let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
6496        let mut push_buffer =
6497            HashMap::from([(replay_key, VecDeque::from([completion_frame("b2-replay")]))]);
6498        let (writer_tx, mut writer_rx) = mpsc::channel(8);
6499        let metrics = Arc::new(DispatchPathMetrics::new());
6500
6501        handle_route_bind_completion(
6502            &writer_tx,
6503            completion,
6504            &mut HashMap::new(),
6505            &mut HashMap::new(),
6506            &mut HashMap::new(),
6507            &mut push_buffer,
6508            &mut HashMap::new(),
6509            &mut pending_binds,
6510            &mut installed_route_epochs,
6511            &Arc::new(Executor::new()),
6512            &Arc::new(Notify::new()),
6513            &metrics,
6514        )
6515        .await
6516        .unwrap();
6517
6518        let ack = writer_rx.try_recv().expect("RouteBindAck");
6519        assert_eq!(ack.header.ty, FrameType::Response);
6520        assert_eq!((ack.header.channel, ack.header.epoch), (0, 0));
6521        let route_frame = writer_rx.try_recv().expect("post-ack route frame");
6522        assert_eq!(route_frame.header.ty, FrameType::Push);
6523        assert_eq!(
6524            (route_frame.header.channel, route_frame.header.epoch),
6525            (route.channel, route.epoch)
6526        );
6527    }
6528}