Skip to main content

aft/subc/
mod.rs

1//! subc daemon attach — transport edge.
2//!
3//! When AFT is launched as `aft --subc <connection-file>`, it does NOT run the
4//! standalone NDJSON-over-stdin loop. Instead it connects to a running subc
5//! daemon over loopback TCP, authenticates with the pre-envelope HMAC handshake
6//! (`subc-transport`), then speaks the subc frame protocol (`subc-protocol`):
7//! ModuleHello → HelloAck (register as a tool provider), then a channel-0
8//! control loop (Ping/Pong, RouteBind) plus route-channel tool calls.
9//!
10//! Concurrency: subc routes tool calls through the executor. The tokio
11//! edge never dispatches against `AppContext` inline; per-actor executor lanes
12//! own the reader/mutator epoch, while a writer task serializes outbound frames.
13
14use std::collections::{HashMap, HashSet, VecDeque};
15use std::fmt;
16use std::io;
17use std::net::{IpAddr, SocketAddr};
18use std::ops::Deref;
19use std::path::{Path, PathBuf};
20use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
21use std::sync::{Arc, LazyLock};
22use std::time::{Duration, Instant};
23
24use serde::Deserialize;
25use serde_json::{json, Value};
26
27use crate::config::Config;
28use crate::config_resolve::ConfigTier;
29use crate::context::{App, AppContext, ProgressSender, RootHealthSnapshot};
30use crate::executor::{Executor, Lane};
31use crate::jsonc::strip_jsonc;
32use crate::log_ctx;
33use crate::path_identity::ProjectRootId;
34use crate::protocol::{ProgressKind, PushFrame, RawRequest, Response};
35use crate::run_tool_call::{
36    run_tool_call, strip_agent_preview_arg_owned, PhaseTrace, ToolCallContext, ToolCallOutcome,
37    ToolCallResult,
38};
39use crate::runtime_drain;
40use crate::sandbox_spawn::{AuthenticatedPrincipal, PrincipalTrust};
41
42use subc_protocol::manifest::{
43    Bindings, Concurrency, ExecutionMode, IdentityBinding, IdentityScope, ModuleManifest,
44    ProviderRole, StorageBinding, StorageKind, StorageScope, Tool, TrustTier,
45};
46use subc_protocol::session::{
47    HealthReport, HealthStatus, ModuleControlRequest, ModuleControlResponse,
48    MODULE_CONTROL_OP_HEALTH_CHECK,
49};
50use subc_protocol::{
51    ErrorBody, Flags, Frame, FrameType, ModuleHelloBody, Principal, Priority, PROTOCOL_VERSION,
52};
53use subc_transport::{authenticate_client, connection_file, read_frame, write_frame};
54use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
55use tokio::net::TcpStream;
56use tokio::sync::{mpsc, oneshot, Notify};
57use tokio::task::JoinHandle;
58
59/// Per-attempt handshake deadline. The initial attach loop has a separate total
60/// budget so a stalled peer cannot consume an unbounded supervisor launch window.
61const AUTH_DEADLINE: Duration = Duration::from_secs(5);
62const ATTACH_RETRY_BUDGET: Duration = Duration::from_secs(60);
63const ATTACH_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(250);
64const ATTACH_RETRY_MAX_BACKOFF: Duration = Duration::from_secs(5);
65const ATTACH_RETRY_JITTER_PERCENT: u64 = 20;
66
67/// Correlation id for the initial ModuleHello (channel 0).
68const HELLO_CORR: u64 = 1;
69
70/// Per-session in-memory replay cap for must-deliver Push frames. This covers
71/// detach/re-attach while AFT stays alive; cross-restart replay is phased later.
72const PUSH_BUFFER_MAX_PER_KEY: usize = 256;
73
74/// Bounded guard for control-frame sends. If the daemon stops reading and the
75/// writer queue stays full, tear the subc edge down instead of stalling the
76/// route loop indefinitely.
77const CONTROL_SEND_TIMEOUT: Duration = Duration::from_millis(250);
78
79/// Cadence for the loop's deadline-driven drain work (retry-buffer flush,
80/// bg-wake emission, maintenance submission). Checked at the top of every
81/// loop turn so busy select arms cannot starve it.
82const DRAIN_TICK_PERIOD: Duration = Duration::from_millis(250);
83
84/// Root-scoped stores and watcher runtimes are reopened lazily after this
85/// period without tool traffic. Keeping the value fixed avoids per-client
86/// eviction policies competing inside the module loop.
87const IDLE_ROOT_TTL: Duration = Duration::from_secs(30 * 60);
88
89const WRITER_QUEUE_CAPACITY: usize = 256;
90
91/// Keep reliable Push bursts from monopolizing the current-thread subc loop;
92/// any remaining must-deliver frames stay queued for the next loop turn.
93const RELIABLE_PUSH_DRAIN_BUDGET: usize = 32;
94
95/// Limit maintenance submissions per tick so background drains cannot delay
96/// control-plane work such as completed RouteBind acknowledgements.
97///
98/// The decomposed maintenance pass charges this budget by Mutating job, not by
99/// root. Size the default burst for one maintenance pass over eight live roots,
100/// while follow-up batches still re-enter the capped queue instead of bypassing
101/// the budget.
102const MAINTENANCE_SUBMIT_BUDGET: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len() * 8;
103const INITIAL_MAINTENANCE_DRAIN_KINDS: [MaintenanceDrainKind; 4] = [
104    MaintenanceDrainKind::Watcher,
105    MaintenanceDrainKind::Lsp,
106    MaintenanceDrainKind::ConfigureTail,
107    MaintenanceDrainKind::CompletionDrains,
108];
109#[cfg(test)]
110const INITIAL_MAINTENANCE_JOB_COUNT: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len();
111
112const RELIABLE_WRITER_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(10);
113const RELIABLE_WRITER_RETRY_MAX_BACKOFF: Duration = Duration::from_millis(250);
114
115const DISPATCH_PATH_BIND_WARN_AFTER: Duration = Duration::from_secs(6);
116const ROUTE_BIND_DEADLINE: Duration = Duration::from_secs(12);
117
118/// Small bounded memory of completed task ids used to suppress stale lossy
119/// long-running reminders that arrive after their reliable completion event.
120const COMPLETED_TASK_SUPPRESSION_MAX: usize = 4096;
121
122/// Bash foreground orchestration polls detached tasks with short read-lane jobs.
123/// The sleep between polls is outside the executor so no read or write worker is
124/// pinned while a foreground command is still running.
125const PENDING_POLL_INTERVAL: Duration = Duration::from_millis(100);
126
127/// Host elicitation asks fail closed if the MCP facade does not answer promptly.
128const BASH_ELICITATION_TIMEOUT: Duration = Duration::from_secs(60);
129const BASH_ELICITATION_CREATE_METHOD: &str = "elicitation/create";
130
131#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
132struct RouteChannel {
133    channel: u16,
134    epoch: u32,
135}
136
137impl fmt::Display for RouteChannel {
138    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
139        write!(f, "{}@{}", self.channel, self.epoch)
140    }
141}
142
143type PushEnvelope = (ProjectRootId, PushFrame);
144type LossyPushEnvelope = (u64, ProjectRootId, PushFrame);
145type RetryBuffer = HashMap<RouteChannel, VecDeque<(push::ReplayKey, PushFrame)>>;
146mod bash;
147mod health;
148mod manifest;
149mod push;
150mod wire;
151
152use self::health::{
153    build_health_report, warn_slow_pending_binds, DispatchPathMetrics, ReapBlockerCensus,
154    ResponseTaskGuard,
155};
156use self::manifest::{
157    build_manifest, command_lane, control_flags, control_ops, is_bash_family_tool,
158    is_subc_agent_core_tool, is_subc_native_plumbing_tool,
159};
160pub use self::wire::SubcError;
161
162/// Test-only view of the fail-closed tool-call gate: would `name` be admitted
163/// on a bound route (as an agent tool or native plumbing)? Used by the
164/// plugin-send drift guard in `subc_plumbing_drift_test.rs`.
165pub fn is_tool_call_admitted_for_test(name: &str) -> bool {
166    manifest::is_subc_agent_core_tool(name) || manifest::is_subc_native_plumbing_tool(name)
167}
168use self::wire::{
169    build_error_frame, build_goodbye_frame, build_tool_response_frame, decrement_counted_channel,
170    response_is_fatal_panic, response_message, send_counted_channel, send_frame,
171    send_reliable_writer_frame, send_traced_tool_response_frame, ToolResponseWriteTrace,
172    WriterFrame, WriterSender,
173};
174
175struct DecodedFrame {
176    frame: Frame,
177    phase_trace: PhaseTrace,
178}
179
180struct ToolCallCompletion {
181    text: String,
182    phase_trace: PhaseTrace,
183}
184
185#[derive(Clone)]
186struct PushSenders {
187    lossy_tx: mpsc::Sender<LossyPushEnvelope>,
188    reliable_tx: mpsc::UnboundedSender<PushEnvelope>,
189    lossy_overflow: Arc<push::LossyOverflow>,
190    lossy_seq: Arc<AtomicU64>,
191}
192
193#[derive(Clone)]
194struct PersistentCancelSignal {
195    inner: Arc<PersistentCancelInner>,
196}
197
198struct PersistentCancelInner {
199    cancelled: AtomicBool,
200    notify: Notify,
201}
202
203impl PersistentCancelSignal {
204    fn new() -> Self {
205        Self {
206            inner: Arc::new(PersistentCancelInner {
207                cancelled: AtomicBool::new(false),
208                notify: Notify::new(),
209            }),
210        }
211    }
212
213    fn cancel(&self) {
214        if !self.inner.cancelled.swap(true, Ordering::SeqCst) {
215            self.inner.notify.notify_waiters();
216        }
217    }
218
219    fn is_cancelled(&self) -> bool {
220        self.inner.cancelled.load(Ordering::SeqCst)
221    }
222
223    async fn cancelled(&self) {
224        // `enable()` REGISTERS this waiter before we read the flag, closing the
225        // lost-wakeup window: `notify_waiters()` only wakes already-registered
226        // waiters and stores no permit, so without enable() a `cancel()` firing
227        // between the flag read and `.await` would be missed and the future
228        // would park forever (cancel() fires only once). With enable(), a cancel
229        // racing the flag read still wakes the registered waiter. The loop is a
230        // belt-and-suspenders re-check on spurious wakeups.
231        loop {
232            let notified = self.inner.notify.notified();
233            tokio::pin!(notified);
234            notified.as_mut().enable();
235            if self.is_cancelled() {
236                return;
237            }
238            notified.await;
239        }
240    }
241}
242
243#[derive(Debug, Clone, Copy, PartialEq, Eq)]
244pub(crate) enum BindTrust {
245    FirstParty,
246    Untrusted,
247}
248
249impl BindTrust {
250    fn allows_bash_observation(self) -> bool {
251        matches!(self, Self::FirstParty)
252    }
253
254    fn label(self) -> &'static str {
255        match self {
256            Self::FirstParty => "first_party",
257            Self::Untrusted => "untrusted",
258        }
259    }
260
261    fn sandbox_trust(self) -> PrincipalTrust {
262        match self {
263            Self::FirstParty => PrincipalTrust::FirstParty,
264            Self::Untrusted => PrincipalTrust::Untrusted,
265        }
266    }
267}
268
269pub(super) fn trust_for_principal(principal: &Option<Principal>) -> BindTrust {
270    match principal {
271        Some(Principal::Direct) => BindTrust::FirstParty,
272        // Module renames are flag-days: the daemon registry refuses duplicate
273        // active ids, so a renaming module cannot advertise both names during
274        // its transition. This allowlist is DIALLED, not dialling — it must
275        // accept a module's NEW name in a released binary before the module
276        // starts using it, and the old name stays until the flip has settled.
277        // That is why transitional pairs appear here: llm-runner/broca was the
278        // previous rename, alfonso-core/prefrontal is the current one. When
279        // retiring an old name, confirm the fleet no longer spawns it — a
280        // stale entry here is inert, but a missing one silently downgrades a
281        // first-party module to Untrusted and revokes its bash access.
282        Some(Principal::Reserved { module_id })
283            if module_id == "llm-runner"
284                || module_id == "aft"
285                || module_id == "broca"
286                || module_id == "alfonso-core"
287                || module_id == "prefrontal"
288                || module_id == "prefrontal-core" =>
289        {
290            BindTrust::FirstParty
291        }
292        Some(Principal::Reserved { .. }) | Some(Principal::Unverified) | None => {
293            BindTrust::Untrusted
294        }
295    }
296}
297
298fn harness_forces_untrusted(harness: &str) -> bool {
299    harness.starts_with("fed:")
300}
301
302pub(super) fn trust_for_bind(harness: &str, principal: &Option<Principal>) -> BindTrust {
303    if harness_forces_untrusted(harness) {
304        BindTrust::Untrusted
305    } else {
306        trust_for_principal(principal)
307    }
308}
309
310fn principal_id(principal: &Option<Principal>) -> Option<String> {
311    match principal {
312        Some(Principal::Direct) => Some("direct".to_string()),
313        Some(Principal::Reserved { module_id }) => Some(format!("reserved:{module_id}")),
314        Some(Principal::Unverified) => Some("unverified".to_string()),
315        None => None,
316    }
317}
318
319fn principal_label(principal: &Option<Principal>) -> String {
320    principal_id(principal).unwrap_or_else(|| "absent".to_string())
321}
322
323#[derive(Debug)]
324/// Per-root route metadata owned by the subc loop. The `active_bash_waits` field
325/// counts detached bash processes that are still being observed for this root.
326/// Any future logic that evicts roots based on idle time must not evict a root
327/// while this count is greater than zero, because a foreground bash response may
328/// still arrive later.
329struct RootMeta {
330    maintenance_pending: bool,
331    maintenance_jobs_in_flight: usize,
332    maintenance_queued_kinds: VecDeque<MaintenanceDrainKind>,
333    maintenance_last_submitted: Option<Instant>,
334    maintenance_poisoned: bool,
335    last_touched: Instant,
336    diagnostics_on_edit: bool,
337    active_bash_waits: usize,
338    idle_artifacts_evicted: bool,
339    unbound_quiesced: bool,
340    consecutive_missing_sweeps: u8,
341}
342
343#[derive(Debug)]
344struct PendingBind {
345    bind_root_id: ProjectRootId,
346    inserted_new_actor: bool,
347    cancelled: bool,
348    configure_request_id: String,
349    started_at: Instant,
350    warned_half_deadline: bool,
351    deadline_reported: bool,
352    corr: u64,
353    ver: u8,
354    flags: Flags,
355    /// Exact-job cancellation for the submitted configure: Goodbye and
356    /// deadline expiry cancel the executor job operationally (queued jobs are
357    /// removed, running configures return at their next checkpoint) instead of
358    /// only marking bookkeeping.
359    cancellation: crate::executor::JobCancellation,
360}
361
362struct RouteBindCompletion {
363    route: RouteChannel,
364    identity: RouteIdentity,
365    bind_root_id: ProjectRootId,
366    inserted_new_actor: bool,
367    configure_response: Response,
368    diagnostics_on_edit: bool,
369    ver: u8,
370    corr: u64,
371    flags: Flags,
372}
373
374#[derive(Debug, Clone)]
375struct RouteIdentity(Arc<RouteIdentityData>);
376
377#[derive(Debug)]
378struct RouteIdentityData {
379    root: ProjectRootId,
380    project_root: PathBuf,
381    harness: String,
382    session: String,
383    trust: BindTrust,
384    spawn_principal: AuthenticatedPrincipal,
385    consumer_elicitation_capable: bool,
386}
387
388impl Deref for RouteIdentity {
389    type Target = RouteIdentityData;
390
391    fn deref(&self) -> &Self::Target {
392        &self.0
393    }
394}
395
396#[derive(Debug, Clone)]
397struct RetainedSessionIdentity {
398    harness: String,
399    trust: BindTrust,
400}
401
402#[derive(Clone, Copy)]
403struct BgSub {
404    corr: u64,
405    ver: u8,
406    flags: Flags,
407}
408
409struct MaintenanceCompletion {
410    root_id: ProjectRootId,
411    kind: MaintenanceDrainKind,
412    response: Response,
413    empty_bg_sessions: Vec<(String, u64)>,
414    requeue_kind: Option<MaintenanceDrainKind>,
415}
416
417#[derive(Clone, Copy, Debug, PartialEq, Eq)]
418enum MaintenanceDrainKind {
419    Watcher,
420    Lsp,
421    ConfigureTail,
422    CompletionDrains,
423}
424
425impl MaintenanceDrainKind {
426    fn label(self) -> &'static str {
427        match self {
428            Self::Watcher => "watcher",
429            Self::Lsp => "lsp",
430            Self::ConfigureTail => "configure-tail",
431            Self::CompletionDrains => "completion-drains",
432        }
433    }
434}
435
436#[derive(Debug, Default)]
437struct MaintenanceJobOutcome {
438    empty_bg_sessions: Vec<(String, u64)>,
439    requeue_kind: Option<MaintenanceDrainKind>,
440}
441
442#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
443struct ReverseCorrKey {
444    route: RouteChannel,
445    corr: u64,
446}
447
448struct PendingBashAsk {
449    route: RouteChannel,
450    tool_corr: u64,
451    tool_flags: Flags,
452    tool_ver: u8,
453    root: ProjectRootId,
454    project_root: PathBuf,
455    session_id: String,
456    spawn_principal: AuthenticatedPrincipal,
457    request_id: String,
458    arguments: Value,
459    format_context: crate::subc_format::FormatContext,
460    cancel: bash::BashWaitCancel,
461    grants: Vec<String>,
462    expires_at: Instant,
463}
464
465impl RootMeta {
466    fn new(now: Instant) -> Self {
467        Self {
468            maintenance_pending: false,
469            maintenance_jobs_in_flight: 0,
470            maintenance_queued_kinds: VecDeque::new(),
471            maintenance_last_submitted: None,
472            maintenance_poisoned: false,
473            last_touched: now,
474            diagnostics_on_edit: false,
475            active_bash_waits: 0,
476            idle_artifacts_evicted: false,
477            unbound_quiesced: false,
478            consecutive_missing_sweeps: 0,
479        }
480    }
481
482    fn note_activity(&mut self) {
483        self.last_touched = Instant::now();
484    }
485
486    fn reactivate_bound(&mut self) {
487        self.note_activity();
488        self.idle_artifacts_evicted = false;
489        self.unbound_quiesced = false;
490    }
491}
492
493fn due_maintenance_jobs(
494    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
495    executor: Option<&Executor>,
496    bg_sub_by_session: &HashMap<(ProjectRootId, String), RouteChannel>,
497    bg_wake_pending: &HashSet<RouteChannel>,
498    budget: usize,
499    pending_bind_roots: &HashSet<ProjectRootId>,
500) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
501    let mut jobs = Vec::new();
502    let mut deferred = false;
503    let mut roots = live_roots.keys().cloned().collect::<Vec<_>>();
504    roots.sort_by(|left, right| {
505        let left_last = live_roots
506            .get(left)
507            .and_then(|meta| meta.maintenance_last_submitted);
508        let right_last = live_roots
509            .get(right)
510            .and_then(|meta| meta.maintenance_last_submitted);
511        left_last
512            .cmp(&right_last)
513            .then_with(|| left.as_path().cmp(right.as_path()))
514    });
515
516    for root_id in roots {
517        let Some(meta) = live_roots.get_mut(&root_id) else {
518            continue;
519        };
520        if meta.maintenance_poisoned {
521            continue;
522        }
523
524        if pending_bind_roots.contains(&root_id) {
525            if meta.maintenance_pending || !meta.maintenance_queued_kinds.is_empty() {
526                deferred = true;
527            }
528            continue;
529        }
530
531        if !meta.maintenance_pending {
532            if jobs.len() >= budget {
533                deferred = true;
534                continue;
535            }
536            // Only enqueue kinds with pending work. Probes are cheap and
537            // fail-open (contended sources count as pending), so an idle root
538            // costs four probes per tick instead of four dispatched jobs.
539            let executor_actor_context =
540                executor.and_then(|executor| executor.actor_context(&root_id));
541            let root_has_pending_bg_wake =
542                bg_sub_by_session.iter().any(|((sub_root, _), channel)| {
543                    sub_root == &root_id && bg_wake_pending.contains(channel)
544                });
545            let kinds_with_work: Vec<MaintenanceDrainKind> = match executor_actor_context {
546                Some(ctx) => INITIAL_MAINTENANCE_DRAIN_KINDS
547                    .into_iter()
548                    .filter(|kind| {
549                        if meta.unbound_quiesced && !matches!(kind, MaintenanceDrainKind::Lsp) {
550                            return false;
551                        }
552                        match kind {
553                            MaintenanceDrainKind::Watcher => ctx.watcher_drain_has_work(),
554                            MaintenanceDrainKind::Lsp => ctx.lsp_drain_has_work(),
555                            MaintenanceDrainKind::ConfigureTail => ctx.configure_tail_has_work(),
556                            // Every CompletionDrains source is visible at this enqueue site:
557                            // AppContext probes completion queues, this loop owns bg wakes,
558                            // and queued continuations bypass probing via maintenance_pending.
559                            // New drain sources must expose a probe here rather than making
560                            // every subscribed root fail open again.
561                            MaintenanceDrainKind::CompletionDrains => {
562                                root_has_pending_bg_wake || ctx.completion_drains_have_work()
563                            }
564                        }
565                    })
566                    .collect(),
567                None if meta.unbound_quiesced => Vec::new(),
568                // No context handle (actor gone mid-tick): enqueue everything.
569                None => INITIAL_MAINTENANCE_DRAIN_KINDS.to_vec(),
570            };
571            if kinds_with_work.is_empty() {
572                continue;
573            }
574            meta.maintenance_pending = true;
575            meta.maintenance_queued_kinds.extend(kinds_with_work);
576        }
577
578        while let Some(kind) = meta.maintenance_queued_kinds.pop_front() {
579            if jobs.len() >= budget {
580                meta.maintenance_queued_kinds.push_front(kind);
581                deferred = true;
582                break;
583            }
584            meta.maintenance_jobs_in_flight += 1;
585            meta.maintenance_last_submitted = Some(Instant::now());
586            jobs.push((root_id.clone(), kind));
587        }
588
589        meta.maintenance_pending =
590            meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
591    }
592
593    (jobs, deferred)
594}
595
596fn eviction_estimate_label(estimate: &crate::memory::MemoryEstimate) -> String {
597    match estimate.estimated_bytes {
598        Some(bytes) => format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
599        None if estimate.status == "busy" => "busy".to_string(),
600        None => "not estimated".to_string(),
601    }
602}
603
604fn optional_memory_label(bytes: Option<u64>) -> String {
605    bytes.map_or_else(
606        || "not estimated".to_string(),
607        |bytes| format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
608    )
609}
610
611fn pressure_relief_label(relief: &crate::memory::AllocatorPressureRelief) -> String {
612    format!(
613        "; allocator pressure relief: RSS {} -> {}, in-use {} -> {}, allocated {} -> {}, slack {} -> {}, allocator reported {:.1} MB released",
614        optional_memory_label(relief.rss_before_bytes),
615        optional_memory_label(relief.rss_after_bytes),
616        optional_memory_label(relief.allocator_before.bytes_in_use),
617        optional_memory_label(relief.allocator_after.bytes_in_use),
618        optional_memory_label(relief.allocator_before.size_allocated),
619        optional_memory_label(relief.allocator_after.size_allocated),
620        optional_memory_label(relief.allocator_before.retained_slack_bytes),
621        optional_memory_label(relief.allocator_after.retained_slack_bytes),
622        relief.bytes_released as f64 / (1024.0 * 1024.0),
623    )
624}
625
626fn idle_root_eviction_message(
627    root_id: &ProjectRootId,
628    memory: &crate::memory::RootMemorySnapshot,
629    pressure_relief: Option<&crate::memory::AllocatorPressureRelief>,
630) -> String {
631    // Bash, LSP, and parser state remain resident. The freed total is deliberately
632    // only the known-byte portion of handles eviction actually drops.
633    let freed_bytes = [
634        &memory.semantic,
635        &memory.trigram,
636        &memory.symbols,
637        &memory.callgraph,
638        &memory.inspect,
639    ]
640    .iter()
641    .filter_map(|estimate| estimate.estimated_bytes)
642    .fold(0u64, u64::saturating_add);
643    let mut message = format!(
644        "evicted idle root {}: freed ~{:.1} MB (semantic {}, trigram {}, symbols {}, callgraph {}, inspect {}; retained: bash {}, lsp {}, parser_pool {})",
645        root_id.as_path().display(),
646        freed_bytes as f64 / (1024.0 * 1024.0),
647        eviction_estimate_label(&memory.semantic),
648        eviction_estimate_label(&memory.trigram),
649        eviction_estimate_label(&memory.symbols),
650        eviction_estimate_label(&memory.callgraph),
651        eviction_estimate_label(&memory.inspect),
652        eviction_estimate_label(&memory.bash),
653        eviction_estimate_label(&memory.lsp),
654        eviction_estimate_label(&memory.parser_pool),
655    );
656    if let Some(pressure_relief) = pressure_relief {
657        message.push_str(&pressure_relief_label(pressure_relief));
658    }
659    message
660}
661
662fn process_has_been_idle(now: Instant, live_roots: &HashMap<ProjectRootId, RootMeta>) -> bool {
663    !live_roots.is_empty()
664        && live_roots.values().all(|meta| {
665            now.saturating_duration_since(meta.last_touched) >= IDLE_ROOT_TTL
666                && meta.active_bash_waits == 0
667                && !meta.maintenance_pending
668                && meta.maintenance_queued_kinds.is_empty()
669        })
670}
671
672fn allocator_pressure_relief_after_idle_sweep(
673    now: Instant,
674    live_roots: &HashMap<ProjectRootId, RootMeta>,
675    executor: &Executor,
676) -> Option<crate::memory::AllocatorPressureRelief> {
677    if !process_has_been_idle(now, live_roots)
678        || live_roots.keys().any(|root_id| {
679            executor
680                .actor_context(root_id)
681                .is_some_and(|ctx| ctx.artifact_eviction_blocked())
682        })
683    {
684        return None;
685    }
686
687    #[cfg(target_os = "macos")]
688    {
689        Some(crate::memory::relieve_allocator_pressure())
690    }
691    #[cfg(not(target_os = "macos"))]
692    {
693        None
694    }
695}
696
697fn quiesce_unbound_root(
698    root_id: &ProjectRootId,
699    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
700    executor: &Arc<Executor>,
701) {
702    let Some(meta) = live_roots.get_mut(root_id) else {
703        return;
704    };
705
706    let ctx = executor.actor_context(root_id);
707    if let Some(ctx) = ctx.as_ref() {
708        // Close lifecycle admission before touching scheduler queues. A running
709        // ConfigureTail cannot release gates, install a watcher, or reserve a
710        // callgraph build after this transition becomes visible.
711        ctx.mark_subc_unbound();
712    }
713    let cancelled = executor.cancel_queued_maintenance(root_id);
714    // Transient unbind keeps the root WARM: the watcher stays running (its
715    // events accumulate and replay on rebind, so no unobserved gap exists) and
716    // resident artifacts stay resident. Host restarts unbind every root and
717    // rebind seconds later; stopping the watcher here would force strict
718    // re-verification plus a full callgraph rebuild on every restart. The
719    // expensive teardown (watcher stop + gap invalidation) belongs to the
720    // idle-TTL reaper and the root-deleted path.
721    let discarded = ctx
722        .map(|ctx| crate::commands::configure::cancel_deferred_configure_maintenance(&ctx))
723        .unwrap_or(0);
724    meta.unbound_quiesced = true;
725    meta.maintenance_queued_kinds.clear();
726    meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
727    log::debug!(
728        "subc attach: quiesced unbound root {} (cancelled {} queued maintenance job(s), cancelled {} configure maintenance job(s))",
729        root_id.as_path().display(),
730        cancelled,
731        discarded
732    );
733}
734
735#[allow(clippy::too_many_arguments)]
736fn quiesce_connection_roots(
737    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
738    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
739    routes: &mut HashMap<RouteChannel, RouteIdentity>,
740    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
741    installed_route_epochs: &mut HashMap<u16, u32>,
742    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
743    executor: &Arc<Executor>,
744) {
745    for cancel in route_bash_cancels.values() {
746        cancel.token.cancel();
747    }
748    route_bash_cancels.clear();
749
750    let mut roots = live_roots.keys().cloned().collect::<HashSet<_>>();
751    for pending in pending_binds.values_mut() {
752        pending.cancelled = true;
753        roots.insert(pending.bind_root_id.clone());
754        let _ = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
755    }
756
757    // A connection exit abandons every installed route at once. Close lifecycle
758    // admission before cancelling maintenance so no deferred worker can restore
759    // root activity after the loop-owned route tables disappear.
760    for root_id in roots {
761        if live_roots.contains_key(&root_id) {
762            quiesce_unbound_root(&root_id, live_roots, executor);
763        } else if let Some(ctx) = executor.actor_context(&root_id) {
764            ctx.mark_subc_unbound();
765            executor.cancel_queued_maintenance(&root_id);
766            crate::commands::configure::cancel_deferred_configure_maintenance(&ctx);
767        }
768    }
769
770    routes.clear();
771    root_channels.clear();
772    installed_route_epochs.clear();
773}
774
775/// Per-channel epoch watermarks for roots reclaimed without a client Goodbye.
776/// The 16-bit channel space bounds this map, and no root identity or resource
777/// handle is retained. It exists only so late requests receive a typed error.
778#[derive(Debug, Default)]
779struct ReclaimedRoutes {
780    highest_epoch_by_channel: HashMap<u16, u32>,
781}
782
783impl ReclaimedRoutes {
784    fn insert(&mut self, route: RouteChannel) {
785        self.highest_epoch_by_channel
786            .entry(route.channel)
787            .and_modify(|epoch| *epoch = (*epoch).max(route.epoch))
788            .or_insert(route.epoch);
789    }
790
791    fn contains(&self, route: RouteChannel) -> bool {
792        self.highest_epoch_by_channel
793            .get(&route.channel)
794            .is_some_and(|epoch| route.epoch <= *epoch)
795    }
796}
797
798#[derive(Debug, Default)]
799struct IdleReapOutcome {
800    evicted: usize,
801    forgotten_deleted_roots: Vec<ProjectRootId>,
802}
803
804fn reap_idle_roots(
805    now: Instant,
806    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
807    pending_binds: &HashMap<RouteChannel, PendingBind>,
808    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
809    executor: &Arc<Executor>,
810    metrics: &DispatchPathMetrics,
811) -> IdleReapOutcome {
812    let pending_bind_roots = pending_binds
813        .values()
814        .map(|pending| pending.bind_root_id.clone())
815        .collect::<HashSet<_>>();
816    let mut census = ReapBlockerCensus::default();
817    let mut candidates = Vec::new();
818
819    for (root_id, meta) in live_roots.iter_mut() {
820        let deleted = !root_id.as_path().exists();
821        if deleted {
822            // A missing directory makes a bound route obsolete, but one failed
823            // lookup is not enough evidence to tear down a client-visible actor.
824            // Requiring two maintenance sweeps protects atomic replacement and
825            // transient filesystem failures; observing the path resets the proof.
826            // Absence also covers renames: a task's cwd handle can follow the
827            // moved directory while the registered path disappears. The old
828            // path is deliberately treated as a retired root identity, so the
829            // reaper accepts killing such tasks; rename a project only with no
830            // live tasks rather than relying on cwd-resolution heuristics.
831            meta.consecutive_missing_sweeps = meta.consecutive_missing_sweeps.saturating_add(1);
832        } else {
833            meta.consecutive_missing_sweeps = 0;
834        }
835        let deletion_confirmed = meta.consecutive_missing_sweeps >= 2;
836        let has_bound_route = root_channels
837            .get(root_id)
838            .is_some_and(|channels| !channels.is_empty());
839        let has_pending_bind = pending_bind_roots.contains(root_id);
840
841        if deleted {
842            let mut retained = false;
843            if !deletion_confirmed {
844                census.absence_unconfirmed += 1;
845                retained = true;
846            }
847            // Once absence is confirmed, the directory cannot serve this route
848            // again. Neither a stale route nor the lack of normal unbind cleanup
849            // justifies retaining the root; purge removes the route after retirement.
850            if meta.active_bash_waits > 0 {
851                census.bash_waits += 1;
852                retained = true;
853            }
854            if meta.maintenance_pending {
855                census.maintenance_pending += 1;
856                retained = true;
857            }
858            if !meta.maintenance_queued_kinds.is_empty() {
859                census.maintenance_queued += 1;
860                retained = true;
861            }
862            if has_pending_bind {
863                census.pending_binds += 1;
864                retained = true;
865            }
866            match executor.try_actor_is_idle(root_id) {
867                Some(true) => {}
868                Some(false) => {
869                    census.actor_busy += 1;
870                    retained = true;
871                }
872                None => {
873                    census.actor_state_busy += 1;
874                    retained = true;
875                }
876            }
877            if retained {
878                census.deleted_retained += 1;
879                continue;
880            }
881        } else {
882            // Route teardown marks the lifecycle admission gate before the last
883            // channel disappears. Requiring zero bound channels and a quiesced
884            // lifecycle prevents a still-bound root from losing its watcher.
885            if has_bound_route
886                || !meta.unbound_quiesced
887                || meta.idle_artifacts_evicted
888                || now.saturating_duration_since(meta.last_touched) < IDLE_ROOT_TTL
889                || meta.active_bash_waits > 0
890                || meta.maintenance_pending
891                || !meta.maintenance_queued_kinds.is_empty()
892                || has_pending_bind
893                || !executor.actor_is_idle(root_id)
894            {
895                continue;
896            }
897        }
898        candidates.push((root_id.clone(), deleted));
899    }
900
901    let mut reaped = Vec::new();
902    let mut forgotten_deleted_roots = Vec::new();
903    for (root_id, deleted) in candidates {
904        let Some(ctx) = executor.actor_context(&root_id) else {
905            if deleted {
906                census.deleted_retained += 1;
907                census.actor_busy += 1;
908            }
909            continue;
910        };
911        // A TTL-aged unbound root retained its watcher-derived pending paths
912        // across the transient-unbind window. Strict gap invalidation subsumes
913        // them, but every abort path must restore them because a rebind can
914        // still happen until eviction commits.
915        //
916        // After two consecutive directory-absence scans confirm that the
917        // root is gone, terminate its background task before checking the
918        // artifact-eviction gate. The task can otherwise keep the root's
919        // artifacts in use; cleanup first lets confirmed reclamation finish
920        // without weakening the gate for unrelated active work.
921        if deleted {
922            ctx.bash_background()
923                .kill_running_tasks_for_root(root_id.as_path());
924        }
925        let taken_pending = Some(ctx.take_pending_reconciliation_state());
926        if ctx.artifact_eviction_blocked() {
927            if let Some(pending) = taken_pending {
928                ctx.restore_pending_reconciliation_state(pending);
929            }
930            if deleted {
931                census.deleted_retained += 1;
932                census.artifact_eviction_blocked += 1;
933            }
934            continue;
935        }
936        let memory_before = ctx.memory_root_snapshot();
937        if !ctx.evict_idle_artifacts() {
938            if let Some(pending) = taken_pending {
939                ctx.restore_pending_reconciliation_state(pending);
940            }
941            if deleted {
942                census.deleted_retained += 1;
943                census.artifact_eviction_failed += 1;
944            }
945            continue;
946        }
947        drop(taken_pending);
948        ctx.stop_watcher_runtime_in_background();
949        // Edits during watcher downtime are unobserved. Advance publication
950        // epochs and force strict verification before any later warm reload.
951        ctx.invalidate_artifacts_after_watcher_gap();
952
953        if deleted {
954            if executor.retire_idle_actor_in_background(&root_id) {
955                live_roots.remove(&root_id);
956                forgotten_deleted_roots.push(root_id.clone());
957            } else {
958                census.deleted_retained += 1;
959                census.actor_busy += 1;
960            }
961        } else {
962            if let Some(meta) = live_roots.get_mut(&root_id) {
963                meta.idle_artifacts_evicted = true;
964            }
965            ctx.release_idle_reopenable_resources_in_background();
966        }
967        reaped.push((root_id, memory_before));
968    }
969
970    metrics.record_reap(census);
971    if census.deleted_retained > 0 {
972        log::info!(
973            "subc attach: retained {} deleted root(s) during idle reap; blockers={}",
974            census.deleted_retained,
975            census.blocker_histogram()
976        );
977    }
978
979    let pressure_relief = (!reaped.is_empty())
980        .then(|| allocator_pressure_relief_after_idle_sweep(now, live_roots, executor))
981        .flatten();
982    for (root_id, memory_before) in &reaped {
983        log::info!(
984            "{}",
985            idle_root_eviction_message(root_id, memory_before, pressure_relief.as_ref())
986        );
987    }
988    IdleReapOutcome {
989        evicted: reaped.len(),
990        forgotten_deleted_roots,
991    }
992}
993
994#[allow(clippy::too_many_arguments)]
995fn purge_deleted_root_residents(
996    root_id: &ProjectRootId,
997    routes: &mut HashMap<RouteChannel, RouteIdentity>,
998    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
999    installed_route_epochs: &mut HashMap<u16, u32>,
1000    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1001    retry_buffer: &mut RetryBuffer,
1002    reclaimed_routes: &mut ReclaimedRoutes,
1003    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1004    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1005    bg_subs: &mut HashMap<RouteChannel, BgSub>,
1006    bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1007    bg_wake_pending: &mut HashSet<RouteChannel>,
1008    bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
1009    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1010) {
1011    let mut stale_routes = root_channels.get(root_id).cloned().unwrap_or_default();
1012    stale_routes.extend(
1013        routes
1014            .iter()
1015            .filter_map(|(route, identity)| (&identity.root == root_id).then_some(*route)),
1016    );
1017    stale_routes.extend(
1018        bg_sub_by_session
1019            .iter()
1020            .filter_map(|((root, _), route)| (root == root_id).then_some(*route)),
1021    );
1022    stale_routes.extend(
1023        pending_bash_asks
1024            .values()
1025            .filter_map(|ask| (&ask.root == root_id).then_some(ask.route)),
1026    );
1027
1028    for route in stale_routes {
1029        reclaimed_routes.insert(route);
1030        remove_installed_route(installed_route_epochs, route);
1031        remove_route_channel(routes, root_channels, route);
1032        if let Some(cancel) = route_bash_cancels.remove(&route) {
1033            cancel.token.cancel();
1034        }
1035        retry_buffer.remove(&route);
1036        bg_subs.remove(&route);
1037        bg_wake_pending.remove(&route);
1038    }
1039    root_channels.remove(root_id);
1040    session_identity.retain(|(root, _), _| root != root_id);
1041    push_buffer.retain(|key, _| &key.root != root_id);
1042    bg_wake_epoch.retain(|(root, _), _| root != root_id);
1043    pending_bash_asks.retain(|_, ask| &ask.root != root_id);
1044    bg_sub_by_session.retain(|(root, _), _| root != root_id);
1045
1046    log::info!(
1047        "subc attach: fully forgot deleted root {}",
1048        root_id.as_path().display()
1049    );
1050}
1051
1052#[allow(clippy::too_many_arguments)]
1053fn submit_due_maintenance_jobs(
1054    executor: &Arc<Executor>,
1055    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1056    pending_binds: &HashMap<RouteChannel, PendingBind>,
1057    bg_sub_by_session: &HashMap<(ProjectRootId, String), RouteChannel>,
1058    bg_wake_pending: &HashSet<RouteChannel>,
1059    bg_wake_epoch: &HashMap<(ProjectRootId, String), u64>,
1060    maintenance_tx: &mpsc::Sender<MaintenanceCompletion>,
1061    metrics: &Arc<DispatchPathMetrics>,
1062) {
1063    let pending_bind_roots = pending_binds
1064        .values()
1065        .map(|pending| pending.bind_root_id.clone())
1066        .collect::<HashSet<_>>();
1067    let (due_jobs, deferred_jobs) = due_maintenance_jobs(
1068        live_roots,
1069        Some(executor),
1070        bg_sub_by_session,
1071        bg_wake_pending,
1072        MAINTENANCE_SUBMIT_BUDGET,
1073        &pending_bind_roots,
1074    );
1075    if deferred_jobs {
1076        metrics
1077            .maintenance_budget_deferrals
1078            .fetch_add(1, Ordering::Relaxed);
1079    }
1080    for (root_id, kind) in due_jobs {
1081        let bg_sessions_to_check = if kind == MaintenanceDrainKind::CompletionDrains {
1082            bg_sub_by_session
1083                .iter()
1084                .filter_map(|((root, session), _)| {
1085                    if root == &root_id {
1086                        Some((
1087                            session.clone(),
1088                            bg_wake_epoch
1089                                .get(&(root_id.clone(), session.clone()))
1090                                .copied()
1091                                .unwrap_or(0),
1092                        ))
1093                    } else {
1094                        None
1095                    }
1096                })
1097                .collect()
1098        } else {
1099            Vec::new()
1100        };
1101        submit_maintenance_job(
1102            executor,
1103            root_id,
1104            kind,
1105            bg_sessions_to_check,
1106            maintenance_tx,
1107            metrics,
1108        );
1109    }
1110}
1111
1112fn should_requiesce_after_maintenance(
1113    meta: &RootMeta,
1114    completed_kind: MaintenanceDrainKind,
1115    bind_pending: bool,
1116) -> bool {
1117    meta.unbound_quiesced && completed_kind != MaintenanceDrainKind::Lsp && !bind_pending
1118}
1119
1120fn note_maintenance_completion(
1121    meta: &mut RootMeta,
1122    requeue_kind: Option<MaintenanceDrainKind>,
1123    fatal: bool,
1124    defer_requeue: bool,
1125) {
1126    if fatal {
1127        meta.maintenance_poisoned = true;
1128    }
1129
1130    if let Some(kind) = requeue_kind.filter(|_| !meta.maintenance_poisoned && !defer_requeue) {
1131        meta.maintenance_queued_kinds.push_back(kind);
1132    }
1133
1134    meta.maintenance_jobs_in_flight = meta.maintenance_jobs_in_flight.saturating_sub(1);
1135    meta.maintenance_pending =
1136        meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
1137}
1138
1139fn route_key(channel: u16, epoch: u32) -> RouteChannel {
1140    RouteChannel { channel, epoch }
1141}
1142
1143fn remove_installed_route(installed_epochs: &mut HashMap<u16, u32>, route: RouteChannel) {
1144    if installed_epochs.get(&route.channel).copied() == Some(route.epoch) {
1145        installed_epochs.remove(&route.channel);
1146    }
1147}
1148
1149fn ingress_route_should_be_processed(
1150    installed_epochs: &HashMap<u16, u32>,
1151    reclaimed_routes: &ReclaimedRoutes,
1152    frame: &Frame,
1153) -> bool {
1154    if frame.header.channel == 0
1155        || installed_epochs.get(&frame.header.channel).copied() == Some(frame.header.epoch)
1156    {
1157        return true;
1158    }
1159
1160    // A late request for a reclaimed root reaches the normal unknown-route
1161    // handler, which returns the typed `route_not_bound` error. Other stale or
1162    // never-installed generations remain silent so they cannot affect a newer
1163    // route or change the protocol's rejected-bind behavior.
1164    frame.header.ty == FrameType::Request
1165        && reclaimed_routes.contains(route_key(frame.header.channel, frame.header.epoch))
1166}
1167
1168fn bash_elicitation_timeout() -> Duration {
1169    if cfg!(debug_assertions) {
1170        if let Ok(raw) = std::env::var("AFT_TEST_SUBC_BASH_ELICITATION_TTL_MS") {
1171            if let Ok(ms) = raw.parse::<u64>() {
1172                if ms > 0 {
1173                    return Duration::from_millis(ms);
1174                }
1175            }
1176        }
1177    }
1178    BASH_ELICITATION_TIMEOUT
1179}
1180
1181fn allocate_reverse_corr(
1182    pending_bash_asks: &HashMap<ReverseCorrKey, PendingBashAsk>,
1183    route: RouteChannel,
1184    next_corr: &mut u64,
1185) -> u64 {
1186    loop {
1187        let corr = *next_corr;
1188        *next_corr = (*next_corr).wrapping_add(1).max(1);
1189        if !pending_bash_asks.contains_key(&ReverseCorrKey { route, corr }) {
1190            return corr;
1191        }
1192    }
1193}
1194
1195fn bash_permission_kind_label(kind: &crate::bash_permissions::PermissionKind) -> &'static str {
1196    match kind {
1197        crate::bash_permissions::PermissionKind::ExternalDirectory => "external directory",
1198        crate::bash_permissions::PermissionKind::Bash => "bash",
1199    }
1200}
1201
1202fn bash_elicitation_patterns(asks: &[crate::bash_permissions::PermissionAsk]) -> Vec<String> {
1203    let mut patterns = Vec::new();
1204    let mut seen = HashSet::new();
1205    for ask in asks {
1206        for pattern in ask.patterns.iter().chain(ask.always.iter()) {
1207            if seen.insert(pattern.clone()) {
1208                patterns.push(pattern.clone());
1209            }
1210        }
1211    }
1212    patterns
1213}
1214
1215fn bash_elicitation_message(
1216    command: &str,
1217    asks: &[crate::bash_permissions::PermissionAsk],
1218) -> String {
1219    let command = command.split_whitespace().collect::<Vec<_>>().join(" ");
1220    let patterns = bash_elicitation_patterns(asks);
1221    let pattern_text = if patterns.is_empty() {
1222        "no matched permission patterns".to_string()
1223    } else {
1224        patterns.join(", ")
1225    };
1226    let ask_kinds = asks
1227        .iter()
1228        .map(|ask| bash_permission_kind_label(&ask.kind))
1229        .collect::<HashSet<_>>()
1230        .into_iter()
1231        .collect::<Vec<_>>()
1232        .join(", ");
1233    if ask_kinds.is_empty() {
1234        format!("Allow bash command `{command}`? Matched patterns: {pattern_text}")
1235    } else {
1236        format!("Allow bash command `{command}`? Matched {ask_kinds} patterns: {pattern_text}")
1237    }
1238}
1239
1240fn bash_elicitation_request_body(
1241    command: &str,
1242    asks: &[crate::bash_permissions::PermissionAsk],
1243) -> Value {
1244    json!({
1245        "method": BASH_ELICITATION_CREATE_METHOD,
1246        "params": {
1247            "mode": "form",
1248            "message": bash_elicitation_message(command, asks),
1249            "requestedSchema": {
1250                "type": "object",
1251                "properties": {
1252                    "decision": {
1253                        "type": "string",
1254                        "enum": ["allow", "deny"],
1255                        "description": "Choose allow to run this bash command once, or deny to block it."
1256                    }
1257                },
1258                "required": ["decision"],
1259                "additionalProperties": false
1260            },
1261            "_meta": {
1262                "aft": {
1263                    "tool": "bash",
1264                    "command": command,
1265                    "asks": asks
1266                }
1267            }
1268        }
1269    })
1270}
1271
1272fn build_bash_elicitation_request_frame(
1273    ver: u8,
1274    route: RouteChannel,
1275    corr: u64,
1276    flags: Flags,
1277    command: &str,
1278    asks: &[crate::bash_permissions::PermissionAsk],
1279) -> Result<Frame, SubcError> {
1280    let body = bash_elicitation_request_body(command, asks);
1281    Frame::build_with_version(
1282        ver,
1283        FrameType::Request,
1284        flags,
1285        route.channel,
1286        route.epoch,
1287        corr,
1288        serde_json::to_vec(&body).map_err(SubcError::Json)?,
1289    )
1290    .map_err(SubcError::FrameBuild)
1291}
1292
1293fn bash_elicitation_reply_is_allow(body: &[u8]) -> bool {
1294    let Ok(value) = serde_json::from_slice::<Value>(body) else {
1295        return false;
1296    };
1297    flat_bash_elicitation_reply_is_allow(&value) || mcp_bash_elicitation_reply_is_allow(&value)
1298}
1299
1300fn flat_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1301    let Some(object) = value.as_object() else {
1302        return false;
1303    };
1304    object.len() == 1 && object.get("decision").and_then(Value::as_str) == Some("allow")
1305}
1306
1307fn mcp_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1308    let Some(object) = value.as_object() else {
1309        return false;
1310    };
1311    if object.len() != 2 || object.get("action").and_then(Value::as_str) != Some("accept") {
1312        return false;
1313    }
1314    let Some(content) = object.get("content").and_then(Value::as_object) else {
1315        return false;
1316    };
1317    content.len() == 1 && content.get("decision").and_then(Value::as_str) == Some("allow")
1318}
1319
1320#[allow(clippy::too_many_arguments)]
1321async fn settle_pending_bash_ask_denied(
1322    tx: &WriterSender,
1323    pending: PendingBashAsk,
1324    routes: &HashMap<RouteChannel, RouteIdentity>,
1325    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1326    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1327    shutdown: &Arc<Notify>,
1328    metrics: &DispatchPathMetrics,
1329) -> Result<(), SubcError> {
1330    let completion = bash::bash_denied_untrusted_completion(
1331        pending.route,
1332        pending.tool_corr,
1333        pending.tool_flags,
1334        pending.tool_ver,
1335        pending.root,
1336        pending.request_id,
1337        pending.format_context,
1338    );
1339    bash::handle_bash_deferred_completion(
1340        tx,
1341        completion,
1342        routes,
1343        live_roots,
1344        route_bash_cancels,
1345        shutdown,
1346        metrics,
1347    )
1348    .await
1349}
1350
1351fn take_pending_bash_asks_for_route(
1352    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1353    route: RouteChannel,
1354) -> Vec<PendingBashAsk> {
1355    let keys = pending_bash_asks
1356        .keys()
1357        .copied()
1358        .filter(|key| key.route == route)
1359        .collect::<Vec<_>>();
1360    keys.into_iter()
1361        .filter_map(|key| pending_bash_asks.remove(&key))
1362        .collect()
1363}
1364
1365#[allow(clippy::too_many_arguments)]
1366async fn settle_pending_bash_asks_for_route(
1367    tx: &WriterSender,
1368    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1369    route: RouteChannel,
1370    routes: &HashMap<RouteChannel, RouteIdentity>,
1371    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1372    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1373    shutdown: &Arc<Notify>,
1374    metrics: &DispatchPathMetrics,
1375) -> Result<(), SubcError> {
1376    for pending in take_pending_bash_asks_for_route(pending_bash_asks, route) {
1377        settle_pending_bash_ask_denied(
1378            tx,
1379            pending,
1380            routes,
1381            live_roots,
1382            route_bash_cancels,
1383            shutdown,
1384            metrics,
1385        )
1386        .await?;
1387    }
1388    Ok(())
1389}
1390
1391#[allow(clippy::too_many_arguments)]
1392async fn settle_all_pending_bash_asks(
1393    tx: &WriterSender,
1394    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1395    routes: &HashMap<RouteChannel, RouteIdentity>,
1396    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1397    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1398    shutdown: &Arc<Notify>,
1399    metrics: &DispatchPathMetrics,
1400) -> Result<(), SubcError> {
1401    let pending = pending_bash_asks
1402        .drain()
1403        .map(|(_, pending)| pending)
1404        .collect::<Vec<_>>();
1405    for pending in pending {
1406        settle_pending_bash_ask_denied(
1407            tx,
1408            pending,
1409            routes,
1410            live_roots,
1411            route_bash_cancels,
1412            shutdown,
1413            metrics,
1414        )
1415        .await?;
1416    }
1417    Ok(())
1418}
1419
1420#[allow(clippy::too_many_arguments)]
1421async fn expire_pending_bash_asks(
1422    tx: &WriterSender,
1423    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1424    routes: &HashMap<RouteChannel, RouteIdentity>,
1425    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1426    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1427    shutdown: &Arc<Notify>,
1428    metrics: &DispatchPathMetrics,
1429) -> Result<(), SubcError> {
1430    let now = Instant::now();
1431    let expired = pending_bash_asks
1432        .iter()
1433        .filter_map(|(key, pending)| (pending.expires_at <= now).then_some(*key))
1434        .collect::<Vec<_>>();
1435    for key in expired {
1436        if let Some(pending) = pending_bash_asks.remove(&key) {
1437            log::debug!(
1438                "subc attach: bash elicitation request {} on route {} expired fail-closed",
1439                key.corr,
1440                pending.route
1441            );
1442            settle_pending_bash_ask_denied(
1443                tx,
1444                pending,
1445                routes,
1446                live_roots,
1447                route_bash_cancels,
1448                shutdown,
1449                metrics,
1450            )
1451            .await?;
1452        }
1453    }
1454    Ok(())
1455}
1456
1457#[allow(clippy::too_many_arguments)]
1458async fn handle_bash_elicitation_reply(
1459    tx: &WriterSender,
1460    frame: &Frame,
1461    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1462    routes: &HashMap<RouteChannel, RouteIdentity>,
1463    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1464    executor: &Arc<Executor>,
1465    shutdown: &Arc<Notify>,
1466    bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
1467    bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
1468    metrics: &Arc<DispatchPathMetrics>,
1469    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1470    dispatch: DispatchFn,
1471) -> Result<(), SubcError> {
1472    let key = ReverseCorrKey {
1473        route: route_key(frame.header.channel, frame.header.epoch),
1474        corr: frame.header.corr,
1475    };
1476    let Some(pending) = pending_bash_asks.remove(&key) else {
1477        return Ok(());
1478    };
1479
1480    if frame.header.ty == FrameType::Response && bash_elicitation_reply_is_allow(&frame.body) {
1481        if routes.contains_key(&key.route) {
1482            bash::submit_deferred_bash(
1483                executor,
1484                bash_deferred_tx,
1485                bash_poll_touch_tx,
1486                metrics,
1487                dispatch,
1488                pending.root,
1489                pending.project_root,
1490                pending.session_id,
1491                pending.request_id,
1492                pending.route,
1493                pending.tool_corr,
1494                pending.tool_flags,
1495                pending.tool_ver,
1496                pending.arguments,
1497                pending.format_context,
1498                pending.cancel,
1499                BindTrust::Untrusted,
1500                pending.spawn_principal,
1501                Some(pending.grants),
1502            );
1503            return Ok(());
1504        }
1505        log::debug!(
1506            "subc attach: dropping allowed bash elicitation reply {} for unbound route {}",
1507            key.corr,
1508            pending.route
1509        );
1510    }
1511
1512    settle_pending_bash_ask_denied(
1513        tx,
1514        pending,
1515        routes,
1516        live_roots,
1517        route_bash_cancels,
1518        shutdown,
1519        metrics,
1520    )
1521    .await
1522}
1523
1524#[allow(clippy::too_many_arguments)]
1525async fn cancel_pending_bash_ask_for_tool_call(
1526    tx: &WriterSender,
1527    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1528    route: RouteChannel,
1529    tool_corr: u64,
1530    routes: &HashMap<RouteChannel, RouteIdentity>,
1531    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1532    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1533    shutdown: &Arc<Notify>,
1534    metrics: &DispatchPathMetrics,
1535) -> Result<(), SubcError> {
1536    let keys = pending_bash_asks
1537        .iter()
1538        .filter_map(|(key, pending)| {
1539            (key.route == route && pending.tool_corr == tool_corr).then_some(*key)
1540        })
1541        .collect::<Vec<_>>();
1542    for key in keys {
1543        if let Some(pending) = pending_bash_asks.remove(&key) {
1544            settle_pending_bash_ask_denied(
1545                tx,
1546                pending,
1547                routes,
1548                live_roots,
1549                route_bash_cancels,
1550                shutdown,
1551                metrics,
1552            )
1553            .await?;
1554        }
1555    }
1556    Ok(())
1557}
1558
1559fn remove_root_channel(
1560    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1561    root: &ProjectRootId,
1562    channel: RouteChannel,
1563) {
1564    let remove_root = if let Some(channels) = root_channels.get_mut(root) {
1565        channels.remove(&channel);
1566        channels.is_empty()
1567    } else {
1568        false
1569    };
1570    if remove_root {
1571        root_channels.remove(root);
1572    }
1573}
1574
1575fn remove_route_channel(
1576    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1577    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1578    channel: RouteChannel,
1579) -> Option<RouteIdentity> {
1580    let removed = routes.remove(&channel);
1581    if let Some(identity) = &removed {
1582        remove_root_channel(root_channels, &identity.root, channel);
1583    }
1584    removed
1585}
1586
1587fn insert_route_channel(
1588    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1589    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1590    channel: RouteChannel,
1591    identity: RouteIdentity,
1592) {
1593    if let Some(previous) = routes.insert(channel, identity.clone()) {
1594        remove_root_channel(root_channels, &previous.root, channel);
1595    }
1596    root_channels
1597        .entry(identity.root.clone())
1598        .or_default()
1599        .insert(channel);
1600}
1601
1602fn remove_bg_subscription_index(
1603    bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1604    channel: RouteChannel,
1605    identity: Option<&RouteIdentity>,
1606) {
1607    if let Some(identity) = identity {
1608        let key = (identity.root.clone(), identity.session.clone());
1609        if bg_sub_by_session.get(&key).copied() == Some(channel) {
1610            bg_sub_by_session.remove(&key);
1611        }
1612    } else {
1613        bg_sub_by_session.retain(|_, mapped_channel| *mapped_channel != channel);
1614    }
1615}
1616
1617fn route_removal_will_quiesce_root(
1618    root: &ProjectRootId,
1619    route: RouteChannel,
1620    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1621    has_pending_bind: bool,
1622    replacement_root: Option<&ProjectRootId>,
1623) -> bool {
1624    let removes_last_route = root_channels
1625        .get(root)
1626        .is_some_and(|channels| channels.len() == 1 && channels.contains(&route));
1627    removes_last_route && !has_pending_bind && replacement_root != Some(root)
1628}
1629
1630fn should_quiesce_removed_root(
1631    root: &ProjectRootId,
1632    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1633    has_pending_bind: bool,
1634    replacement_root: Option<&ProjectRootId>,
1635) -> bool {
1636    !root_channels.contains_key(root) && !has_pending_bind && replacement_root != Some(root)
1637}
1638
1639async fn end_bg_subscription(
1640    writer_tx: &WriterSender,
1641    metrics: &DispatchPathMetrics,
1642    bg_subs: &mut HashMap<RouteChannel, BgSub>,
1643    bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1644    bg_wake_pending: &mut HashSet<RouteChannel>,
1645    channel: RouteChannel,
1646    identity: Option<&RouteIdentity>,
1647) -> Result<(), SubcError> {
1648    if let Some(sub) = bg_subs.remove(&channel) {
1649        bg_wake_pending.remove(&channel);
1650        remove_bg_subscription_index(bg_sub_by_session, channel, identity);
1651        push::send_reliable_bg_stream_end(writer_tx, metrics, channel, &sub).await?;
1652    }
1653    Ok(())
1654}
1655
1656#[allow(clippy::too_many_arguments)]
1657async fn teardown_installed_route(
1658    tx: &WriterSender,
1659    metrics: &DispatchPathMetrics,
1660    executor: &Arc<Executor>,
1661    channel: RouteChannel,
1662    cancellation_reason: &str,
1663    replacement_root: Option<&ProjectRootId>,
1664    installed_route_epochs: &mut HashMap<u16, u32>,
1665    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1666    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1667    bg_subs: &mut HashMap<RouteChannel, BgSub>,
1668    bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1669    bg_wake_pending: &mut HashSet<RouteChannel>,
1670    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1671    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1672    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1673    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
1674    retry_buffer: &mut RetryBuffer,
1675    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1676    shutdown: &Arc<Notify>,
1677) -> Result<(), SubcError> {
1678    remove_installed_route(installed_route_epochs, channel);
1679    end_bg_subscription(
1680        tx,
1681        metrics,
1682        bg_subs,
1683        bg_sub_by_session,
1684        bg_wake_pending,
1685        channel,
1686        routes.get(&channel),
1687    )
1688    .await?;
1689    settle_pending_bash_asks_for_route(
1690        tx,
1691        pending_bash_asks,
1692        channel,
1693        routes,
1694        live_roots,
1695        route_bash_cancels,
1696        shutdown,
1697        metrics,
1698    )
1699    .await?;
1700    if let Some(cancel) = route_bash_cancels.remove(&channel) {
1701        cancel.token.cancel();
1702    }
1703    if let Some(pending) = pending_binds.get_mut(&channel) {
1704        pending.cancelled = true;
1705        let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
1706        log::debug!(
1707            "subc attach: cancelled pending RouteBind for route {} on {cancellation_reason} (configure job: {outcome:?})",
1708            channel.channel
1709        );
1710    }
1711    let migrated = push::migrate_retry_buffer_to_push_buffer(retry_buffer, channel, push_buffer);
1712    if let Some(identity) = routes.get(&channel) {
1713        let has_pending_bind = pending_binds
1714            .values()
1715            .any(|pending| pending.bind_root_id == identity.root);
1716        if route_removal_will_quiesce_root(
1717            &identity.root,
1718            channel,
1719            root_channels,
1720            has_pending_bind,
1721            replacement_root,
1722        ) {
1723            if let Some(ctx) = executor.actor_context(&identity.root) {
1724                // Fence deferred admissions before the final route disappears
1725                // from the loop-owned routing tables.
1726                ctx.mark_subc_unbound();
1727            }
1728        }
1729    }
1730    if let Some(identity) = remove_route_channel(routes, root_channels, channel) {
1731        if migrated > 0 {
1732            log::debug!(
1733                "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from route {} into detach replay",
1734                channel.channel
1735            );
1736        }
1737        if let Some(meta) = live_roots.get_mut(&identity.root) {
1738            let idle_for = meta.last_touched.elapsed();
1739            meta.note_activity();
1740            log::debug!(
1741                "subc attach: route {} torn down for root {} harness {} session {} (last touched {:?} ago)",
1742                channel.channel,
1743                identity.root.as_path().display(),
1744                identity.harness,
1745                identity.session,
1746                idle_for
1747            );
1748        } else {
1749            log::debug!(
1750                "subc attach: route {} torn down for root {} harness {} session {}",
1751                channel.channel,
1752                identity.root.as_path().display(),
1753                identity.harness,
1754                identity.session
1755            );
1756        }
1757        let has_pending_bind = pending_binds
1758            .values()
1759            .any(|pending| pending.bind_root_id == identity.root);
1760        if should_quiesce_removed_root(
1761            &identity.root,
1762            root_channels,
1763            has_pending_bind,
1764            replacement_root,
1765        ) {
1766            quiesce_unbound_root(&identity.root, live_roots, executor);
1767        }
1768    } else {
1769        if migrated > 0 {
1770            log::debug!(
1771                "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from unbound route {} into detach replay",
1772                channel.channel
1773            );
1774        }
1775        log::debug!("subc attach: unbound route {} torn down", channel.channel);
1776    }
1777    Ok(())
1778}
1779
1780fn remember_session_identity(
1781    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1782    identity: &RouteIdentity,
1783) {
1784    let key = (identity.root.clone(), identity.session.clone());
1785    if matches!(identity.trust, BindTrust::Untrusted)
1786        && session_identity
1787            .get(&key)
1788            .is_some_and(|retained| matches!(retained.trust, BindTrust::FirstParty))
1789    {
1790        return;
1791    }
1792
1793    // Retained after route Goodbye so reliable session-scoped frames emitted while
1794    // the session is detached can still be keyed by the full (root,harness,session)
1795    // replay triple. Untrusted binds never overwrite a retained first-party
1796    // session identity, because bash completion replay is an observation channel.
1797    session_identity.insert(
1798        key,
1799        RetainedSessionIdentity {
1800            harness: identity.harness.clone(),
1801            trust: identity.trust,
1802        },
1803    );
1804}
1805
1806fn replay_key_for_session(
1807    session_identity: &HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1808    root: &ProjectRootId,
1809    session: &str,
1810) -> Option<(push::ReplayKey, BindTrust)> {
1811    let retained = session_identity.get(&(root.clone(), session.to_string()))?;
1812    Some((
1813        push::ReplayKey {
1814            root: root.clone(),
1815            harness: retained.harness.clone(),
1816            session: session.to_string(),
1817        },
1818        retained.trust,
1819    ))
1820}
1821/// Sync command dispatch, passed in from `main` (the binary owns the command
1822/// table). Invoked only inside executor jobs in subc mode.
1823pub type DispatchFn = fn(RawRequest, &AppContext) -> Response;
1824
1825#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1826enum ModuleLoopExit {
1827    Graceful,
1828    SkipSearchFlush,
1829}
1830
1831/// Entry point for `aft --subc <connection-file>`. Synchronous on the outside;
1832/// owns an isolated current-thread tokio runtime for the async transport.
1833/// Returns `Err` (fail-loud) on any connect/auth/protocol failure — we never
1834/// fall back to the standalone loop, to avoid split-brain index state.
1835pub fn run_subc_mode(
1836    connection_file_path: &Path,
1837    ctx: Arc<AppContext>,
1838    executor: Arc<Executor>,
1839    dispatch: DispatchFn,
1840    user_config_path: Option<PathBuf>,
1841) -> Result<(), SubcError> {
1842    // Production NEVER allows non-manifest tool names on route channels: AFT
1843    // fails closed and does not trust subc to enforce the manifest. The
1844    // test-only harness sets this through `run_subc_mode_for_test`.
1845    run_subc_mode_inner(
1846        connection_file_path,
1847        ctx,
1848        executor,
1849        dispatch,
1850        user_config_path,
1851        false,
1852    )
1853}
1854
1855fn run_subc_mode_inner(
1856    connection_file_path: &Path,
1857    ctx: Arc<AppContext>,
1858    executor: Arc<Executor>,
1859    dispatch: DispatchFn,
1860    user_config_path: Option<PathBuf>,
1861    allow_native_passthrough: bool,
1862) -> Result<(), SubcError> {
1863    let runtime = tokio::runtime::Builder::new_current_thread()
1864        .enable_all()
1865        .build()
1866        .map_err(SubcError::Runtime)?;
1867
1868    let executor_for_loop = Arc::clone(&executor);
1869    let loop_result = runtime.block_on(async move {
1870        let shared_app = ctx.app();
1871        drop(ctx);
1872        let stream = connect_and_authenticate(connection_file_path).await?;
1873        log::info!(
1874            "subc attach: authenticated to daemon via {}",
1875            connection_file_path.display()
1876        );
1877        let (read_half, write_half) = tokio::io::split(stream);
1878        run_module_loop(
1879            read_half,
1880            write_half,
1881            shared_app,
1882            executor_for_loop,
1883            dispatch,
1884            user_config_path,
1885            allow_native_passthrough,
1886        )
1887        .await
1888    });
1889
1890    let actor_contexts = executor.actor_contexts();
1891    if matches!(loop_result, Ok(ModuleLoopExit::Graceful)) {
1892        // EOF/Goodbye teardown flushes each root's index deltas and queued
1893        // callgraph refreshes. Fatal/panic teardown skips this best-effort work.
1894        flush_actor_indexes_on_graceful_shutdown(&actor_contexts);
1895    }
1896    for actor_ctx in &actor_contexts {
1897        actor_ctx.lsp().shutdown_all();
1898        actor_ctx.bash_background().detach();
1899    }
1900
1901    loop_result.map(|_| ())
1902}
1903
1904fn flush_actor_indexes_on_graceful_shutdown(actor_contexts: &[Arc<AppContext>]) {
1905    for actor_ctx in actor_contexts {
1906        let _ = actor_ctx.flush_search_index_on_graceful_shutdown();
1907    }
1908    let _ = crate::callgraph_store::flush_callgraph_store_refreshes_on_graceful_shutdown();
1909}
1910
1911/// Test-only entry that enables the non-manifest native-command passthrough on
1912/// route channels. Integration tests drive synthetic native commands (`glob`,
1913/// `callers`, `subc_test_echo_session`, …) through the executor to exercise
1914/// mechanics; production callers use [`run_subc_mode`], which fails closed.
1915#[doc(hidden)]
1916pub fn run_subc_mode_for_test(
1917    connection_file_path: &Path,
1918    ctx: Arc<AppContext>,
1919    executor: Arc<Executor>,
1920    dispatch: DispatchFn,
1921    user_config_path: Option<PathBuf>,
1922) -> Result<(), SubcError> {
1923    run_subc_mode_inner(
1924        connection_file_path,
1925        ctx,
1926        executor,
1927        dispatch,
1928        user_config_path,
1929        true,
1930    )
1931}
1932
1933#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1934enum AttachErrorClass {
1935    Transient,
1936    Permanent,
1937}
1938
1939impl fmt::Display for AttachErrorClass {
1940    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1941        match self {
1942            Self::Transient => f.write_str("transient"),
1943            Self::Permanent => f.write_str("permanent"),
1944        }
1945    }
1946}
1947
1948#[derive(Clone, Copy)]
1949struct AttachRetryPolicy {
1950    budget: Duration,
1951    initial_backoff: Duration,
1952    max_backoff: Duration,
1953    jitter_percent: u64,
1954}
1955
1956const ATTACH_RETRY_POLICY: AttachRetryPolicy = AttachRetryPolicy {
1957    budget: ATTACH_RETRY_BUDGET,
1958    initial_backoff: ATTACH_RETRY_INITIAL_BACKOFF,
1959    max_backoff: ATTACH_RETRY_MAX_BACKOFF,
1960    jitter_percent: ATTACH_RETRY_JITTER_PERCENT,
1961};
1962
1963/// Retry only failures that can be caused by a daemon bounce or an interrupted
1964/// handshake. Protocol and credential failures are permanent for this process.
1965fn classify_attach_error(error: &SubcError) -> AttachErrorClass {
1966    let transient = match error {
1967        SubcError::Connect { source, .. } => is_transient_attach_io(source.kind()),
1968        SubcError::Auth { source, .. } => match source {
1969            subc_transport::AuthError::Timeout { .. }
1970            | subc_transport::AuthError::UnexpectedEof { .. } => true,
1971            subc_transport::AuthError::Io { source, .. } => is_transient_attach_io(source.kind()),
1972            _ => false,
1973        },
1974        _ => false,
1975    };
1976    if transient {
1977        AttachErrorClass::Transient
1978    } else {
1979        AttachErrorClass::Permanent
1980    }
1981}
1982
1983fn is_transient_attach_io(kind: io::ErrorKind) -> bool {
1984    matches!(
1985        kind,
1986        io::ErrorKind::ConnectionRefused
1987            | io::ErrorKind::TimedOut
1988            | io::ErrorKind::ConnectionReset
1989            | io::ErrorKind::ConnectionAborted
1990            | io::ErrorKind::BrokenPipe
1991            | io::ErrorKind::UnexpectedEof
1992    )
1993}
1994
1995/// Read the connection file → resolve the first endpoint → TCP connect → HMAC
1996/// handshake. Transient initial-attach failures retry on fresh sockets and reread
1997/// the file so a daemon bounce can publish a new endpoint or authentication key.
1998async fn connect_and_authenticate(connection_file_path: &Path) -> Result<TcpStream, SubcError> {
1999    connect_and_authenticate_with_policy(connection_file_path, ATTACH_RETRY_POLICY).await
2000}
2001
2002async fn connect_and_authenticate_with_policy(
2003    connection_file_path: &Path,
2004    policy: AttachRetryPolicy,
2005) -> Result<TcpStream, SubcError> {
2006    let started_at = Instant::now();
2007    let deadline = started_at + policy.budget;
2008    let mut attempt = 0_u32;
2009    let mut backoff = policy.initial_backoff;
2010    let mut history = Vec::new();
2011
2012    loop {
2013        attempt = attempt.saturating_add(1);
2014        let error = match connect_and_authenticate_once(connection_file_path, deadline).await {
2015            Ok(stream) => return Ok(stream),
2016            Err(error) => error,
2017        };
2018        let class = classify_attach_error(&error);
2019        let error_text = error.to_string().lines().collect::<Vec<_>>().join(" ");
2020        history.push(format!("attempt {attempt} [{class}]: {error_text}"));
2021
2022        if class == AttachErrorClass::Permanent {
2023            log_attach_final_failure(started_at.elapsed(), &history);
2024            return Err(error);
2025        }
2026
2027        let remaining = deadline.saturating_duration_since(Instant::now());
2028        if remaining.is_zero() {
2029            log_attach_final_failure(started_at.elapsed(), &history);
2030            return Err(error);
2031        }
2032
2033        let delay = jittered_attach_delay(backoff, policy.jitter_percent, attempt).min(remaining);
2034        log::info!(
2035            "subc attach retry: attempt {attempt} failed; error_class={class}; error={error_text}; next_delay={delay:?}"
2036        );
2037        tokio::time::sleep(delay).await;
2038
2039        if Instant::now() >= deadline {
2040            log_attach_final_failure(started_at.elapsed(), &history);
2041            return Err(error);
2042        }
2043        backoff = backoff.saturating_mul(2).min(policy.max_backoff);
2044    }
2045}
2046
2047fn jittered_attach_delay(base: Duration, jitter_percent: u64, attempt: u32) -> Duration {
2048    let jitter_percent = jitter_percent.min(100);
2049    if jitter_percent == 0 {
2050        return base;
2051    }
2052
2053    let mut random_bytes = [0_u8; 8];
2054    let random = if getrandom::fill(&mut random_bytes).is_ok() {
2055        u64::from_le_bytes(random_bytes)
2056    } else {
2057        let timestamp = std::time::SystemTime::now()
2058            .duration_since(std::time::UNIX_EPOCH)
2059            .unwrap_or_default()
2060            .subsec_nanos();
2061        u64::from(timestamp) ^ u64::from(attempt)
2062    };
2063    let span = jitter_percent.saturating_mul(2).saturating_add(1);
2064    let multiplier_percent = 100 - jitter_percent + random % span;
2065    let base_millis = u64::try_from(base.as_millis()).unwrap_or(u64::MAX);
2066    Duration::from_millis(base_millis.saturating_mul(multiplier_percent) / 100)
2067}
2068
2069fn log_attach_final_failure(elapsed: Duration, history: &[String]) {
2070    log::error!(
2071        "subc initial attach failed after {} attempt(s) in {elapsed:?}; attempt history: {}",
2072        history.len(),
2073        history.join(" | ")
2074    );
2075}
2076
2077async fn connect_and_authenticate_once(
2078    connection_file_path: &Path,
2079    deadline: Instant,
2080) -> Result<TcpStream, SubcError> {
2081    // This read intentionally lives inside the per-attempt function. The daemon
2082    // publishes connection files atomically and may change both port and key.
2083    let conn = connection_file::read_for_client(connection_file_path).map_err(|source| {
2084        SubcError::ConnectionFile {
2085            path: connection_file_path.to_path_buf(),
2086            source,
2087        }
2088    })?;
2089
2090    let endpoint = conn
2091        .endpoints
2092        .first()
2093        .ok_or_else(|| SubcError::NoEndpoint {
2094            path: connection_file_path.to_path_buf(),
2095        })?;
2096    let endpoint_label = format!("{}:{}", endpoint.host, endpoint.port);
2097    let ip = endpoint
2098        .host
2099        .parse::<IpAddr>()
2100        .map_err(|_| SubcError::InvalidEndpoint {
2101            path: connection_file_path.to_path_buf(),
2102            endpoint: endpoint_label.clone(),
2103        })?;
2104    let addr = SocketAddr::new(ip, endpoint.port);
2105
2106    let connect_budget = deadline.saturating_duration_since(Instant::now());
2107    let mut stream = tokio::time::timeout(connect_budget, TcpStream::connect(addr))
2108        .await
2109        .map_err(|_| SubcError::Connect {
2110            endpoint: endpoint_label.clone(),
2111            source: io::Error::new(
2112                io::ErrorKind::TimedOut,
2113                "initial subc attach retry budget elapsed during TCP connect",
2114            ),
2115        })?
2116        .map_err(|source| SubcError::Connect {
2117            endpoint: endpoint_label.clone(),
2118            source,
2119        })?;
2120    stream
2121        .set_nodelay(true)
2122        .map_err(|source| SubcError::Connect {
2123            endpoint: endpoint_label.clone(),
2124            source,
2125        })?;
2126
2127    let auth_budget = AUTH_DEADLINE.min(deadline.saturating_duration_since(Instant::now()));
2128    authenticate_client(&mut stream, &conn, auth_budget)
2129        .await
2130        .map_err(|source| SubcError::Auth {
2131            endpoint: endpoint_label,
2132            source,
2133        })?;
2134
2135    Ok(stream)
2136}
2137
2138#[allow(clippy::too_many_arguments)]
2139async fn process_route_bind_completion(
2140    writer_tx: &WriterSender,
2141    completion: RouteBindCompletion,
2142    routes: &mut HashMap<RouteChannel, RouteIdentity>,
2143    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2144    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2145    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2146    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2147    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2148    installed_route_epochs: &mut HashMap<u16, u32>,
2149    executor: &Arc<Executor>,
2150    shutdown: &Arc<Notify>,
2151    metrics: &Arc<DispatchPathMetrics>,
2152) -> Result<(), SubcError> {
2153    decrement_counted_channel(&metrics.control_completion_queued);
2154    handle_route_bind_completion(
2155        writer_tx,
2156        completion,
2157        routes,
2158        root_channels,
2159        session_identity,
2160        push_buffer,
2161        live_roots,
2162        pending_binds,
2163        installed_route_epochs,
2164        executor,
2165        shutdown,
2166        metrics,
2167    )
2168    .await
2169}
2170
2171#[allow(clippy::too_many_arguments)]
2172async fn drain_pending_route_bind_completions(
2173    control_completion_rx: &mut mpsc::Receiver<RouteBindCompletion>,
2174    writer_tx: &WriterSender,
2175    routes: &mut HashMap<RouteChannel, RouteIdentity>,
2176    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2177    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2178    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2179    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2180    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2181    installed_route_epochs: &mut HashMap<u16, u32>,
2182    executor: &Arc<Executor>,
2183    shutdown: &Arc<Notify>,
2184    metrics: &Arc<DispatchPathMetrics>,
2185) -> Result<usize, SubcError> {
2186    let mut drained = 0;
2187    while let Ok(completion) = control_completion_rx.try_recv() {
2188        process_route_bind_completion(
2189            writer_tx,
2190            completion,
2191            routes,
2192            root_channels,
2193            session_identity,
2194            push_buffer,
2195            live_roots,
2196            pending_binds,
2197            installed_route_epochs,
2198            executor,
2199            shutdown,
2200            metrics,
2201        )
2202        .await?;
2203        drained += 1;
2204    }
2205    Ok(drained)
2206}
2207
2208/// ModuleHello → HelloAck → control/route loop. Runs until the daemon closes
2209/// the connection (EOF), sends channel-0 Goodbye, or a fatal mutating executor
2210/// response requests whole-connection teardown.
2211async fn run_module_loop<R, W>(
2212    mut read: R,
2213    mut write: W,
2214    shared_app: Arc<App>,
2215    executor: Arc<Executor>,
2216    dispatch: DispatchFn,
2217    user_config_path: Option<PathBuf>,
2218    allow_native_passthrough: bool,
2219) -> Result<ModuleLoopExit, SubcError>
2220where
2221    R: AsyncRead + Unpin + Send + 'static,
2222    W: AsyncWrite + Unpin + Send + 'static,
2223{
2224    // ModuleHello: register as a tool provider and advertise the supported control-plane operations.
2225    // Echo the one-time launch nonce the daemon injected via SUBC_LAUNCH_NONCE so a
2226    // reserved module_id's HELLO is accepted; absent for non-reserved/self-connect.
2227    let hello = ModuleHelloBody {
2228        manifest: build_manifest(),
2229        protocol_ver: PROTOCOL_VERSION,
2230        control_ops: control_ops(),
2231        launch_nonce: std::env::var("SUBC_LAUNCH_NONCE").ok(),
2232    };
2233    let hello_frame = Frame::build(
2234        FrameType::Hello,
2235        control_flags(),
2236        0,
2237        0,
2238        HELLO_CORR,
2239        serde_json::to_vec(&hello).map_err(SubcError::Json)?,
2240    )
2241    .map_err(SubcError::FrameBuild)?;
2242    write_frame(&mut write, &hello_frame)
2243        .await
2244        .map_err(SubcError::FrameIo)?;
2245
2246    // Expect HelloAck (registered) or a channel-0 Error (manifest/version reject).
2247    match read_frame(&mut read).await.map_err(SubcError::FrameIo)? {
2248        None => return Err(SubcError::ClosedBeforeHelloAck),
2249        Some(frame) => match frame.header.ty {
2250            FrameType::HelloAck => {
2251                log::info!("subc attach: registered (HelloAck received)");
2252            }
2253            FrameType::Error => {
2254                let body = serde_json::from_slice::<ErrorBody>(&frame.body).ok();
2255                return Err(SubcError::HelloRejected { body });
2256            }
2257            other => return Err(SubcError::UnexpectedFrame { ty: other }),
2258        },
2259    }
2260
2261    let dispatch_path_metrics = Arc::new(DispatchPathMetrics::new());
2262    let (writer_tx, writer_rx) = mpsc::channel::<WriterFrame>(WRITER_QUEUE_CAPACITY);
2263    let writer_task = spawn_writer_task(write, writer_rx, Arc::clone(&dispatch_path_metrics));
2264    // `read_frame` is NOT cancellation-safe, so it must never sit directly inside
2265    // the `select!` below: a drain-interval tick (or shutdown) firing while a
2266    // frame is mid-transit would drop the partially-consumed bytes and desync the
2267    // stream (the next read would parse a body byte as a frame header). A
2268    // dedicated reader task owns the socket, reads whole frames sequentially, and
2269    // forwards them over a channel; the loop selects on the cancel-safe `recv()`.
2270    let (reader_tx, mut reader_rx) = mpsc::channel::<Result<DecodedFrame, SubcError>>(256);
2271    let reader_task = spawn_reader_task(read, reader_tx);
2272    let shutdown = Arc::new(Notify::new());
2273    // Drain-tick deadline is tracked manually and checked at the TOP of every
2274    // loop turn rather than as an Interval select arm: the select below is
2275    // `biased` (bind completions first), and biased polling means a saturated
2276    // higher arm (sustained lossy push traffic keeps lossy_rx always-ready)
2277    // would starve every arm below it, including a timer arm — leaving
2278    // backpressured reliable frames parked in the retry buffer past their
2279    // delivery deadline. The pre-turn check cannot be starved by arm order;
2280    // the sleep_until arm below only exists to wake an otherwise-idle loop.
2281    let mut next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2282    let mut next_maintenance_at = next_drain_at;
2283    // Rate-limit stamp for opportunistic macOS allocator slack relief (checked
2284    // on the maintenance tick; policy shared with standalone via memory.rs).
2285    #[cfg(target_os = "macos")]
2286    let mut last_slack_relief: Option<std::time::Instant> = None;
2287    let (maintenance_tx, mut maintenance_rx) = mpsc::channel::<MaintenanceCompletion>(256);
2288    let (bash_deferred_tx, mut bash_deferred_rx) =
2289        mpsc::channel::<bash::BashDeferredCompletion>(256);
2290    let (bash_poll_touch_tx, mut bash_poll_touch_rx) = mpsc::channel::<ProjectRootId>(256);
2291    let (control_completion_tx, mut control_completion_rx) =
2292        mpsc::channel::<RouteBindCompletion>(256);
2293    let (lossy_tx, mut lossy_rx) = mpsc::channel::<LossyPushEnvelope>(1024);
2294    let lossy_overflow = Arc::new(push::LossyOverflow::default());
2295    let lossy_seq = Arc::new(AtomicU64::new(0));
2296    let (reliable_tx, mut reliable_rx) = mpsc::unbounded_channel::<PushEnvelope>();
2297    let push_senders = PushSenders {
2298        lossy_tx,
2299        reliable_tx,
2300        lossy_overflow: Arc::clone(&lossy_overflow),
2301        lossy_seq,
2302    };
2303    let connection_cancel = PersistentCancelSignal::new();
2304    let mut installed_route_epochs: HashMap<u16, u32> = HashMap::new();
2305    let mut routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
2306    let mut bg_subs: HashMap<RouteChannel, BgSub> = HashMap::new();
2307    let mut bg_sub_by_session: HashMap<(ProjectRootId, String), RouteChannel> = HashMap::new();
2308    let mut bg_wake_pending: HashSet<RouteChannel> = HashSet::new();
2309    let mut bg_wake_epoch: HashMap<(ProjectRootId, String), u64> = HashMap::new();
2310    let mut root_channels: HashMap<ProjectRootId, HashSet<RouteChannel>> = HashMap::new();
2311    let mut session_identity: HashMap<(ProjectRootId, String), RetainedSessionIdentity> =
2312        HashMap::new();
2313    let mut push_buffer: HashMap<push::ReplayKey, VecDeque<PushFrame>> = HashMap::new();
2314    let mut retry_buffer: RetryBuffer = HashMap::new();
2315    let mut reclaimed_routes = ReclaimedRoutes::default();
2316    let mut completed_tasks = push::CompletedTaskIds::default();
2317    let mut live_roots: HashMap<ProjectRootId, RootMeta> = HashMap::new();
2318    let mut pending_binds: HashMap<RouteChannel, PendingBind> = HashMap::new();
2319    let mut pending_bash_asks: HashMap<ReverseCorrKey, PendingBashAsk> = HashMap::new();
2320    let mut next_bash_ask_corr: u64 = 1;
2321    let mut route_bash_cancels: HashMap<RouteChannel, bash::RouteBashCancel> = HashMap::new();
2322
2323    let loop_result: Result<ModuleLoopExit, SubcError> = loop {
2324        shared_app.set_open_route_count(routes.len());
2325        crate::logging::perf_tick(Some(&executor));
2326        dispatch_path_metrics.mark_frame_loop_tick();
2327        if let Err(error) = expire_pending_bash_asks(
2328            &writer_tx,
2329            &mut pending_bash_asks,
2330            &routes,
2331            &mut live_roots,
2332            &mut route_bash_cancels,
2333            &shutdown,
2334            &dispatch_path_metrics,
2335        )
2336        .await
2337        {
2338            break Err(error);
2339        }
2340
2341        // RouteBind completions are control-plane unblockers. Drain any completed
2342        // binds before entering other branch work so Push and maintenance bursts
2343        // can only add one loop-turn of latency.
2344        match drain_pending_route_bind_completions(
2345            &mut control_completion_rx,
2346            &writer_tx,
2347            &mut routes,
2348            &mut root_channels,
2349            &mut session_identity,
2350            &mut push_buffer,
2351            &mut live_roots,
2352            &mut pending_binds,
2353            &mut installed_route_epochs,
2354            &executor,
2355            &shutdown,
2356            &dispatch_path_metrics,
2357        )
2358        .await
2359        {
2360            Ok(drained) => {
2361                if drained > 0 {
2362                    next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2363                }
2364            }
2365            Err(error) => break Err(error),
2366        }
2367
2368        if tokio::time::Instant::now() >= next_drain_at {
2369            push::emit_bg_event_wakes(
2370                &writer_tx,
2371                &dispatch_path_metrics,
2372                &bg_subs,
2373                &mut bg_wake_pending,
2374            );
2375            warn_slow_pending_binds(&mut pending_binds, &executor);
2376            if let Err(error) = expire_overdue_route_binds(
2377                &writer_tx,
2378                &executor,
2379                &mut pending_binds,
2380                &mut installed_route_epochs,
2381                &dispatch_path_metrics,
2382            )
2383            .await
2384            {
2385                break Err(error);
2386            }
2387
2388            let retried = push::drain_retry_buffers_for_bound_routes(
2389                &writer_tx,
2390                &dispatch_path_metrics,
2391                &routes,
2392                &mut retry_buffer,
2393            );
2394            if retried > 0 {
2395                log::debug!(
2396                    "subc attach: retried {retried} reliable Push frame(s) after writer backpressure"
2397                );
2398            }
2399
2400            next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2401        }
2402
2403        // A lossy emitter may place its newest update in the overflow buffer
2404        // when the bounded channel is full, while this receive loop is draining
2405        // the channel. Drain overflow before selecting again so that raced
2406        // update is delivered on the next timer tick instead of waiting for
2407        // another lossy enqueue.
2408        let overflow_batch = lossy_overflow.drain();
2409        if !overflow_batch.is_empty() {
2410            let (_, deferred) = push::drain_reliable_push_turn(
2411                &writer_tx,
2412                &dispatch_path_metrics,
2413                &routes,
2414                &root_channels,
2415                &session_identity,
2416                &mut retry_buffer,
2417                &mut push_buffer,
2418                &mut completed_tasks,
2419                &bg_sub_by_session,
2420                &mut bg_wake_pending,
2421                &mut bg_wake_epoch,
2422                &mut reliable_rx,
2423                None,
2424            );
2425            if deferred {
2426                tokio::task::yield_now().await;
2427            }
2428
2429            let mut batch = Vec::new();
2430            while let Ok(item) = lossy_rx.try_recv() {
2431                batch.push(item);
2432            }
2433            batch.extend(overflow_batch);
2434            push::process_lossy_push_envelope_batch(
2435                &writer_tx,
2436                &dispatch_path_metrics,
2437                &routes,
2438                &root_channels,
2439                &completed_tasks,
2440                batch,
2441            );
2442        }
2443
2444        tokio::select! {
2445            biased;
2446            Some(completion) = control_completion_rx.recv() => {
2447                if let Err(error) = process_route_bind_completion(
2448                    &writer_tx,
2449                    completion,
2450                    &mut routes,
2451                    &mut root_channels,
2452                    &mut session_identity,
2453                    &mut push_buffer,
2454                    &mut live_roots,
2455                    &mut pending_binds,
2456                    &mut installed_route_epochs,
2457                    &executor,
2458                    &shutdown,
2459                    &dispatch_path_metrics,
2460                )
2461                .await
2462                {
2463                    break Err(error);
2464                }
2465                next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2466            }
2467            _ = shutdown.notified() => {
2468                log::warn!("subc attach: fatal executor response requested teardown");
2469                break Ok(ModuleLoopExit::SkipSearchFlush);
2470            }
2471            maybe_frame = reader_rx.recv() => {
2472                let frame = match maybe_frame {
2473                    None => {
2474                        log::info!("subc attach: daemon closed connection");
2475                        break Ok(ModuleLoopExit::Graceful);
2476                    }
2477                    Some(Err(error)) => break Err(error),
2478                    Some(Ok(frame)) => frame,
2479                };
2480                let phase_trace = frame.phase_trace;
2481                let frame = frame.frame;
2482
2483                if !ingress_route_should_be_processed(
2484                    &installed_route_epochs,
2485                    &reclaimed_routes,
2486                    &frame,
2487                ) {
2488                    log::debug!(
2489                        "subc attach: silently dropping {:?} for uninstalled route {}@{}",
2490                        frame.header.ty,
2491                        frame.header.channel,
2492                        frame.header.epoch
2493                    );
2494                    continue;
2495                }
2496
2497                match frame.header.ty {
2498                    FrameType::Ping if frame.header.channel == 0 => {
2499                        let pong = match Frame::build_with_version(
2500                            frame.header.ver,
2501                            FrameType::Pong,
2502                            frame.header.flags,
2503                            0,
2504                            0,
2505                            frame.header.corr,
2506                            Vec::new(),
2507                        ) {
2508                            Ok(pong) => pong,
2509                            Err(error) => break Err(SubcError::FrameBuild(error)),
2510                        };
2511                        if let Err(error) = send_frame(&writer_tx, &dispatch_path_metrics, pong).await {
2512                            break Err(error);
2513                        }
2514                    }
2515                    FrameType::Goodbye if frame.header.channel == 0 => {
2516                        log::info!("subc attach: received channel-0 Goodbye");
2517                        break Ok(ModuleLoopExit::Graceful);
2518                    }
2519                    FrameType::Goodbye => {
2520                        let channel = route_key(frame.header.channel, frame.header.epoch);
2521                        if let Err(error) = teardown_installed_route(
2522                            &writer_tx,
2523                            &dispatch_path_metrics,
2524                            &executor,
2525                            channel,
2526                            "Goodbye",
2527                            None,
2528                            &mut installed_route_epochs,
2529                            &mut routes,
2530                            &mut root_channels,
2531                            &mut bg_subs,
2532                            &mut bg_sub_by_session,
2533                            &mut bg_wake_pending,
2534                            &mut pending_bash_asks,
2535                            &mut live_roots,
2536                            &mut route_bash_cancels,
2537                            &mut pending_binds,
2538                            &mut retry_buffer,
2539                            &mut push_buffer,
2540                            &shutdown,
2541                        )
2542                        .await
2543                        {
2544                            break Err(error);
2545                        }
2546                    }
2547                    FrameType::Response | FrameType::Error if frame.header.channel != 0 => {
2548                        if let Err(error) = handle_bash_elicitation_reply(
2549                            &writer_tx,
2550                            &frame,
2551                            &mut pending_bash_asks,
2552                            &routes,
2553                            &mut live_roots,
2554                            &executor,
2555                            &shutdown,
2556                            &bash_deferred_tx,
2557                            &bash_poll_touch_tx,
2558                            &dispatch_path_metrics,
2559                            &mut route_bash_cancels,
2560                            dispatch,
2561                        )
2562                        .await
2563                        {
2564                            break Err(error);
2565                        }
2566                    }
2567                    FrameType::Request if frame.header.channel == 0 => {
2568                        if let Err(error) = handle_control_request(
2569                            &writer_tx,
2570                            &frame,
2571                            &shared_app,
2572                            &executor,
2573                            &mut live_roots,
2574                            &mut pending_binds,
2575                            &mut installed_route_epochs,
2576                            &mut routes,
2577                            &mut root_channels,
2578                            &mut bg_subs,
2579                            &mut bg_sub_by_session,
2580                            &mut bg_wake_pending,
2581                            &mut pending_bash_asks,
2582                            &mut route_bash_cancels,
2583                            &mut retry_buffer,
2584                            &mut push_buffer,
2585                            &shutdown,
2586                            &control_completion_tx,
2587                            &dispatch_path_metrics,
2588                            &push_senders,
2589                            dispatch,
2590                            user_config_path.as_deref(),
2591                        )
2592                        .await
2593                        {
2594                            break Err(error);
2595                        }
2596                    }
2597                    FrameType::Request => {
2598                        if let Err(error) = handle_tool_call(
2599                            &writer_tx,
2600                            &frame,
2601                            phase_trace,
2602                            &routes,
2603                            &pending_binds,
2604                            &mut live_roots,
2605                            &executor,
2606                            &shutdown,
2607                            &connection_cancel,
2608                            &bash_deferred_tx,
2609                            &bash_poll_touch_tx,
2610                            &dispatch_path_metrics,
2611                            &mut route_bash_cancels,
2612                            &mut pending_bash_asks,
2613                            &mut next_bash_ask_corr,
2614                            &mut bg_subs,
2615                            &mut bg_sub_by_session,
2616                            &mut bg_wake_pending,
2617                            &mut bg_wake_epoch,
2618                            dispatch,
2619                            allow_native_passthrough,
2620                        )
2621                        .await
2622                        {
2623                            break Err(error);
2624                        }
2625                    }
2626                    FrameType::Cancel => {
2627                        let channel = route_key(frame.header.channel, frame.header.epoch);
2628                        if bg_subs.contains_key(&channel) {
2629                            if let Err(error) = end_bg_subscription(
2630                                &writer_tx,
2631                                &dispatch_path_metrics,
2632                                &mut bg_subs,
2633                                &mut bg_sub_by_session,
2634                                &mut bg_wake_pending,
2635                                channel,
2636                                routes.get(&channel),
2637                            )
2638                            .await
2639                            {
2640                                break Err(error);
2641                            }
2642                        }
2643                        if let Err(error) = cancel_pending_bash_ask_for_tool_call(
2644                            &writer_tx,
2645                            &mut pending_bash_asks,
2646                            channel,
2647                            frame.header.corr,
2648                            &routes,
2649                            &mut live_roots,
2650                            &mut route_bash_cancels,
2651                            &shutdown,
2652                            &dispatch_path_metrics,
2653                        )
2654                        .await
2655                        {
2656                            break Err(error);
2657                        }
2658                    }
2659                    // Incoming push messages are ignored here. Cancel frames only
2660                    // stop pending bash elicitation requests; executor-level
2661                    // cancellation for tool calls that are already running is not
2662                    // implemented.
2663                    _ => {}
2664                }
2665            }
2666            Some((root_id, frame)) = reliable_rx.recv() => {
2667                // Reliable Push frames are FIFO and must-deliver, but draining an
2668                // unbounded burst in one current-thread turn can starve RouteBind
2669                // completions. The budget defers excess frames, never drops them.
2670                let (_, deferred) = push::drain_reliable_push_turn(
2671                    &writer_tx,
2672                    &dispatch_path_metrics,
2673                    &routes,
2674                    &root_channels,
2675                    &session_identity,
2676                    &mut retry_buffer,
2677                    &mut push_buffer,
2678                    &mut completed_tasks,
2679                    &bg_sub_by_session,
2680                    &mut bg_wake_pending,
2681                    &mut bg_wake_epoch,
2682                    &mut reliable_rx,
2683                    Some((root_id, frame)),
2684                );
2685                if deferred {
2686                    tokio::task::yield_now().await;
2687                }
2688            }
2689            Some((order, root_id, frame)) = lossy_rx.recv() => {
2690                // When both push lanes have work, handle a small reliable slice before lossy work.
2691                // That ordering lets completed task ids suppress stale BashLongRunning frames.
2692                // The slice stays bounded so reliable bursts cannot monopolize this loop turn.
2693                let (_, deferred) = push::drain_reliable_push_turn(
2694                    &writer_tx,
2695                    &dispatch_path_metrics,
2696                    &routes,
2697                    &root_channels,
2698                    &session_identity,
2699                    &mut retry_buffer,
2700                    &mut push_buffer,
2701                    &mut completed_tasks,
2702                    &bg_sub_by_session,
2703                    &mut bg_wake_pending,
2704                    &mut bg_wake_epoch,
2705                    &mut reliable_rx,
2706                    None,
2707                );
2708                if deferred {
2709                    tokio::task::yield_now().await;
2710                }
2711
2712                // Drain the currently queued burst in one loop turn so lossy
2713                // status/progress updates can be merged before reaching subc's
2714                // shared egress queue. Each lossy frame gets a sequence number
2715                // before it goes to the channel or overflow buffer, so the
2716                // combined batch is sorted back into producer order before
2717                // coalescing drops stale updates for the same key.
2718                let mut batch = vec![(order, root_id, frame)];
2719                while let Ok(item) = lossy_rx.try_recv() {
2720                    batch.push(item);
2721                }
2722                batch.extend(lossy_overflow.drain());
2723                push::process_lossy_push_envelope_batch(
2724                    &writer_tx,
2725                    &dispatch_path_metrics,
2726                    &routes,
2727                    &root_channels,
2728                    &completed_tasks,
2729                    batch,
2730                );
2731            }
2732            Some(done) = bash_deferred_rx.recv() => {
2733                decrement_counted_channel(&dispatch_path_metrics.bash_deferred_queued);
2734                if let Err(error) = bash::handle_bash_deferred_completion(
2735                    &writer_tx,
2736                    done,
2737                    &routes,
2738                    &mut live_roots,
2739                    &mut route_bash_cancels,
2740                    &shutdown,
2741                    &dispatch_path_metrics,
2742                )
2743                .await
2744                {
2745                    break Err(error);
2746                }
2747            }
2748            Some(root_id) = bash_poll_touch_rx.recv() => {
2749                decrement_counted_channel(&dispatch_path_metrics.bash_poll_touch_queued);
2750                if let Some(meta) = live_roots.get_mut(&root_id) {
2751                    meta.note_activity();
2752                }
2753            }
2754            Some(completion) = maintenance_rx.recv() => {
2755                decrement_counted_channel(&dispatch_path_metrics.maintenance_queued);
2756                let root_id = completion.root_id.clone();
2757                let response = completion.response;
2758                let response_is_fatal = response_is_fatal_panic(&response);
2759                let bind_pending = pending_binds
2760                    .values()
2761                    .any(|pending| pending.bind_root_id == root_id);
2762                let requiesce = if let Some(meta) = live_roots.get_mut(&root_id) {
2763                    let defer_requeue = meta.unbound_quiesced || bind_pending;
2764                    note_maintenance_completion(
2765                        meta,
2766                        completion.requeue_kind,
2767                        response_is_fatal,
2768                        defer_requeue,
2769                    );
2770                    should_requiesce_after_maintenance(meta, completion.kind, bind_pending)
2771                } else {
2772                    false
2773                };
2774                if requiesce {
2775                    quiesce_unbound_root(&root_id, &mut live_roots, &executor);
2776                }
2777                push::clear_stale_bg_wakes_for_empty_sessions(
2778                    &root_id,
2779                    &completion.empty_bg_sessions,
2780                    &bg_sub_by_session,
2781                    &mut bg_wake_pending,
2782                    &bg_wake_epoch,
2783                );
2784                if response_is_fatal {
2785                    if let Some(meta) = live_roots.get_mut(&root_id) {
2786                        meta.maintenance_poisoned = true;
2787                    }
2788                    log::warn!(
2789                        "subc attach: maintenance drain observed a fatal actor; deferring teardown until a route request can receive actor_fatal"
2790                    );
2791                }
2792            }
2793            _ = tokio::time::sleep_until(next_drain_at) => {
2794                // Wakes an otherwise-idle loop so the pre-turn drain check
2795                // above runs on schedule; the drain work itself lives there.
2796            }
2797            _ = tokio::time::sleep_until(next_maintenance_at) => {
2798                // Delay cache-draining maintenance until any already-ready
2799                // inbound route/control messages and push completions have run,
2800                // so maintenance does not block the actor from handling the
2801                // first request that arrives after a route bind is acknowledged.
2802                let reaped_lsp_children = shared_app
2803                    .lsp_child_registry()
2804                    .reap_children_with_gone_cwd();
2805                if reaped_lsp_children > 0 {
2806                    log::warn!(
2807                        "subc attach: reaped {reaped_lsp_children} LSP child process group(s) whose cwd no longer exists"
2808                    );
2809                }
2810                let reap = reap_idle_roots(
2811                    Instant::now(),
2812                    &mut live_roots,
2813                    &pending_binds,
2814                    &root_channels,
2815                    &executor,
2816                    &dispatch_path_metrics,
2817                );
2818                for root_id in &reap.forgotten_deleted_roots {
2819                    purge_deleted_root_residents(
2820                        root_id,
2821                        &mut routes,
2822                        &mut root_channels,
2823                        &mut installed_route_epochs,
2824                        &mut route_bash_cancels,
2825                        &mut retry_buffer,
2826                        &mut reclaimed_routes,
2827                        &mut session_identity,
2828                        &mut push_buffer,
2829                        &mut bg_subs,
2830                        &mut bg_sub_by_session,
2831                        &mut bg_wake_pending,
2832                        &mut bg_wake_epoch,
2833                        &mut pending_bash_asks,
2834                    );
2835                }
2836                if reap.evicted > 0 {
2837                    log::debug!("subc attach: reaped {} idle root(s)", reap.evicted);
2838                }
2839                submit_due_maintenance_jobs(
2840                    &executor,
2841                    &mut live_roots,
2842                    &pending_binds,
2843                    &bg_sub_by_session,
2844                    &bg_wake_pending,
2845                    &bg_wake_epoch,
2846                    &maintenance_tx,
2847                    &dispatch_path_metrics,
2848                );
2849                // Opportunistic allocator relief, independent of the idle
2850                // sweep: the sweep's whole-process idle gate never opens while
2851                // any session stays active, which let freed warm-up arenas sit
2852                // resident for the process lifetime (5.1 GB RSS over ~600 MB
2853                // live). Slack threshold + spacing live in memory.rs; the pass
2854                // itself runs on a detached thread.
2855                #[cfg(target_os = "macos")]
2856                {
2857                    let now_std = std::time::Instant::now();
2858                    if crate::memory::spawn_allocator_slack_relief_if_due(
2859                        last_slack_relief,
2860                        now_std,
2861                    ) {
2862                        last_slack_relief = Some(now_std);
2863                    }
2864                }
2865                next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2866            }
2867        }
2868    };
2869
2870    shared_app.set_open_route_count(0);
2871
2872    connection_cancel.cancel();
2873    // Channel-0 Goodbye, EOF, and fatal exits bypass per-route Goodbye. Settle
2874    // their root lifecycle state before loop-owned routing metadata is dropped.
2875    quiesce_connection_roots(
2876        &mut live_roots,
2877        &mut pending_binds,
2878        &mut routes,
2879        &mut root_channels,
2880        &mut installed_route_epochs,
2881        &mut route_bash_cancels,
2882        &executor,
2883    );
2884
2885    let mut loop_result = loop_result;
2886    if !pending_bash_asks.is_empty() {
2887        let no_routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
2888        if let Err(error) = settle_all_pending_bash_asks(
2889            &writer_tx,
2890            &mut pending_bash_asks,
2891            &no_routes,
2892            &mut live_roots,
2893            &mut route_bash_cancels,
2894            &shutdown,
2895            &dispatch_path_metrics,
2896        )
2897        .await
2898        {
2899            loop_result = loop_result.and(Err(error));
2900        }
2901    }
2902
2903    // The reader task may be parked on `read_frame`; abort it (we are done with
2904    // the connection) and flush the writer.
2905    reader_task.abort();
2906    drop(writer_tx);
2907    let writer_result = finish_writer_task(writer_task).await;
2908    loop_result.and_then(|exit| writer_result.map(|_| exit))
2909}
2910
2911fn spawn_writer_task<W>(
2912    mut write: W,
2913    mut rx: mpsc::Receiver<WriterFrame>,
2914    metrics: Arc<DispatchPathMetrics>,
2915) -> JoinHandle<Result<(), subc_transport::FrameIoError>>
2916where
2917    W: AsyncWrite + Unpin + Send + 'static,
2918{
2919    tokio::spawn(async move {
2920        let mut write_buffer = Vec::new();
2921        while let Some(mut queued) = rx.recv().await {
2922            let measure = queued.tool_response_trace.is_some();
2923            let dequeued = measure.then(Instant::now);
2924            metrics.writer_active.store(true, Ordering::Relaxed);
2925            decrement_counted_channel(&metrics.writer_queued);
2926            let write_timing = write_frame_contiguous(
2927                &mut write,
2928                queued.frame(),
2929                queued.body(),
2930                &mut write_buffer,
2931                measure,
2932            )
2933            .await;
2934            metrics.writer_active.store(false, Ordering::Relaxed);
2935            let write_timing = write_timing?;
2936
2937            if let (Some(trace), Some(dequeued), Some(write_timing)) =
2938                (queued.tool_response_trace.take(), dequeued, write_timing)
2939            {
2940                if let Some(completed) = trace.finish(
2941                    dequeued,
2942                    write_timing.write_started,
2943                    write_timing.write_finished,
2944                    write_timing.frame_bytes,
2945                ) {
2946                    log_ctx::with_session(Some(completed.session), || {
2947                        crate::logging::note_tool_call_trace(
2948                            &completed.name,
2949                            &completed.root,
2950                            completed.channel,
2951                            completed.corr,
2952                            completed.phases,
2953                        );
2954                    });
2955                }
2956            }
2957        }
2958        Ok(())
2959    })
2960}
2961
2962struct FrameWriteTiming {
2963    write_started: Instant,
2964    write_finished: Instant,
2965    frame_bytes: usize,
2966}
2967
2968/// Encode one complete frame into the existing reusable buffer and write it
2969/// without interleaving bytes from another channel. Timing is collected only
2970/// for tool responses, so Push and control frames add no clock reads.
2971async fn write_frame_contiguous<W>(
2972    writer: &mut W,
2973    frame: &Frame,
2974    body: &[u8],
2975    buffer: &mut Vec<u8>,
2976    measure: bool,
2977) -> Result<Option<FrameWriteTiming>, subc_transport::FrameIoError>
2978where
2979    W: AsyncWrite + Unpin,
2980{
2981    if frame.header.len as usize != body.len() {
2982        return Err(subc_transport::FrameIoError::BodyLengthMismatch {
2983            header_len: frame.header.len,
2984            body_len: body.len(),
2985        });
2986    }
2987
2988    let header = frame.header.encode();
2989    buffer.clear();
2990    buffer.reserve(header.len() + body.len());
2991    buffer.extend_from_slice(&header);
2992    buffer.extend_from_slice(body);
2993    let write_started = measure.then(Instant::now);
2994    writer
2995        .write_all(buffer)
2996        .await
2997        .map_err(subc_transport::FrameIoError::Io)?;
2998    Ok(write_started.map(|write_started| FrameWriteTiming {
2999        write_started,
3000        write_finished: Instant::now(),
3001        frame_bytes: buffer.len(),
3002    }))
3003}
3004
3005fn spawn_reader_task<R>(
3006    mut read: R,
3007    tx: mpsc::Sender<Result<DecodedFrame, SubcError>>,
3008) -> JoinHandle<()>
3009where
3010    R: AsyncRead + Unpin + Send + 'static,
3011{
3012    tokio::spawn(async move {
3013        loop {
3014            match read_frame(&mut read).await {
3015                Ok(Some(frame)) => {
3016                    let decoded = DecodedFrame {
3017                        frame,
3018                        phase_trace: PhaseTrace::new(Instant::now()),
3019                    };
3020                    if tx.send(Ok(decoded)).await.is_err() {
3021                        return;
3022                    }
3023                }
3024                Ok(None) => {
3025                    // EOF: let the loop observe channel close as "daemon closed".
3026                    return;
3027                }
3028                Err(error) => {
3029                    // A killed daemon surfaces as ConnectionReset (RST) on
3030                    // Windows where Unix delivers a clean EOF (FIN); a
3031                    // mid-teardown daemon can also abort the socket. Both mean
3032                    // "daemon went away", not a wire fault — normalize them to
3033                    // the clean-close path so module exit behavior matches
3034                    // across platforms (same class subc-core fixed in d33d9a71).
3035                    if let subc_transport::FrameIoError::Io(io_error) = &error {
3036                        if matches!(
3037                            io_error.kind(),
3038                            std::io::ErrorKind::ConnectionReset
3039                                | std::io::ErrorKind::ConnectionAborted
3040                        ) {
3041                            log::info!(
3042                                "subc attach: connection reset by daemon; treating as close"
3043                            );
3044                            return;
3045                        }
3046                    }
3047                    let _ = tx.send(Err(SubcError::FrameIo(error))).await;
3048                    return;
3049                }
3050            }
3051        }
3052    })
3053}
3054
3055async fn finish_writer_task(
3056    mut writer_task: JoinHandle<Result<(), subc_transport::FrameIoError>>,
3057) -> Result<(), SubcError> {
3058    match tokio::time::timeout(Duration::from_millis(100), &mut writer_task).await {
3059        Ok(Ok(Ok(()))) => Ok(()),
3060        Ok(Ok(Err(error))) => Err(SubcError::FrameIo(error)),
3061        Ok(Err(error)) => Err(SubcError::WriterJoin(error)),
3062        Err(_) => {
3063            writer_task.abort();
3064            Ok(())
3065        }
3066    }
3067}
3068
3069fn register_actor_for_bind(
3070    shared_app: &Arc<App>,
3071    executor: &Arc<Executor>,
3072    push_senders: &PushSenders,
3073    bind_root_id: &ProjectRootId,
3074    route_channel: u16,
3075    root_was_live: bool,
3076) -> bool {
3077    if executor.actor_registered(bind_root_id) {
3078        log::debug!(
3079            "subc attach: reusing actor for route {} root {}",
3080            route_channel,
3081            bind_root_id.as_path().display()
3082        );
3083        return false;
3084    }
3085
3086    if root_was_live {
3087        log::warn!(
3088            "subc attach: recreating missing actor for live root {} on route {}",
3089            bind_root_id.as_path().display(),
3090            route_channel
3091        );
3092    }
3093
3094    let actor_ctx = Arc::new(AppContext::from_app(
3095        Arc::clone(shared_app),
3096        Config::default(),
3097    ));
3098    install_bash_compressor(&actor_ctx);
3099    actor_ctx.set_progress_sender(Some(push::progress_sender_for_root(
3100        push_senders.clone(),
3101        bind_root_id.clone(),
3102    )));
3103    let inserted = executor.register_actor(bind_root_id.clone(), Arc::clone(&actor_ctx));
3104    drop(actor_ctx);
3105    if inserted {
3106        // Do not insert into live_roots until configure succeeds: live_roots
3107        // drives maintenance, and a half-configured new actor must not be
3108        // maintenance-eligible before its route/session identity exists.
3109        log::debug!(
3110            "subc attach: registered actor for route {} root {}",
3111            route_channel,
3112            bind_root_id.as_path().display()
3113        );
3114    } else {
3115        log::debug!(
3116            "subc attach: actor appeared while binding route {} root {}; reusing it",
3117            route_channel,
3118            bind_root_id.as_path().display()
3119        );
3120    }
3121    inserted
3122}
3123
3124fn rollback_pending_bind_actor(
3125    executor: &Arc<Executor>,
3126    live_roots: &HashMap<ProjectRootId, RootMeta>,
3127    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3128    root_id: &ProjectRootId,
3129    inserted_new_actor: bool,
3130) {
3131    if !inserted_new_actor || live_roots.contains_key(root_id) {
3132        return;
3133    }
3134
3135    if let Some((route, pending)) = pending_binds
3136        .iter_mut()
3137        .find(|(_, pending)| &pending.bind_root_id == root_id)
3138    {
3139        pending.inserted_new_actor = true;
3140        log::debug!(
3141            "subc attach: transferred rollback ownership for root {} to pending route {}",
3142            root_id.as_path().display(),
3143            route
3144        );
3145        return;
3146    }
3147
3148    executor.remove_actor(root_id);
3149}
3150
3151fn route_bind_error_code_for_configure_response(response: &Response) -> &'static str {
3152    match response.data.get("code").and_then(|code| code.as_str()) {
3153        // Preserve typed configure rejections across the bind boundary: a
3154        // malformed fed fingerprint means a federation-module bug or
3155        // fingerprint-format drift, and the fed side matches on the code rather
3156        // than parsing prose.
3157        Some("bad_harness_fingerprint") => "bad_harness_fingerprint",
3158        // Cache-key probe failures are transient (fd pressure, git spawn
3159        // contention); the client retries the bind rather than treating the
3160        // root as permanently divergent.
3161        Some("cache_key_probe_failed") => "cache_key_probe_failed",
3162        // Actor lifecycle gaps are transient from the daemon/client viewpoint:
3163        // a fresh bind can create or join a healthy actor, so do not classify
3164        // them as permanent config divergence.
3165        Some("actor_not_registered" | "actor_fatal") => "actor_not_ready",
3166        _ => "config_divergence",
3167    }
3168}
3169
3170fn queue_post_bind_configure_and_completion_maintenance(
3171    root_id: &ProjectRootId,
3172    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3173) {
3174    let Some(meta) = live_roots.get_mut(root_id) else {
3175        return;
3176    };
3177    if meta.maintenance_poisoned || meta.maintenance_pending {
3178        return;
3179    }
3180
3181    meta.maintenance_pending = true;
3182    meta.maintenance_queued_kinds
3183        .push_back(MaintenanceDrainKind::ConfigureTail);
3184    meta.maintenance_queued_kinds
3185        .push_back(MaintenanceDrainKind::CompletionDrains);
3186}
3187
3188#[allow(clippy::too_many_arguments)]
3189async fn handle_route_bind_completion(
3190    tx: &WriterSender,
3191    completion: RouteBindCompletion,
3192    routes: &mut HashMap<RouteChannel, RouteIdentity>,
3193    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
3194    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
3195    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
3196    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3197    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3198    installed_route_epochs: &mut HashMap<u16, u32>,
3199    executor: &Arc<Executor>,
3200    shutdown: &Arc<Notify>,
3201    metrics: &Arc<DispatchPathMetrics>,
3202) -> Result<(), SubcError> {
3203    let route_id = completion.route;
3204    let Some(pending) = pending_binds.remove(&route_id) else {
3205        log::warn!(
3206            "subc attach: dropping RouteBind completion for non-pending route {}",
3207            completion.route
3208        );
3209        rollback_pending_bind_actor(
3210            executor,
3211            live_roots,
3212            pending_binds,
3213            &completion.bind_root_id,
3214            completion.inserted_new_actor,
3215        );
3216        let has_pending_bind = pending_binds
3217            .values()
3218            .any(|pending| pending.bind_root_id == completion.bind_root_id);
3219        if !root_channels
3220            .get(&completion.bind_root_id)
3221            .is_some_and(|channels| !channels.is_empty())
3222            && !has_pending_bind
3223        {
3224            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3225        }
3226        remove_installed_route(installed_route_epochs, route_id);
3227        return Ok(());
3228    };
3229
3230    if pending.bind_root_id != completion.bind_root_id {
3231        log::warn!(
3232            "subc attach: pending RouteBind root mismatch for route {} (pending {} completion {})",
3233            completion.route,
3234            pending.bind_root_id.as_path().display(),
3235            completion.bind_root_id.as_path().display()
3236        );
3237    }
3238
3239    let inserted_new_actor = pending.inserted_new_actor || completion.inserted_new_actor;
3240    if pending.cancelled {
3241        rollback_pending_bind_actor(
3242            executor,
3243            live_roots,
3244            pending_binds,
3245            &completion.bind_root_id,
3246            inserted_new_actor,
3247        );
3248        let has_pending_bind = pending_binds
3249            .values()
3250            .any(|pending| pending.bind_root_id == completion.bind_root_id);
3251        if !root_channels
3252            .get(&completion.bind_root_id)
3253            .is_some_and(|channels| !channels.is_empty())
3254            && !has_pending_bind
3255        {
3256            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3257        }
3258        log::debug!(
3259            "subc attach: discarded completed RouteBind for cancelled route {} root {}",
3260            completion.route,
3261            completion.bind_root_id.as_path().display()
3262        );
3263        remove_installed_route(installed_route_epochs, route_id);
3264        return Ok(());
3265    }
3266
3267    let failure = if !completion.configure_response.success {
3268        Some((
3269            &completion.configure_response,
3270            "configure failed during route bind",
3271        ))
3272    } else {
3273        None
3274    };
3275
3276    if let Some((response, fallback)) = failure {
3277        rollback_pending_bind_actor(
3278            executor,
3279            live_roots,
3280            pending_binds,
3281            &completion.bind_root_id,
3282            inserted_new_actor,
3283        );
3284        let has_pending_bind = pending_binds
3285            .values()
3286            .any(|pending| pending.bind_root_id == completion.bind_root_id);
3287        if !root_channels
3288            .get(&completion.bind_root_id)
3289            .is_some_and(|channels| !channels.is_empty())
3290            && !has_pending_bind
3291        {
3292            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3293        }
3294        let message = response_message(response, fallback);
3295        let fatal = response_is_fatal_panic(response);
3296        let error_code = route_bind_error_code_for_configure_response(response);
3297        send_route_bind_error_parts(
3298            tx,
3299            completion.ver,
3300            completion.corr,
3301            completion.flags,
3302            error_code,
3303            &message,
3304            metrics,
3305        )
3306        .await?;
3307        remove_installed_route(installed_route_epochs, route_id);
3308        if fatal {
3309            signal_fatal_teardown(
3310                tx,
3311                Some(completion.route),
3312                completion.ver,
3313                completion.corr,
3314                shutdown,
3315                metrics,
3316            )
3317            .await;
3318        }
3319        return Ok(());
3320    }
3321
3322    remember_session_identity(session_identity, &completion.identity);
3323    let replay_key = push::ReplayKey::from_identity(&completion.identity);
3324    let bind_trust = completion.identity.trust;
3325    insert_route_channel(routes, root_channels, route_id, completion.identity);
3326    let restore_watcher = live_roots
3327        .get(&completion.bind_root_id)
3328        .is_some_and(|meta| meta.idle_artifacts_evicted || meta.unbound_quiesced);
3329    live_roots
3330        .entry(completion.bind_root_id.clone())
3331        .and_modify(|meta| {
3332            meta.reactivate_bound();
3333            meta.diagnostics_on_edit = completion.diagnostics_on_edit;
3334            meta.maintenance_poisoned = false;
3335        })
3336        .or_insert_with(|| RootMeta::new(Instant::now()));
3337    if let Some(meta) = live_roots.get_mut(&completion.bind_root_id) {
3338        meta.diagnostics_on_edit = completion.diagnostics_on_edit;
3339        meta.maintenance_poisoned = false;
3340    }
3341    if let Some(ctx) = executor.actor_context(&completion.bind_root_id) {
3342        ctx.mark_subc_bound();
3343        if restore_watcher {
3344            crate::commands::configure::ensure_project_watcher(&ctx);
3345        }
3346    }
3347
3348    let ack =
3349        serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).map_err(SubcError::Json)?;
3350    let response = Frame::build_with_version(
3351        completion.ver,
3352        FrameType::Response,
3353        control_flags(),
3354        0,
3355        0,
3356        completion.corr,
3357        ack,
3358    )
3359    .map_err(SubcError::FrameBuild)?;
3360    send_reliable_writer_frame(tx, metrics, response, "RouteBindAck").await?;
3361    queue_post_bind_configure_and_completion_maintenance(&completion.bind_root_id, live_roots);
3362    let replayed = push::replay_buffered_push_frames(
3363        tx,
3364        metrics,
3365        route_id,
3366        push_buffer,
3367        &replay_key,
3368        bind_trust,
3369    );
3370    if replayed > 0 {
3371        log::debug!(
3372            "subc attach: replayed {} buffered Push frame(s) to route {} root {} harness {} session {}",
3373            replayed,
3374            completion.route,
3375            replay_key.root.as_path().display(),
3376            replay_key.harness,
3377            replay_key.session
3378        );
3379    }
3380    log::info!(
3381        "subc attach: route {} bound to root {}",
3382        completion.route,
3383        completion.bind_root_id.as_path().display()
3384    );
3385    Ok(())
3386}
3387
3388async fn expire_overdue_route_binds(
3389    tx: &WriterSender,
3390    executor: &Arc<Executor>,
3391    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3392    installed_route_epochs: &mut HashMap<u16, u32>,
3393    metrics: &DispatchPathMetrics,
3394) -> Result<(), SubcError> {
3395    let now = Instant::now();
3396    let expired: Vec<_> = pending_binds
3397        .iter()
3398        .filter_map(|(route, pending)| {
3399            let age = now.saturating_duration_since(pending.started_at);
3400            (!pending.deadline_reported && age >= ROUTE_BIND_DEADLINE).then(|| {
3401                (
3402                    *route,
3403                    pending.corr,
3404                    pending.ver,
3405                    pending.flags,
3406                    pending.bind_root_id.clone(),
3407                    pending.configure_request_id.clone(),
3408                    age,
3409                )
3410            })
3411        })
3412        .collect();
3413
3414    for (route, corr, ver, flags, root_id, configure_request_id, age) in expired {
3415        if let Some(pending) = pending_binds.get_mut(&route) {
3416            pending.cancelled = true;
3417            pending.deadline_reported = true;
3418            let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
3419            log::debug!(
3420                "subc attach: cancelled overdue RouteBind configure for route {route} ({outcome:?})"
3421            );
3422        }
3423        remove_installed_route(installed_route_epochs, route);
3424        let age_ms = age.as_millis().min(u128::from(u64::MAX)) as u64;
3425        let deadline_ms = ROUTE_BIND_DEADLINE.as_millis();
3426        send_route_bind_error_parts(
3427            tx,
3428            ver,
3429            corr,
3430            flags,
3431            "actor_not_ready",
3432            &format!("route bind deadline exceeded after {age_ms}ms (deadline {deadline_ms}ms)"),
3433            metrics,
3434        )
3435        .await?;
3436        log::warn!(
3437            "subc attach: route {} bind for root {} exceeded {}ms deadline (configure_request_id={})",
3438            route,
3439            root_id.as_path().display(),
3440            deadline_ms,
3441            configure_request_id
3442        );
3443    }
3444
3445    Ok(())
3446}
3447
3448/// channel-0 control requests: RouteBind plus the cached health probe. RouteBind
3449/// still reconciles the route's RootConfig through the executor's Mutating lane
3450/// and resolves completion on a loop-owned control-completion channel so slow
3451/// configure jobs do not block the transport loop.
3452#[allow(clippy::too_many_arguments)]
3453async fn handle_control_request(
3454    tx: &WriterSender,
3455    frame: &Frame,
3456    shared_app: &Arc<App>,
3457    executor: &Arc<Executor>,
3458    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3459    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3460    installed_route_epochs: &mut HashMap<u16, u32>,
3461    routes: &mut HashMap<RouteChannel, RouteIdentity>,
3462    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
3463    bg_subs: &mut HashMap<RouteChannel, BgSub>,
3464    bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
3465    bg_wake_pending: &mut HashSet<RouteChannel>,
3466    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
3467    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
3468    retry_buffer: &mut RetryBuffer,
3469    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
3470    shutdown: &Arc<Notify>,
3471    control_completion_tx: &mpsc::Sender<RouteBindCompletion>,
3472    metrics: &Arc<DispatchPathMetrics>,
3473    push_senders: &PushSenders,
3474    dispatch: DispatchFn,
3475    user_config_path: Option<&Path>,
3476) -> Result<(), SubcError> {
3477    let request =
3478        serde_json::from_slice::<ModuleControlRequest>(&frame.body).map_err(SubcError::Json)?;
3479    match request {
3480        ModuleControlRequest::RouteBind {
3481            route_channel,
3482            epoch,
3483            target: _,
3484            identity,
3485            principal,
3486            consumer_capabilities,
3487        } => {
3488            let route_id = route_key(route_channel, epoch);
3489            if epoch == 0 {
3490                return send_route_bind_error(
3491                    tx,
3492                    frame,
3493                    "config_divergence",
3494                    "route bind uses an invalid channel generation",
3495                    metrics,
3496                )
3497                .await;
3498            }
3499            let mut bind_root_id = None;
3500            if let Some(installed_epoch) = installed_route_epochs.get(&route_channel).copied() {
3501                if installed_epoch >= epoch {
3502                    return send_route_bind_error(
3503                        tx,
3504                        frame,
3505                        "config_divergence",
3506                        "route bind generation is not newer than the installed generation",
3507                        metrics,
3508                    )
3509                    .await;
3510                }
3511
3512                let replacement_root = match ProjectRootId::from_path(&identity.project_root) {
3513                    Ok(root_id) => root_id,
3514                    Err(error) => {
3515                        return send_route_bind_error(
3516                            tx,
3517                            frame,
3518                            "config_divergence",
3519                            &format!("invalid route project root: {error}"),
3520                            metrics,
3521                        )
3522                        .await;
3523                    }
3524                };
3525                teardown_installed_route(
3526                    tx,
3527                    metrics,
3528                    executor,
3529                    route_key(route_channel, installed_epoch),
3530                    "higher-epoch RouteBind",
3531                    Some(&replacement_root),
3532                    installed_route_epochs,
3533                    routes,
3534                    root_channels,
3535                    bg_subs,
3536                    bg_sub_by_session,
3537                    bg_wake_pending,
3538                    pending_bash_asks,
3539                    live_roots,
3540                    route_bash_cancels,
3541                    pending_binds,
3542                    retry_buffer,
3543                    push_buffer,
3544                    shutdown,
3545                )
3546                .await?;
3547                bind_root_id = Some(replacement_root);
3548            }
3549            if pending_binds.contains_key(&route_id) {
3550                return send_route_bind_error(
3551                    tx,
3552                    frame,
3553                    "config_divergence",
3554                    "route bind is already pending for channel",
3555                    metrics,
3556                )
3557                .await;
3558            }
3559            let bind_root_id = match bind_root_id {
3560                Some(root_id) => root_id,
3561                None => match ProjectRootId::from_path(&identity.project_root) {
3562                    Ok(root_id) => root_id,
3563                    Err(error) => {
3564                        return send_route_bind_error(
3565                            tx,
3566                            frame,
3567                            "config_divergence",
3568                            &format!("invalid route project root: {error}"),
3569                            metrics,
3570                        )
3571                        .await;
3572                    }
3573                },
3574            };
3575
3576            // Reconcile RootConfig: build a configure request from the bind
3577            // identity + forwarded config tiers and run it through the executor.
3578            let request_id = format!("subc-bind-{route_channel}");
3579            let bind_project_root = identity.project_root.clone();
3580            let bind_harness = identity.harness.clone();
3581            let bind_session = identity.session.clone();
3582            let bind_trust = trust_for_bind(&bind_harness, &principal);
3583            let bind_principal_id = principal_id(&principal);
3584            // Typed capability declaration from the consumer: the facade stamps it
3585            // from the MCP host's initialize-advertised capabilities. Absent
3586            // means no reverse-request capability — flat deny, fail-closed. A
3587            // consumer over-declaring only earns asks that TTL-deny.
3588            let consumer_elicitation_capable = consumer_capabilities
3589                .as_ref()
3590                .is_some_and(|capabilities| capabilities.iter().any(|c| c == "elicitation"));
3591            log::info!(
3592                "subc attach: route {} harness={} principal={} trust={} elicitation={}",
3593                route_channel,
3594                bind_harness,
3595                principal_label(&principal),
3596                bind_trust.label(),
3597                consumer_elicitation_capable
3598            );
3599
3600            // Config is single-per-project, read by AFT directly from the
3601            // CortexKit config files (user: ~/.config/cortexkit/aft.jsonc,
3602            // project: <root>/.cortexkit/aft.jsonc). Wire-relayed config tiers are
3603            // IGNORED entirely: a front (runner, mcp:*, or fed:*) cannot push config over
3604            // the wire. This is what makes config harness-INDEPENDENT — every
3605            // harness binding a project gets the identical on-disk config, so two
3606            // trust domains sharing the per-root actor can never diverge or
3607            // inherit each other's capabilities (the cross-bind escalation class).
3608            // Wire-relayed config tiers (if the protocol still carries them) are
3609            // ignored entirely; the per-tier trust boundary (user trusted, project
3610            // privileged-dropped) is applied to the FILE tiers in handle_configure.
3611            let local_tiers = crate::subc_config::read_local_cortexkit_config_tiers(
3612                user_config_path,
3613                Path::new(&bind_project_root),
3614            );
3615            let config_tiers: Vec<Value> = local_tiers
3616                .iter()
3617                .map(|t| json!({ "tier": t.tier, "source": t.source, "doc": t.doc }))
3618                .collect();
3619            let diagnostics_on_edit = diagnostics_on_edit_from_tiers(&local_tiers);
3620            let configure_json = json!({
3621                "id": request_id,
3622                "command": "configure",
3623                "project_root": bind_project_root,
3624                "harness": bind_harness,
3625                "session_id": bind_session.clone(),
3626                "config": config_tiers,
3627            });
3628            let configure_req = match serde_json::from_value::<RawRequest>(configure_json) {
3629                Ok(req) => req,
3630                Err(error) => {
3631                    return send_route_bind_error(
3632                        tx,
3633                        frame,
3634                        "config_divergence",
3635                        &format!("failed to build configure request: {error}"),
3636                        metrics,
3637                    )
3638                    .await;
3639                }
3640            };
3641
3642            let route_identity = RouteIdentity(Arc::new(RouteIdentityData {
3643                root: bind_root_id.clone(),
3644                project_root: PathBuf::from(&bind_project_root),
3645                harness: bind_harness.clone(),
3646                session: bind_session.clone(),
3647                trust: bind_trust,
3648                spawn_principal: AuthenticatedPrincipal::RouteBind {
3649                    trust: bind_trust.sandbox_trust(),
3650                    route_channel,
3651                    route_epoch: epoch,
3652                    project_root: PathBuf::from(&bind_project_root),
3653                    harness: bind_harness.clone(),
3654                    session_id: bind_session.clone(),
3655                    principal_id: bind_principal_id,
3656                },
3657                consumer_elicitation_capable,
3658            }));
3659            let configure_session = route_identity.session.clone();
3660            let root_was_live = live_roots.contains_key(&bind_root_id);
3661            let inserted_new_actor = register_actor_for_bind(
3662                shared_app,
3663                executor,
3664                push_senders,
3665                &bind_root_id,
3666                route_channel,
3667                root_was_live,
3668            );
3669
3670            let configure_request_id = configure_req.id.clone();
3671            installed_route_epochs.insert(route_channel, epoch);
3672            if let Some(meta) = live_roots.get_mut(&bind_root_id) {
3673                meta.maintenance_queued_kinds.clear();
3674                meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
3675            }
3676            let (configure_rx, configure_cancellation) = executor.submit_cancellable_async(
3677                bind_root_id.clone(),
3678                Lane::Mutating,
3679                configure_request_id.clone(),
3680                Box::new(move |ctx| {
3681                    log_ctx::with_session(Some(configure_session.clone()), || {
3682                        dispatch(configure_req, ctx)
3683                    })
3684                }),
3685            );
3686            pending_binds.insert(
3687                route_id,
3688                PendingBind {
3689                    bind_root_id: bind_root_id.clone(),
3690                    inserted_new_actor,
3691                    cancelled: false,
3692                    configure_request_id: configure_request_id.clone(),
3693                    started_at: Instant::now(),
3694                    warned_half_deadline: false,
3695                    deadline_reported: false,
3696                    corr: frame.header.corr,
3697                    ver: frame.header.ver,
3698                    flags: frame.header.flags,
3699                    cancellation: configure_cancellation,
3700                },
3701            );
3702
3703            let completion_tx = control_completion_tx.clone();
3704            let completion_identity = route_identity;
3705            let completion_root = bind_root_id.clone();
3706            let completion_route_channel = route_channel;
3707            let completion_ver = frame.header.ver;
3708            let completion_corr = frame.header.corr;
3709            let completion_flags = frame.header.flags;
3710            let completion_metrics = Arc::clone(metrics);
3711            tokio::spawn(async move {
3712                let _response_task = ResponseTaskGuard::new(&completion_metrics);
3713                let configure_response =
3714                    await_executor_response(configure_rx, configure_request_id.clone()).await;
3715                // Send the route-bind acknowledgment as soon as configure succeeds.
3716                // Installing completed search or callgraph builds only refreshes cached
3717                // read data, so a later maintenance pass can do it without delaying the
3718                // daemon's confirmation that the route is usable.
3719                let completion = RouteBindCompletion {
3720                    route: route_key(completion_route_channel, epoch),
3721                    identity: completion_identity,
3722                    bind_root_id: completion_root,
3723                    inserted_new_actor,
3724                    configure_response,
3725                    diagnostics_on_edit,
3726                    ver: completion_ver,
3727                    corr: completion_corr,
3728                    flags: completion_flags,
3729                };
3730                if send_counted_channel(
3731                    &completion_tx,
3732                    &completion_metrics.control_completion_queued,
3733                    completion,
3734                )
3735                .await
3736                .is_err()
3737                {
3738                    log::debug!(
3739                        "subc attach: dropped RouteBind completion for route {} after loop exit",
3740                        completion_route_channel
3741                    );
3742                }
3743            });
3744
3745            Ok(())
3746        }
3747        ModuleControlRequest::HealthCheck {} => {
3748            let report = build_health_report(executor, pending_binds, metrics, shared_app);
3749            let body = serde_json::to_vec(&ModuleControlResponse::from(report))
3750                .map_err(SubcError::Json)?;
3751            let response = Frame::build_with_version(
3752                frame.header.ver,
3753                FrameType::Response,
3754                frame.header.flags,
3755                0,
3756                0,
3757                frame.header.corr,
3758                body,
3759            )
3760            .map_err(SubcError::FrameBuild)?;
3761            send_frame(tx, metrics, response).await
3762        }
3763    }
3764}
3765
3766fn install_bash_compressor(ctx: &AppContext) {
3767    // Mirrors main.rs per-actor compressor installation for subc-created actors.
3768    let filter_registry_handle = ctx.shared_filter_registry();
3769    let compress_flag = ctx.bash_compress_flag();
3770    ctx.bash_background().set_compressor_with_exit_code(
3771        move |command: &str, output: String, exit_code: Option<i32>| {
3772            if !compress_flag.load(std::sync::atomic::Ordering::Relaxed) {
3773                return crate::compress::CompressionResult::new(output);
3774            }
3775            let registry_guard = match filter_registry_handle.read() {
3776                Ok(g) => g,
3777                Err(poisoned) => poisoned.into_inner(),
3778            };
3779            crate::compress::compress_with_registry_exit_code(
3780                command,
3781                &output,
3782                exit_code,
3783                &registry_guard,
3784            )
3785        },
3786    );
3787}
3788
3789fn diagnostics_on_edit_from_tiers(tiers: &[ConfigTier]) -> bool {
3790    let mut diagnostics_on_edit = false;
3791    for tier in tiers {
3792        if let Some(value) = diagnostics_on_edit_from_doc(&tier.doc) {
3793            diagnostics_on_edit = value;
3794        }
3795    }
3796    diagnostics_on_edit
3797}
3798
3799fn diagnostics_on_edit_from_doc(doc: &str) -> Option<bool> {
3800    let stripped = strip_jsonc(doc);
3801    let value = serde_json::from_str::<Value>(&stripped).ok()?;
3802    value
3803        .get("lsp")
3804        .and_then(Value::as_object)?
3805        .get("diagnostics_on_edit")
3806        .and_then(Value::as_bool)
3807}
3808
3809async fn send_route_bind_error(
3810    tx: &WriterSender,
3811    frame: &Frame,
3812    code: &str,
3813    message: &str,
3814    metrics: &DispatchPathMetrics,
3815) -> Result<(), SubcError> {
3816    send_route_bind_error_parts(
3817        tx,
3818        frame.header.ver,
3819        frame.header.corr,
3820        frame.header.flags,
3821        code,
3822        message,
3823        metrics,
3824    )
3825    .await
3826}
3827
3828async fn send_route_bind_error_parts(
3829    tx: &WriterSender,
3830    ver: u8,
3831    corr: u64,
3832    flags: Flags,
3833    code: &str,
3834    message: &str,
3835    metrics: &DispatchPathMetrics,
3836) -> Result<(), SubcError> {
3837    let response = build_error_frame(ver, 0, 0, corr, flags, code, message)?;
3838    send_reliable_writer_frame(tx, metrics, response, "RouteBind error").await?;
3839    log::warn!("subc attach: route bind rejected ({code}): {message}");
3840    Ok(())
3841}
3842
3843/// Route-channel tool call: `{name, arguments}` → executor lane → dispatch to
3844/// the sync command core → wrap the structured Response in a CallToolResult
3845/// `{content, isError}`. Tool-result mapping: the whole `{success, ...}` Response
3846/// serialized into ONE text block; `isError` carries `success == false`.
3847async fn handle_tool_call(
3848    tx: &WriterSender,
3849    frame: &Frame,
3850    mut phase_trace: PhaseTrace,
3851    routes: &HashMap<RouteChannel, RouteIdentity>,
3852    pending_binds: &HashMap<RouteChannel, PendingBind>,
3853    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3854    executor: &Arc<Executor>,
3855    shutdown: &Arc<Notify>,
3856    connection_cancel: &PersistentCancelSignal,
3857    bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
3858    bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
3859    metrics: &Arc<DispatchPathMetrics>,
3860    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
3861    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
3862    next_bash_ask_corr: &mut u64,
3863    bg_subs: &mut HashMap<RouteChannel, BgSub>,
3864    bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
3865    bg_wake_pending: &mut HashSet<RouteChannel>,
3866    bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
3867    dispatch: DispatchFn,
3868    allow_native_passthrough: bool,
3869) -> Result<(), SubcError> {
3870    let route_id = route_key(frame.header.channel, frame.header.epoch);
3871    if pending_binds.contains_key(&route_id) {
3872        let error = build_error_frame(
3873            frame.header.ver,
3874            frame.header.channel,
3875            frame.header.epoch,
3876            frame.header.corr,
3877            frame.header.flags,
3878            "route_not_bound",
3879            "route is not bound before tool call",
3880        )?;
3881        return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
3882    }
3883
3884    let Some(identity) = routes.get(&route_id).cloned() else {
3885        let error = build_error_frame(
3886            frame.header.ver,
3887            frame.header.channel,
3888            frame.header.epoch,
3889            frame.header.corr,
3890            frame.header.flags,
3891            "route_not_bound",
3892            "route is not bound before tool call",
3893        )?;
3894        return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
3895    };
3896    let restore_watcher = live_roots
3897        .get(&identity.root)
3898        .is_some_and(|meta| meta.idle_artifacts_evicted);
3899    if let Some(meta) = live_roots.get_mut(&identity.root) {
3900        meta.reactivate_bound();
3901    }
3902    if restore_watcher {
3903        if let Some(ctx) = executor.actor_context(&identity.root) {
3904            crate::commands::configure::ensure_project_watcher(&ctx);
3905        }
3906    }
3907
3908    let route_request =
3909        serde_json::from_slice::<RouteRequest>(&frame.body).map_err(SubcError::Json)?;
3910    if matches!(
3911        route_request,
3912        RouteRequest::BgEvents(BgEventsRequest {
3913            op: BgEventsOp::BgEvents
3914        })
3915    ) {
3916        if let Some(old_sub) = bg_subs.get(&route_id).copied() {
3917            push::send_reliable_bg_stream_end(tx, metrics, route_id, &old_sub).await?;
3918        }
3919        if !identity.trust.allows_bash_observation() {
3920            bg_subs.remove(&route_id);
3921            bg_wake_pending.remove(&route_id);
3922            remove_bg_subscription_index(bg_sub_by_session, route_id, Some(&identity));
3923            let denied_sub = BgSub {
3924                corr: frame.header.corr,
3925                ver: frame.header.ver,
3926                flags: frame.header.flags,
3927            };
3928            push::send_reliable_bg_stream_end(tx, metrics, route_id, &denied_sub).await?;
3929            return Ok(());
3930        }
3931        bg_subs.insert(
3932            route_id,
3933            BgSub {
3934                corr: frame.header.corr,
3935                ver: frame.header.ver,
3936                flags: frame.header.flags,
3937            },
3938        );
3939        bg_sub_by_session.insert((identity.root.clone(), identity.session.clone()), route_id);
3940        push::arm_bg_wake(
3941            identity.root.clone(),
3942            identity.session.clone(),
3943            route_id,
3944            bg_wake_pending,
3945            bg_wake_epoch,
3946        );
3947        return Ok(());
3948    }
3949
3950    let RouteRequest::ToolCall(call) = route_request else {
3951        unreachable!("background event subscription returned above")
3952    };
3953    let bare_name = call.name;
3954    let arguments = strip_agent_preview_arg_owned(call.arguments);
3955    let format_context = crate::subc_format::FormatContext::from_tool_call(
3956        &bare_name,
3957        &arguments,
3958        identity.project_root.as_path(),
3959    );
3960
3961    let request_id = format!("subc-{}-{}", frame.header.channel, frame.header.corr);
3962    let bind_trust = identity.trust;
3963    let diagnostics_on_edit = live_roots
3964        .get(&identity.root)
3965        .map(|meta| meta.diagnostics_on_edit)
3966        .unwrap_or(false);
3967
3968    let requests_host = bare_name == "bash"
3969        && arguments
3970            .get("sandbox")
3971            .or_else(|| {
3972                arguments
3973                    .get("params")
3974                    .and_then(|params| params.get("sandbox"))
3975            })
3976            .and_then(Value::as_str)
3977            == Some("host");
3978    if matches!(bind_trust, BindTrust::Untrusted) && requests_host {
3979        let response = Response::error(
3980            request_id.clone(),
3981            "sandbox_escalation_denied",
3982            "sandbox host escalation is unavailable to untrusted principals",
3983        );
3984        let text = crate::subc_format::format_response_with_context(
3985            &bare_name,
3986            &response,
3987            &format_context,
3988        );
3989        let result = ToolCallResult { text, response };
3990        let response_frame = build_tool_response_frame(
3991            frame.header.ver,
3992            route_id,
3993            frame.header.corr,
3994            frame.header.flags,
3995            &result,
3996            bind_trust,
3997        )?;
3998        return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
3999    }
4000
4001    if matches!(bind_trust, BindTrust::Untrusted)
4002        && is_bash_family_tool(&bare_name)
4003        && (bare_name != "bash" || !identity.consumer_elicitation_capable)
4004    {
4005        let response = bash::bash_denied_untrusted_response(request_id.clone());
4006        let text = crate::subc_format::format_response_with_context(
4007            &bare_name,
4008            &response,
4009            &format_context,
4010        );
4011        let result = ToolCallResult { text, response };
4012        let response_frame = build_tool_response_frame(
4013            frame.header.ver,
4014            route_id,
4015            frame.header.corr,
4016            frame.header.flags,
4017            &result,
4018            bind_trust,
4019        )?;
4020        return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4021    }
4022
4023    // A non-core name is NOT in the tool manifest. AFT fails closed and
4024    // does not trust subc to enforce the manifest: rejecting here is the
4025    // defense-in-depth backstop that prevents a forwarded native command
4026    // (e.g. `configure`, which would reach handle_configure and bypass
4027    // the RouteBind config-trust cap) from ever reaching dispatch. Only
4028    // the integration-test harness (run_subc_mode_for_test) opens this to
4029    // drive synthetic native commands through the executor.
4030    if !is_subc_agent_core_tool(&bare_name)
4031        && !is_subc_native_plumbing_tool(&bare_name)
4032        && !allow_native_passthrough
4033    {
4034        log::warn!(
4035            "subc tool call: rejecting non-manifest tool name {:?} on route {} (fail-closed)",
4036            bare_name,
4037            frame.header.channel
4038        );
4039        let response = Response::error(
4040            request_id.clone(),
4041            "unknown_tool",
4042            format!("tool {:?} is not in the AFT tool manifest", bare_name),
4043        );
4044        let text = crate::subc_format::format_response_with_context(
4045            &bare_name,
4046            &response,
4047            &format_context,
4048        );
4049        let result = ToolCallResult { text, response };
4050        let response_frame = build_tool_response_frame(
4051            frame.header.ver,
4052            route_id,
4053            frame.header.corr,
4054            frame.header.flags,
4055            &result,
4056            bind_trust,
4057        )?;
4058        return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4059    }
4060
4061    if bare_name == "bash" {
4062        if matches!(bind_trust, BindTrust::Untrusted) {
4063            let plan = match bash::prepare_bash_elicitation_plan(
4064                &arguments,
4065                identity.project_root.as_path(),
4066            ) {
4067                Ok(plan) => plan,
4068                Err(error) => {
4069                    let response = Response::error(request_id.clone(), error.code, error.message);
4070                    let text = crate::subc_format::format_response_with_context(
4071                        &bare_name,
4072                        &response,
4073                        &format_context,
4074                    );
4075                    let result = ToolCallResult { text, response };
4076                    let response_frame = build_tool_response_frame(
4077                        frame.header.ver,
4078                        route_id,
4079                        frame.header.corr,
4080                        frame.header.flags,
4081                        &result,
4082                        bind_trust,
4083                    )?;
4084                    return send_reliable_writer_frame(
4085                        tx,
4086                        metrics,
4087                        response_frame,
4088                        "tool response",
4089                    )
4090                    .await;
4091                }
4092            };
4093
4094            let reverse_corr =
4095                allocate_reverse_corr(pending_bash_asks, route_id, next_bash_ask_corr);
4096            let ask_frame = build_bash_elicitation_request_frame(
4097                frame.header.ver,
4098                route_id,
4099                reverse_corr,
4100                frame.header.flags,
4101                &plan.command,
4102                &plan.asks,
4103            )?;
4104
4105            let meta = live_roots
4106                .entry(identity.root.clone())
4107                .or_insert_with(|| RootMeta::new(Instant::now()));
4108            meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
4109            meta.reactivate_bound();
4110
4111            let route_cancel =
4112                route_bash_cancels
4113                    .entry(route_id)
4114                    .or_insert_with(|| bash::RouteBashCancel {
4115                        token: PersistentCancelSignal::new(),
4116                        active_waits: 0,
4117                    });
4118            route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
4119            let cancel = bash::BashWaitCancel {
4120                connection: connection_cancel.clone(),
4121                route: route_cancel.token.clone(),
4122            };
4123            pending_bash_asks.insert(
4124                ReverseCorrKey {
4125                    route: route_id,
4126                    corr: reverse_corr,
4127                },
4128                PendingBashAsk {
4129                    route: route_id,
4130                    tool_corr: frame.header.corr,
4131                    tool_flags: frame.header.flags,
4132                    tool_ver: frame.header.ver,
4133                    root: identity.root.clone(),
4134                    project_root: identity.project_root.clone(),
4135                    session_id: identity.session.clone(),
4136                    spawn_principal: identity.spawn_principal.clone(),
4137                    request_id,
4138                    arguments,
4139                    format_context,
4140                    cancel,
4141                    grants: plan.grants,
4142                    expires_at: Instant::now() + bash_elicitation_timeout(),
4143                },
4144            );
4145            return send_reliable_writer_frame(tx, metrics, ask_frame, "bash elicitation request")
4146                .await;
4147        }
4148
4149        let meta = live_roots
4150            .entry(identity.root.clone())
4151            .or_insert_with(|| RootMeta::new(Instant::now()));
4152        meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
4153        meta.reactivate_bound();
4154
4155        let route_cancel =
4156            route_bash_cancels
4157                .entry(route_id)
4158                .or_insert_with(|| bash::RouteBashCancel {
4159                    token: PersistentCancelSignal::new(),
4160                    active_waits: 0,
4161                });
4162        route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
4163        let cancel = bash::BashWaitCancel {
4164            connection: connection_cancel.clone(),
4165            route: route_cancel.token.clone(),
4166        };
4167
4168        bash::submit_deferred_bash(
4169            executor,
4170            bash_deferred_tx,
4171            bash_poll_touch_tx,
4172            metrics,
4173            dispatch,
4174            identity.root.clone(),
4175            identity.project_root.clone(),
4176            identity.session.clone(),
4177            request_id,
4178            route_id,
4179            frame.header.corr,
4180            frame.header.flags,
4181            frame.header.ver,
4182            arguments,
4183            format_context,
4184            cancel,
4185            bind_trust,
4186            identity.spawn_principal.clone(),
4187            None,
4188        );
4189        return Ok(());
4190    }
4191
4192    let lane = command_lane(&bare_name);
4193    let tool_call_context = ToolCallContext {
4194        project_root: identity.project_root.clone(),
4195        session_id: Some(identity.session.clone()),
4196        request_id: request_id.clone(),
4197        diagnostics_on_edit,
4198        preview: call.preview,
4199    };
4200    let bare_name_for_frame = bare_name.clone();
4201    let identity_for_run = identity.clone();
4202    let completion_session = identity.session.clone();
4203    let completion_root = identity.project_root.clone();
4204    let request_id_for_force = request_id.clone();
4205    let format_context_for_frame = format_context.clone();
4206    let (tool_call_tx, tool_call_rx) = oneshot::channel::<ToolCallCompletion>();
4207    phase_trace.mark_executor_submitted();
4208    let rx = executor.submit_async(
4209        identity.root.clone(),
4210        lane,
4211        request_id.clone(),
4212        Box::new(move |ctx| {
4213            phase_trace.mark_job_admitted();
4214            log_ctx::with_session(Some(identity_for_run.session.clone()), || {
4215                let run = || {
4216                    let finalizer = |response: &mut Response| {
4217                        crate::response_finalize::finalize_response_with_bg_completions(
4218                            response,
4219                            ctx,
4220                            &identity_for_run.session,
4221                            &bare_name,
4222                            bind_trust.allows_bash_observation(),
4223                        );
4224                    };
4225                    match run_tool_call(
4226                        &bare_name,
4227                        arguments,
4228                        &format_context,
4229                        &tool_call_context,
4230                        ctx,
4231                        &dispatch,
4232                        Some(&finalizer),
4233                        Some(&mut phase_trace),
4234                    ) {
4235                        ToolCallOutcome::Unary(result) => {
4236                            let response = result.response;
4237                            let _ = tool_call_tx.send(ToolCallCompletion {
4238                                text: result.text,
4239                                phase_trace,
4240                            });
4241                            response
4242                        }
4243                    }
4244                };
4245                if matches!(bind_trust, BindTrust::Untrusted) {
4246                    ctx.with_force_restrict(&request_id_for_force, run)
4247                } else {
4248                    run()
4249                }
4250            })
4251        }),
4252    );
4253    let completion_tx = tx.clone();
4254    let completion_shutdown = Arc::clone(shutdown);
4255    let route = route_id;
4256    let corr = frame.header.corr;
4257    let flags = frame.header.flags;
4258    let ver = frame.header.ver;
4259    let completion_metrics = Arc::clone(metrics);
4260    tokio::spawn(async move {
4261        let _response_task = ResponseTaskGuard::new(&completion_metrics);
4262        let response = await_executor_response(rx, request_id.clone()).await;
4263        let (text, phase_trace) = match tool_call_rx.await {
4264            Ok(completion) => (completion.text, Some(completion.phase_trace)),
4265            Err(_) => (
4266                crate::subc_format::format_response_with_context(
4267                    &bare_name_for_frame,
4268                    &response,
4269                    &format_context_for_frame,
4270                ),
4271                None,
4272            ),
4273        };
4274        let result = ToolCallResult { text, response };
4275        let fatal = response_is_fatal_panic(&result.response);
4276        match build_tool_response_frame(ver, route, corr, flags, &result, bind_trust) {
4277            Ok(response_frame) => {
4278                let send_result = if let Some(phase_trace) = phase_trace {
4279                    let trace = ToolResponseWriteTrace::new(
4280                        phase_trace,
4281                        bare_name_for_frame,
4282                        completion_root,
4283                        completion_session,
4284                        route.channel,
4285                        corr,
4286                    );
4287                    send_traced_tool_response_frame(
4288                        &completion_tx,
4289                        &completion_metrics,
4290                        response_frame,
4291                        trace,
4292                    )
4293                    .await
4294                } else {
4295                    send_reliable_writer_frame(
4296                        &completion_tx,
4297                        &completion_metrics,
4298                        response_frame,
4299                        "tool response",
4300                    )
4301                    .await
4302                };
4303                if let Err(error) = send_result {
4304                    log::warn!("subc attach: failed to queue tool response frame: {error}");
4305                }
4306            }
4307            Err(error) => {
4308                log::error!("subc attach: failed to build tool response frame: {error}");
4309            }
4310        }
4311        if fatal {
4312            signal_fatal_teardown(
4313                &completion_tx,
4314                Some(route),
4315                ver,
4316                corr,
4317                &completion_shutdown,
4318                &completion_metrics,
4319            )
4320            .await;
4321        }
4322    });
4323    Ok(())
4324}
4325
4326fn submit_maintenance_job(
4327    executor: &Arc<Executor>,
4328    root_id: ProjectRootId,
4329    kind: MaintenanceDrainKind,
4330    bg_sessions_to_check: Vec<(String, u64)>,
4331    completion_tx: &mpsc::Sender<MaintenanceCompletion>,
4332    metrics: &Arc<DispatchPathMetrics>,
4333) {
4334    let request_id = format!(
4335        "subc-maintenance-drain-{}-{}",
4336        kind.label(),
4337        root_id.as_path().to_string_lossy()
4338    );
4339    let response_id = request_id.clone();
4340    let completion_root_id = root_id.clone();
4341    let maintenance_generation = executor
4342        .actor_context(&root_id)
4343        .map(|ctx| ctx.configure_generation())
4344        .unwrap_or(0);
4345    let (outcome_tx, outcome_rx) = oneshot::channel::<MaintenanceJobOutcome>();
4346    // ConfigureTail runs deferred configure mutations and needs the actor
4347    // epoch write gate. The other drain kinds only mutate subsystem state
4348    // behind that subsystem's own lock, so they run on MaintenanceCommit and
4349    // overlap interactive reads instead of excluding them.
4350    let lane = match kind {
4351        MaintenanceDrainKind::ConfigureTail => Lane::Mutating,
4352        MaintenanceDrainKind::Watcher
4353        | MaintenanceDrainKind::Lsp
4354        | MaintenanceDrainKind::CompletionDrains => Lane::MaintenanceCommit,
4355    };
4356    let rx = executor.submit_maintenance_async(
4357        root_id,
4358        lane,
4359        request_id.clone(),
4360        Box::new(move |ctx| {
4361            let outcome = match kind {
4362                MaintenanceDrainKind::Watcher => {
4363                    let drained = runtime_drain::drain_watcher_events_bounded(
4364                        ctx,
4365                        runtime_drain::WATCHER_PATH_DRAIN_BATCH_CAP,
4366                    );
4367                    MaintenanceJobOutcome {
4368                        empty_bg_sessions: Vec::new(),
4369                        requeue_kind: drained.has_more.then_some(kind),
4370                    }
4371                }
4372                MaintenanceDrainKind::Lsp => {
4373                    let drained = runtime_drain::drain_lsp_events_bounded(
4374                        ctx,
4375                        runtime_drain::LSP_EVENT_DRAIN_BATCH_CAP,
4376                    );
4377                    MaintenanceJobOutcome {
4378                        empty_bg_sessions: Vec::new(),
4379                        requeue_kind: drained.has_more.then_some(kind),
4380                    }
4381                }
4382                MaintenanceDrainKind::ConfigureTail => {
4383                    runtime_drain::drain_deferred_configure_maintenance(ctx);
4384                    runtime_drain::drain_configure_warning_events(ctx);
4385                    MaintenanceJobOutcome::default()
4386                }
4387                MaintenanceDrainKind::CompletionDrains => {
4388                    runtime_drain::drain_search_index_events(ctx);
4389                    runtime_drain::drain_callgraph_store_events(ctx);
4390                    runtime_drain::drain_semantic_index_events(ctx);
4391                    runtime_drain::drain_semantic_refresh_events(ctx);
4392                    runtime_drain::drain_inspect_events_for_generation(ctx, maintenance_generation);
4393                    let empty_bg_sessions = bg_sessions_to_check
4394                        .into_iter()
4395                        .filter(|(session, _)| {
4396                            !ctx.bash_background()
4397                                .has_completions_for_session(Some(session.as_str()))
4398                        })
4399                        .collect();
4400                    MaintenanceJobOutcome {
4401                        empty_bg_sessions,
4402                        requeue_kind: None,
4403                    }
4404                }
4405            };
4406            let requeued = outcome.requeue_kind.is_some();
4407            let _ = outcome_tx.send(outcome);
4408            Response::success(
4409                response_id,
4410                json!({ "drained": true, "kind": kind.label(), "requeued": requeued }),
4411            )
4412        }),
4413    );
4414    let completion_tx = completion_tx.clone();
4415    let completion_metrics = Arc::clone(metrics);
4416    tokio::spawn(async move {
4417        let _response_task = ResponseTaskGuard::new(&completion_metrics);
4418        let response = await_executor_response(rx, request_id).await;
4419        let outcome = outcome_rx.await.unwrap_or_default();
4420        let _ = send_counted_channel(
4421            &completion_tx,
4422            &completion_metrics.maintenance_queued,
4423            MaintenanceCompletion {
4424                root_id: completion_root_id,
4425                kind,
4426                response,
4427                empty_bg_sessions: outcome.empty_bg_sessions,
4428                requeue_kind: outcome.requeue_kind,
4429            },
4430        )
4431        .await;
4432    });
4433}
4434
4435async fn await_executor_response(rx: oneshot::Receiver<Response>, request_id: String) -> Response {
4436    rx.await
4437        .unwrap_or_else(|_| Response::error(request_id, "internal_error", "executor dropped"))
4438}
4439async fn signal_fatal_teardown(
4440    tx: &WriterSender,
4441    route: Option<RouteChannel>,
4442    ver: u8,
4443    corr: u64,
4444    shutdown: &Arc<Notify>,
4445    metrics: &DispatchPathMetrics,
4446) {
4447    if let Some(route) = route {
4448        if let Ok(frame) = build_goodbye_frame(ver, route.channel, route.epoch, corr) {
4449            if let Err(error) = send_frame(tx, metrics, frame).await {
4450                log::warn!(
4451                    "subc attach: failed to queue fatal route Goodbye for route {route}: {error}"
4452                );
4453            }
4454        }
4455    }
4456    if let Ok(frame) = build_goodbye_frame(ver, 0, 0, 0) {
4457        if let Err(error) = send_frame(tx, metrics, frame).await {
4458            log::warn!("subc attach: failed to queue fatal channel-0 Goodbye: {error}");
4459        }
4460    }
4461    shutdown.notify_one();
4462}
4463#[derive(Debug, Deserialize)]
4464#[serde(untagged)]
4465enum RouteRequest {
4466    BgEvents(BgEventsRequest),
4467    ToolCall(ToolCallRequest),
4468}
4469
4470#[derive(Debug, Deserialize)]
4471struct BgEventsRequest {
4472    op: BgEventsOp,
4473}
4474
4475#[derive(Debug, Deserialize)]
4476#[serde(rename_all = "snake_case")]
4477enum BgEventsOp {
4478    BgEvents,
4479}
4480
4481#[derive(Debug, Deserialize)]
4482struct ToolCallRequest {
4483    name: String,
4484    #[serde(default)]
4485    arguments: Value,
4486    /// Server-owned preview control (B1c-0): the plugin's mutation flow is
4487    /// preview -> permission ask -> apply. Dropping this field made "preview"
4488    /// calls mutate disk before the permission prompt and the subsequent
4489    /// apply fail with not-found.
4490    #[serde(default)]
4491    preview: bool,
4492}
4493
4494#[cfg(test)]
4495pub(crate) mod test_support {
4496    use super::*;
4497    use crate::bash_background::BgTaskStatus;
4498    use crate::protocol::{
4499        BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, ConfigureWarningsFrame,
4500        ProgressFrame, StatusChangedFrame,
4501    };
4502    use serde_json::json;
4503
4504    pub(super) fn test_root(name: &str) -> (tempfile::TempDir, ProjectRootId) {
4505        let dir = tempfile::Builder::new()
4506            .prefix(name)
4507            .tempdir()
4508            .expect("temp root");
4509        let root = ProjectRootId::from_path(dir.path()).expect("project root id");
4510        (dir, root)
4511    }
4512
4513    pub(super) fn test_ctx() -> Arc<AppContext> {
4514        Arc::new(AppContext::new(
4515            Box::new(crate::parser::TreeSitterProvider::new()),
4516            crate::config::Config::default(),
4517        ))
4518    }
4519
4520    pub(super) fn wait_for_watcher_count(ctx: &AppContext, expected: usize) {
4521        let deadline = Instant::now() + Duration::from_secs(30);
4522        loop {
4523            let observed = ctx.watcher_registry_count();
4524            if observed == expected {
4525                return;
4526            }
4527            assert!(
4528                Instant::now() < deadline,
4529                "watcher count did not settle before deadline: expected={expected}, observed={observed}"
4530            );
4531            std::thread::sleep(Duration::from_millis(50));
4532        }
4533    }
4534
4535    /// Sweep until `root` is forgotten, mirroring how production reaps.
4536    ///
4537    /// `reap_idle_roots` probes actor idleness with a try-lock and retains the
4538    /// root when the scheduler holds that lock — correct behavior, since the
4539    /// real caller sweeps on a timer and simply catches the root next tick. A
4540    /// test that asserts a single sweep succeeds is therefore asserting it wins
4541    /// a lock race that nothing in production depends on: `register_actor` wakes
4542    /// the scheduler, which grabs the same lock, and on a loaded runner that
4543    /// window is wide enough to lose. Sweep to the outcome instead.
4544    pub(super) fn reap_until_forgotten(
4545        root: &ProjectRootId,
4546        live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4547        pending_binds: &HashMap<RouteChannel, PendingBind>,
4548        root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
4549        executor: &Arc<Executor>,
4550        metrics: &DispatchPathMetrics,
4551    ) -> IdleReapOutcome {
4552        let deadline = Instant::now() + Duration::from_secs(30);
4553        loop {
4554            let outcome = reap_idle_roots(
4555                Instant::now(),
4556                live_roots,
4557                pending_binds,
4558                root_channels,
4559                executor,
4560                metrics,
4561            );
4562            if outcome.forgotten_deleted_roots.contains(root) {
4563                return outcome;
4564            }
4565            assert!(
4566                Instant::now() < deadline,
4567                "deleted root was never forgotten: {root:?}"
4568            );
4569            std::thread::sleep(Duration::from_millis(10));
4570        }
4571    }
4572
4573    pub(super) fn wait_for_actor_root_count(app: &App, expected: usize) {
4574        let deadline = Instant::now() + Duration::from_secs(30);
4575        loop {
4576            let observed = app.actor_root_count();
4577            if observed == expected {
4578                return;
4579            }
4580            assert!(
4581                Instant::now() < deadline,
4582                "actor root count did not settle before deadline: expected={expected}, observed={observed}"
4583            );
4584            std::thread::sleep(Duration::from_millis(50));
4585        }
4586    }
4587
4588    pub(super) fn status_frame(seq: u64) -> PushFrame {
4589        status_frame_with_session(seq, None)
4590    }
4591
4592    pub(super) fn status_frame_with_session(seq: u64, session_id: Option<&str>) -> PushFrame {
4593        PushFrame::StatusChanged(StatusChangedFrame {
4594            frame_type: "status_changed",
4595            session_id: session_id.map(str::to_string),
4596            snapshot: json!({ "seq": seq }),
4597        })
4598    }
4599
4600    pub(super) fn completion_frame(task_id: &str) -> PushFrame {
4601        completion_frame_with_session(task_id, "session-1")
4602    }
4603
4604    pub(super) fn completion_frame_with_session(task_id: &str, session_id: &str) -> PushFrame {
4605        PushFrame::BashCompleted(BashCompletedFrame {
4606            frame_type: "bash_completed",
4607            task_id: task_id.to_string(),
4608            session_id: session_id.to_string(),
4609            status: BgTaskStatus::Completed,
4610            exit_code: Some(0),
4611            command: format!("echo {task_id}"),
4612            output_preview: String::new(),
4613            output_truncated: false,
4614            original_tokens: None,
4615            compressed_tokens: None,
4616            tokens_skipped: false,
4617            status_reason: None,
4618        })
4619    }
4620
4621    pub(super) fn long_running_frame(task_id: &str, elapsed_ms: u64) -> PushFrame {
4622        long_running_frame_with_session(task_id, "session-1", elapsed_ms)
4623    }
4624
4625    pub(super) fn long_running_frame_with_session(
4626        task_id: &str,
4627        session_id: &str,
4628        elapsed_ms: u64,
4629    ) -> PushFrame {
4630        PushFrame::BashLongRunning(BashLongRunningFrame {
4631            frame_type: "bash_long_running",
4632            task_id: task_id.to_string(),
4633            session_id: session_id.to_string(),
4634            command: format!("sleep {elapsed_ms}"),
4635            elapsed_ms,
4636        })
4637    }
4638
4639    pub(super) fn pattern_match_frame(session_id: &str) -> PushFrame {
4640        PushFrame::BashPatternMatch(BashPatternMatchFrame {
4641            frame_type: "bash_pattern_match",
4642            task_id: "task-pattern".to_string(),
4643            session_id: session_id.to_string(),
4644            watch_id: "watch-1".to_string(),
4645            match_text: "needle".to_string(),
4646            match_offset: 7,
4647            context: "haystack needle".to_string(),
4648            once: true,
4649            reason: "pattern_match",
4650        })
4651    }
4652
4653    pub(super) fn configure_warnings_frame(session_id: Option<&str>) -> PushFrame {
4654        PushFrame::ConfigureWarnings(ConfigureWarningsFrame {
4655            frame_type: "configure_warnings",
4656            session_id: session_id.map(str::to_string),
4657            project_root: "/tmp/subc-test".to_string(),
4658            warnings: Vec::new(),
4659        })
4660    }
4661
4662    pub(super) fn route_identity(root: &ProjectRootId, session_id: &str) -> RouteIdentity {
4663        route_identity_with_trust(root, session_id, BindTrust::FirstParty)
4664    }
4665
4666    pub(super) fn route_identity_with_trust(
4667        root: &ProjectRootId,
4668        session_id: &str,
4669        trust: BindTrust,
4670    ) -> RouteIdentity {
4671        RouteIdentity(Arc::new(RouteIdentityData {
4672            root: root.clone(),
4673            project_root: root.as_path().to_path_buf(),
4674            harness: "opencode".to_string(),
4675            session: session_id.to_string(),
4676            trust,
4677            spawn_principal: AuthenticatedPrincipal::RouteBind {
4678                trust: trust.sandbox_trust(),
4679                route_channel: 0,
4680                route_epoch: 0,
4681                project_root: root.as_path().to_path_buf(),
4682                harness: "opencode".to_string(),
4683                session_id: session_id.to_string(),
4684                principal_id: Some(match trust {
4685                    BindTrust::FirstParty => "direct".to_string(),
4686                    BindTrust::Untrusted => "unverified".to_string(),
4687                }),
4688            },
4689            consumer_elicitation_capable: false,
4690        }))
4691    }
4692
4693    pub(super) fn progress_frame(request_id: &str, kind: ProgressKind, chunk: &str) -> PushFrame {
4694        PushFrame::Progress(ProgressFrame::new(request_id, kind, chunk))
4695    }
4696
4697    pub(super) fn status_seq(frame: &PushFrame) -> Option<u64> {
4698        match frame {
4699            PushFrame::StatusChanged(status) => status.snapshot.get("seq").and_then(|v| v.as_u64()),
4700            _ => None,
4701        }
4702    }
4703
4704    pub(super) fn completion_task(frame: &PushFrame) -> Option<&str> {
4705        match frame {
4706            PushFrame::BashCompleted(completion) => Some(completion.task_id.as_str()),
4707            _ => None,
4708        }
4709    }
4710
4711    pub(super) fn push_frame_task_id(frame: &Frame) -> Option<String> {
4712        let body: serde_json::Value = serde_json::from_slice(&frame.body).expect("push body");
4713        body.get("task_id")
4714            .and_then(serde_json::Value::as_str)
4715            .map(str::to_string)
4716    }
4717}
4718
4719#[cfg(test)]
4720mod tests {
4721    use super::test_support::{
4722        completion_frame, reap_until_forgotten, route_identity, test_ctx, test_root,
4723        wait_for_actor_root_count, wait_for_watcher_count,
4724    };
4725    use super::*;
4726    use crate::bash_background::BgTaskStatus;
4727
4728    fn attach_error(kind: io::ErrorKind) -> SubcError {
4729        SubcError::Connect {
4730            endpoint: "127.0.0.1:1".to_string(),
4731            source: io::Error::new(kind, "constructed attach failure"),
4732        }
4733    }
4734
4735    fn auth_io_error(kind: io::ErrorKind) -> SubcError {
4736        SubcError::Auth {
4737            endpoint: "127.0.0.1:1".to_string(),
4738            source: subc_transport::AuthError::Io {
4739                stage: subc_transport::AuthStage::ServerProof,
4740                source: io::Error::new(kind, "constructed auth failure"),
4741            },
4742        }
4743    }
4744
4745    #[test]
4746    fn initial_attach_error_classifier_distinguishes_transient_and_permanent_failures() {
4747        let transient_errors = vec![
4748            attach_error(io::ErrorKind::ConnectionRefused),
4749            attach_error(io::ErrorKind::TimedOut),
4750            attach_error(io::ErrorKind::ConnectionReset),
4751            auth_io_error(io::ErrorKind::ConnectionAborted),
4752            auth_io_error(io::ErrorKind::BrokenPipe),
4753            SubcError::Auth {
4754                endpoint: "127.0.0.1:1".to_string(),
4755                source: subc_transport::AuthError::UnexpectedEof {
4756                    stage: subc_transport::AuthStage::ServerProof,
4757                    expected: 4,
4758                    actual: 0,
4759                },
4760            },
4761            SubcError::Auth {
4762                endpoint: "127.0.0.1:1".to_string(),
4763                source: subc_transport::AuthError::Timeout {
4764                    stage: subc_transport::AuthStage::ServerProof,
4765                    deadline: AUTH_DEADLINE,
4766                },
4767            },
4768        ];
4769        for error in &transient_errors {
4770            assert_eq!(
4771                classify_attach_error(error),
4772                AttachErrorClass::Transient,
4773                "expected transient: {error}"
4774            );
4775        }
4776
4777        let permanent_errors = vec![
4778            attach_error(io::ErrorKind::PermissionDenied),
4779            auth_io_error(io::ErrorKind::InvalidData),
4780            SubcError::Auth {
4781                endpoint: "127.0.0.1:1".to_string(),
4782                source: subc_transport::AuthError::InvalidServerProof,
4783            },
4784            SubcError::Auth {
4785                endpoint: "127.0.0.1:1".to_string(),
4786                source: subc_transport::AuthError::DaemonIdMismatch,
4787            },
4788            SubcError::ConnectionFile {
4789                path: PathBuf::from("subc-connection.json"),
4790                source: subc_transport::ConnectionFileError::Invalid {
4791                    reason: "constructed invalid file".to_string(),
4792                },
4793            },
4794            SubcError::NoEndpoint {
4795                path: PathBuf::from("subc-connection.json"),
4796            },
4797            SubcError::InvalidEndpoint {
4798                path: PathBuf::from("subc-connection.json"),
4799                endpoint: "not-an-ip:1234".to_string(),
4800            },
4801        ];
4802        for error in &permanent_errors {
4803            assert_eq!(
4804                classify_attach_error(error),
4805                AttachErrorClass::Permanent,
4806                "expected permanent: {error}"
4807            );
4808        }
4809    }
4810
4811    #[test]
4812    fn incompatible_wire_version_is_rejected_before_tcp_connect() {
4813        let conn_dir = tempfile::tempdir().expect("connection tempdir");
4814        let conn_path = conn_dir.path().join("subc-connection.json");
4815        let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind listener");
4816        listener
4817            .set_nonblocking(true)
4818            .expect("set listener nonblocking");
4819        let port = listener.local_addr().expect("listener addr").port();
4820        connection_file::write_atomic(
4821            &conn_path,
4822            &connection_file::ConnectionInfo {
4823                schema: connection_file::SCHEMA_VERSION,
4824                wire_version: Some(PROTOCOL_VERSION.wrapping_add(1)),
4825                endpoints: vec![connection_file::Endpoint {
4826                    host: "127.0.0.1".to_string(),
4827                    port,
4828                }],
4829                key: vec![0x42; subc_transport::KEY_LEN],
4830                daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
4831                pid: std::process::id(),
4832                daemon_ver: "subc-test".to_string(),
4833            },
4834        )
4835        .expect("write connection file");
4836
4837        let runtime = tokio::runtime::Builder::new_current_thread()
4838            .enable_all()
4839            .build()
4840            .expect("test runtime");
4841        let result = runtime.block_on(connect_and_authenticate_with_policy(
4842            &conn_path,
4843            AttachRetryPolicy {
4844                budget: Duration::from_secs(1),
4845                initial_backoff: Duration::from_millis(5),
4846                max_backoff: Duration::from_millis(10),
4847                jitter_percent: 0,
4848            },
4849        ));
4850        assert!(matches!(
4851            result,
4852            Err(SubcError::ConnectionFile {
4853                source: connection_file::ConnectionFileError::WireVersionMismatch { .. },
4854                ..
4855            })
4856        ));
4857        assert!(matches!(
4858            listener.accept(),
4859            Err(error) if error.kind() == std::io::ErrorKind::WouldBlock
4860        ));
4861    }
4862
4863    #[test]
4864    fn initial_attach_unreachable_endpoint_retries_until_budget_then_fails_loud() {
4865        let conn_dir = tempfile::tempdir().expect("connection tempdir");
4866        let conn_path = conn_dir.path().join("subc-connection.json");
4867        let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve port");
4868        let port = listener.local_addr().expect("reserved addr").port();
4869        drop(listener);
4870        connection_file::write_atomic(
4871            &conn_path,
4872            &connection_file::ConnectionInfo {
4873                schema: connection_file::SCHEMA_VERSION,
4874                wire_version: Some(PROTOCOL_VERSION),
4875                endpoints: vec![connection_file::Endpoint {
4876                    host: "127.0.0.1".to_string(),
4877                    port,
4878                }],
4879                key: vec![0x42; subc_transport::KEY_LEN],
4880                daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
4881                pid: std::process::id(),
4882                daemon_ver: "subc-test".to_string(),
4883            },
4884        )
4885        .expect("write connection file");
4886
4887        let policy = AttachRetryPolicy {
4888            budget: Duration::from_millis(40),
4889            initial_backoff: Duration::from_millis(5),
4890            max_backoff: Duration::from_millis(10),
4891            jitter_percent: 0,
4892        };
4893        let runtime = tokio::runtime::Builder::new_current_thread()
4894            .enable_all()
4895            .build()
4896            .expect("test runtime");
4897        let started_at = Instant::now();
4898        let result = runtime.block_on(connect_and_authenticate_with_policy(&conn_path, policy));
4899        let elapsed = started_at.elapsed();
4900        let error = match result {
4901            Ok(_) => panic!("unreachable endpoint unexpectedly attached"),
4902            Err(error) => error,
4903        };
4904
4905        assert!(matches!(error, SubcError::Connect { .. }), "{error}");
4906        assert!(
4907            elapsed >= Duration::from_millis(35),
4908            "retry budget ended too early: {elapsed:?}"
4909        );
4910        assert!(
4911            elapsed < Duration::from_secs(1),
4912            "retry budget was not bounded: {elapsed:?}"
4913        );
4914    }
4915
4916    fn due_maintenance_jobs_without_actor_context(
4917        live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4918        budget: usize,
4919        pending_bind_roots: &HashSet<ProjectRootId>,
4920    ) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
4921        due_maintenance_jobs(
4922            live_roots,
4923            None,
4924            &HashMap::new(),
4925            &HashSet::new(),
4926            budget,
4927            pending_bind_roots,
4928        )
4929    }
4930
4931    fn actor_ctx_with_dirty_search_index(
4932        root: &Path,
4933        storage: &Path,
4934        file_name: &str,
4935        old_contents: &str,
4936        new_contents: &str,
4937    ) -> (Arc<AppContext>, PathBuf, PathBuf) {
4938        let file = root.join(file_name);
4939        std::fs::write(&file, old_contents).expect("write source");
4940        let canonical_root = std::fs::canonicalize(root).expect("canonical root");
4941        let ctx = Arc::new(AppContext::new(
4942            Box::new(crate::parser::TreeSitterProvider::new()),
4943            Config {
4944                project_root: Some(root.to_path_buf()),
4945                storage_dir: Some(storage.to_path_buf()),
4946                ..Config::default()
4947            },
4948        ));
4949        ctx.set_canonical_cache_root(canonical_root.clone());
4950
4951        let cache_dir = crate::search_index::resolve_cache_dir(&canonical_root, Some(storage));
4952        let mut index = crate::search_index::SearchIndex::build(&canonical_root);
4953        let git_head = index.stored_git_head().map(str::to_owned);
4954        index.write_to_disk(&cache_dir, git_head.as_deref());
4955
4956        std::fs::write(&file, new_contents).expect("edit source");
4957        index.update_file(&file);
4958        *ctx.search_index()
4959            .write()
4960            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
4961        (ctx, canonical_root, cache_dir)
4962    }
4963
4964    #[test]
4965    fn graceful_shutdown_flushes_every_actor_search_index() {
4966        let storage = tempfile::tempdir().expect("storage tempdir");
4967        let (root1_dir, root1) = test_root("shutdown-flush-root-1");
4968        let (root2_dir, root2) = test_root("shutdown-flush-root-2");
4969        let (ctx1, canonical_root1, cache_dir1) = actor_ctx_with_dirty_search_index(
4970            root1_dir.path(),
4971            storage.path(),
4972            "alpha.txt",
4973            "old actor one token\n",
4974            "new actor one token\n",
4975        );
4976        let (ctx2, canonical_root2, cache_dir2) = actor_ctx_with_dirty_search_index(
4977            root2_dir.path(),
4978            storage.path(),
4979            "beta.txt",
4980            "old actor two token\n",
4981            "new actor two token\n",
4982        );
4983
4984        let executor = Executor::new();
4985        assert!(executor.register_actor(root1.clone(), Arc::clone(&ctx1)));
4986        assert!(executor.register_actor(root2.clone(), Arc::clone(&ctx2)));
4987
4988        flush_actor_indexes_on_graceful_shutdown(&executor.actor_contexts());
4989
4990        let mut restored1 =
4991            crate::search_index::SearchIndex::read_from_disk(&cache_dir1, &canonical_root1)
4992                .expect("load flushed root one index");
4993        restored1.ready = true;
4994        assert_eq!(
4995            restored1
4996                .grep("new actor one token", true, &[], &[], &canonical_root1, 10)
4997                .matches
4998                .len(),
4999            1,
5000            "graceful subc shutdown should flush the first root's trigram delta"
5001        );
5002
5003        let mut restored2 =
5004            crate::search_index::SearchIndex::read_from_disk(&cache_dir2, &canonical_root2)
5005                .expect("load flushed root two index");
5006        restored2.ready = true;
5007        assert_eq!(
5008            restored2
5009                .grep("new actor two token", true, &[], &[], &canonical_root2, 10)
5010                .matches
5011                .len(),
5012            1,
5013            "graceful subc shutdown should flush every registered root"
5014        );
5015    }
5016
5017    #[test]
5018    fn idle_root_reaper_closes_artifacts_and_stops_watcher() {
5019        let _ = env_logger::builder().is_test(true).try_init();
5020        let (root_dir, root) = test_root("idle-root-reaper");
5021        let storage = tempfile::tempdir().expect("storage tempdir");
5022        std::fs::write(
5023            root_dir.path().join("main.rs"),
5024            "fn entry() { leaf(); }\nfn leaf() {}\n",
5025        )
5026        .expect("source file");
5027        let canonical_root = std::fs::canonicalize(root_dir.path()).expect("canonical root");
5028        let app = App::default_shared();
5029        let ctx = Arc::new(AppContext::from_app(
5030            Arc::clone(&app),
5031            Config {
5032                project_root: Some(canonical_root.clone()),
5033                storage_dir: Some(storage.path().to_path_buf()),
5034                callgraph_store: true,
5035                search_index: true,
5036                ..Config::default()
5037            },
5038        ));
5039        ctx.set_canonical_cache_root(canonical_root.clone());
5040        assert!(ctx
5041            .ensure_callgraph_store()
5042            .expect("build callgraph store")
5043            .is_some());
5044
5045        let cache_dir =
5046            crate::search_index::resolve_cache_dir(&canonical_root, Some(storage.path()));
5047        let mut index = crate::search_index::SearchIndex::build(&canonical_root);
5048        let git_head = index.stored_git_head().map(str::to_owned);
5049        index.write_to_disk(&cache_dir, git_head.as_deref());
5050        *ctx.search_index()
5051            .write()
5052            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
5053        // Seed a completed warm verification so the test can prove eviction
5054        // downgrades it to Strict rather than passing vacuously.
5055        let seeded_generation =
5056            crate::cache_freshness::artifact_generation(&cache_dir.join("cache.bin"))
5057                .expect("seeded artifact generation");
5058        crate::cache_freshness::record_verify_completed(
5059            &canonical_root,
5060            crate::cache_freshness::VerifyArtifact::Search,
5061            Some(seeded_generation),
5062        );
5063        assert!(
5064            matches!(
5065                crate::cache_freshness::warm_verify_plan(
5066                    canonical_root.as_path(),
5067                    crate::cache_freshness::VerifyArtifact::Search,
5068                    Some(seeded_generation),
5069                ),
5070                crate::cache_freshness::WarmVerifyPlan::Skip
5071            ),
5072            "memo must be warm before eviction for the downgrade assertion to bite"
5073        );
5074
5075        let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
5076        let _dispatch_tx = dispatch_tx;
5077        let shutdown = Arc::new(AtomicBool::new(false));
5078        let thread_shutdown = Arc::clone(&shutdown);
5079        let join = std::thread::spawn(move || {
5080            while !thread_shutdown.load(Ordering::SeqCst) {
5081                std::thread::yield_now();
5082            }
5083        });
5084        ctx.install_watcher_runtime(
5085            dispatch_rx,
5086            crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
5087        );
5088        wait_for_watcher_count(&ctx, 1);
5089
5090        let executor = Arc::new(Executor::new());
5091        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5092        ctx.mark_subc_unbound();
5093        let mut live_roots = HashMap::new();
5094        let mut meta = RootMeta::new(Instant::now());
5095        meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
5096        meta.unbound_quiesced = true;
5097        live_roots.insert(root.clone(), meta);
5098
5099        let message = idle_root_eviction_message(&root, &ctx.memory_root_snapshot(), None);
5100        assert!(message.contains("evicted idle root"));
5101        assert!(message.contains("freed ~"));
5102        assert!(message.contains("semantic"));
5103        assert!(!message.contains("semantic not estimated retained"));
5104        assert!(message.contains("trigram"));
5105        assert!(message.contains("retained: bash"));
5106        assert!(message.contains("parser_pool"));
5107
5108        assert_eq!(
5109            reap_idle_roots(
5110                Instant::now(),
5111                &mut live_roots,
5112                &HashMap::new(),
5113                &HashMap::new(),
5114                &executor,
5115                &DispatchPathMetrics::new(),
5116            )
5117            .evicted,
5118            1
5119        );
5120        assert!(ctx.search_index().read().unwrap().is_none());
5121        wait_for_watcher_count(&ctx, 0);
5122        // The watcher stopped with the eviction, so the idle interval is
5123        // unobserved: the pre-seeded warm-verify memo (Skip) must fall back to
5124        // strict content verification (stat-first would miss same-size,
5125        // preserved-mtime edits made while nobody was watching).
5126        assert!(
5127            matches!(
5128                crate::cache_freshness::warm_verify_plan(
5129                    canonical_root.as_path(),
5130                    crate::cache_freshness::VerifyArtifact::Search,
5131                    Some(seeded_generation),
5132                ),
5133                crate::cache_freshness::WarmVerifyPlan::Strict
5134            ),
5135            "idle eviction must force strict re-verification"
5136        );
5137        assert!(
5138            crate::search_index::SearchIndex::read_from_disk(&cache_dir, &canonical_root).is_some()
5139        );
5140        ctx.mark_subc_bound();
5141        assert!(ctx
5142            .ensure_callgraph_store()
5143            .expect("reopen callgraph store")
5144            .is_some());
5145        assert!(live_roots[&root].idle_artifacts_evicted);
5146    }
5147
5148    #[test]
5149    fn idle_root_reaper_applies_ttl_to_unbound_roots() {
5150        let (_root_dir, root) = test_root("idle-root-ttl-gate");
5151        let ctx = test_ctx();
5152        let executor = Arc::new(Executor::new());
5153        assert!(executor.register_actor(root.clone(), ctx));
5154        let ctx = executor.actor_context(&root).expect("actor context");
5155        ctx.mark_subc_unbound();
5156        let now = Instant::now();
5157        let mut meta = RootMeta::new(now);
5158        meta.unbound_quiesced = true;
5159        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5160
5161        // A recently-unbound root stays warm: a transient unbind (host
5162        // restart) must not pay the strict-verify + forced-rebuild teardown
5163        // on the next maintenance sweep.
5164        assert_eq!(
5165            reap_idle_roots(
5166                now,
5167                &mut live_roots,
5168                &HashMap::new(),
5169                &HashMap::new(),
5170                &executor,
5171                &DispatchPathMetrics::new(),
5172            )
5173            .evicted,
5174            0
5175        );
5176        assert!(!live_roots[&root].idle_artifacts_evicted);
5177
5178        // Past the TTL the same unbound root pays the full teardown. The
5179        // pending reconciliation paths retained across the transient-unbind
5180        // window would block eviction forever through
5181        // `artifact_eviction_blocked`; the reaper disposes them because the
5182        // strict gap invalidation subsumes their purpose.
5183        ctx.add_pending_search_index_paths([root.as_path().join("retained.rs")]);
5184        assert_eq!(
5185            reap_idle_roots(
5186                now + IDLE_ROOT_TTL,
5187                &mut live_roots,
5188                &HashMap::new(),
5189                &HashMap::new(),
5190                &executor,
5191                &DispatchPathMetrics::new(),
5192            )
5193            .evicted,
5194            1
5195        );
5196        assert!(live_roots[&root].idle_artifacts_evicted);
5197        assert!(
5198            ctx.take_pending_search_index_paths().is_empty(),
5199            "TTL eviction must dispose retained pending reconciliation paths"
5200        );
5201    }
5202
5203    #[test]
5204    fn blocked_ttl_eviction_restores_taken_pending_reconciliation_state() {
5205        let (_root_dir, root) = test_root("ttl-eviction-blocked-restore");
5206        let ctx = test_ctx();
5207        let executor = Arc::new(Executor::new());
5208        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5209        ctx.mark_subc_unbound();
5210
5211        // Retained pending path from the transient-unbind window plus a
5212        // SECONDARY eviction blocker (a non-ready resident search index, the
5213        // dirty-index blocker in artifact_eviction_blocked). Disposal must be
5214        // transactional: the blocked eviction may be followed by a rebind, and
5215        // the path is the only repair record for its consumed watcher event.
5216        let pending = root.as_path().join("edited-while-unbound.rs");
5217        ctx.add_pending_search_index_paths([pending.clone()]);
5218        let dirty_source = root.as_path().join("dirty.rs");
5219        std::fs::write(&dirty_source, "fn dirty() {}\n").expect("dirty source");
5220        let mut dirty = crate::search_index::SearchIndex::new();
5221        dirty.ready = true;
5222        dirty.update_file(&dirty_source);
5223        *ctx.search_index()
5224            .write()
5225            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(dirty);
5226        assert!(ctx.artifact_eviction_blocked());
5227
5228        let mut live_roots = HashMap::new();
5229        let mut meta = RootMeta::new(Instant::now());
5230        meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
5231        meta.unbound_quiesced = true;
5232        live_roots.insert(root.clone(), meta);
5233
5234        assert_eq!(
5235            reap_idle_roots(
5236                Instant::now(),
5237                &mut live_roots,
5238                &HashMap::new(),
5239                &HashMap::new(),
5240                &executor,
5241                &DispatchPathMetrics::new(),
5242            )
5243            .evicted,
5244            0,
5245            "the dirty index must still block this eviction"
5246        );
5247        assert_eq!(
5248            ctx.take_pending_search_index_paths(),
5249            vec![pending],
5250            "a blocked eviction must restore the taken pending paths"
5251        );
5252    }
5253
5254    #[test]
5255    fn idle_reap_with_bound_route_keeps_watcher_running() {
5256        let (_root_dir, root) = test_root("bound-root-reap-gate");
5257        let ctx = test_ctx();
5258        let executor = Arc::new(Executor::new());
5259        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5260
5261        let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
5262        let _dispatch_tx = dispatch_tx;
5263        let shutdown = Arc::new(AtomicBool::new(false));
5264        let thread_shutdown = Arc::clone(&shutdown);
5265        let join = std::thread::spawn(move || {
5266            while !thread_shutdown.load(Ordering::SeqCst) {
5267                std::thread::yield_now();
5268            }
5269        });
5270        ctx.install_watcher_runtime(
5271            dispatch_rx,
5272            crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
5273        );
5274
5275        let mut meta = RootMeta::new(Instant::now());
5276        meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
5277        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5278        let bound = HashMap::from([(root, HashSet::from([route_key(7, 1)]))]);
5279        assert_eq!(
5280            reap_idle_roots(
5281                Instant::now(),
5282                &mut live_roots,
5283                &HashMap::new(),
5284                &bound,
5285                &executor,
5286                &DispatchPathMetrics::new(),
5287            )
5288            .evicted,
5289            0
5290        );
5291        wait_for_watcher_count(&ctx, 1);
5292        ctx.stop_watcher_runtime_in_background();
5293        wait_for_watcher_count(&ctx, 0);
5294    }
5295
5296    #[test]
5297    fn deleted_root_with_bound_route_is_reclaimed_after_confirmation_and_routes_are_purged() {
5298        let (root_dir, root) = test_root("deleted-bound-root-reap");
5299        let executor = Arc::new(Executor::new());
5300        assert!(executor.register_actor(root.clone(), test_ctx()));
5301        root_dir.close().expect("delete project root");
5302
5303        let route = route_key(19, 3);
5304        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
5305        let cancel_signal = PersistentCancelSignal::new();
5306        let mut routes = HashMap::from([(route, route_identity(&root, "deleted-route"))]);
5307        let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
5308        let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
5309        let mut route_bash_cancels = HashMap::from([(
5310            route,
5311            bash::RouteBashCancel {
5312                token: cancel_signal.clone(),
5313                active_waits: 0,
5314            },
5315        )]);
5316        let metrics = DispatchPathMetrics::new();
5317
5318        let first = reap_idle_roots(
5319            Instant::now(),
5320            &mut live_roots,
5321            &HashMap::new(),
5322            &root_channels,
5323            &executor,
5324            &metrics,
5325        );
5326        assert!(first.forgotten_deleted_roots.is_empty());
5327        assert!(executor.actor_registered(&root));
5328
5329        let mut forgotten = Vec::new();
5330        for _ in 0..100 {
5331            let outcome = reap_idle_roots(
5332                Instant::now(),
5333                &mut live_roots,
5334                &HashMap::new(),
5335                &root_channels,
5336                &executor,
5337                &metrics,
5338            );
5339            if !outcome.forgotten_deleted_roots.is_empty() {
5340                forgotten = outcome.forgotten_deleted_roots;
5341                break;
5342            }
5343            std::thread::sleep(Duration::from_millis(10));
5344        }
5345        assert_eq!(forgotten, vec![root.clone()]);
5346        assert!(!executor.actor_registered(&root));
5347
5348        let mut retry_buffer = HashMap::new();
5349        let mut reclaimed_routes = ReclaimedRoutes::default();
5350        let mut session_identity = HashMap::new();
5351        let mut push_buffer = HashMap::new();
5352        let mut bg_subs = HashMap::new();
5353        let mut bg_sub_by_session = HashMap::new();
5354        let mut bg_wake_pending = HashSet::new();
5355        let mut bg_wake_epoch = HashMap::new();
5356        let mut pending_bash_asks = HashMap::new();
5357        purge_deleted_root_residents(
5358            &root,
5359            &mut routes,
5360            &mut root_channels,
5361            &mut installed_route_epochs,
5362            &mut route_bash_cancels,
5363            &mut retry_buffer,
5364            &mut reclaimed_routes,
5365            &mut session_identity,
5366            &mut push_buffer,
5367            &mut bg_subs,
5368            &mut bg_sub_by_session,
5369            &mut bg_wake_pending,
5370            &mut bg_wake_epoch,
5371            &mut pending_bash_asks,
5372        );
5373
5374        assert!(routes.is_empty());
5375        assert!(root_channels.is_empty());
5376        assert!(installed_route_epochs.is_empty());
5377        assert!(route_bash_cancels.is_empty());
5378        assert!(reclaimed_routes.contains(route));
5379        assert!(cancel_signal.is_cancelled());
5380    }
5381
5382    /// The control for the deleted-root reclamation above: a root whose
5383    /// directory still EXISTS must stay retained while it holds a bound route,
5384    /// even with every other reap precondition satisfied. Without this, the
5385    /// suite cannot tell "reclaim roots that are provably gone" apart from
5386    /// "reap any root that looks idle" — the second would tear down live
5387    /// sessions, and both satisfy the deleted-root tests.
5388    #[test]
5389    fn live_root_with_bound_route_is_never_reclaimed() {
5390        let (_root_dir, root) = test_root("live-bound-root-retained");
5391        let ctx = test_ctx();
5392        ctx.mark_subc_unbound();
5393        let executor = Arc::new(Executor::new());
5394        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5395
5396        let route = RouteChannel {
5397            channel: 7,
5398            epoch: 1,
5399        };
5400        let mut meta = RootMeta::new(Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1));
5401        meta.unbound_quiesced = true;
5402        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5403        let root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
5404
5405        // Sweep three times because reclamation requires the root path to be
5406        // observed missing on two CONSECUTIVE sweeps. A single sweep would pass
5407        // here even if reclamation were wrongly unconditional, since the first
5408        // absence never reclaims on its own.
5409        for _ in 0..3 {
5410            let outcome = reap_idle_roots(
5411                Instant::now(),
5412                &mut live_roots,
5413                &HashMap::new(),
5414                &root_channels,
5415                &executor,
5416                &DispatchPathMetrics::new(),
5417            );
5418            assert!(
5419                outcome.forgotten_deleted_roots.is_empty(),
5420                "a root whose directory exists must never be forgotten"
5421            );
5422        }
5423
5424        assert!(live_roots.contains_key(&root), "live root must be retained");
5425        assert!(
5426            executor.actor_registered(&root),
5427            "live root's actor must survive"
5428        );
5429        assert!(
5430            root.as_path().exists(),
5431            "test vehicle must keep the directory alive; otherwise this control proves nothing"
5432        );
5433    }
5434
5435    #[test]
5436    fn deleted_root_is_not_reclaimed_on_first_absence_observation() {
5437        let (root_dir, root) = test_root("deleted-root-first-observation");
5438        let ctx = test_ctx();
5439        ctx.mark_subc_unbound();
5440        let executor = Arc::new(Executor::new());
5441        assert!(executor.register_actor(root.clone(), ctx));
5442        root_dir.close().expect("delete project root");
5443
5444        let mut meta = RootMeta::new(Instant::now());
5445        meta.unbound_quiesced = true;
5446        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5447        let outcome = reap_idle_roots(
5448            Instant::now(),
5449            &mut live_roots,
5450            &HashMap::new(),
5451            &HashMap::new(),
5452            &executor,
5453            &DispatchPathMetrics::new(),
5454        );
5455
5456        assert!(outcome.forgotten_deleted_roots.is_empty());
5457        assert!(live_roots.contains_key(&root));
5458        assert!(executor.actor_registered(&root));
5459    }
5460
5461    fn spawn_background_for_root(
5462        ctx: &AppContext,
5463        root: &ProjectRootId,
5464        storage: &tempfile::TempDir,
5465        session_id: &str,
5466    ) -> (String, u32) {
5467        // Windows refuses to delete a directory that is a running process's
5468        // cwd (ERROR_SHARING_VIOLATION), so the task must not live inside the
5469        // project root these tests delete. The kill path matches on the task's
5470        // registered project_root, not its cwd, so pointing the cwd at task
5471        // storage keeps the association under test intact.
5472        let command = if cfg!(windows) {
5473            // timeout.exe requires a console; ping is the standard sleep shim.
5474            "ping -n 31 127.0.0.1 > nul"
5475        } else {
5476            "sleep 30"
5477        };
5478        let task_id = ctx
5479            .bash_background()
5480            .spawn(
5481                crate::sandbox_spawn::SpawnPlan::Unsandboxed,
5482                command,
5483                session_id.to_string(),
5484                storage.path().to_path_buf(),
5485                HashMap::new(),
5486                Some(Duration::from_secs(60)),
5487                storage.path().to_path_buf(),
5488                8,
5489                true,
5490                false,
5491                Some(root.as_path().to_path_buf()),
5492            )
5493            .expect("spawn background task");
5494        let snapshot = ctx
5495            .bash_background()
5496            .status(
5497                &task_id,
5498                session_id,
5499                Some(root.as_path()),
5500                Some(storage.path()),
5501                0,
5502            )
5503            .expect("background task status");
5504        (task_id, snapshot.child_pid.expect("background child pid"))
5505    }
5506
5507    fn wait_for_background_exit(pid: u32) {
5508        let deadline = Instant::now() + Duration::from_secs(5);
5509        while crate::bash_background::process::is_process_alive(pid) {
5510            assert!(
5511                Instant::now() < deadline,
5512                "background task process survived kill"
5513            );
5514            std::thread::sleep(Duration::from_millis(20));
5515        }
5516    }
5517
5518    #[test]
5519    fn deleted_root_reclaims_background_task_after_two_absence_sweeps() {
5520        let (root_dir, root) = test_root("deleted-root-background-task");
5521        let storage = tempfile::tempdir().expect("task storage");
5522        let ctx = test_ctx();
5523        let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "reclaim-session");
5524        assert!(crate::bash_background::process::is_process_alive(pid));
5525
5526        let executor = Arc::new(Executor::new());
5527        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5528        assert!(executor.actor_is_idle(&root));
5529        root_dir.close().expect("delete project root");
5530        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
5531        let pending_binds = HashMap::new();
5532        let root_channels = HashMap::new();
5533        let metrics = DispatchPathMetrics::new();
5534
5535        let first = reap_idle_roots(
5536            Instant::now(),
5537            &mut live_roots,
5538            &pending_binds,
5539            &root_channels,
5540            &executor,
5541            &metrics,
5542        );
5543        assert!(first.forgotten_deleted_roots.is_empty());
5544        assert!(crate::bash_background::process::is_process_alive(pid));
5545
5546        let outcome = reap_until_forgotten(
5547            &root,
5548            &mut live_roots,
5549            &pending_binds,
5550            &root_channels,
5551            &executor,
5552            &metrics,
5553        );
5554        assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
5555        wait_for_background_exit(pid);
5556
5557        let snapshot = ctx
5558            .bash_background()
5559            .status(
5560                &task_id,
5561                "reclaim-session",
5562                Some(root.as_path()),
5563                Some(storage.path()),
5564                0,
5565            )
5566            .expect("reclaimed task status");
5567        assert_eq!(snapshot.info.status, BgTaskStatus::Killed);
5568        assert_eq!(
5569            snapshot.info.status_reason.as_deref(),
5570            Some(crate::bash_background::registry::ROOT_RECLAIMED_REASON)
5571        );
5572        assert_eq!(
5573            serde_json::to_value(&snapshot).expect("serialize bash status")["status_reason"],
5574            crate::bash_background::registry::ROOT_RECLAIMED_REASON
5575        );
5576        let completion = ctx
5577            .bash_background()
5578            .drain_completions_for_session(Some("reclaim-session"))
5579            .pop()
5580            .expect("reclaimed task completion");
5581        assert_eq!(
5582            completion.status_reason.as_deref(),
5583            Some(crate::bash_background::registry::ROOT_RECLAIMED_REASON)
5584        );
5585    }
5586
5587    #[test]
5588    fn existing_unbound_root_keeps_background_task_alive_across_sweeps() {
5589        let (root_dir, root) = test_root("existing-root-background-task");
5590        let storage = tempfile::tempdir().expect("task storage");
5591        let ctx = test_ctx();
5592        ctx.mark_subc_unbound();
5593        let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "existing-session");
5594
5595        let executor = Arc::new(Executor::new());
5596        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5597        let mut meta = RootMeta::new(
5598            Instant::now()
5599                .checked_sub(IDLE_ROOT_TTL + Duration::from_secs(1))
5600                .expect("old root timestamp"),
5601        );
5602        meta.unbound_quiesced = true;
5603        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5604        let pending_binds = HashMap::new();
5605        let root_channels = HashMap::new();
5606        let metrics = DispatchPathMetrics::new();
5607
5608        for _ in 0..8 {
5609            reap_idle_roots(
5610                Instant::now(),
5611                &mut live_roots,
5612                &pending_binds,
5613                &root_channels,
5614                &executor,
5615                &metrics,
5616            );
5617            std::thread::sleep(Duration::from_millis(10));
5618        }
5619        assert!(root_dir.path().exists());
5620        assert!(crate::bash_background::process::is_process_alive(pid));
5621        let snapshot = ctx
5622            .bash_background()
5623            .status(
5624                &task_id,
5625                "existing-session",
5626                Some(root.as_path()),
5627                Some(storage.path()),
5628                0,
5629            )
5630            .expect("existing task status");
5631        assert_eq!(snapshot.info.status, BgTaskStatus::Running);
5632        let _ = ctx.bash_background().kill(&task_id, "existing-session");
5633        wait_for_background_exit(pid);
5634    }
5635
5636    #[test]
5637    fn restored_root_between_absence_sweeps_keeps_background_task_alive() {
5638        let (root_dir, root) = test_root("restored-root-background-task");
5639        let storage = tempfile::tempdir().expect("task storage");
5640        let ctx = test_ctx();
5641        let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "restored-session");
5642
5643        let executor = Arc::new(Executor::new());
5644        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5645        root_dir.close().expect("delete project root");
5646        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
5647        let pending_binds = HashMap::new();
5648        let root_channels = HashMap::new();
5649        let metrics = DispatchPathMetrics::new();
5650
5651        let first = reap_idle_roots(
5652            Instant::now(),
5653            &mut live_roots,
5654            &pending_binds,
5655            &root_channels,
5656            &executor,
5657            &metrics,
5658        );
5659        assert!(first.forgotten_deleted_roots.is_empty());
5660        std::fs::create_dir_all(root.as_path()).expect("restore project root");
5661        let second = reap_idle_roots(
5662            Instant::now(),
5663            &mut live_roots,
5664            &pending_binds,
5665            &root_channels,
5666            &executor,
5667            &metrics,
5668        );
5669        assert!(second.forgotten_deleted_roots.is_empty());
5670        assert!(crate::bash_background::process::is_process_alive(pid));
5671        let _ = ctx.bash_background().kill(&task_id, "restored-session");
5672        wait_for_background_exit(pid);
5673    }
5674
5675    #[test]
5676    fn observing_root_again_resets_deleted_sweep_confirmation() {
5677        let (root_dir, root) = test_root("deleted-root-observation-reset");
5678        let ctx = test_ctx();
5679        ctx.mark_subc_unbound();
5680        let executor = Arc::new(Executor::new());
5681        assert!(executor.register_actor(root.clone(), ctx));
5682        root_dir.close().expect("delete project root");
5683
5684        let mut meta = RootMeta::new(Instant::now());
5685        meta.unbound_quiesced = true;
5686        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5687        let pending_binds = HashMap::new();
5688        let root_channels = HashMap::new();
5689        let metrics = DispatchPathMetrics::new();
5690
5691        let first = reap_idle_roots(
5692            Instant::now(),
5693            &mut live_roots,
5694            &pending_binds,
5695            &root_channels,
5696            &executor,
5697            &metrics,
5698        );
5699        assert!(first.forgotten_deleted_roots.is_empty());
5700
5701        std::fs::create_dir_all(root.as_path()).expect("restore project root");
5702        reap_idle_roots(
5703            Instant::now(),
5704            &mut live_roots,
5705            &pending_binds,
5706            &root_channels,
5707            &executor,
5708            &metrics,
5709        );
5710        std::fs::remove_dir_all(root.as_path()).expect("delete project root again");
5711
5712        let after_reset = reap_idle_roots(
5713            Instant::now(),
5714            &mut live_roots,
5715            &pending_binds,
5716            &root_channels,
5717            &executor,
5718            &metrics,
5719        );
5720        assert!(after_reset.forgotten_deleted_roots.is_empty());
5721        assert!(live_roots.contains_key(&root));
5722        assert!(executor.actor_registered(&root));
5723    }
5724
5725    #[test]
5726    fn deleted_idle_root_is_fully_forgotten_and_status_counts_drop() {
5727        let (root_dir, root) = test_root("deleted-root-reap");
5728        let app = App::default_shared();
5729        let ctx = Arc::new(AppContext::from_app(
5730            Arc::clone(&app),
5731            Config {
5732                project_root: Some(root.as_path().to_path_buf()),
5733                ..Config::default()
5734            },
5735        ));
5736        ctx.set_canonical_cache_root(root.as_path().to_path_buf());
5737        ctx.mark_subc_unbound();
5738        let executor = Arc::new(Executor::new());
5739        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5740        assert_eq!(app.actor_root_count(), 1);
5741        drop(ctx);
5742        root_dir.close().expect("delete project root");
5743
5744        let mut meta = RootMeta::new(Instant::now());
5745        meta.unbound_quiesced = true;
5746        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5747        let outcome = reap_until_forgotten(
5748            &root,
5749            &mut live_roots,
5750            &HashMap::new(),
5751            &HashMap::new(),
5752            &executor,
5753            &DispatchPathMetrics::new(),
5754        );
5755        assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
5756        assert!(!executor.actor_registered(&root));
5757        assert!(!live_roots.contains_key(&root));
5758        wait_for_actor_root_count(&app, 0);
5759
5760        let status_ctx = AppContext::from_app(app, Config::default());
5761        let status = status_ctx.build_status_snapshot();
5762        assert_eq!(status["runtime"]["live_actor_roots"], 0);
5763        assert_eq!(status["runtime"]["open_routes"], 0);
5764    }
5765
5766    #[test]
5767    fn deleted_root_reap_blocker_census_is_exposed_in_health_metrics() {
5768        let (root_dir, root) = test_root("deleted-root-reap-census");
5769        let executor = Arc::new(Executor::new());
5770        assert!(executor.register_actor(root.clone(), test_ctx()));
5771        root_dir.close().expect("delete project root");
5772
5773        let mut live_roots = HashMap::from([(root, RootMeta::new(Instant::now()))]);
5774        let metrics = DispatchPathMetrics::new();
5775        let outcome = reap_idle_roots(
5776            Instant::now(),
5777            &mut live_roots,
5778            &HashMap::new(),
5779            &HashMap::new(),
5780            &executor,
5781            &metrics,
5782        );
5783        assert_eq!(outcome.evicted, 0);
5784
5785        let report = build_health_report(
5786            &executor,
5787            &HashMap::new(),
5788            &metrics,
5789            &crate::context::App::default_shared(),
5790        );
5791        let reap = report
5792            .metrics
5793            .as_ref()
5794            .and_then(|metrics| metrics.get("reap"))
5795            .expect("reap health metrics");
5796        assert_eq!(reap["deleted_retained"].as_u64(), Some(1));
5797        assert_eq!(reap["blockers"]["absence_unconfirmed"].as_u64(), Some(1));
5798        assert_eq!(reap["blockers"]["unbound_quiesced"].as_u64(), Some(0));
5799        assert_eq!(reap["blockers"]["actor_busy"].as_u64(), Some(0));
5800    }
5801
5802    #[test]
5803    fn connection_exit_quiesces_queued_maintenance_and_deleted_root_is_purged() {
5804        let (root_dir, root) = test_root("connection-exit-deleted-root");
5805        let executor = Arc::new(Executor::new());
5806        assert!(executor.register_actor(root.clone(), test_ctx()));
5807
5808        let route = route_key(11, 1);
5809        let mut meta = RootMeta::new(Instant::now());
5810        meta.maintenance_pending = true;
5811        meta.maintenance_queued_kinds
5812            .push_back(MaintenanceDrainKind::CompletionDrains);
5813        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5814        let mut pending_binds = HashMap::new();
5815        let mut routes = HashMap::from([(route, route_identity(&root, "abandoned"))]);
5816        let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
5817        let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
5818        let mut route_bash_cancels = HashMap::new();
5819
5820        quiesce_connection_roots(
5821            &mut live_roots,
5822            &mut pending_binds,
5823            &mut routes,
5824            &mut root_channels,
5825            &mut installed_route_epochs,
5826            &mut route_bash_cancels,
5827            &executor,
5828        );
5829        assert!(live_roots[&root].unbound_quiesced);
5830        assert!(!live_roots[&root].maintenance_pending);
5831        assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
5832        assert!(routes.is_empty());
5833        assert!(root_channels.is_empty());
5834
5835        root_dir.close().expect("delete project root");
5836        let metrics = DispatchPathMetrics::new();
5837        let outcome = reap_until_forgotten(
5838            &root,
5839            &mut live_roots,
5840            &pending_binds,
5841            &root_channels,
5842            &executor,
5843            &metrics,
5844        );
5845        let mut session_identity = HashMap::new();
5846        let mut push_buffer = HashMap::new();
5847        let mut bg_subs = HashMap::new();
5848        let mut bg_sub_by_session = HashMap::new();
5849        let mut bg_wake_pending = HashSet::new();
5850        let mut bg_wake_epoch = HashMap::new();
5851        let mut pending_bash_asks = HashMap::new();
5852        let mut retry_buffer = HashMap::new();
5853        let mut reclaimed_routes = ReclaimedRoutes::default();
5854        for forgotten in &outcome.forgotten_deleted_roots {
5855            purge_deleted_root_residents(
5856                forgotten,
5857                &mut routes,
5858                &mut root_channels,
5859                &mut installed_route_epochs,
5860                &mut route_bash_cancels,
5861                &mut retry_buffer,
5862                &mut reclaimed_routes,
5863                &mut session_identity,
5864                &mut push_buffer,
5865                &mut bg_subs,
5866                &mut bg_sub_by_session,
5867                &mut bg_wake_pending,
5868                &mut bg_wake_epoch,
5869                &mut pending_bash_asks,
5870            );
5871        }
5872
5873        assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
5874        assert!(!executor.actor_registered(&root));
5875        assert!(!live_roots.contains_key(&root));
5876    }
5877
5878    #[test]
5879    fn unbound_root_quiesces_maintenance_without_removing_actor() {
5880        let (_root_dir, root) = test_root("unbound-root-quiesce");
5881        let ctx = test_ctx();
5882        let executor = Arc::new(Executor::new());
5883        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5884        let mut meta = RootMeta::new(Instant::now());
5885        meta.maintenance_pending = true;
5886        meta.maintenance_jobs_in_flight = 1;
5887        meta.maintenance_queued_kinds
5888            .push_back(MaintenanceDrainKind::ConfigureTail);
5889        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5890        // Warm state planted before the unbind: quiesce must keep it.
5891        *ctx.search_index()
5892            .write()
5893            .unwrap_or_else(std::sync::PoisonError::into_inner) =
5894            Some(crate::search_index::SearchIndex::new());
5895        ctx.set_cache_writer_capabilities(true, true);
5896        let pending = root.as_path().join("pending.rs");
5897        ctx.add_pending_search_index_paths([pending.clone()]);
5898        // A warm verify memo must survive the transient unbind: the watcher
5899        // keeps running, so no unobserved window exists and the next warm
5900        // reload must not pay a strict full-corpus re-hash.
5901        let canonical_root = root.as_path().to_path_buf();
5902        let artifact = canonical_root.join("cache.bin");
5903        std::fs::write(&artifact, b"warm-artifact").expect("write artifact");
5904        let seeded_generation = crate::cache_freshness::artifact_generation(&artifact);
5905        crate::cache_freshness::record_verify_completed(
5906            &canonical_root,
5907            crate::cache_freshness::VerifyArtifact::Search,
5908            seeded_generation,
5909        );
5910        assert!(matches!(
5911            crate::cache_freshness::warm_verify_plan(
5912                &canonical_root,
5913                crate::cache_freshness::VerifyArtifact::Search,
5914                seeded_generation,
5915            ),
5916            crate::cache_freshness::WarmVerifyPlan::Skip
5917        ));
5918        // A live watcher runtime must survive quiesce (its events accumulate
5919        // for the rebind replay).
5920        let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
5921        let _dispatch_tx = dispatch_tx;
5922        let shutdown = Arc::new(AtomicBool::new(false));
5923        let thread_shutdown = Arc::clone(&shutdown);
5924        let join = std::thread::spawn(move || {
5925            while !thread_shutdown.load(Ordering::SeqCst) {
5926                std::thread::yield_now();
5927            }
5928        });
5929        ctx.install_watcher_runtime(
5930            dispatch_rx,
5931            crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
5932        );
5933        assert!(ctx.watcher_runtime_active());
5934
5935        quiesce_unbound_root(&root, &mut live_roots, &executor);
5936        let meta = &live_roots[&root];
5937        assert!(meta.unbound_quiesced);
5938        assert!(ctx.subc_unbound_quiesced());
5939        assert!(meta.maintenance_pending);
5940        assert!(meta.maintenance_queued_kinds.is_empty());
5941        assert!(executor.actor_registered(&root));
5942        // Transient unbind keeps the root warm: resident artifacts stay
5943        // resident, no forced callgraph rebuild is planted, and pending
5944        // reconciliation paths survive for the rebind replay.
5945        assert!(
5946            ctx.search_index()
5947                .read()
5948                .unwrap_or_else(std::sync::PoisonError::into_inner)
5949                .is_some(),
5950            "quiesce must not evict resident artifacts"
5951        );
5952        assert_eq!(
5953            ctx.pending_callgraph_store_force_token(),
5954            None,
5955            "quiesce must not force a callgraph rebuild"
5956        );
5957        assert_eq!(
5958            ctx.take_pending_search_index_paths(),
5959            vec![pending],
5960            "quiesce must retain pending watcher-derived paths"
5961        );
5962        assert!(
5963            matches!(
5964                crate::cache_freshness::warm_verify_plan(
5965                    &canonical_root,
5966                    crate::cache_freshness::VerifyArtifact::Search,
5967                    seeded_generation,
5968                ),
5969                crate::cache_freshness::WarmVerifyPlan::Skip
5970            ),
5971            "quiesce must not invalidate the warm verify memo"
5972        );
5973        assert!(
5974            ctx.watcher_runtime_active(),
5975            "quiesce must not stop a running watcher"
5976        );
5977        ctx.stop_watcher_runtime();
5978
5979        let meta = live_roots.get_mut(&root).expect("root metadata");
5980        note_maintenance_completion(
5981            meta,
5982            Some(MaintenanceDrainKind::ConfigureTail),
5983            false,
5984            meta.unbound_quiesced,
5985        );
5986        assert!(!meta.maintenance_pending);
5987        assert!(meta.maintenance_queued_kinds.is_empty());
5988    }
5989
5990    #[test]
5991    fn same_root_higher_epoch_replacement_does_not_quiesce_between_generations() {
5992        let (_dir, root) = test_root("same-root-replacement");
5993        let route = route_key(7, 1);
5994        let installed_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
5995        let root_channels = HashMap::new();
5996
5997        assert!(!route_removal_will_quiesce_root(
5998            &root,
5999            route,
6000            &installed_channels,
6001            false,
6002            Some(&root),
6003        ));
6004        assert!(route_removal_will_quiesce_root(
6005            &root,
6006            route,
6007            &installed_channels,
6008            false,
6009            None,
6010        ));
6011        assert!(!should_quiesce_removed_root(
6012            &root,
6013            &root_channels,
6014            false,
6015            Some(&root),
6016        ));
6017        assert!(should_quiesce_removed_root(
6018            &root,
6019            &root_channels,
6020            false,
6021            None,
6022        ));
6023        assert!(!should_quiesce_removed_root(
6024            &root,
6025            &root_channels,
6026            true,
6027            None,
6028        ));
6029    }
6030
6031    #[test]
6032    fn root_quiesces_only_after_its_last_route_is_removed_and_reactivates_on_bind() {
6033        let (_root_dir, root) = test_root("unbound-root-route-count");
6034        let executor = Arc::new(Executor::new());
6035        assert!(executor.register_actor(root.clone(), test_ctx()));
6036        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
6037        let mut root_channels = HashMap::from([(
6038            root.clone(),
6039            HashSet::from([route_key(7, 1), route_key(8, 1)]),
6040        )]);
6041
6042        remove_root_channel(&mut root_channels, &root, route_key(7, 1));
6043        if !root_channels.contains_key(&root) {
6044            quiesce_unbound_root(&root, &mut live_roots, &executor);
6045        }
6046        assert!(!live_roots[&root].unbound_quiesced);
6047
6048        remove_root_channel(&mut root_channels, &root, route_key(8, 1));
6049        if !root_channels.contains_key(&root) {
6050            quiesce_unbound_root(&root, &mut live_roots, &executor);
6051        }
6052        assert!(live_roots[&root].unbound_quiesced);
6053
6054        live_roots
6055            .get_mut(&root)
6056            .expect("root metadata")
6057            .note_activity();
6058        assert!(
6059            live_roots[&root].unbound_quiesced,
6060            "late asynchronous activity must not reactivate an unbound root"
6061        );
6062
6063        live_roots
6064            .get_mut(&root)
6065            .expect("root metadata")
6066            .reactivate_bound();
6067        assert!(!live_roots[&root].unbound_quiesced);
6068    }
6069
6070    #[test]
6071    fn allocator_pressure_relief_requires_every_root_to_be_idle() {
6072        let (_idle_dir, idle_root) = test_root("allocator-relief-idle");
6073        let (_active_dir, active_root) = test_root("allocator-relief-active");
6074        let now = Instant::now();
6075        let mut live_roots = HashMap::new();
6076        let mut idle = RootMeta::new(now);
6077        idle.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
6078        live_roots.insert(idle_root, idle);
6079        assert!(process_has_been_idle(now, &live_roots));
6080
6081        live_roots.insert(active_root.clone(), RootMeta::new(now));
6082        assert!(!process_has_been_idle(now, &live_roots));
6083
6084        let active = live_roots
6085            .get_mut(&active_root)
6086            .expect("active root metadata");
6087        active.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
6088        active.active_bash_waits = 1;
6089        assert!(!process_has_been_idle(now, &live_roots));
6090    }
6091
6092    #[test]
6093    fn pressure_relief_log_reports_before_and_after_measurements() {
6094        let allocator = crate::memory::AllocatorMemorySnapshot {
6095            status: "measured",
6096            bytes_in_use: Some(8 * 1024 * 1024),
6097            size_allocated: Some(12 * 1024 * 1024),
6098            retained_slack_bytes: Some(4 * 1024 * 1024),
6099            not_estimated: None,
6100        };
6101        let relief = crate::memory::AllocatorPressureRelief {
6102            bytes_released: 3 * 1024 * 1024,
6103            rss_before_bytes: Some(20 * 1024 * 1024),
6104            rss_after_bytes: Some(17 * 1024 * 1024),
6105            allocator_before: allocator.clone(),
6106            allocator_after: crate::memory::AllocatorMemorySnapshot {
6107                size_allocated: Some(9 * 1024 * 1024),
6108                retained_slack_bytes: Some(1024 * 1024),
6109                ..allocator
6110            },
6111        };
6112        let message = pressure_relief_label(&relief);
6113        assert!(message.contains("RSS 20.0 MB -> 17.0 MB"));
6114        assert!(message.contains("allocated 12.0 MB -> 9.0 MB"));
6115        assert!(message.contains("slack 4.0 MB -> 1.0 MB"));
6116        assert!(message.contains("reported 3.0 MB released"));
6117    }
6118
6119    #[test]
6120    fn due_maintenance_jobs_skip_poisoned_roots() {
6121        let (_healthy_dir, healthy_root) = test_root("maintenance-healthy");
6122        let (_poisoned_dir, poisoned_root) = test_root("maintenance-poisoned");
6123        let mut live_roots = HashMap::new();
6124        live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
6125        let mut poisoned_meta = RootMeta::new(Instant::now());
6126        poisoned_meta.maintenance_poisoned = true;
6127        live_roots.insert(poisoned_root.clone(), poisoned_meta);
6128
6129        let (due, deferred) = due_maintenance_jobs_without_actor_context(
6130            &mut live_roots,
6131            MAINTENANCE_SUBMIT_BUDGET,
6132            &HashSet::new(),
6133        );
6134
6135        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6136        assert!(due.iter().all(|(root, _)| root == &healthy_root));
6137        assert!(!deferred);
6138        assert!(live_roots[&healthy_root].maintenance_pending);
6139        assert_eq!(
6140            live_roots[&healthy_root].maintenance_jobs_in_flight,
6141            INITIAL_MAINTENANCE_JOB_COUNT
6142        );
6143        assert!(!live_roots[&poisoned_root].maintenance_pending);
6144    }
6145
6146    #[test]
6147    fn due_maintenance_jobs_do_not_restart_quiesced_root_work() {
6148        let (_dir, root) = test_root("maintenance-unbound");
6149        let mut meta = RootMeta::new(Instant::now());
6150        meta.unbound_quiesced = true;
6151        let mut live_roots = HashMap::from([(root.clone(), meta)]);
6152
6153        let (due, deferred) = due_maintenance_jobs_without_actor_context(
6154            &mut live_roots,
6155            MAINTENANCE_SUBMIT_BUDGET,
6156            &HashSet::new(),
6157        );
6158
6159        assert!(due.is_empty());
6160        assert!(!deferred);
6161        assert!(!live_roots[&root].maintenance_pending);
6162    }
6163
6164    #[test]
6165    fn idle_bg_subscription_queues_no_jobs_until_a_wake_arrives() {
6166        let (_dir, root) = test_root("maintenance-idle-bg-subscription");
6167        let ctx = test_ctx();
6168        assert!(!ctx.completion_drains_have_work());
6169
6170        let executor = Executor::new();
6171        assert!(executor.register_actor(root.clone(), ctx));
6172        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
6173        let session = "idle-session".to_string();
6174        let channel = route_key(17, 1);
6175        let bg_sub_by_session = HashMap::from([((root.clone(), session.clone()), channel)]);
6176        let mut bg_wake_pending = HashSet::new();
6177
6178        let (idle_tick_jobs, deferred) = due_maintenance_jobs(
6179            &mut live_roots,
6180            Some(&executor),
6181            &bg_sub_by_session,
6182            &bg_wake_pending,
6183            MAINTENANCE_SUBMIT_BUDGET,
6184            &HashSet::new(),
6185        );
6186        assert!(idle_tick_jobs.is_empty());
6187        assert!(!deferred);
6188        assert!(!live_roots[&root].maintenance_pending);
6189
6190        // A completion can arm its wake after the idle tick's probes. The wake
6191        // remains loop-owned state, so the following tick must observe it.
6192        let mut bg_wake_epoch = HashMap::new();
6193        push::arm_bg_wake(
6194            root.clone(),
6195            session,
6196            channel,
6197            &mut bg_wake_pending,
6198            &mut bg_wake_epoch,
6199        );
6200        let (next_tick_jobs, deferred) = due_maintenance_jobs(
6201            &mut live_roots,
6202            Some(&executor),
6203            &bg_sub_by_session,
6204            &bg_wake_pending,
6205            MAINTENANCE_SUBMIT_BUDGET,
6206            &HashSet::new(),
6207        );
6208        assert_eq!(
6209            next_tick_jobs,
6210            vec![(root, MaintenanceDrainKind::CompletionDrains)]
6211        );
6212        assert!(!deferred);
6213    }
6214
6215    #[tokio::test]
6216    async fn subc_configure_tail_precedes_completed_search_install() {
6217        let root_dir = tempfile::tempdir().unwrap();
6218        let storage = tempfile::tempdir().unwrap();
6219        let root = ProjectRootId::from_path(root_dir.path()).unwrap();
6220        let (ctx, ignored_path) =
6221            runtime_drain::configure_search_order_context_for_test(root_dir.path(), storage.path());
6222        let ctx = Arc::new(ctx);
6223        assert!(!runtime_drain::watcher_path_is_ignored_by_current_matcher(
6224            &ctx,
6225            &ignored_path
6226        ));
6227
6228        let executor = Arc::new(Executor::new());
6229        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6230        let metrics = Arc::new(DispatchPathMetrics::new());
6231        let (completion_tx, mut completion_rx) = mpsc::channel(4);
6232        submit_maintenance_job(
6233            &executor,
6234            root.clone(),
6235            MaintenanceDrainKind::ConfigureTail,
6236            Vec::new(),
6237            &completion_tx,
6238            &metrics,
6239        );
6240        submit_maintenance_job(
6241            &executor,
6242            root,
6243            MaintenanceDrainKind::CompletionDrains,
6244            Vec::new(),
6245            &completion_tx,
6246            &metrics,
6247        );
6248
6249        let first = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
6250            .await
6251            .expect("configure-tail completion timed out")
6252            .expect("configure-tail completion channel closed");
6253        let second = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
6254            .await
6255            .expect("completion-drains completion timed out")
6256            .expect("completion-drains completion channel closed");
6257        assert!(first.response.id.contains("configure-tail"));
6258        assert!(second.response.id.contains("completion-drains"));
6259        assert!(runtime_drain::watcher_path_is_ignored_by_current_matcher(
6260            &ctx,
6261            &ignored_path
6262        ));
6263        assert_eq!(
6264            ctx.search_index()
6265                .read()
6266                .unwrap_or_else(std::sync::PoisonError::into_inner)
6267                .as_ref()
6268                .expect("completed search index installed")
6269                .file_count(),
6270            0,
6271            "configure must install the ignore matcher before pending paths replay"
6272        );
6273        ctx.stop_watcher_runtime();
6274    }
6275
6276    #[test]
6277    fn post_bind_configure_and_completion_jobs_are_queued_in_order() {
6278        let (_dir, root) = test_root("maintenance-post-bind");
6279        let mut live_roots = HashMap::new();
6280        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6281
6282        queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
6283        queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
6284
6285        let meta = live_roots.get(&root).expect("root metadata");
6286        assert!(meta.maintenance_pending);
6287        assert_eq!(meta.maintenance_jobs_in_flight, 0);
6288        assert_eq!(
6289            meta.maintenance_queued_kinds
6290                .iter()
6291                .copied()
6292                .collect::<Vec<_>>(),
6293            vec![
6294                MaintenanceDrainKind::ConfigureTail,
6295                MaintenanceDrainKind::CompletionDrains,
6296            ]
6297        );
6298
6299        let (due, deferred) = due_maintenance_jobs_without_actor_context(
6300            &mut live_roots,
6301            MAINTENANCE_SUBMIT_BUDGET,
6302            &HashSet::new(),
6303        );
6304
6305        assert_eq!(
6306            due,
6307            vec![
6308                (root.clone(), MaintenanceDrainKind::ConfigureTail),
6309                (root.clone(), MaintenanceDrainKind::CompletionDrains),
6310            ]
6311        );
6312        assert!(!deferred);
6313        assert_eq!(live_roots[&root].maintenance_jobs_in_flight, 2);
6314        assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
6315    }
6316
6317    #[test]
6318    fn due_maintenance_jobs_defers_unsubmitted_roots_without_marking_pending() {
6319        let mut live_roots = HashMap::new();
6320        let mut root_ids = Vec::new();
6321        let mut _dirs = Vec::new();
6322        for index in 0..4 {
6323            let (dir, root_id) = test_root(&format!("maintenance-budget-{index}"));
6324            live_roots.insert(root_id.clone(), RootMeta::new(Instant::now()));
6325            root_ids.push(root_id);
6326            _dirs.push(dir);
6327        }
6328
6329        let small_budget = INITIAL_MAINTENANCE_JOB_COUNT + 1;
6330        let (first_due, first_deferred) = due_maintenance_jobs_without_actor_context(
6331            &mut live_roots,
6332            small_budget,
6333            &HashSet::new(),
6334        );
6335
6336        assert_eq!(first_due.len(), small_budget);
6337        assert!(first_deferred);
6338        let first_due_set: HashSet<_> = first_due.into_iter().map(|(root, _)| root).collect();
6339        assert!(first_due_set
6340            .iter()
6341            .all(|root| live_roots[root].maintenance_pending));
6342        assert!(first_due_set
6343            .iter()
6344            .any(|root| !live_roots[root].maintenance_queued_kinds.is_empty()));
6345
6346        let all_roots: HashSet<_> = root_ids.into_iter().collect();
6347        let deferred_roots: HashSet<_> = all_roots.difference(&first_due_set).cloned().collect();
6348        assert!(deferred_roots
6349            .iter()
6350            .all(|root| !live_roots[root].maintenance_pending));
6351    }
6352
6353    #[test]
6354    fn due_maintenance_jobs_defers_pending_bind_roots() {
6355        let (_bind_dir, bind_root) = test_root("maintenance-pending-bind");
6356        let (_healthy_dir, healthy_root) = test_root("maintenance-no-bind");
6357        let mut live_roots = HashMap::new();
6358        live_roots.insert(bind_root.clone(), RootMeta::new(Instant::now()));
6359        live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
6360        let pending_bind_roots = HashSet::from([bind_root.clone()]);
6361
6362        let (due, deferred) = due_maintenance_jobs_without_actor_context(
6363            &mut live_roots,
6364            usize::MAX,
6365            &pending_bind_roots,
6366        );
6367
6368        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6369        assert!(due.iter().all(|(root, _)| root == &healthy_root));
6370        assert!(!deferred);
6371        assert!(!live_roots[&bind_root].maintenance_pending);
6372        assert!(live_roots[&bind_root].maintenance_queued_kinds.is_empty());
6373    }
6374
6375    #[test]
6376    fn maintenance_pending_survives_requeue_and_clears_after_final_batch() {
6377        let (_dir, root) = test_root("maintenance-requeue");
6378        let mut live_roots = HashMap::new();
6379        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6380        let (due, deferred) = due_maintenance_jobs_without_actor_context(
6381            &mut live_roots,
6382            usize::MAX,
6383            &HashSet::new(),
6384        );
6385        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6386        assert!(due.iter().all(|(due_root, _)| due_root == &root));
6387        assert!(!deferred);
6388
6389        let meta = live_roots.get_mut(&root).unwrap();
6390        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, false);
6391        assert!(meta.maintenance_pending);
6392        assert_eq!(
6393            meta.maintenance_jobs_in_flight,
6394            INITIAL_MAINTENANCE_JOB_COUNT - 1
6395        );
6396        assert_eq!(meta.maintenance_queued_kinds.len(), 1);
6397
6398        let (requeued, deferred) =
6399            due_maintenance_jobs_without_actor_context(&mut live_roots, 1, &HashSet::new());
6400        assert_eq!(
6401            requeued,
6402            vec![(root.clone(), MaintenanceDrainKind::Watcher)]
6403        );
6404        assert!(!deferred);
6405        let meta = live_roots.get_mut(&root).unwrap();
6406        assert_eq!(
6407            meta.maintenance_jobs_in_flight,
6408            INITIAL_MAINTENANCE_JOB_COUNT
6409        );
6410        assert!(meta.maintenance_queued_kinds.is_empty());
6411
6412        for _ in 0..INITIAL_MAINTENANCE_JOB_COUNT {
6413            note_maintenance_completion(meta, None, false, false);
6414        }
6415        assert!(!meta.maintenance_pending);
6416        assert_eq!(meta.maintenance_jobs_in_flight, 0);
6417    }
6418
6419    #[test]
6420    fn maintenance_requeue_drops_while_bind_is_pending() {
6421        let (_dir, root) = test_root("maintenance-bind-requeue");
6422        let mut live_roots = HashMap::new();
6423        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6424        let (due, _) = due_maintenance_jobs_without_actor_context(
6425            &mut live_roots,
6426            usize::MAX,
6427            &HashSet::new(),
6428        );
6429        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6430
6431        let meta = live_roots.get_mut(&root).unwrap();
6432        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, true);
6433
6434        assert_eq!(
6435            meta.maintenance_jobs_in_flight,
6436            INITIAL_MAINTENANCE_JOB_COUNT - 1
6437        );
6438        assert!(meta.maintenance_queued_kinds.is_empty());
6439        assert!(meta.maintenance_pending);
6440    }
6441
6442    #[test]
6443    fn parked_lsp_completion_never_requiesces_or_cancels_a_pending_bind() {
6444        let mut meta = RootMeta::new(Instant::now());
6445        meta.unbound_quiesced = true;
6446
6447        assert!(!should_requiesce_after_maintenance(
6448            &meta,
6449            MaintenanceDrainKind::Lsp,
6450            false,
6451        ));
6452        assert!(!should_requiesce_after_maintenance(
6453            &meta,
6454            MaintenanceDrainKind::ConfigureTail,
6455            true,
6456        ));
6457        assert!(should_requiesce_after_maintenance(
6458            &meta,
6459            MaintenanceDrainKind::ConfigureTail,
6460            false,
6461        ));
6462    }
6463
6464    #[test]
6465    fn maintenance_pending_clears_and_poison_stops_requeue_after_fatal() {
6466        let (_dir, root) = test_root("maintenance-fatal");
6467        let mut live_roots = HashMap::new();
6468        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6469        let (due, _) = due_maintenance_jobs_without_actor_context(
6470            &mut live_roots,
6471            usize::MAX,
6472            &HashSet::new(),
6473        );
6474        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6475
6476        let meta = live_roots.get_mut(&root).unwrap();
6477        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), true, false);
6478        assert!(meta.maintenance_poisoned);
6479        assert!(meta.maintenance_queued_kinds.is_empty());
6480
6481        for _ in 1..INITIAL_MAINTENANCE_JOB_COUNT {
6482            note_maintenance_completion(meta, None, false, false);
6483        }
6484        assert!(!meta.maintenance_pending);
6485        assert_eq!(meta.maintenance_jobs_in_flight, 0);
6486    }
6487
6488    #[test]
6489    fn trust_for_principal_matrix() {
6490        assert_eq!(
6491            trust_for_principal(&Some(Principal::Direct)),
6492            BindTrust::FirstParty
6493        );
6494        // Every first-party reserved id is asserted BY NAME, in one loop over
6495        // the full set, so two failure classes stay distinguishable: an empty
6496        // or broken allowlist reddens every name at once, while a dropped
6497        // single entry (the rename hazard) reddens exactly the missing name.
6498        // Both halves of each transitional rename pair stay listed until the
6499        // flip settles (see the allowlist comment).
6500        for module_id in [
6501            "llm-runner",
6502            "aft",
6503            "broca",
6504            "alfonso-core",
6505            "prefrontal",
6506            "prefrontal-core",
6507        ] {
6508            assert_eq!(
6509                trust_for_principal(&Some(Principal::Reserved {
6510                    module_id: module_id.to_string(),
6511                })),
6512                BindTrust::FirstParty,
6513                "reserved module id '{module_id}' must resolve to first-party trust"
6514            );
6515        }
6516        assert_eq!(
6517            trust_for_principal(&Some(Principal::Reserved {
6518                module_id: "subc-mcp".to_string(),
6519            })),
6520            BindTrust::Untrusted
6521        );
6522        assert_eq!(
6523            trust_for_principal(&Some(Principal::Reserved {
6524                module_id: "anything-unknown".to_string(),
6525            })),
6526            BindTrust::Untrusted
6527        );
6528        assert_eq!(
6529            trust_for_principal(&Some(Principal::Unverified)),
6530            BindTrust::Untrusted
6531        );
6532        assert_eq!(trust_for_principal(&None), BindTrust::Untrusted);
6533    }
6534
6535    #[test]
6536    fn fed_harness_class_maps_to_untrusted_regardless_of_fingerprint_value() {
6537        let principal = Some(Principal::Direct);
6538        let fingerprint_a = "fed:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
6539        let fingerprint_b = "fed:0123456789abcdef111111111111111111111111111111111111111111111111";
6540
6541        assert_eq!(
6542            trust_for_bind(fingerprint_a, &principal),
6543            BindTrust::Untrusted
6544        );
6545        assert_eq!(
6546            trust_for_bind(fingerprint_b, &principal),
6547            BindTrust::Untrusted
6548        );
6549    }
6550
6551    /// The table above proves `trust_for_principal` maps correctly, and the
6552    /// test above proves the `fed:` harness override wins — but neither
6553    /// exercises the ordinary path, so an implementation that ignored the
6554    /// principal entirely for non-fed harnesses would satisfy both. Pin the
6555    /// delegation itself: on a normal harness the verdict must still come from
6556    /// the principal, in both directions.
6557    #[test]
6558    fn trust_for_bind_delegates_to_the_principal_on_ordinary_harnesses() {
6559        for harness in ["opencode", "pi", "runner", "mcp:claude"] {
6560            assert_eq!(
6561                trust_for_bind(harness, &Some(Principal::Direct)),
6562                BindTrust::FirstParty,
6563                "a direct principal must stay first-party on {harness}"
6564            );
6565            assert_eq!(
6566                trust_for_bind(harness, &Some(Principal::Unverified)),
6567                BindTrust::Untrusted,
6568                "an unverified principal must stay untrusted on {harness}"
6569            );
6570            assert_eq!(
6571                trust_for_bind(harness, &None),
6572                BindTrust::Untrusted,
6573                "an absent principal must fail closed on {harness}"
6574            );
6575            assert_eq!(
6576                trust_for_bind(
6577                    harness,
6578                    &Some(Principal::Reserved {
6579                        module_id: "subc-mcp".to_string(),
6580                    })
6581                ),
6582                BindTrust::Untrusted,
6583                "a non-allowlisted reserved module must stay untrusted on {harness}"
6584            );
6585        }
6586    }
6587
6588    #[tokio::test]
6589    async fn persistent_cancel_resolves_when_fired_before_await() {
6590        // The lost-wakeup guard: cancel() fires exactly once via notify_waiters()
6591        // (no stored permit). A waiter that registers AFTER the cancel must still
6592        // observe it via the flag; a waiter racing the cancel must still be woken.
6593        let signal = PersistentCancelSignal::new();
6594        signal.cancel();
6595        // Fired before we ever call cancelled() — must return immediately, not park.
6596        tokio::time::timeout(Duration::from_secs(1), signal.cancelled())
6597            .await
6598            .expect("cancelled() must resolve when cancel fired beforehand");
6599
6600        // A fresh signal cancelled concurrently with an in-flight cancelled().
6601        let racing = PersistentCancelSignal::new();
6602        let racing_for_task = racing.clone();
6603        let waiter = tokio::spawn(async move { racing_for_task.cancelled().await });
6604        racing.cancel();
6605        tokio::time::timeout(Duration::from_secs(1), waiter)
6606            .await
6607            .expect("cancelled() must resolve when cancel races the await")
6608            .expect("waiter task panicked");
6609    }
6610
6611    #[test]
6612    fn ingress_epoch_validation_rejects_reclaimed_requests_and_drops_other_stale_epochs() {
6613        let installed = HashMap::from([(7, 9)]);
6614        let mut reclaimed = ReclaimedRoutes::default();
6615        reclaimed.insert(route_key(8, 1));
6616        for ty in [
6617            FrameType::Request,
6618            FrameType::Response,
6619            FrameType::Error,
6620            FrameType::Push,
6621            FrameType::Cancel,
6622            FrameType::Goodbye,
6623        ] {
6624            let body = if ty.is_pure_header() {
6625                Vec::new()
6626            } else {
6627                br#"{}"#.to_vec()
6628            };
6629            let stale = Frame::build(ty, control_flags(), 7, 8, 41, body).unwrap();
6630            assert!(
6631                !ingress_route_should_be_processed(&installed, &reclaimed, &stale),
6632                "{ty:?}"
6633            );
6634        }
6635
6636        let reclaimed_request = Frame::build(
6637            FrameType::Request,
6638            control_flags(),
6639            8,
6640            1,
6641            42,
6642            br#"{}"#.to_vec(),
6643        )
6644        .unwrap();
6645        assert!(ingress_route_should_be_processed(
6646            &installed,
6647            &reclaimed,
6648            &reclaimed_request
6649        ));
6650
6651        let never_installed = Frame::build(
6652            FrameType::Request,
6653            control_flags(),
6654            9,
6655            1,
6656            43,
6657            br#"{}"#.to_vec(),
6658        )
6659        .unwrap();
6660        assert!(!ingress_route_should_be_processed(
6661            &installed,
6662            &reclaimed,
6663            &never_installed
6664        ));
6665
6666        let current = Frame::build(
6667            FrameType::Request,
6668            control_flags(),
6669            7,
6670            9,
6671            43,
6672            br#"{}"#.to_vec(),
6673        )
6674        .unwrap();
6675        let control = Frame::build(FrameType::Ping, control_flags(), 0, 0, 44, Vec::new()).unwrap();
6676        assert!(ingress_route_should_be_processed(
6677            &installed, &reclaimed, &current
6678        ));
6679        assert!(ingress_route_should_be_processed(
6680            &installed, &reclaimed, &control
6681        ));
6682        assert_eq!(installed, HashMap::from([(7, 9)]));
6683    }
6684
6685    #[tokio::test]
6686    async fn route_bind_ack_precedes_route_egress_in_writer_queue() {
6687        let (_dir, root) = test_root("route-bind-b2-ordering");
6688        let route = route_key(7, 3);
6689        let identity = RouteIdentity(Arc::new(RouteIdentityData {
6690            root: root.clone(),
6691            project_root: root.as_path().to_path_buf(),
6692            harness: "opencode".to_string(),
6693            session: "b2-session".to_string(),
6694            trust: BindTrust::FirstParty,
6695            spawn_principal: AuthenticatedPrincipal::FirstParty,
6696            consumer_elicitation_capable: false,
6697        }));
6698        let replay_key = push::ReplayKey::from_identity(&identity);
6699        let completion = RouteBindCompletion {
6700            route,
6701            identity,
6702            bind_root_id: root.clone(),
6703            inserted_new_actor: false,
6704            configure_response: Response::success("subc-bind-7", json!({})),
6705            diagnostics_on_edit: false,
6706            ver: PROTOCOL_VERSION,
6707            corr: 91,
6708            flags: control_flags(),
6709        };
6710        let mut pending_binds = HashMap::from([(
6711            route,
6712            PendingBind {
6713                bind_root_id: root,
6714                inserted_new_actor: false,
6715                cancelled: false,
6716                configure_request_id: "subc-bind-7".to_string(),
6717                started_at: Instant::now(),
6718                warned_half_deadline: false,
6719                deadline_reported: false,
6720                corr: 91,
6721                ver: PROTOCOL_VERSION,
6722                flags: control_flags(),
6723                cancellation: crate::executor::JobCancellation::new(),
6724            },
6725        )]);
6726        let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
6727        let mut push_buffer =
6728            HashMap::from([(replay_key, VecDeque::from([completion_frame("b2-replay")]))]);
6729        let (writer_tx, mut writer_rx) = mpsc::channel(8);
6730        let metrics = Arc::new(DispatchPathMetrics::new());
6731
6732        handle_route_bind_completion(
6733            &writer_tx,
6734            completion,
6735            &mut HashMap::new(),
6736            &mut HashMap::new(),
6737            &mut HashMap::new(),
6738            &mut push_buffer,
6739            &mut HashMap::new(),
6740            &mut pending_binds,
6741            &mut installed_route_epochs,
6742            &Arc::new(Executor::new()),
6743            &Arc::new(Notify::new()),
6744            &metrics,
6745        )
6746        .await
6747        .unwrap();
6748
6749        let ack = writer_rx.try_recv().expect("RouteBindAck");
6750        assert_eq!(ack.header.ty, FrameType::Response);
6751        assert_eq!((ack.header.channel, ack.header.epoch), (0, 0));
6752        let route_frame = writer_rx.try_recv().expect("post-ack route frame");
6753        assert_eq!(route_frame.header.ty, FrameType::Push);
6754        assert_eq!(
6755            (route_frame.header.channel, route_frame.header.epoch),
6756            (route.channel, route.epoch)
6757        );
6758    }
6759}