Skip to main content

aft/subc/
mod.rs

1//! subc daemon attach — transport edge.
2//!
3//! When AFT is launched as `aft --subc <connection-file>`, it does NOT run the
4//! standalone NDJSON-over-stdin loop. Instead it connects to a running subc
5//! daemon over loopback TCP, authenticates with the pre-envelope HMAC handshake
6//! (`subc-transport`), then speaks the subc frame protocol (`subc-protocol`):
7//! ModuleHello → HelloAck (register as a tool provider), then a channel-0
8//! control loop (Ping/Pong, RouteBind) plus route-channel tool calls.
9//!
10//! Concurrency: subc routes tool calls through the executor. The tokio
11//! edge never dispatches against `AppContext` inline; per-actor executor lanes
12//! own the reader/mutator epoch, while a writer task serializes outbound frames.
13
14use std::collections::{HashMap, HashSet, VecDeque};
15use std::fmt;
16use std::io;
17use std::net::{IpAddr, SocketAddr};
18use std::ops::Deref;
19use std::path::{Path, PathBuf};
20use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
21use std::sync::{Arc, LazyLock, Mutex as StdMutex, OnceLock};
22use std::time::{Duration, Instant};
23
24use serde::Deserialize;
25use serde_json::{json, Value};
26
27use crate::config::Config;
28use crate::config_resolve::ConfigTier;
29use crate::context::{App, AppContext, ProgressSender, RootHealthSnapshot};
30use crate::executor::{Executor, Lane};
31use crate::jsonc::strip_jsonc;
32use crate::log_ctx;
33use crate::path_identity::ProjectRootId;
34use crate::protocol::{ProgressKind, PushFrame, RawRequest, Response};
35use crate::run_tool_call::{
36    run_tool_call, strip_agent_preview_arg_owned, PhaseTrace, ToolCallContext, ToolCallOutcome,
37    ToolCallResult,
38};
39use crate::runtime_drain;
40use crate::sandbox_spawn::{AuthenticatedPrincipal, PrincipalTrust};
41
42use subc_protocol::manifest::{
43    Bindings, Concurrency, ExecutionMode, IdentityBinding, IdentityScope, ModuleManifest,
44    ProviderRole, StorageBinding, StorageKind, StorageScope, Tool, TrustTier,
45};
46use subc_protocol::session::{
47    HealthReport, HealthStatus, ModuleControlRequest, ModuleControlResponse,
48    MODULE_CONTROL_OP_HEALTH_CHECK,
49};
50use subc_protocol::{
51    ErrorBody, Flags, Frame, FrameType, ModuleHelloBody, Principal, Priority, PROTOCOL_VERSION,
52};
53use subc_transport::{authenticate_client, connection_file, read_frame, write_frame};
54use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
55use tokio::net::TcpStream;
56use tokio::sync::{mpsc, oneshot, Notify};
57use tokio::task::JoinHandle;
58
59/// Per-attempt handshake deadline. The initial attach loop has a separate total
60/// budget so a stalled peer cannot consume an unbounded supervisor launch window.
61const AUTH_DEADLINE: Duration = Duration::from_secs(5);
62const ATTACH_RETRY_BUDGET: Duration = Duration::from_secs(60);
63const ATTACH_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(250);
64const ATTACH_RETRY_MAX_BACKOFF: Duration = Duration::from_secs(5);
65const ATTACH_RETRY_JITTER_PERCENT: u64 = 20;
66
67/// Correlation id for the initial ModuleHello (channel 0).
68const HELLO_CORR: u64 = 1;
69
70/// Per-session in-memory replay cap for must-deliver Push frames. This covers
71/// detach/re-attach while AFT stays alive; cross-restart replay is phased later.
72const PUSH_BUFFER_MAX_PER_KEY: usize = 256;
73
74/// Bounded guard for control-frame sends. If the daemon stops reading and the
75/// writer queue stays full, tear the subc edge down instead of stalling the
76/// route loop indefinitely.
77const CONTROL_SEND_TIMEOUT: Duration = Duration::from_millis(250);
78
79/// Cadence for the loop's deadline-driven drain work (retry-buffer flush,
80/// bg-wake emission, maintenance submission). Checked at the top of every
81/// loop turn so busy select arms cannot starve it.
82const DRAIN_TICK_PERIOD: Duration = Duration::from_millis(250);
83
84/// Root-scoped stores and watcher runtimes are reopened lazily after this
85/// period without tool traffic. Keeping the value fixed avoids per-client
86/// eviction policies competing inside the module loop.
87const IDLE_ROOT_TTL: Duration = Duration::from_secs(30 * 60);
88
89const WRITER_QUEUE_CAPACITY: usize = 256;
90
91/// Keep reliable Push bursts from monopolizing the current-thread subc loop;
92/// any remaining must-deliver frames stay queued for the next loop turn.
93const RELIABLE_PUSH_DRAIN_BUDGET: usize = 32;
94
95/// Limit maintenance submissions per tick so background drains cannot delay
96/// control-plane work such as completed RouteBind acknowledgements.
97///
98/// The decomposed maintenance pass charges this budget by Mutating job, not by
99/// root. Size the default burst for one maintenance pass over eight live roots,
100/// while follow-up batches still re-enter the capped queue instead of bypassing
101/// the budget.
102const MAINTENANCE_SUBMIT_BUDGET: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len() * 8;
103const INITIAL_MAINTENANCE_DRAIN_KINDS: [MaintenanceDrainKind; 4] = [
104    MaintenanceDrainKind::Watcher,
105    MaintenanceDrainKind::Lsp,
106    MaintenanceDrainKind::ConfigureTail,
107    MaintenanceDrainKind::CompletionDrains,
108];
109#[cfg(test)]
110const INITIAL_MAINTENANCE_JOB_COUNT: usize = INITIAL_MAINTENANCE_DRAIN_KINDS.len();
111
112const RELIABLE_WRITER_RETRY_INITIAL_BACKOFF: Duration = Duration::from_millis(10);
113const RELIABLE_WRITER_RETRY_MAX_BACKOFF: Duration = Duration::from_millis(250);
114
115const DISPATCH_PATH_BIND_WARN_AFTER: Duration = Duration::from_secs(6);
116const ROUTE_BIND_DEADLINE: Duration = Duration::from_secs(12);
117
118/// Small bounded memory of completed task ids used to suppress stale lossy
119/// long-running reminders that arrive after their reliable completion event.
120const COMPLETED_TASK_SUPPRESSION_MAX: usize = 4096;
121
122/// Bash foreground orchestration polls detached tasks with short read-lane jobs.
123/// The sleep between polls is outside the executor so no read or write worker is
124/// pinned while a foreground command is still running.
125const PENDING_POLL_INTERVAL: Duration = Duration::from_millis(100);
126
127/// Host elicitation asks fail closed if the MCP facade does not answer promptly.
128const BASH_ELICITATION_TIMEOUT: Duration = Duration::from_secs(60);
129const BASH_ELICITATION_CREATE_METHOD: &str = "elicitation/create";
130
131#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
132struct RouteChannel {
133    channel: u16,
134    epoch: u32,
135}
136
137impl fmt::Display for RouteChannel {
138    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
139        write!(f, "{}@{}", self.channel, self.epoch)
140    }
141}
142
143type PushEnvelope = (ProjectRootId, PushFrame);
144type LossyPushEnvelope = (u64, ProjectRootId, PushFrame);
145type RetryBuffer = HashMap<RouteChannel, VecDeque<(push::ReplayKey, PushFrame)>>;
146mod bash;
147mod health;
148mod manifest;
149mod push;
150mod wire;
151
152use self::health::{
153    build_health_report, warn_slow_pending_binds, DispatchPathMetrics, ReapBlockerCensus,
154    ResponseTaskGuard,
155};
156use self::manifest::{
157    build_manifest, command_lane, control_flags, control_ops, is_bash_family_tool,
158    is_subc_agent_core_tool, is_subc_native_plumbing_tool,
159};
160pub use self::wire::SubcError;
161
162/// Test-only view of the fail-closed tool-call gate: would `name` be admitted
163/// on a bound route (as an agent tool or native plumbing)? Used by the
164/// plugin-send drift guard in `subc_plumbing_drift_test.rs`.
165pub fn is_tool_call_admitted_for_test(name: &str) -> bool {
166    manifest::is_subc_agent_core_tool(name) || manifest::is_subc_native_plumbing_tool(name)
167}
168use self::wire::{
169    build_error_frame, build_goodbye_frame, build_tool_response_frame, decrement_counted_channel,
170    response_is_fatal_panic, response_message, send_counted_channel, send_frame,
171    send_reliable_writer_frame, send_traced_tool_response_frame, ToolResponseWriteTrace,
172    WriterFrame, WriterSender,
173};
174
175struct DecodedFrame {
176    frame: Frame,
177    phase_trace: PhaseTrace,
178}
179
180struct ToolCallCompletion {
181    text: String,
182    phase_trace: PhaseTrace,
183}
184
185#[derive(Clone)]
186struct PushSenders {
187    lossy_tx: mpsc::Sender<LossyPushEnvelope>,
188    reliable_tx: mpsc::UnboundedSender<PushEnvelope>,
189    lossy_overflow: Arc<push::LossyOverflow>,
190    lossy_seq: Arc<AtomicU64>,
191}
192
193#[derive(Clone)]
194struct PersistentCancelSignal {
195    inner: Arc<PersistentCancelInner>,
196}
197
198struct PersistentCancelInner {
199    cancelled: AtomicBool,
200    notify: Notify,
201}
202
203impl PersistentCancelSignal {
204    fn new() -> Self {
205        Self {
206            inner: Arc::new(PersistentCancelInner {
207                cancelled: AtomicBool::new(false),
208                notify: Notify::new(),
209            }),
210        }
211    }
212
213    fn cancel(&self) {
214        if !self.inner.cancelled.swap(true, Ordering::SeqCst) {
215            self.inner.notify.notify_waiters();
216        }
217    }
218
219    fn is_cancelled(&self) -> bool {
220        self.inner.cancelled.load(Ordering::SeqCst)
221    }
222
223    async fn cancelled(&self) {
224        // `enable()` REGISTERS this waiter before we read the flag, closing the
225        // lost-wakeup window: `notify_waiters()` only wakes already-registered
226        // waiters and stores no permit, so without enable() a `cancel()` firing
227        // between the flag read and `.await` would be missed and the future
228        // would park forever (cancel() fires only once). With enable(), a cancel
229        // racing the flag read still wakes the registered waiter. The loop is a
230        // belt-and-suspenders re-check on spurious wakeups.
231        loop {
232            let notified = self.inner.notify.notified();
233            tokio::pin!(notified);
234            notified.as_mut().enable();
235            if self.is_cancelled() {
236                return;
237            }
238            notified.await;
239        }
240    }
241}
242
243#[derive(Debug, Clone, Copy, PartialEq, Eq)]
244pub(crate) enum BindTrust {
245    FirstParty,
246    Untrusted,
247}
248
249impl BindTrust {
250    fn allows_bash_observation(self) -> bool {
251        matches!(self, Self::FirstParty)
252    }
253
254    fn label(self) -> &'static str {
255        match self {
256            Self::FirstParty => "first_party",
257            Self::Untrusted => "untrusted",
258        }
259    }
260
261    fn sandbox_trust(self) -> PrincipalTrust {
262        match self {
263            Self::FirstParty => PrincipalTrust::FirstParty,
264            Self::Untrusted => PrincipalTrust::Untrusted,
265        }
266    }
267}
268
269pub(super) fn trust_for_principal(principal: &Option<Principal>) -> BindTrust {
270    match principal {
271        Some(Principal::Direct) => BindTrust::FirstParty,
272        // Module renames are flag-days: the daemon registry refuses duplicate
273        // active ids, so a renaming module cannot advertise both names during
274        // its transition. This allowlist is DIALLED, not dialling — it must
275        // accept a module's NEW name in a released binary before the module
276        // starts using it, and the old name stays until the flip has settled.
277        // That is why transitional pairs appear here: llm-runner/broca was the
278        // previous rename, alfonso-core/prefrontal is the current one. When
279        // retiring an old name, confirm the fleet no longer spawns it — a
280        // stale entry here is inert, but a missing one silently downgrades a
281        // first-party module to Untrusted and revokes its bash access.
282        Some(Principal::Reserved { module_id })
283            if module_id == "llm-runner"
284                || module_id == "aft"
285                || module_id == "broca"
286                || module_id == "alfonso-core"
287                || module_id == "prefrontal"
288                || module_id == "prefrontal-core" =>
289        {
290            BindTrust::FirstParty
291        }
292        Some(Principal::Reserved { .. }) | Some(Principal::Unverified) | None => {
293            BindTrust::Untrusted
294        }
295    }
296}
297
298fn harness_forces_untrusted(harness: &str) -> bool {
299    harness.starts_with("fed:")
300}
301
302pub(super) fn trust_for_bind(harness: &str, principal: &Option<Principal>) -> BindTrust {
303    if harness_forces_untrusted(harness) {
304        BindTrust::Untrusted
305    } else {
306        trust_for_principal(principal)
307    }
308}
309
310fn principal_id(principal: &Option<Principal>) -> Option<String> {
311    match principal {
312        Some(Principal::Direct) => Some("direct".to_string()),
313        Some(Principal::Reserved { module_id }) => Some(format!("reserved:{module_id}")),
314        Some(Principal::Unverified) => Some("unverified".to_string()),
315        None => None,
316    }
317}
318
319fn principal_label(principal: &Option<Principal>) -> String {
320    principal_id(principal).unwrap_or_else(|| "absent".to_string())
321}
322
323#[derive(Debug)]
324/// Per-root route metadata owned by the subc loop. The `active_bash_waits` field
325/// counts detached bash processes that are still being observed for this root.
326/// Any future logic that evicts roots based on idle time must not evict a root
327/// while this count is greater than zero, because a foreground bash response may
328/// still arrive later.
329struct RootMeta {
330    maintenance_pending: bool,
331    maintenance_jobs_in_flight: usize,
332    maintenance_queued_kinds: VecDeque<MaintenanceDrainKind>,
333    maintenance_last_submitted: Option<Instant>,
334    maintenance_poisoned: bool,
335    last_touched: Instant,
336    diagnostics_on_edit: bool,
337    active_bash_waits: usize,
338    idle_artifacts_evicted: bool,
339    unbound_quiesced: bool,
340    consecutive_missing_sweeps: u8,
341}
342
343#[derive(Debug)]
344struct PendingBind {
345    bind_root_id: ProjectRootId,
346    inserted_new_actor: bool,
347    cancelled: bool,
348    configure_request_id: String,
349    started_at: Instant,
350    warned_half_deadline: bool,
351    deadline_reported: bool,
352    corr: u64,
353    ver: u8,
354    flags: Flags,
355    /// Exact-job cancellation for the submitted configure: Goodbye and
356    /// deadline expiry cancel the executor job operationally (queued jobs are
357    /// removed, running configures return at their next checkpoint) instead of
358    /// only marking bookkeeping.
359    cancellation: crate::executor::JobCancellation,
360}
361
362struct RouteBindCompletion {
363    route: RouteChannel,
364    identity: RouteIdentity,
365    bind_root_id: ProjectRootId,
366    inserted_new_actor: bool,
367    configure_response: Response,
368    diagnostics_on_edit: bool,
369    ver: u8,
370    corr: u64,
371    flags: Flags,
372}
373
374#[derive(Debug, Clone)]
375struct RouteIdentity(Arc<RouteIdentityData>);
376
377#[derive(Debug)]
378struct RouteIdentityData {
379    root: ProjectRootId,
380    project_root: PathBuf,
381    harness: String,
382    session: String,
383    trust: BindTrust,
384    spawn_principal: AuthenticatedPrincipal,
385    consumer_elicitation_capable: bool,
386}
387
388impl Deref for RouteIdentity {
389    type Target = RouteIdentityData;
390
391    fn deref(&self) -> &Self::Target {
392        &self.0
393    }
394}
395
396#[derive(Debug, Clone)]
397struct RetainedSessionIdentity {
398    harness: String,
399    trust: BindTrust,
400}
401
402#[derive(Clone, Copy)]
403struct BgSub {
404    corr: u64,
405    ver: u8,
406    flags: Flags,
407}
408
409struct MaintenanceCompletion {
410    root_id: ProjectRootId,
411    kind: MaintenanceDrainKind,
412    response: Response,
413    empty_bg_sessions: Vec<(String, u64)>,
414    requeue_kind: Option<MaintenanceDrainKind>,
415}
416
417#[derive(Clone, Copy, Debug, PartialEq, Eq)]
418enum MaintenanceDrainKind {
419    Watcher,
420    Lsp,
421    ConfigureTail,
422    CompletionDrains,
423}
424
425impl MaintenanceDrainKind {
426    fn label(self) -> &'static str {
427        match self {
428            Self::Watcher => "watcher",
429            Self::Lsp => "lsp",
430            Self::ConfigureTail => "configure-tail",
431            Self::CompletionDrains => "completion-drains",
432        }
433    }
434}
435
436#[derive(Debug, Default)]
437struct MaintenanceJobOutcome {
438    empty_bg_sessions: Vec<(String, u64)>,
439    requeue_kind: Option<MaintenanceDrainKind>,
440}
441
442#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
443struct ReverseCorrKey {
444    route: RouteChannel,
445    corr: u64,
446}
447
448struct PendingBashAsk {
449    route: RouteChannel,
450    tool_corr: u64,
451    tool_flags: Flags,
452    tool_ver: u8,
453    root: ProjectRootId,
454    project_root: PathBuf,
455    session_id: String,
456    spawn_principal: AuthenticatedPrincipal,
457    request_id: String,
458    arguments: Value,
459    format_context: crate::subc_format::FormatContext,
460    cancel: bash::BashWaitCancel,
461    grants: Vec<String>,
462    expires_at: Instant,
463}
464
465impl RootMeta {
466    fn new(now: Instant) -> Self {
467        Self {
468            maintenance_pending: false,
469            maintenance_jobs_in_flight: 0,
470            maintenance_queued_kinds: VecDeque::new(),
471            maintenance_last_submitted: None,
472            maintenance_poisoned: false,
473            last_touched: now,
474            diagnostics_on_edit: false,
475            active_bash_waits: 0,
476            idle_artifacts_evicted: false,
477            unbound_quiesced: false,
478            consecutive_missing_sweeps: 0,
479        }
480    }
481
482    fn note_activity(&mut self) {
483        self.last_touched = Instant::now();
484    }
485
486    fn reactivate_bound(&mut self) {
487        self.note_activity();
488        self.idle_artifacts_evicted = false;
489        self.unbound_quiesced = false;
490    }
491}
492
493fn due_maintenance_jobs(
494    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
495    executor: Option<&Executor>,
496    bg_sub_by_session: &HashMap<(ProjectRootId, String), RouteChannel>,
497    bg_wake_pending: &HashSet<RouteChannel>,
498    budget: usize,
499    pending_bind_roots: &HashSet<ProjectRootId>,
500) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
501    let mut jobs = Vec::new();
502    let mut deferred = false;
503    let mut roots = live_roots.keys().cloned().collect::<Vec<_>>();
504    roots.sort_by(|left, right| {
505        let left_last = live_roots
506            .get(left)
507            .and_then(|meta| meta.maintenance_last_submitted);
508        let right_last = live_roots
509            .get(right)
510            .and_then(|meta| meta.maintenance_last_submitted);
511        left_last
512            .cmp(&right_last)
513            .then_with(|| left.as_path().cmp(right.as_path()))
514    });
515
516    for root_id in roots {
517        let Some(meta) = live_roots.get_mut(&root_id) else {
518            continue;
519        };
520        if meta.maintenance_poisoned {
521            continue;
522        }
523
524        if pending_bind_roots.contains(&root_id) {
525            if meta.maintenance_pending || !meta.maintenance_queued_kinds.is_empty() {
526                deferred = true;
527            }
528            continue;
529        }
530
531        if !meta.maintenance_pending {
532            if jobs.len() >= budget {
533                deferred = true;
534                continue;
535            }
536            // Only enqueue kinds with pending work. Probes are cheap and
537            // fail-open (contended sources count as pending), so an idle root
538            // costs four probes per tick instead of four dispatched jobs.
539            let executor_actor_context =
540                executor.and_then(|executor| executor.actor_context(&root_id));
541            let root_has_pending_bg_wake =
542                bg_sub_by_session.iter().any(|((sub_root, _), channel)| {
543                    sub_root == &root_id && bg_wake_pending.contains(channel)
544                });
545            let kinds_with_work: Vec<MaintenanceDrainKind> = match executor_actor_context {
546                Some(ctx) => INITIAL_MAINTENANCE_DRAIN_KINDS
547                    .into_iter()
548                    .filter(|kind| {
549                        if meta.unbound_quiesced && !matches!(kind, MaintenanceDrainKind::Lsp) {
550                            return false;
551                        }
552                        match kind {
553                            MaintenanceDrainKind::Watcher => ctx.watcher_drain_has_work(),
554                            MaintenanceDrainKind::Lsp => ctx.lsp_drain_has_work(),
555                            MaintenanceDrainKind::ConfigureTail => ctx.configure_tail_has_work(),
556                            // Every CompletionDrains source is visible at this enqueue site:
557                            // AppContext probes completion queues, this loop owns bg wakes,
558                            // and queued continuations bypass probing via maintenance_pending.
559                            // New drain sources must expose a probe here rather than making
560                            // every subscribed root fail open again.
561                            MaintenanceDrainKind::CompletionDrains => {
562                                root_has_pending_bg_wake || ctx.completion_drains_have_work()
563                            }
564                        }
565                    })
566                    .collect(),
567                None if meta.unbound_quiesced => Vec::new(),
568                // No context handle (actor gone mid-tick): enqueue everything.
569                None => INITIAL_MAINTENANCE_DRAIN_KINDS.to_vec(),
570            };
571            if kinds_with_work.is_empty() {
572                continue;
573            }
574            meta.maintenance_pending = true;
575            meta.maintenance_queued_kinds.extend(kinds_with_work);
576        }
577
578        while let Some(kind) = meta.maintenance_queued_kinds.pop_front() {
579            if jobs.len() >= budget {
580                meta.maintenance_queued_kinds.push_front(kind);
581                deferred = true;
582                break;
583            }
584            meta.maintenance_jobs_in_flight += 1;
585            meta.maintenance_last_submitted = Some(Instant::now());
586            jobs.push((root_id.clone(), kind));
587        }
588
589        meta.maintenance_pending =
590            meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
591    }
592
593    (jobs, deferred)
594}
595
596fn eviction_estimate_label(estimate: &crate::memory::MemoryEstimate) -> String {
597    match estimate.estimated_bytes {
598        Some(bytes) => format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
599        None if estimate.status == "busy" => "busy".to_string(),
600        None => "not estimated".to_string(),
601    }
602}
603
604fn optional_memory_label(bytes: Option<u64>) -> String {
605    bytes.map_or_else(
606        || "not estimated".to_string(),
607        |bytes| format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)),
608    )
609}
610
611fn pressure_relief_label(relief: &crate::memory::AllocatorPressureRelief) -> String {
612    format!(
613        "; allocator pressure relief: RSS {} -> {}, in-use {} -> {}, allocated {} -> {}, slack {} -> {}, allocator reported {:.1} MB released",
614        optional_memory_label(relief.rss_before_bytes),
615        optional_memory_label(relief.rss_after_bytes),
616        optional_memory_label(relief.allocator_before.bytes_in_use),
617        optional_memory_label(relief.allocator_after.bytes_in_use),
618        optional_memory_label(relief.allocator_before.size_allocated),
619        optional_memory_label(relief.allocator_after.size_allocated),
620        optional_memory_label(relief.allocator_before.retained_slack_bytes),
621        optional_memory_label(relief.allocator_after.retained_slack_bytes),
622        relief.bytes_released as f64 / (1024.0 * 1024.0),
623    )
624}
625
626fn idle_root_eviction_message(
627    root_id: &ProjectRootId,
628    memory: &crate::memory::RootMemorySnapshot,
629    pressure_relief: Option<&crate::memory::AllocatorPressureRelief>,
630) -> String {
631    // Bash, LSP, and parser state remain resident. The freed total is deliberately
632    // only the known-byte portion of handles eviction actually drops.
633    let freed_bytes = [
634        &memory.semantic,
635        &memory.trigram,
636        &memory.symbols,
637        &memory.callgraph,
638        &memory.inspect,
639    ]
640    .iter()
641    .filter_map(|estimate| estimate.estimated_bytes)
642    .fold(0u64, u64::saturating_add);
643    let mut message = format!(
644        "evicted idle root {}: freed ~{:.1} MB (semantic {}, trigram {}, symbols {}, callgraph {}, inspect {}; retained: bash {}, lsp {}, parser_pool {})",
645        root_id.as_path().display(),
646        freed_bytes as f64 / (1024.0 * 1024.0),
647        eviction_estimate_label(&memory.semantic),
648        eviction_estimate_label(&memory.trigram),
649        eviction_estimate_label(&memory.symbols),
650        eviction_estimate_label(&memory.callgraph),
651        eviction_estimate_label(&memory.inspect),
652        eviction_estimate_label(&memory.bash),
653        eviction_estimate_label(&memory.lsp),
654        eviction_estimate_label(&memory.parser_pool),
655    );
656    if let Some(pressure_relief) = pressure_relief {
657        message.push_str(&pressure_relief_label(pressure_relief));
658    }
659    message
660}
661
662fn process_has_been_idle(now: Instant, live_roots: &HashMap<ProjectRootId, RootMeta>) -> bool {
663    !live_roots.is_empty()
664        && live_roots.values().all(|meta| {
665            now.saturating_duration_since(meta.last_touched) >= IDLE_ROOT_TTL
666                && meta.active_bash_waits == 0
667                && !meta.maintenance_pending
668                && meta.maintenance_queued_kinds.is_empty()
669        })
670}
671
672fn allocator_pressure_relief_after_idle_sweep(
673    now: Instant,
674    live_roots: &HashMap<ProjectRootId, RootMeta>,
675    executor: &Executor,
676) -> Option<crate::memory::AllocatorPressureRelief> {
677    if !process_has_been_idle(now, live_roots)
678        || live_roots.keys().any(|root_id| {
679            executor
680                .actor_context(root_id)
681                .is_some_and(|ctx| ctx.artifact_eviction_blocked())
682        })
683    {
684        return None;
685    }
686
687    #[cfg(target_os = "macos")]
688    {
689        Some(crate::memory::relieve_allocator_pressure())
690    }
691    #[cfg(not(target_os = "macos"))]
692    {
693        None
694    }
695}
696
697fn quiesce_unbound_root(
698    root_id: &ProjectRootId,
699    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
700    executor: &Arc<Executor>,
701) {
702    let Some(meta) = live_roots.get_mut(root_id) else {
703        return;
704    };
705
706    let ctx = executor.actor_context(root_id);
707    if let Some(ctx) = ctx.as_ref() {
708        // Close lifecycle admission before touching scheduler queues. A running
709        // ConfigureTail cannot release gates, install a watcher, or reserve a
710        // callgraph build after this transition becomes visible.
711        ctx.mark_subc_unbound();
712    }
713    let cancelled = executor.cancel_queued_maintenance(root_id);
714    // Transient unbind keeps the root WARM: the watcher stays running (its
715    // events accumulate and replay on rebind, so no unobserved gap exists) and
716    // resident artifacts stay resident. Host restarts unbind every root and
717    // rebind seconds later; stopping the watcher here would force strict
718    // re-verification plus a full callgraph rebuild on every restart. The
719    // expensive teardown (watcher stop + gap invalidation) belongs to the
720    // idle-TTL reaper and the root-deleted path.
721    let discarded = ctx
722        .map(|ctx| crate::commands::configure::cancel_deferred_configure_maintenance(&ctx))
723        .unwrap_or(0);
724    meta.unbound_quiesced = true;
725    meta.maintenance_queued_kinds.clear();
726    meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
727    log::info!(
728        "subc attach: quiesced unbound root {} (cancelled {} queued maintenance job(s), cancelled {} configure maintenance job(s)); cause=goodbye_unbound",
729        root_id.as_path().display(),
730        cancelled,
731        discarded
732    );
733}
734
735#[allow(clippy::too_many_arguments)]
736fn quiesce_connection_roots(
737    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
738    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
739    routes: &mut HashMap<RouteChannel, RouteIdentity>,
740    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
741    installed_route_epochs: &mut HashMap<u16, u32>,
742    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
743    executor: &Arc<Executor>,
744) {
745    for cancel in route_bash_cancels.values() {
746        cancel.token.cancel();
747    }
748    route_bash_cancels.clear();
749
750    let mut roots = live_roots.keys().cloned().collect::<HashSet<_>>();
751    for pending in pending_binds.values_mut() {
752        pending.cancelled = true;
753        roots.insert(pending.bind_root_id.clone());
754        let _ = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
755    }
756
757    // A connection exit abandons every installed route at once. Close lifecycle
758    // admission before cancelling maintenance so no deferred worker can restore
759    // root activity after the loop-owned route tables disappear.
760    for root_id in roots {
761        if live_roots.contains_key(&root_id) {
762            quiesce_unbound_root(&root_id, live_roots, executor);
763        } else if let Some(ctx) = executor.actor_context(&root_id) {
764            ctx.mark_subc_unbound();
765            executor.cancel_queued_maintenance(&root_id);
766            crate::commands::configure::cancel_deferred_configure_maintenance(&ctx);
767        }
768    }
769
770    routes.clear();
771    root_channels.clear();
772    installed_route_epochs.clear();
773}
774
775/// Per-channel epoch watermarks for roots reclaimed without a client Goodbye.
776/// The 16-bit channel space bounds this map, and no root identity or resource
777/// handle is retained. It exists only so late requests receive a typed error.
778#[derive(Debug, Default)]
779struct ReclaimedRoutes {
780    highest_epoch_by_channel: HashMap<u16, u32>,
781}
782
783impl ReclaimedRoutes {
784    fn insert(&mut self, route: RouteChannel) {
785        self.highest_epoch_by_channel
786            .entry(route.channel)
787            .and_modify(|epoch| *epoch = (*epoch).max(route.epoch))
788            .or_insert(route.epoch);
789    }
790
791    fn contains(&self, route: RouteChannel) -> bool {
792        self.highest_epoch_by_channel
793            .get(&route.channel)
794            .is_some_and(|epoch| route.epoch <= *epoch)
795    }
796}
797
798#[derive(Debug, Default)]
799struct IdleReapOutcome {
800    evicted: usize,
801    forgotten_deleted_roots: Vec<ProjectRootId>,
802}
803
804fn reap_idle_roots(
805    now: Instant,
806    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
807    pending_binds: &HashMap<RouteChannel, PendingBind>,
808    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
809    executor: &Arc<Executor>,
810    metrics: &DispatchPathMetrics,
811) -> IdleReapOutcome {
812    let pending_bind_roots = pending_binds
813        .values()
814        .map(|pending| pending.bind_root_id.clone())
815        .collect::<HashSet<_>>();
816    let mut census = ReapBlockerCensus::default();
817    let mut candidates = Vec::new();
818
819    for (root_id, meta) in live_roots.iter_mut() {
820        let deleted = !root_id.as_path().exists();
821        if deleted {
822            // A missing directory makes a bound route obsolete, but one failed
823            // lookup is not enough evidence to tear down a client-visible actor.
824            // Requiring two maintenance sweeps protects atomic replacement and
825            // transient filesystem failures; observing the path resets the proof.
826            // Absence also covers renames: a task's cwd handle can follow the
827            // moved directory while the registered path disappears. The old
828            // path is deliberately treated as a retired root identity, so the
829            // reaper accepts killing such tasks; rename a project only with no
830            // live tasks rather than relying on cwd-resolution heuristics.
831            meta.consecutive_missing_sweeps = meta.consecutive_missing_sweeps.saturating_add(1);
832        } else {
833            meta.consecutive_missing_sweeps = 0;
834        }
835        let deletion_confirmed = meta.consecutive_missing_sweeps >= 2;
836        let has_bound_route = root_channels
837            .get(root_id)
838            .is_some_and(|channels| !channels.is_empty());
839        let has_pending_bind = pending_bind_roots.contains(root_id);
840
841        if deleted {
842            let mut retained = false;
843            if !deletion_confirmed {
844                census.absence_unconfirmed += 1;
845                retained = true;
846            }
847            // Once absence is confirmed, the directory cannot serve this route
848            // again. Neither a stale route nor the lack of normal unbind cleanup
849            // justifies retaining the root; purge removes the route after retirement.
850            if meta.active_bash_waits > 0 {
851                census.bash_waits += 1;
852                retained = true;
853            }
854            if meta.maintenance_pending {
855                census.maintenance_pending += 1;
856                retained = true;
857            }
858            if !meta.maintenance_queued_kinds.is_empty() {
859                census.maintenance_queued += 1;
860                retained = true;
861            }
862            if has_pending_bind {
863                census.pending_binds += 1;
864                retained = true;
865            }
866            match executor.try_actor_is_idle(root_id) {
867                Some(true) => {}
868                Some(false) => {
869                    census.actor_busy += 1;
870                    retained = true;
871                }
872                None => {
873                    census.actor_state_busy += 1;
874                    retained = true;
875                }
876            }
877            if retained {
878                census.deleted_retained += 1;
879                continue;
880            }
881        } else {
882            // Route teardown marks the lifecycle admission gate before the last
883            // channel disappears. Requiring zero bound channels and a quiesced
884            // lifecycle prevents a still-bound root from losing its watcher.
885            if has_bound_route
886                || !meta.unbound_quiesced
887                || meta.idle_artifacts_evicted
888                || now.saturating_duration_since(meta.last_touched) < IDLE_ROOT_TTL
889                || meta.active_bash_waits > 0
890                || meta.maintenance_pending
891                || !meta.maintenance_queued_kinds.is_empty()
892                || has_pending_bind
893                || !executor.actor_is_idle(root_id)
894            {
895                continue;
896            }
897        }
898        candidates.push((root_id.clone(), deleted));
899    }
900
901    let mut reaped = Vec::new();
902    let mut forgotten_deleted_roots = Vec::new();
903    for (root_id, deleted) in candidates {
904        let Some(ctx) = executor.actor_context(&root_id) else {
905            if deleted {
906                census.deleted_retained += 1;
907                census.actor_busy += 1;
908            }
909            continue;
910        };
911        // A TTL-aged unbound root retained its watcher-derived pending paths
912        // across the transient-unbind window. Strict gap invalidation subsumes
913        // them, but every abort path must restore them because a rebind can
914        // still happen until eviction commits.
915        //
916        // After two consecutive directory-absence scans confirm that the
917        // root is gone, terminate its background task before checking the
918        // artifact-eviction gate. The task can otherwise keep the root's
919        // artifacts in use; cleanup first lets confirmed reclamation finish
920        // without weakening the gate for unrelated active work.
921        if deleted {
922            ctx.bash_background()
923                .kill_running_tasks_for_root(root_id.as_path());
924        }
925        let taken_pending = Some(ctx.take_pending_reconciliation_state());
926        if ctx.artifact_eviction_blocked() {
927            if let Some(pending) = taken_pending {
928                ctx.restore_pending_reconciliation_state(pending);
929            }
930            if deleted {
931                census.deleted_retained += 1;
932                census.artifact_eviction_blocked += 1;
933            }
934            continue;
935        }
936        let memory_before = ctx.memory_root_snapshot();
937        if !ctx.evict_idle_artifacts() {
938            if let Some(pending) = taken_pending {
939                ctx.restore_pending_reconciliation_state(pending);
940            }
941            if deleted {
942                census.deleted_retained += 1;
943                census.artifact_eviction_failed += 1;
944            }
945            continue;
946        }
947        drop(taken_pending);
948        ctx.stop_watcher_runtime_in_background();
949        // Edits during watcher downtime are unobserved. Advance publication
950        // epochs and force strict verification before any later warm reload.
951        ctx.invalidate_artifacts_after_watcher_gap();
952
953        if deleted {
954            if executor.retire_idle_actor_in_background(&root_id) {
955                live_roots.remove(&root_id);
956                forgotten_deleted_roots.push(root_id.clone());
957            } else {
958                census.deleted_retained += 1;
959                census.actor_busy += 1;
960            }
961        } else {
962            if let Some(meta) = live_roots.get_mut(&root_id) {
963                meta.idle_artifacts_evicted = true;
964            }
965            ctx.release_idle_reopenable_resources_in_background();
966        }
967        reaped.push((root_id, memory_before));
968    }
969
970    metrics.record_reap(census);
971    if census.deleted_retained > 0 {
972        log::info!(
973            "subc attach: retained {} deleted root(s) during idle reap; blockers={}",
974            census.deleted_retained,
975            census.blocker_histogram()
976        );
977    }
978
979    let pressure_relief = (!reaped.is_empty())
980        .then(|| allocator_pressure_relief_after_idle_sweep(now, live_roots, executor))
981        .flatten();
982    for (root_id, memory_before) in &reaped {
983        log::info!(
984            "{}",
985            idle_root_eviction_message(root_id, memory_before, pressure_relief.as_ref())
986        );
987    }
988    IdleReapOutcome {
989        evicted: reaped.len(),
990        forgotten_deleted_roots,
991    }
992}
993
994#[allow(clippy::too_many_arguments)]
995fn purge_deleted_root_residents(
996    root_id: &ProjectRootId,
997    routes: &mut HashMap<RouteChannel, RouteIdentity>,
998    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
999    installed_route_epochs: &mut HashMap<u16, u32>,
1000    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1001    retry_buffer: &mut RetryBuffer,
1002    reclaimed_routes: &mut ReclaimedRoutes,
1003    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1004    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1005    bg_subs: &mut HashMap<RouteChannel, BgSub>,
1006    bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1007    bg_wake_pending: &mut HashSet<RouteChannel>,
1008    bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
1009    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1010) {
1011    let mut stale_routes = root_channels.get(root_id).cloned().unwrap_or_default();
1012    stale_routes.extend(
1013        routes
1014            .iter()
1015            .filter_map(|(route, identity)| (&identity.root == root_id).then_some(*route)),
1016    );
1017    stale_routes.extend(
1018        bg_sub_by_session
1019            .iter()
1020            .filter_map(|((root, _), route)| (root == root_id).then_some(*route)),
1021    );
1022    stale_routes.extend(
1023        pending_bash_asks
1024            .values()
1025            .filter_map(|ask| (&ask.root == root_id).then_some(ask.route)),
1026    );
1027
1028    for route in stale_routes {
1029        reclaimed_routes.insert(route);
1030        remove_installed_route(installed_route_epochs, route);
1031        remove_route_channel(routes, root_channels, route);
1032        if let Some(cancel) = route_bash_cancels.remove(&route) {
1033            cancel.token.cancel();
1034        }
1035        retry_buffer.remove(&route);
1036        bg_subs.remove(&route);
1037        bg_wake_pending.remove(&route);
1038    }
1039    root_channels.remove(root_id);
1040    session_identity.retain(|(root, _), _| root != root_id);
1041    push_buffer.retain(|key, _| &key.root != root_id);
1042    bg_wake_epoch.retain(|(root, _), _| root != root_id);
1043    pending_bash_asks.retain(|_, ask| &ask.root != root_id);
1044    bg_sub_by_session.retain(|(root, _), _| root != root_id);
1045
1046    log::info!(
1047        "subc attach: fully forgot deleted root {}; cause=absence_reclaim",
1048        root_id.as_path().display()
1049    );
1050}
1051
1052#[allow(clippy::too_many_arguments)]
1053fn submit_due_maintenance_jobs(
1054    executor: &Arc<Executor>,
1055    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1056    pending_binds: &HashMap<RouteChannel, PendingBind>,
1057    bg_sub_by_session: &HashMap<(ProjectRootId, String), RouteChannel>,
1058    bg_wake_pending: &HashSet<RouteChannel>,
1059    bg_wake_epoch: &HashMap<(ProjectRootId, String), u64>,
1060    maintenance_tx: &mpsc::Sender<MaintenanceCompletion>,
1061    metrics: &Arc<DispatchPathMetrics>,
1062) {
1063    let pending_bind_roots = pending_binds
1064        .values()
1065        .map(|pending| pending.bind_root_id.clone())
1066        .collect::<HashSet<_>>();
1067    let (due_jobs, deferred_jobs) = due_maintenance_jobs(
1068        live_roots,
1069        Some(executor),
1070        bg_sub_by_session,
1071        bg_wake_pending,
1072        MAINTENANCE_SUBMIT_BUDGET,
1073        &pending_bind_roots,
1074    );
1075    if deferred_jobs {
1076        metrics
1077            .maintenance_budget_deferrals
1078            .fetch_add(1, Ordering::Relaxed);
1079    }
1080    for (root_id, kind) in due_jobs {
1081        let bg_sessions_to_check = if kind == MaintenanceDrainKind::CompletionDrains {
1082            bg_sub_by_session
1083                .iter()
1084                .filter_map(|((root, session), _)| {
1085                    if root == &root_id {
1086                        Some((
1087                            session.clone(),
1088                            bg_wake_epoch
1089                                .get(&(root_id.clone(), session.clone()))
1090                                .copied()
1091                                .unwrap_or(0),
1092                        ))
1093                    } else {
1094                        None
1095                    }
1096                })
1097                .collect()
1098        } else {
1099            Vec::new()
1100        };
1101        submit_maintenance_job(
1102            executor,
1103            root_id,
1104            kind,
1105            bg_sessions_to_check,
1106            maintenance_tx,
1107            metrics,
1108        );
1109    }
1110}
1111
1112fn should_requiesce_after_maintenance(
1113    meta: &RootMeta,
1114    completed_kind: MaintenanceDrainKind,
1115    bind_pending: bool,
1116) -> bool {
1117    meta.unbound_quiesced && completed_kind != MaintenanceDrainKind::Lsp && !bind_pending
1118}
1119
1120fn note_maintenance_completion(
1121    meta: &mut RootMeta,
1122    requeue_kind: Option<MaintenanceDrainKind>,
1123    fatal: bool,
1124    defer_requeue: bool,
1125) {
1126    if fatal {
1127        meta.maintenance_poisoned = true;
1128    }
1129
1130    if let Some(kind) = requeue_kind.filter(|_| !meta.maintenance_poisoned && !defer_requeue) {
1131        meta.maintenance_queued_kinds.push_back(kind);
1132    }
1133
1134    meta.maintenance_jobs_in_flight = meta.maintenance_jobs_in_flight.saturating_sub(1);
1135    meta.maintenance_pending =
1136        meta.maintenance_jobs_in_flight > 0 || !meta.maintenance_queued_kinds.is_empty();
1137}
1138
1139fn route_key(channel: u16, epoch: u32) -> RouteChannel {
1140    RouteChannel { channel, epoch }
1141}
1142
1143fn remove_installed_route(installed_epochs: &mut HashMap<u16, u32>, route: RouteChannel) {
1144    if installed_epochs.get(&route.channel).copied() == Some(route.epoch) {
1145        installed_epochs.remove(&route.channel);
1146    }
1147}
1148
1149fn ingress_route_should_be_processed(
1150    installed_epochs: &HashMap<u16, u32>,
1151    reclaimed_routes: &ReclaimedRoutes,
1152    frame: &Frame,
1153) -> bool {
1154    if frame.header.channel == 0
1155        || installed_epochs.get(&frame.header.channel).copied() == Some(frame.header.epoch)
1156    {
1157        return true;
1158    }
1159
1160    // A late request for a reclaimed root reaches the normal unknown-route
1161    // handler, which returns the typed `route_not_bound` error. Other stale or
1162    // never-installed generations remain silent so they cannot affect a newer
1163    // route or change the protocol's rejected-bind behavior.
1164    frame.header.ty == FrameType::Request
1165        && reclaimed_routes.contains(route_key(frame.header.channel, frame.header.epoch))
1166}
1167
1168fn bash_elicitation_timeout() -> Duration {
1169    if cfg!(debug_assertions) {
1170        if let Ok(raw) = std::env::var("AFT_TEST_SUBC_BASH_ELICITATION_TTL_MS") {
1171            if let Ok(ms) = raw.parse::<u64>() {
1172                if ms > 0 {
1173                    return Duration::from_millis(ms);
1174                }
1175            }
1176        }
1177    }
1178    BASH_ELICITATION_TIMEOUT
1179}
1180
1181fn allocate_reverse_corr(
1182    pending_bash_asks: &HashMap<ReverseCorrKey, PendingBashAsk>,
1183    route: RouteChannel,
1184    next_corr: &mut u64,
1185) -> u64 {
1186    loop {
1187        let corr = *next_corr;
1188        *next_corr = (*next_corr).wrapping_add(1).max(1);
1189        if !pending_bash_asks.contains_key(&ReverseCorrKey { route, corr }) {
1190            return corr;
1191        }
1192    }
1193}
1194
1195fn bash_permission_kind_label(kind: &crate::bash_permissions::PermissionKind) -> &'static str {
1196    match kind {
1197        crate::bash_permissions::PermissionKind::ExternalDirectory => "external directory",
1198        crate::bash_permissions::PermissionKind::Bash => "bash",
1199    }
1200}
1201
1202fn bash_elicitation_patterns(asks: &[crate::bash_permissions::PermissionAsk]) -> Vec<String> {
1203    let mut patterns = Vec::new();
1204    let mut seen = HashSet::new();
1205    for ask in asks {
1206        for pattern in ask.patterns.iter().chain(ask.always.iter()) {
1207            if seen.insert(pattern.clone()) {
1208                patterns.push(pattern.clone());
1209            }
1210        }
1211    }
1212    patterns
1213}
1214
1215fn bash_elicitation_message(
1216    command: &str,
1217    asks: &[crate::bash_permissions::PermissionAsk],
1218) -> String {
1219    let command = command.split_whitespace().collect::<Vec<_>>().join(" ");
1220    let patterns = bash_elicitation_patterns(asks);
1221    let pattern_text = if patterns.is_empty() {
1222        "no matched permission patterns".to_string()
1223    } else {
1224        patterns.join(", ")
1225    };
1226    let ask_kinds = asks
1227        .iter()
1228        .map(|ask| bash_permission_kind_label(&ask.kind))
1229        .collect::<HashSet<_>>()
1230        .into_iter()
1231        .collect::<Vec<_>>()
1232        .join(", ");
1233    if ask_kinds.is_empty() {
1234        format!("Allow bash command `{command}`? Matched patterns: {pattern_text}")
1235    } else {
1236        format!("Allow bash command `{command}`? Matched {ask_kinds} patterns: {pattern_text}")
1237    }
1238}
1239
1240fn bash_elicitation_request_body(
1241    command: &str,
1242    asks: &[crate::bash_permissions::PermissionAsk],
1243) -> Value {
1244    json!({
1245        "method": BASH_ELICITATION_CREATE_METHOD,
1246        "params": {
1247            "mode": "form",
1248            "message": bash_elicitation_message(command, asks),
1249            "requestedSchema": {
1250                "type": "object",
1251                "properties": {
1252                    "decision": {
1253                        "type": "string",
1254                        "enum": ["allow", "deny"],
1255                        "description": "Choose allow to run this bash command once, or deny to block it."
1256                    }
1257                },
1258                "required": ["decision"],
1259                "additionalProperties": false
1260            },
1261            "_meta": {
1262                "aft": {
1263                    "tool": "bash",
1264                    "command": command,
1265                    "asks": asks
1266                }
1267            }
1268        }
1269    })
1270}
1271
1272fn build_bash_elicitation_request_frame(
1273    ver: u8,
1274    route: RouteChannel,
1275    corr: u64,
1276    flags: Flags,
1277    command: &str,
1278    asks: &[crate::bash_permissions::PermissionAsk],
1279) -> Result<Frame, SubcError> {
1280    let body = bash_elicitation_request_body(command, asks);
1281    Frame::build_with_version(
1282        ver,
1283        FrameType::Request,
1284        flags,
1285        route.channel,
1286        route.epoch,
1287        corr,
1288        serde_json::to_vec(&body).map_err(SubcError::Json)?,
1289    )
1290    .map_err(SubcError::FrameBuild)
1291}
1292
1293fn bash_elicitation_reply_is_allow(body: &[u8]) -> bool {
1294    let Ok(value) = serde_json::from_slice::<Value>(body) else {
1295        return false;
1296    };
1297    flat_bash_elicitation_reply_is_allow(&value) || mcp_bash_elicitation_reply_is_allow(&value)
1298}
1299
1300fn flat_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1301    let Some(object) = value.as_object() else {
1302        return false;
1303    };
1304    object.len() == 1 && object.get("decision").and_then(Value::as_str) == Some("allow")
1305}
1306
1307fn mcp_bash_elicitation_reply_is_allow(value: &Value) -> bool {
1308    let Some(object) = value.as_object() else {
1309        return false;
1310    };
1311    if object.len() != 2 || object.get("action").and_then(Value::as_str) != Some("accept") {
1312        return false;
1313    }
1314    let Some(content) = object.get("content").and_then(Value::as_object) else {
1315        return false;
1316    };
1317    content.len() == 1 && content.get("decision").and_then(Value::as_str) == Some("allow")
1318}
1319
1320#[allow(clippy::too_many_arguments)]
1321async fn settle_pending_bash_ask_denied(
1322    tx: &WriterSender,
1323    pending: PendingBashAsk,
1324    routes: &HashMap<RouteChannel, RouteIdentity>,
1325    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1326    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1327    shutdown: &Arc<Notify>,
1328    metrics: &DispatchPathMetrics,
1329) -> Result<(), SubcError> {
1330    let completion = bash::bash_denied_untrusted_completion(
1331        pending.route,
1332        pending.tool_corr,
1333        pending.tool_flags,
1334        pending.tool_ver,
1335        pending.root,
1336        pending.request_id,
1337        pending.format_context,
1338    );
1339    bash::handle_bash_deferred_completion(
1340        tx,
1341        completion,
1342        routes,
1343        live_roots,
1344        route_bash_cancels,
1345        shutdown,
1346        metrics,
1347    )
1348    .await
1349}
1350
1351fn take_pending_bash_asks_for_route(
1352    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1353    route: RouteChannel,
1354) -> Vec<PendingBashAsk> {
1355    let keys = pending_bash_asks
1356        .keys()
1357        .copied()
1358        .filter(|key| key.route == route)
1359        .collect::<Vec<_>>();
1360    keys.into_iter()
1361        .filter_map(|key| pending_bash_asks.remove(&key))
1362        .collect()
1363}
1364
1365#[allow(clippy::too_many_arguments)]
1366async fn settle_pending_bash_asks_for_route(
1367    tx: &WriterSender,
1368    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1369    route: RouteChannel,
1370    routes: &HashMap<RouteChannel, RouteIdentity>,
1371    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1372    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1373    shutdown: &Arc<Notify>,
1374    metrics: &DispatchPathMetrics,
1375) -> Result<(), SubcError> {
1376    for pending in take_pending_bash_asks_for_route(pending_bash_asks, route) {
1377        settle_pending_bash_ask_denied(
1378            tx,
1379            pending,
1380            routes,
1381            live_roots,
1382            route_bash_cancels,
1383            shutdown,
1384            metrics,
1385        )
1386        .await?;
1387    }
1388    Ok(())
1389}
1390
1391#[allow(clippy::too_many_arguments)]
1392async fn settle_all_pending_bash_asks(
1393    tx: &WriterSender,
1394    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1395    routes: &HashMap<RouteChannel, RouteIdentity>,
1396    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1397    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1398    shutdown: &Arc<Notify>,
1399    metrics: &DispatchPathMetrics,
1400) -> Result<(), SubcError> {
1401    let pending = pending_bash_asks
1402        .drain()
1403        .map(|(_, pending)| pending)
1404        .collect::<Vec<_>>();
1405    for pending in pending {
1406        settle_pending_bash_ask_denied(
1407            tx,
1408            pending,
1409            routes,
1410            live_roots,
1411            route_bash_cancels,
1412            shutdown,
1413            metrics,
1414        )
1415        .await?;
1416    }
1417    Ok(())
1418}
1419
1420#[allow(clippy::too_many_arguments)]
1421async fn expire_pending_bash_asks(
1422    tx: &WriterSender,
1423    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1424    routes: &HashMap<RouteChannel, RouteIdentity>,
1425    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1426    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1427    shutdown: &Arc<Notify>,
1428    metrics: &DispatchPathMetrics,
1429) -> Result<(), SubcError> {
1430    let now = Instant::now();
1431    let expired = pending_bash_asks
1432        .iter()
1433        .filter_map(|(key, pending)| (pending.expires_at <= now).then_some(*key))
1434        .collect::<Vec<_>>();
1435    for key in expired {
1436        if let Some(pending) = pending_bash_asks.remove(&key) {
1437            log::debug!(
1438                "subc attach: bash elicitation request {} on route {} expired fail-closed",
1439                key.corr,
1440                pending.route
1441            );
1442            settle_pending_bash_ask_denied(
1443                tx,
1444                pending,
1445                routes,
1446                live_roots,
1447                route_bash_cancels,
1448                shutdown,
1449                metrics,
1450            )
1451            .await?;
1452        }
1453    }
1454    Ok(())
1455}
1456
1457#[allow(clippy::too_many_arguments)]
1458async fn handle_bash_elicitation_reply(
1459    tx: &WriterSender,
1460    frame: &Frame,
1461    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1462    routes: &HashMap<RouteChannel, RouteIdentity>,
1463    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1464    executor: &Arc<Executor>,
1465    shutdown: &Arc<Notify>,
1466    bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
1467    bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
1468    metrics: &Arc<DispatchPathMetrics>,
1469    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1470    dispatch: DispatchFn,
1471) -> Result<(), SubcError> {
1472    let key = ReverseCorrKey {
1473        route: route_key(frame.header.channel, frame.header.epoch),
1474        corr: frame.header.corr,
1475    };
1476    let Some(pending) = pending_bash_asks.remove(&key) else {
1477        return Ok(());
1478    };
1479
1480    if frame.header.ty == FrameType::Response && bash_elicitation_reply_is_allow(&frame.body) {
1481        if routes.contains_key(&key.route) {
1482            bash::submit_deferred_bash(
1483                executor,
1484                bash_deferred_tx,
1485                bash_poll_touch_tx,
1486                metrics,
1487                dispatch,
1488                pending.root,
1489                pending.project_root,
1490                pending.session_id,
1491                pending.request_id,
1492                pending.route,
1493                pending.tool_corr,
1494                pending.tool_flags,
1495                pending.tool_ver,
1496                pending.arguments,
1497                pending.format_context,
1498                pending.cancel,
1499                BindTrust::Untrusted,
1500                pending.spawn_principal,
1501                Some(pending.grants),
1502            );
1503            return Ok(());
1504        }
1505        log::debug!(
1506            "subc attach: dropping allowed bash elicitation reply {} for unbound route {}",
1507            key.corr,
1508            pending.route
1509        );
1510    }
1511
1512    settle_pending_bash_ask_denied(
1513        tx,
1514        pending,
1515        routes,
1516        live_roots,
1517        route_bash_cancels,
1518        shutdown,
1519        metrics,
1520    )
1521    .await
1522}
1523
1524#[allow(clippy::too_many_arguments)]
1525async fn cancel_pending_bash_ask_for_tool_call(
1526    tx: &WriterSender,
1527    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1528    route: RouteChannel,
1529    tool_corr: u64,
1530    routes: &HashMap<RouteChannel, RouteIdentity>,
1531    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1532    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1533    shutdown: &Arc<Notify>,
1534    metrics: &DispatchPathMetrics,
1535) -> Result<(), SubcError> {
1536    let keys = pending_bash_asks
1537        .iter()
1538        .filter_map(|(key, pending)| {
1539            (key.route == route && pending.tool_corr == tool_corr).then_some(*key)
1540        })
1541        .collect::<Vec<_>>();
1542    for key in keys {
1543        if let Some(pending) = pending_bash_asks.remove(&key) {
1544            settle_pending_bash_ask_denied(
1545                tx,
1546                pending,
1547                routes,
1548                live_roots,
1549                route_bash_cancels,
1550                shutdown,
1551                metrics,
1552            )
1553            .await?;
1554        }
1555    }
1556    Ok(())
1557}
1558
1559fn remove_root_channel(
1560    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1561    root: &ProjectRootId,
1562    channel: RouteChannel,
1563) {
1564    let remove_root = if let Some(channels) = root_channels.get_mut(root) {
1565        channels.remove(&channel);
1566        channels.is_empty()
1567    } else {
1568        false
1569    };
1570    if remove_root {
1571        root_channels.remove(root);
1572    }
1573}
1574
1575fn remove_route_channel(
1576    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1577    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1578    channel: RouteChannel,
1579) -> Option<RouteIdentity> {
1580    let removed = routes.remove(&channel);
1581    if let Some(identity) = &removed {
1582        remove_root_channel(root_channels, &identity.root, channel);
1583    }
1584    removed
1585}
1586
1587fn insert_route_channel(
1588    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1589    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1590    channel: RouteChannel,
1591    identity: RouteIdentity,
1592) {
1593    if let Some(previous) = routes.insert(channel, identity.clone()) {
1594        remove_root_channel(root_channels, &previous.root, channel);
1595    }
1596    root_channels
1597        .entry(identity.root.clone())
1598        .or_default()
1599        .insert(channel);
1600}
1601
1602fn remove_bg_subscription_index(
1603    bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1604    channel: RouteChannel,
1605    identity: Option<&RouteIdentity>,
1606) {
1607    if let Some(identity) = identity {
1608        let key = (identity.root.clone(), identity.session.clone());
1609        if bg_sub_by_session.get(&key).copied() == Some(channel) {
1610            bg_sub_by_session.remove(&key);
1611        }
1612    } else {
1613        bg_sub_by_session.retain(|_, mapped_channel| *mapped_channel != channel);
1614    }
1615}
1616
1617fn route_removal_will_quiesce_root(
1618    root: &ProjectRootId,
1619    route: RouteChannel,
1620    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1621    has_pending_bind: bool,
1622    replacement_root: Option<&ProjectRootId>,
1623) -> bool {
1624    let removes_last_route = root_channels
1625        .get(root)
1626        .is_some_and(|channels| channels.len() == 1 && channels.contains(&route));
1627    removes_last_route && !has_pending_bind && replacement_root != Some(root)
1628}
1629
1630fn should_quiesce_removed_root(
1631    root: &ProjectRootId,
1632    root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
1633    has_pending_bind: bool,
1634    replacement_root: Option<&ProjectRootId>,
1635) -> bool {
1636    !root_channels.contains_key(root) && !has_pending_bind && replacement_root != Some(root)
1637}
1638
1639async fn end_bg_subscription(
1640    writer_tx: &WriterSender,
1641    metrics: &DispatchPathMetrics,
1642    bg_subs: &mut HashMap<RouteChannel, BgSub>,
1643    bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1644    bg_wake_pending: &mut HashSet<RouteChannel>,
1645    channel: RouteChannel,
1646    identity: Option<&RouteIdentity>,
1647) -> Result<(), SubcError> {
1648    if let Some(sub) = bg_subs.remove(&channel) {
1649        bg_wake_pending.remove(&channel);
1650        remove_bg_subscription_index(bg_sub_by_session, channel, identity);
1651        push::send_reliable_bg_stream_end(writer_tx, metrics, channel, &sub).await?;
1652    }
1653    Ok(())
1654}
1655
1656#[allow(clippy::too_many_arguments)]
1657async fn teardown_installed_route(
1658    tx: &WriterSender,
1659    metrics: &DispatchPathMetrics,
1660    executor: &Arc<Executor>,
1661    channel: RouteChannel,
1662    cancellation_reason: &str,
1663    replacement_root: Option<&ProjectRootId>,
1664    installed_route_epochs: &mut HashMap<u16, u32>,
1665    routes: &mut HashMap<RouteChannel, RouteIdentity>,
1666    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
1667    bg_subs: &mut HashMap<RouteChannel, BgSub>,
1668    bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
1669    bg_wake_pending: &mut HashSet<RouteChannel>,
1670    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
1671    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
1672    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
1673    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
1674    retry_buffer: &mut RetryBuffer,
1675    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
1676    shutdown: &Arc<Notify>,
1677) -> Result<(), SubcError> {
1678    remove_installed_route(installed_route_epochs, channel);
1679    end_bg_subscription(
1680        tx,
1681        metrics,
1682        bg_subs,
1683        bg_sub_by_session,
1684        bg_wake_pending,
1685        channel,
1686        routes.get(&channel),
1687    )
1688    .await?;
1689    settle_pending_bash_asks_for_route(
1690        tx,
1691        pending_bash_asks,
1692        channel,
1693        routes,
1694        live_roots,
1695        route_bash_cancels,
1696        shutdown,
1697        metrics,
1698    )
1699    .await?;
1700    if let Some(cancel) = route_bash_cancels.remove(&channel) {
1701        cancel.token.cancel();
1702    }
1703    if let Some(pending) = pending_binds.get_mut(&channel) {
1704        pending.cancelled = true;
1705        let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
1706        log::debug!(
1707            "subc attach: cancelled pending RouteBind for route {} on {cancellation_reason} (configure job: {outcome:?})",
1708            channel.channel
1709        );
1710    }
1711    let migrated = push::migrate_retry_buffer_to_push_buffer(retry_buffer, channel, push_buffer);
1712    if let Some(identity) = routes.get(&channel) {
1713        let has_pending_bind = pending_binds
1714            .values()
1715            .any(|pending| pending.bind_root_id == identity.root);
1716        if route_removal_will_quiesce_root(
1717            &identity.root,
1718            channel,
1719            root_channels,
1720            has_pending_bind,
1721            replacement_root,
1722        ) {
1723            if let Some(ctx) = executor.actor_context(&identity.root) {
1724                // Fence deferred admissions before the final route disappears
1725                // from the loop-owned routing tables.
1726                ctx.mark_subc_unbound();
1727            }
1728        }
1729    }
1730    if let Some(identity) = remove_route_channel(routes, root_channels, channel) {
1731        if migrated > 0 {
1732            log::debug!(
1733                "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from route {} into detach replay",
1734                channel.channel
1735            );
1736        }
1737        if let Some(meta) = live_roots.get_mut(&identity.root) {
1738            let idle_for = meta.last_touched.elapsed();
1739            meta.note_activity();
1740            log::debug!(
1741                "subc attach: route {} torn down for root {} harness {} session {} (last touched {:?} ago)",
1742                channel.channel,
1743                identity.root.as_path().display(),
1744                identity.harness,
1745                identity.session,
1746                idle_for
1747            );
1748        } else {
1749            log::debug!(
1750                "subc attach: route {} torn down for root {} harness {} session {}",
1751                channel.channel,
1752                identity.root.as_path().display(),
1753                identity.harness,
1754                identity.session
1755            );
1756        }
1757        let has_pending_bind = pending_binds
1758            .values()
1759            .any(|pending| pending.bind_root_id == identity.root);
1760        if should_quiesce_removed_root(
1761            &identity.root,
1762            root_channels,
1763            has_pending_bind,
1764            replacement_root,
1765        ) {
1766            quiesce_unbound_root(&identity.root, live_roots, executor);
1767        }
1768    } else {
1769        if migrated > 0 {
1770            log::debug!(
1771                "subc attach: migrated {migrated} retry-buffered reliable Push frame(s) from unbound route {} into detach replay",
1772                channel.channel
1773            );
1774        }
1775        log::debug!("subc attach: unbound route {} torn down", channel.channel);
1776    }
1777    Ok(())
1778}
1779
1780fn remember_session_identity(
1781    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1782    identity: &RouteIdentity,
1783) {
1784    let key = (identity.root.clone(), identity.session.clone());
1785    if matches!(identity.trust, BindTrust::Untrusted)
1786        && session_identity
1787            .get(&key)
1788            .is_some_and(|retained| matches!(retained.trust, BindTrust::FirstParty))
1789    {
1790        return;
1791    }
1792
1793    // Retained after route Goodbye so reliable session-scoped frames emitted while
1794    // the session is detached can still be keyed by the full (root,harness,session)
1795    // replay triple. Untrusted binds never overwrite a retained first-party
1796    // session identity, because bash completion replay is an observation channel.
1797    session_identity.insert(
1798        key,
1799        RetainedSessionIdentity {
1800            harness: identity.harness.clone(),
1801            trust: identity.trust,
1802        },
1803    );
1804}
1805
1806fn replay_key_for_session(
1807    session_identity: &HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
1808    root: &ProjectRootId,
1809    session: &str,
1810) -> Option<(push::ReplayKey, BindTrust)> {
1811    let retained = session_identity.get(&(root.clone(), session.to_string()))?;
1812    Some((
1813        push::ReplayKey {
1814            root: root.clone(),
1815            harness: retained.harness.clone(),
1816            session: session.to_string(),
1817        },
1818        retained.trust,
1819    ))
1820}
1821/// Sync command dispatch, passed in from `main` (the binary owns the command
1822/// table). Invoked only inside executor jobs in subc mode.
1823pub type DispatchFn = fn(RawRequest, &AppContext) -> Response;
1824
1825#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1826enum ModuleLoopExit {
1827    Graceful,
1828    SkipSearchFlush,
1829}
1830
1831/// Entry point for `aft --subc <connection-file>`. Synchronous on the outside;
1832/// owns an isolated current-thread tokio runtime for the async transport.
1833/// Returns `Err` (fail-loud) on any connect/auth/protocol failure — we never
1834/// fall back to the standalone loop, to avoid split-brain index state.
1835pub fn run_subc_mode(
1836    connection_file_path: &Path,
1837    ctx: Arc<AppContext>,
1838    executor: Arc<Executor>,
1839    dispatch: DispatchFn,
1840    user_config_path: Option<PathBuf>,
1841) -> Result<(), SubcError> {
1842    // Production NEVER allows non-manifest tool names on route channels: AFT
1843    // fails closed and does not trust subc to enforce the manifest. The
1844    // test-only harness sets this through `run_subc_mode_for_test`.
1845    run_subc_mode_inner(
1846        connection_file_path,
1847        ctx,
1848        executor,
1849        dispatch,
1850        user_config_path,
1851        false,
1852    )
1853}
1854
1855fn run_subc_mode_inner(
1856    connection_file_path: &Path,
1857    ctx: Arc<AppContext>,
1858    executor: Arc<Executor>,
1859    dispatch: DispatchFn,
1860    user_config_path: Option<PathBuf>,
1861    allow_native_passthrough: bool,
1862) -> Result<(), SubcError> {
1863    let runtime = tokio::runtime::Builder::new_current_thread()
1864        .enable_all()
1865        .build()
1866        .map_err(SubcError::Runtime)?;
1867
1868    let executor_for_loop = Arc::clone(&executor);
1869    let loop_result = runtime.block_on(async move {
1870        let shared_app = ctx.app();
1871        drop(ctx);
1872        let stream = connect_and_authenticate(connection_file_path).await?;
1873        log::info!(
1874            "subc attach: authenticated to daemon via {}",
1875            connection_file_path.display()
1876        );
1877        let (read_half, write_half) = tokio::io::split(stream);
1878        run_module_loop(
1879            read_half,
1880            write_half,
1881            shared_app,
1882            executor_for_loop,
1883            dispatch,
1884            user_config_path,
1885            allow_native_passthrough,
1886        )
1887        .await
1888    });
1889
1890    let actor_contexts = executor.actor_contexts();
1891    if matches!(loop_result, Ok(ModuleLoopExit::Graceful)) {
1892        // EOF/Goodbye teardown flushes each root's index deltas and queued
1893        // callgraph refreshes. Fatal/panic teardown skips this best-effort work.
1894        flush_actor_indexes_on_graceful_shutdown(&actor_contexts);
1895    }
1896    for actor_ctx in &actor_contexts {
1897        actor_ctx.lsp().shutdown_all();
1898        actor_ctx.bash_background().detach();
1899    }
1900
1901    loop_result.map(|_| ())
1902}
1903
1904fn flush_actor_indexes_on_graceful_shutdown(actor_contexts: &[Arc<AppContext>]) {
1905    for actor_ctx in actor_contexts {
1906        let _ = actor_ctx.flush_search_index_on_graceful_shutdown();
1907    }
1908    let _ = crate::callgraph_store::flush_callgraph_store_refreshes_on_graceful_shutdown();
1909}
1910
1911/// Test-only entry that enables the non-manifest native-command passthrough on
1912/// route channels. Integration tests drive synthetic native commands (`glob`,
1913/// `callers`, `subc_test_echo_session`, …) through the executor to exercise
1914/// mechanics; production callers use [`run_subc_mode`], which fails closed.
1915#[doc(hidden)]
1916pub fn run_subc_mode_for_test(
1917    connection_file_path: &Path,
1918    ctx: Arc<AppContext>,
1919    executor: Arc<Executor>,
1920    dispatch: DispatchFn,
1921    user_config_path: Option<PathBuf>,
1922) -> Result<(), SubcError> {
1923    run_subc_mode_inner(
1924        connection_file_path,
1925        ctx,
1926        executor,
1927        dispatch,
1928        user_config_path,
1929        true,
1930    )
1931}
1932
1933#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1934enum AttachErrorClass {
1935    Transient,
1936    Permanent,
1937}
1938
1939impl fmt::Display for AttachErrorClass {
1940    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1941        match self {
1942            Self::Transient => f.write_str("transient"),
1943            Self::Permanent => f.write_str("permanent"),
1944        }
1945    }
1946}
1947
1948#[derive(Clone, Copy)]
1949struct AttachRetryPolicy {
1950    budget: Duration,
1951    initial_backoff: Duration,
1952    max_backoff: Duration,
1953    jitter_percent: u64,
1954}
1955
1956const ATTACH_RETRY_POLICY: AttachRetryPolicy = AttachRetryPolicy {
1957    budget: ATTACH_RETRY_BUDGET,
1958    initial_backoff: ATTACH_RETRY_INITIAL_BACKOFF,
1959    max_backoff: ATTACH_RETRY_MAX_BACKOFF,
1960    jitter_percent: ATTACH_RETRY_JITTER_PERCENT,
1961};
1962
1963/// Retry only failures that can be caused by a daemon bounce or an interrupted
1964/// handshake. Protocol and credential failures are permanent for this process.
1965fn classify_attach_error(error: &SubcError) -> AttachErrorClass {
1966    let transient = match error {
1967        SubcError::Connect { source, .. } => is_transient_attach_io(source.kind()),
1968        SubcError::Auth { source, .. } => match source {
1969            subc_transport::AuthError::Timeout { .. }
1970            | subc_transport::AuthError::UnexpectedEof { .. } => true,
1971            subc_transport::AuthError::Io { source, .. } => is_transient_attach_io(source.kind()),
1972            _ => false,
1973        },
1974        _ => false,
1975    };
1976    if transient {
1977        AttachErrorClass::Transient
1978    } else {
1979        AttachErrorClass::Permanent
1980    }
1981}
1982
1983fn is_transient_attach_io(kind: io::ErrorKind) -> bool {
1984    matches!(
1985        kind,
1986        io::ErrorKind::ConnectionRefused
1987            | io::ErrorKind::TimedOut
1988            | io::ErrorKind::ConnectionReset
1989            | io::ErrorKind::ConnectionAborted
1990            | io::ErrorKind::BrokenPipe
1991            | io::ErrorKind::UnexpectedEof
1992    )
1993}
1994
1995/// Read the connection file → resolve the first endpoint → TCP connect → HMAC
1996/// handshake. Transient initial-attach failures retry on fresh sockets and reread
1997/// the file so a daemon bounce can publish a new endpoint or authentication key.
1998async fn connect_and_authenticate(connection_file_path: &Path) -> Result<TcpStream, SubcError> {
1999    connect_and_authenticate_with_policy(connection_file_path, ATTACH_RETRY_POLICY).await
2000}
2001
2002async fn connect_and_authenticate_with_policy(
2003    connection_file_path: &Path,
2004    policy: AttachRetryPolicy,
2005) -> Result<TcpStream, SubcError> {
2006    let started_at = Instant::now();
2007    let deadline = started_at + policy.budget;
2008    let mut attempt = 0_u32;
2009    let mut backoff = policy.initial_backoff;
2010    let mut history = Vec::new();
2011
2012    loop {
2013        attempt = attempt.saturating_add(1);
2014        let error = match connect_and_authenticate_once(connection_file_path, deadline).await {
2015            Ok(stream) => return Ok(stream),
2016            Err(error) => error,
2017        };
2018        let class = classify_attach_error(&error);
2019        let error_text = error.to_string().lines().collect::<Vec<_>>().join(" ");
2020        history.push(format!("attempt {attempt} [{class}]: {error_text}"));
2021
2022        if class == AttachErrorClass::Permanent {
2023            log_attach_final_failure(started_at.elapsed(), &history);
2024            return Err(error);
2025        }
2026
2027        let remaining = deadline.saturating_duration_since(Instant::now());
2028        if remaining.is_zero() {
2029            log_attach_final_failure(started_at.elapsed(), &history);
2030            return Err(error);
2031        }
2032
2033        let delay = jittered_attach_delay(backoff, policy.jitter_percent, attempt).min(remaining);
2034        log::info!(
2035            "subc attach retry: attempt {attempt} failed; error_class={class}; error={error_text}; next_delay={delay:?}"
2036        );
2037        tokio::time::sleep(delay).await;
2038
2039        if Instant::now() >= deadline {
2040            log_attach_final_failure(started_at.elapsed(), &history);
2041            return Err(error);
2042        }
2043        backoff = backoff.saturating_mul(2).min(policy.max_backoff);
2044    }
2045}
2046
2047fn jittered_attach_delay(base: Duration, jitter_percent: u64, attempt: u32) -> Duration {
2048    let jitter_percent = jitter_percent.min(100);
2049    if jitter_percent == 0 {
2050        return base;
2051    }
2052
2053    let mut random_bytes = [0_u8; 8];
2054    let random = if getrandom::fill(&mut random_bytes).is_ok() {
2055        u64::from_le_bytes(random_bytes)
2056    } else {
2057        let timestamp = std::time::SystemTime::now()
2058            .duration_since(std::time::UNIX_EPOCH)
2059            .unwrap_or_default()
2060            .subsec_nanos();
2061        u64::from(timestamp) ^ u64::from(attempt)
2062    };
2063    let span = jitter_percent.saturating_mul(2).saturating_add(1);
2064    let multiplier_percent = 100 - jitter_percent + random % span;
2065    let base_millis = u64::try_from(base.as_millis()).unwrap_or(u64::MAX);
2066    Duration::from_millis(base_millis.saturating_mul(multiplier_percent) / 100)
2067}
2068
2069fn log_attach_final_failure(elapsed: Duration, history: &[String]) {
2070    log::error!(
2071        "subc initial attach failed after {} attempt(s) in {elapsed:?}; attempt history: {}",
2072        history.len(),
2073        history.join(" | ")
2074    );
2075}
2076
2077async fn connect_and_authenticate_once(
2078    connection_file_path: &Path,
2079    deadline: Instant,
2080) -> Result<TcpStream, SubcError> {
2081    // This read intentionally lives inside the per-attempt function. The daemon
2082    // publishes connection files atomically and may change both port and key.
2083    let conn = connection_file::read_for_client(connection_file_path).map_err(|source| {
2084        SubcError::ConnectionFile {
2085            path: connection_file_path.to_path_buf(),
2086            source,
2087        }
2088    })?;
2089
2090    let endpoint = conn
2091        .endpoints
2092        .first()
2093        .ok_or_else(|| SubcError::NoEndpoint {
2094            path: connection_file_path.to_path_buf(),
2095        })?;
2096    let endpoint_label = format!("{}:{}", endpoint.host, endpoint.port);
2097    let ip = endpoint
2098        .host
2099        .parse::<IpAddr>()
2100        .map_err(|_| SubcError::InvalidEndpoint {
2101            path: connection_file_path.to_path_buf(),
2102            endpoint: endpoint_label.clone(),
2103        })?;
2104    let addr = SocketAddr::new(ip, endpoint.port);
2105
2106    let connect_budget = deadline.saturating_duration_since(Instant::now());
2107    let mut stream = tokio::time::timeout(connect_budget, TcpStream::connect(addr))
2108        .await
2109        .map_err(|_| SubcError::Connect {
2110            endpoint: endpoint_label.clone(),
2111            source: io::Error::new(
2112                io::ErrorKind::TimedOut,
2113                "initial subc attach retry budget elapsed during TCP connect",
2114            ),
2115        })?
2116        .map_err(|source| SubcError::Connect {
2117            endpoint: endpoint_label.clone(),
2118            source,
2119        })?;
2120    stream
2121        .set_nodelay(true)
2122        .map_err(|source| SubcError::Connect {
2123            endpoint: endpoint_label.clone(),
2124            source,
2125        })?;
2126
2127    let auth_budget = AUTH_DEADLINE.min(deadline.saturating_duration_since(Instant::now()));
2128    authenticate_client(&mut stream, &conn, auth_budget)
2129        .await
2130        .map_err(|source| SubcError::Auth {
2131            endpoint: endpoint_label,
2132            source,
2133        })?;
2134
2135    Ok(stream)
2136}
2137
2138#[allow(clippy::too_many_arguments)]
2139async fn process_route_bind_completion(
2140    writer_tx: &WriterSender,
2141    completion: RouteBindCompletion,
2142    routes: &mut HashMap<RouteChannel, RouteIdentity>,
2143    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2144    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2145    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2146    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2147    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2148    installed_route_epochs: &mut HashMap<u16, u32>,
2149    executor: &Arc<Executor>,
2150    shutdown: &Arc<Notify>,
2151    metrics: &Arc<DispatchPathMetrics>,
2152) -> Result<(), SubcError> {
2153    decrement_counted_channel(&metrics.control_completion_queued);
2154    handle_route_bind_completion(
2155        writer_tx,
2156        completion,
2157        routes,
2158        root_channels,
2159        session_identity,
2160        push_buffer,
2161        live_roots,
2162        pending_binds,
2163        installed_route_epochs,
2164        executor,
2165        shutdown,
2166        metrics,
2167    )
2168    .await
2169}
2170
2171#[allow(clippy::too_many_arguments)]
2172async fn drain_pending_route_bind_completions(
2173    control_completion_rx: &mut mpsc::Receiver<RouteBindCompletion>,
2174    writer_tx: &WriterSender,
2175    routes: &mut HashMap<RouteChannel, RouteIdentity>,
2176    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
2177    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
2178    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
2179    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
2180    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
2181    installed_route_epochs: &mut HashMap<u16, u32>,
2182    executor: &Arc<Executor>,
2183    shutdown: &Arc<Notify>,
2184    metrics: &Arc<DispatchPathMetrics>,
2185) -> Result<usize, SubcError> {
2186    let mut drained = 0;
2187    while let Ok(completion) = control_completion_rx.try_recv() {
2188        process_route_bind_completion(
2189            writer_tx,
2190            completion,
2191            routes,
2192            root_channels,
2193            session_identity,
2194            push_buffer,
2195            live_roots,
2196            pending_binds,
2197            installed_route_epochs,
2198            executor,
2199            shutdown,
2200            metrics,
2201        )
2202        .await?;
2203        drained += 1;
2204    }
2205    Ok(drained)
2206}
2207
2208/// ModuleHello → HelloAck → control/route loop. Runs until the daemon closes
2209/// the connection (EOF), sends channel-0 Goodbye, or a fatal mutating executor
2210/// response requests whole-connection teardown.
2211async fn run_module_loop<R, W>(
2212    mut read: R,
2213    mut write: W,
2214    shared_app: Arc<App>,
2215    executor: Arc<Executor>,
2216    dispatch: DispatchFn,
2217    user_config_path: Option<PathBuf>,
2218    allow_native_passthrough: bool,
2219) -> Result<ModuleLoopExit, SubcError>
2220where
2221    R: AsyncRead + Unpin + Send + 'static,
2222    W: AsyncWrite + Unpin + Send + 'static,
2223{
2224    // ModuleHello: register as a tool provider and advertise the supported control-plane operations.
2225    // Echo the one-time launch nonce the daemon injected via SUBC_LAUNCH_NONCE so a
2226    // reserved module_id's HELLO is accepted; absent for non-reserved/self-connect.
2227    let hello = ModuleHelloBody {
2228        manifest: build_manifest(),
2229        protocol_ver: PROTOCOL_VERSION,
2230        control_ops: control_ops(),
2231        launch_nonce: std::env::var("SUBC_LAUNCH_NONCE").ok(),
2232    };
2233    let hello_frame = Frame::build(
2234        FrameType::Hello,
2235        control_flags(),
2236        0,
2237        0,
2238        HELLO_CORR,
2239        serde_json::to_vec(&hello).map_err(SubcError::Json)?,
2240    )
2241    .map_err(SubcError::FrameBuild)?;
2242    write_frame(&mut write, &hello_frame)
2243        .await
2244        .map_err(SubcError::FrameIo)?;
2245
2246    // Expect HelloAck (registered) or a channel-0 Error (manifest/version reject).
2247    match read_frame(&mut read).await.map_err(SubcError::FrameIo)? {
2248        None => return Err(SubcError::ClosedBeforeHelloAck),
2249        Some(frame) => match frame.header.ty {
2250            FrameType::HelloAck => {
2251                log::info!("subc attach: registered (HelloAck received)");
2252            }
2253            FrameType::Error => {
2254                let body = serde_json::from_slice::<ErrorBody>(&frame.body).ok();
2255                return Err(SubcError::HelloRejected { body });
2256            }
2257            other => return Err(SubcError::UnexpectedFrame { ty: other }),
2258        },
2259    }
2260
2261    let dispatch_path_metrics = Arc::new(DispatchPathMetrics::new());
2262    let (writer_tx, writer_rx) = mpsc::channel::<WriterFrame>(WRITER_QUEUE_CAPACITY);
2263    let writer_task = spawn_writer_task(write, writer_rx, Arc::clone(&dispatch_path_metrics));
2264    // `read_frame` is NOT cancellation-safe, so it must never sit directly inside
2265    // the `select!` below: a drain-interval tick (or shutdown) firing while a
2266    // frame is mid-transit would drop the partially-consumed bytes and desync the
2267    // stream (the next read would parse a body byte as a frame header). A
2268    // dedicated reader task owns the socket, reads whole frames sequentially, and
2269    // forwards them over a channel; the loop selects on the cancel-safe `recv()`.
2270    let (reader_tx, mut reader_rx) = mpsc::channel::<Result<DecodedFrame, SubcError>>(256);
2271    let reader_task = spawn_reader_task(read, reader_tx);
2272    let shutdown = Arc::new(Notify::new());
2273    // Drain-tick deadline is tracked manually and checked at the TOP of every
2274    // loop turn rather than as an Interval select arm: the select below is
2275    // `biased` (bind completions first), and biased polling means a saturated
2276    // higher arm (sustained lossy push traffic keeps lossy_rx always-ready)
2277    // would starve every arm below it, including a timer arm — leaving
2278    // backpressured reliable frames parked in the retry buffer past their
2279    // delivery deadline. The pre-turn check cannot be starved by arm order;
2280    // the sleep_until arm below only exists to wake an otherwise-idle loop.
2281    let mut next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2282    let mut next_maintenance_at = next_drain_at;
2283    // Rate-limit stamp for opportunistic macOS allocator slack relief (checked
2284    // on the maintenance tick; policy shared with standalone via memory.rs).
2285    #[cfg(target_os = "macos")]
2286    let mut last_slack_relief: Option<std::time::Instant> = None;
2287    let (maintenance_tx, mut maintenance_rx) = mpsc::channel::<MaintenanceCompletion>(256);
2288    let (bash_deferred_tx, mut bash_deferred_rx) =
2289        mpsc::channel::<bash::BashDeferredCompletion>(256);
2290    let (bash_poll_touch_tx, mut bash_poll_touch_rx) = mpsc::channel::<ProjectRootId>(256);
2291    let (control_completion_tx, mut control_completion_rx) =
2292        mpsc::channel::<RouteBindCompletion>(256);
2293    let (lossy_tx, mut lossy_rx) = mpsc::channel::<LossyPushEnvelope>(1024);
2294    let lossy_overflow = Arc::new(push::LossyOverflow::default());
2295    let lossy_seq = Arc::new(AtomicU64::new(0));
2296    let (reliable_tx, mut reliable_rx) = mpsc::unbounded_channel::<PushEnvelope>();
2297    let push_senders = PushSenders {
2298        lossy_tx,
2299        reliable_tx,
2300        lossy_overflow: Arc::clone(&lossy_overflow),
2301        lossy_seq,
2302    };
2303    let connection_cancel = PersistentCancelSignal::new();
2304    let mut installed_route_epochs: HashMap<u16, u32> = HashMap::new();
2305    let mut routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
2306    let mut bg_subs: HashMap<RouteChannel, BgSub> = HashMap::new();
2307    let mut bg_sub_by_session: HashMap<(ProjectRootId, String), RouteChannel> = HashMap::new();
2308    let mut bg_wake_pending: HashSet<RouteChannel> = HashSet::new();
2309    let mut bg_wake_epoch: HashMap<(ProjectRootId, String), u64> = HashMap::new();
2310    let mut root_channels: HashMap<ProjectRootId, HashSet<RouteChannel>> = HashMap::new();
2311    let mut session_identity: HashMap<(ProjectRootId, String), RetainedSessionIdentity> =
2312        HashMap::new();
2313    let mut push_buffer: HashMap<push::ReplayKey, VecDeque<PushFrame>> = HashMap::new();
2314    let mut retry_buffer: RetryBuffer = HashMap::new();
2315    let mut reclaimed_routes = ReclaimedRoutes::default();
2316    let mut completed_tasks = push::CompletedTaskIds::default();
2317    let mut live_roots: HashMap<ProjectRootId, RootMeta> = HashMap::new();
2318    let mut pending_binds: HashMap<RouteChannel, PendingBind> = HashMap::new();
2319    let mut pending_bash_asks: HashMap<ReverseCorrKey, PendingBashAsk> = HashMap::new();
2320    let mut next_bash_ask_corr: u64 = 1;
2321    let mut route_bash_cancels: HashMap<RouteChannel, bash::RouteBashCancel> = HashMap::new();
2322
2323    let loop_result: Result<ModuleLoopExit, SubcError> = loop {
2324        shared_app.set_open_route_count(routes.len());
2325        crate::logging::perf_tick(Some(&executor));
2326        dispatch_path_metrics.mark_frame_loop_tick();
2327        if let Err(error) = expire_pending_bash_asks(
2328            &writer_tx,
2329            &mut pending_bash_asks,
2330            &routes,
2331            &mut live_roots,
2332            &mut route_bash_cancels,
2333            &shutdown,
2334            &dispatch_path_metrics,
2335        )
2336        .await
2337        {
2338            break Err(error);
2339        }
2340
2341        // RouteBind completions are control-plane unblockers. Drain any completed
2342        // binds before entering other branch work so Push and maintenance bursts
2343        // can only add one loop-turn of latency.
2344        match drain_pending_route_bind_completions(
2345            &mut control_completion_rx,
2346            &writer_tx,
2347            &mut routes,
2348            &mut root_channels,
2349            &mut session_identity,
2350            &mut push_buffer,
2351            &mut live_roots,
2352            &mut pending_binds,
2353            &mut installed_route_epochs,
2354            &executor,
2355            &shutdown,
2356            &dispatch_path_metrics,
2357        )
2358        .await
2359        {
2360            Ok(drained) => {
2361                if drained > 0 {
2362                    next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2363                }
2364            }
2365            Err(error) => break Err(error),
2366        }
2367
2368        if tokio::time::Instant::now() >= next_drain_at {
2369            push::emit_bg_event_wakes(
2370                &writer_tx,
2371                &dispatch_path_metrics,
2372                &bg_subs,
2373                &mut bg_wake_pending,
2374            );
2375            warn_slow_pending_binds(&mut pending_binds, &executor);
2376            if let Err(error) = expire_overdue_route_binds(
2377                &writer_tx,
2378                &executor,
2379                &mut pending_binds,
2380                &mut installed_route_epochs,
2381                &dispatch_path_metrics,
2382            )
2383            .await
2384            {
2385                break Err(error);
2386            }
2387
2388            let retried = push::drain_retry_buffers_for_bound_routes(
2389                &writer_tx,
2390                &dispatch_path_metrics,
2391                &routes,
2392                &mut retry_buffer,
2393            );
2394            if retried > 0 {
2395                log::debug!(
2396                    "subc attach: retried {retried} reliable Push frame(s) after writer backpressure"
2397                );
2398            }
2399
2400            next_drain_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2401        }
2402
2403        // A lossy emitter may place its newest update in the overflow buffer
2404        // when the bounded channel is full, while this receive loop is draining
2405        // the channel. Drain overflow before selecting again so that raced
2406        // update is delivered on the next timer tick instead of waiting for
2407        // another lossy enqueue.
2408        let overflow_batch = lossy_overflow.drain();
2409        if !overflow_batch.is_empty() {
2410            let (_, deferred) = push::drain_reliable_push_turn(
2411                &writer_tx,
2412                &dispatch_path_metrics,
2413                &routes,
2414                &root_channels,
2415                &session_identity,
2416                &mut retry_buffer,
2417                &mut push_buffer,
2418                &mut completed_tasks,
2419                &bg_sub_by_session,
2420                &mut bg_wake_pending,
2421                &mut bg_wake_epoch,
2422                &mut reliable_rx,
2423                None,
2424            );
2425            if deferred {
2426                tokio::task::yield_now().await;
2427            }
2428
2429            let mut batch = Vec::new();
2430            while let Ok(item) = lossy_rx.try_recv() {
2431                batch.push(item);
2432            }
2433            batch.extend(overflow_batch);
2434            push::process_lossy_push_envelope_batch(
2435                &writer_tx,
2436                &dispatch_path_metrics,
2437                &routes,
2438                &root_channels,
2439                &completed_tasks,
2440                batch,
2441            );
2442        }
2443
2444        tokio::select! {
2445            biased;
2446            Some(completion) = control_completion_rx.recv() => {
2447                if let Err(error) = process_route_bind_completion(
2448                    &writer_tx,
2449                    completion,
2450                    &mut routes,
2451                    &mut root_channels,
2452                    &mut session_identity,
2453                    &mut push_buffer,
2454                    &mut live_roots,
2455                    &mut pending_binds,
2456                    &mut installed_route_epochs,
2457                    &executor,
2458                    &shutdown,
2459                    &dispatch_path_metrics,
2460                )
2461                .await
2462                {
2463                    break Err(error);
2464                }
2465                next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2466            }
2467            _ = shutdown.notified() => {
2468                log::warn!("subc attach: fatal executor response requested teardown");
2469                break Ok(ModuleLoopExit::SkipSearchFlush);
2470            }
2471            maybe_frame = reader_rx.recv() => {
2472                let frame = match maybe_frame {
2473                    None => {
2474                        log::info!("subc attach: daemon closed connection");
2475                        break Ok(ModuleLoopExit::Graceful);
2476                    }
2477                    Some(Err(error)) => break Err(error),
2478                    Some(Ok(frame)) => frame,
2479                };
2480                let phase_trace = frame.phase_trace;
2481                let frame = frame.frame;
2482
2483                if !ingress_route_should_be_processed(
2484                    &installed_route_epochs,
2485                    &reclaimed_routes,
2486                    &frame,
2487                ) {
2488                    log::debug!(
2489                        "subc attach: silently dropping {:?} for uninstalled route {}@{}",
2490                        frame.header.ty,
2491                        frame.header.channel,
2492                        frame.header.epoch
2493                    );
2494                    continue;
2495                }
2496
2497                match frame.header.ty {
2498                    FrameType::Ping if frame.header.channel == 0 => {
2499                        let pong = match Frame::build_with_version(
2500                            frame.header.ver,
2501                            FrameType::Pong,
2502                            frame.header.flags,
2503                            0,
2504                            0,
2505                            frame.header.corr,
2506                            Vec::new(),
2507                        ) {
2508                            Ok(pong) => pong,
2509                            Err(error) => break Err(SubcError::FrameBuild(error)),
2510                        };
2511                        if let Err(error) = send_frame(&writer_tx, &dispatch_path_metrics, pong).await {
2512                            break Err(error);
2513                        }
2514                    }
2515                    FrameType::Goodbye if frame.header.channel == 0 => {
2516                        log::info!("subc attach: received channel-0 Goodbye");
2517                        break Ok(ModuleLoopExit::Graceful);
2518                    }
2519                    FrameType::Goodbye => {
2520                        let channel = route_key(frame.header.channel, frame.header.epoch);
2521                        if let Err(error) = teardown_installed_route(
2522                            &writer_tx,
2523                            &dispatch_path_metrics,
2524                            &executor,
2525                            channel,
2526                            "Goodbye",
2527                            None,
2528                            &mut installed_route_epochs,
2529                            &mut routes,
2530                            &mut root_channels,
2531                            &mut bg_subs,
2532                            &mut bg_sub_by_session,
2533                            &mut bg_wake_pending,
2534                            &mut pending_bash_asks,
2535                            &mut live_roots,
2536                            &mut route_bash_cancels,
2537                            &mut pending_binds,
2538                            &mut retry_buffer,
2539                            &mut push_buffer,
2540                            &shutdown,
2541                        )
2542                        .await
2543                        {
2544                            break Err(error);
2545                        }
2546                    }
2547                    FrameType::Response | FrameType::Error if frame.header.channel != 0 => {
2548                        if let Err(error) = handle_bash_elicitation_reply(
2549                            &writer_tx,
2550                            &frame,
2551                            &mut pending_bash_asks,
2552                            &routes,
2553                            &mut live_roots,
2554                            &executor,
2555                            &shutdown,
2556                            &bash_deferred_tx,
2557                            &bash_poll_touch_tx,
2558                            &dispatch_path_metrics,
2559                            &mut route_bash_cancels,
2560                            dispatch,
2561                        )
2562                        .await
2563                        {
2564                            break Err(error);
2565                        }
2566                    }
2567                    FrameType::Request if frame.header.channel == 0 => {
2568                        if let Err(error) = handle_control_request(
2569                            &writer_tx,
2570                            &frame,
2571                            &shared_app,
2572                            &executor,
2573                            &mut live_roots,
2574                            &mut pending_binds,
2575                            &mut installed_route_epochs,
2576                            &mut routes,
2577                            &mut root_channels,
2578                            &mut bg_subs,
2579                            &mut bg_sub_by_session,
2580                            &mut bg_wake_pending,
2581                            &mut pending_bash_asks,
2582                            &mut route_bash_cancels,
2583                            &mut retry_buffer,
2584                            &mut push_buffer,
2585                            &shutdown,
2586                            &control_completion_tx,
2587                            &dispatch_path_metrics,
2588                            &push_senders,
2589                            dispatch,
2590                            user_config_path.as_deref(),
2591                        )
2592                        .await
2593                        {
2594                            break Err(error);
2595                        }
2596                    }
2597                    FrameType::Request => {
2598                        if let Err(error) = handle_tool_call(
2599                            &writer_tx,
2600                            &frame,
2601                            phase_trace,
2602                            &routes,
2603                            &pending_binds,
2604                            &mut live_roots,
2605                            &executor,
2606                            &shutdown,
2607                            &connection_cancel,
2608                            &bash_deferred_tx,
2609                            &bash_poll_touch_tx,
2610                            &dispatch_path_metrics,
2611                            &mut route_bash_cancels,
2612                            &mut pending_bash_asks,
2613                            &mut next_bash_ask_corr,
2614                            &mut bg_subs,
2615                            &mut bg_sub_by_session,
2616                            &mut bg_wake_pending,
2617                            &mut bg_wake_epoch,
2618                            dispatch,
2619                            allow_native_passthrough,
2620                        )
2621                        .await
2622                        {
2623                            break Err(error);
2624                        }
2625                    }
2626                    FrameType::Cancel => {
2627                        let channel = route_key(frame.header.channel, frame.header.epoch);
2628                        if bg_subs.contains_key(&channel) {
2629                            if let Err(error) = end_bg_subscription(
2630                                &writer_tx,
2631                                &dispatch_path_metrics,
2632                                &mut bg_subs,
2633                                &mut bg_sub_by_session,
2634                                &mut bg_wake_pending,
2635                                channel,
2636                                routes.get(&channel),
2637                            )
2638                            .await
2639                            {
2640                                break Err(error);
2641                            }
2642                        }
2643                        if let Err(error) = cancel_pending_bash_ask_for_tool_call(
2644                            &writer_tx,
2645                            &mut pending_bash_asks,
2646                            channel,
2647                            frame.header.corr,
2648                            &routes,
2649                            &mut live_roots,
2650                            &mut route_bash_cancels,
2651                            &shutdown,
2652                            &dispatch_path_metrics,
2653                        )
2654                        .await
2655                        {
2656                            break Err(error);
2657                        }
2658                    }
2659                    // Incoming push messages are ignored here. Cancel frames only
2660                    // stop pending bash elicitation requests; executor-level
2661                    // cancellation for tool calls that are already running is not
2662                    // implemented.
2663                    _ => {}
2664                }
2665            }
2666            Some((root_id, frame)) = reliable_rx.recv() => {
2667                // Reliable Push frames are FIFO and must-deliver, but draining an
2668                // unbounded burst in one current-thread turn can starve RouteBind
2669                // completions. The budget defers excess frames, never drops them.
2670                let (_, deferred) = push::drain_reliable_push_turn(
2671                    &writer_tx,
2672                    &dispatch_path_metrics,
2673                    &routes,
2674                    &root_channels,
2675                    &session_identity,
2676                    &mut retry_buffer,
2677                    &mut push_buffer,
2678                    &mut completed_tasks,
2679                    &bg_sub_by_session,
2680                    &mut bg_wake_pending,
2681                    &mut bg_wake_epoch,
2682                    &mut reliable_rx,
2683                    Some((root_id, frame)),
2684                );
2685                if deferred {
2686                    tokio::task::yield_now().await;
2687                }
2688            }
2689            Some((order, root_id, frame)) = lossy_rx.recv() => {
2690                // When both push lanes have work, handle a small reliable slice before lossy work.
2691                // That ordering lets completed task ids suppress stale BashLongRunning frames.
2692                // The slice stays bounded so reliable bursts cannot monopolize this loop turn.
2693                let (_, deferred) = push::drain_reliable_push_turn(
2694                    &writer_tx,
2695                    &dispatch_path_metrics,
2696                    &routes,
2697                    &root_channels,
2698                    &session_identity,
2699                    &mut retry_buffer,
2700                    &mut push_buffer,
2701                    &mut completed_tasks,
2702                    &bg_sub_by_session,
2703                    &mut bg_wake_pending,
2704                    &mut bg_wake_epoch,
2705                    &mut reliable_rx,
2706                    None,
2707                );
2708                if deferred {
2709                    tokio::task::yield_now().await;
2710                }
2711
2712                // Drain the currently queued burst in one loop turn so lossy
2713                // status/progress updates can be merged before reaching subc's
2714                // shared egress queue. Each lossy frame gets a sequence number
2715                // before it goes to the channel or overflow buffer, so the
2716                // combined batch is sorted back into producer order before
2717                // coalescing drops stale updates for the same key.
2718                let mut batch = vec![(order, root_id, frame)];
2719                while let Ok(item) = lossy_rx.try_recv() {
2720                    batch.push(item);
2721                }
2722                batch.extend(lossy_overflow.drain());
2723                push::process_lossy_push_envelope_batch(
2724                    &writer_tx,
2725                    &dispatch_path_metrics,
2726                    &routes,
2727                    &root_channels,
2728                    &completed_tasks,
2729                    batch,
2730                );
2731            }
2732            Some(done) = bash_deferred_rx.recv() => {
2733                decrement_counted_channel(&dispatch_path_metrics.bash_deferred_queued);
2734                if let Err(error) = bash::handle_bash_deferred_completion(
2735                    &writer_tx,
2736                    done,
2737                    &routes,
2738                    &mut live_roots,
2739                    &mut route_bash_cancels,
2740                    &shutdown,
2741                    &dispatch_path_metrics,
2742                )
2743                .await
2744                {
2745                    break Err(error);
2746                }
2747            }
2748            Some(root_id) = bash_poll_touch_rx.recv() => {
2749                decrement_counted_channel(&dispatch_path_metrics.bash_poll_touch_queued);
2750                if let Some(meta) = live_roots.get_mut(&root_id) {
2751                    meta.note_activity();
2752                }
2753            }
2754            Some(completion) = maintenance_rx.recv() => {
2755                decrement_counted_channel(&dispatch_path_metrics.maintenance_queued);
2756                let root_id = completion.root_id.clone();
2757                let response = completion.response;
2758                let response_is_fatal = response_is_fatal_panic(&response);
2759                let bind_pending = pending_binds
2760                    .values()
2761                    .any(|pending| pending.bind_root_id == root_id);
2762                let requiesce = if let Some(meta) = live_roots.get_mut(&root_id) {
2763                    let defer_requeue = meta.unbound_quiesced || bind_pending;
2764                    note_maintenance_completion(
2765                        meta,
2766                        completion.requeue_kind,
2767                        response_is_fatal,
2768                        defer_requeue,
2769                    );
2770                    should_requiesce_after_maintenance(meta, completion.kind, bind_pending)
2771                } else {
2772                    false
2773                };
2774                if requiesce {
2775                    quiesce_unbound_root(&root_id, &mut live_roots, &executor);
2776                }
2777                push::clear_stale_bg_wakes_for_empty_sessions(
2778                    &root_id,
2779                    &completion.empty_bg_sessions,
2780                    &bg_sub_by_session,
2781                    &mut bg_wake_pending,
2782                    &bg_wake_epoch,
2783                );
2784                if response_is_fatal {
2785                    if let Some(meta) = live_roots.get_mut(&root_id) {
2786                        meta.maintenance_poisoned = true;
2787                    }
2788                    log::warn!(
2789                        "subc attach: maintenance drain observed a fatal actor; deferring teardown until a route request can receive actor_fatal"
2790                    );
2791                }
2792            }
2793            _ = tokio::time::sleep_until(next_drain_at) => {
2794                // Wakes an otherwise-idle loop so the pre-turn drain check
2795                // above runs on schedule; the drain work itself lives there.
2796            }
2797            _ = tokio::time::sleep_until(next_maintenance_at) => {
2798                // Delay cache-draining maintenance until any already-ready
2799                // inbound route/control messages and push completions have run,
2800                // so maintenance does not block the actor from handling the
2801                // first request that arrives after a route bind is acknowledged.
2802                crate::logging::maybe_sweep_logs();
2803                let reaped_lsp_children = shared_app
2804                    .lsp_child_registry()
2805                    .reap_children_with_gone_cwd();
2806                if reaped_lsp_children > 0 {
2807                    log::warn!(
2808                        "subc attach: reaped {reaped_lsp_children} LSP child process group(s) whose cwd no longer exists"
2809                    );
2810                }
2811                let reap = reap_idle_roots(
2812                    Instant::now(),
2813                    &mut live_roots,
2814                    &pending_binds,
2815                    &root_channels,
2816                    &executor,
2817                    &dispatch_path_metrics,
2818                );
2819                for root_id in &reap.forgotten_deleted_roots {
2820                    purge_deleted_root_residents(
2821                        root_id,
2822                        &mut routes,
2823                        &mut root_channels,
2824                        &mut installed_route_epochs,
2825                        &mut route_bash_cancels,
2826                        &mut retry_buffer,
2827                        &mut reclaimed_routes,
2828                        &mut session_identity,
2829                        &mut push_buffer,
2830                        &mut bg_subs,
2831                        &mut bg_sub_by_session,
2832                        &mut bg_wake_pending,
2833                        &mut bg_wake_epoch,
2834                        &mut pending_bash_asks,
2835                    );
2836                }
2837                if reap.evicted > 0 {
2838                    log::debug!("subc attach: reaped {} idle root(s)", reap.evicted);
2839                }
2840                submit_due_maintenance_jobs(
2841                    &executor,
2842                    &mut live_roots,
2843                    &pending_binds,
2844                    &bg_sub_by_session,
2845                    &bg_wake_pending,
2846                    &bg_wake_epoch,
2847                    &maintenance_tx,
2848                    &dispatch_path_metrics,
2849                );
2850                // Opportunistic allocator relief, independent of the idle
2851                // sweep: the sweep's whole-process idle gate never opens while
2852                // any session stays active, which let freed warm-up arenas sit
2853                // resident for the process lifetime (5.1 GB RSS over ~600 MB
2854                // live). Slack threshold + spacing live in memory.rs; the pass
2855                // itself runs on a detached thread.
2856                #[cfg(target_os = "macos")]
2857                {
2858                    let now_std = std::time::Instant::now();
2859                    if crate::memory::spawn_allocator_slack_relief_if_due(
2860                        last_slack_relief,
2861                        now_std,
2862                    ) {
2863                        last_slack_relief = Some(now_std);
2864                    }
2865                }
2866                next_maintenance_at = tokio::time::Instant::now() + DRAIN_TICK_PERIOD;
2867            }
2868        }
2869    };
2870
2871    shared_app.set_open_route_count(0);
2872
2873    connection_cancel.cancel();
2874    // Channel-0 Goodbye, EOF, and fatal exits bypass per-route Goodbye. Settle
2875    // their root lifecycle state before loop-owned routing metadata is dropped.
2876    quiesce_connection_roots(
2877        &mut live_roots,
2878        &mut pending_binds,
2879        &mut routes,
2880        &mut root_channels,
2881        &mut installed_route_epochs,
2882        &mut route_bash_cancels,
2883        &executor,
2884    );
2885
2886    let mut loop_result = loop_result;
2887    if !pending_bash_asks.is_empty() {
2888        let no_routes: HashMap<RouteChannel, RouteIdentity> = HashMap::new();
2889        if let Err(error) = settle_all_pending_bash_asks(
2890            &writer_tx,
2891            &mut pending_bash_asks,
2892            &no_routes,
2893            &mut live_roots,
2894            &mut route_bash_cancels,
2895            &shutdown,
2896            &dispatch_path_metrics,
2897        )
2898        .await
2899        {
2900            loop_result = loop_result.and(Err(error));
2901        }
2902    }
2903
2904    // The reader task may be parked on `read_frame`; abort it (we are done with
2905    // the connection) and flush the writer.
2906    reader_task.abort();
2907    drop(writer_tx);
2908    let writer_result = finish_writer_task(writer_task).await;
2909    loop_result.and_then(|exit| writer_result.map(|_| exit))
2910}
2911
2912fn spawn_writer_task<W>(
2913    mut write: W,
2914    mut rx: mpsc::Receiver<WriterFrame>,
2915    metrics: Arc<DispatchPathMetrics>,
2916) -> JoinHandle<Result<(), subc_transport::FrameIoError>>
2917where
2918    W: AsyncWrite + Unpin + Send + 'static,
2919{
2920    tokio::spawn(async move {
2921        let mut write_buffer = Vec::new();
2922        while let Some(mut queued) = rx.recv().await {
2923            let measure = queued.tool_response_trace.is_some();
2924            let dequeued = measure.then(Instant::now);
2925            metrics.writer_active.store(true, Ordering::Relaxed);
2926            decrement_counted_channel(&metrics.writer_queued);
2927            let write_timing = write_frame_contiguous(
2928                &mut write,
2929                queued.frame(),
2930                queued.body(),
2931                &mut write_buffer,
2932                measure,
2933            )
2934            .await;
2935            metrics.writer_active.store(false, Ordering::Relaxed);
2936            let write_timing = write_timing?;
2937
2938            if let (Some(trace), Some(dequeued), Some(write_timing)) =
2939                (queued.tool_response_trace.take(), dequeued, write_timing)
2940            {
2941                if let Some(completed) = trace.finish(
2942                    dequeued,
2943                    write_timing.write_started,
2944                    write_timing.write_finished,
2945                    write_timing.frame_bytes,
2946                ) {
2947                    log_ctx::with_session(Some(completed.session), || {
2948                        crate::logging::note_tool_call_trace(
2949                            &completed.name,
2950                            &completed.root,
2951                            completed.channel,
2952                            completed.corr,
2953                            completed.phases,
2954                        );
2955                    });
2956                }
2957            }
2958        }
2959        Ok(())
2960    })
2961}
2962
2963struct FrameWriteTiming {
2964    write_started: Instant,
2965    write_finished: Instant,
2966    frame_bytes: usize,
2967}
2968
2969/// Encode one complete frame into the existing reusable buffer and write it
2970/// without interleaving bytes from another channel. Timing is collected only
2971/// for tool responses, so Push and control frames add no clock reads.
2972async fn write_frame_contiguous<W>(
2973    writer: &mut W,
2974    frame: &Frame,
2975    body: &[u8],
2976    buffer: &mut Vec<u8>,
2977    measure: bool,
2978) -> Result<Option<FrameWriteTiming>, subc_transport::FrameIoError>
2979where
2980    W: AsyncWrite + Unpin,
2981{
2982    if frame.header.len as usize != body.len() {
2983        return Err(subc_transport::FrameIoError::BodyLengthMismatch {
2984            header_len: frame.header.len,
2985            body_len: body.len(),
2986        });
2987    }
2988
2989    let header = frame.header.encode();
2990    buffer.clear();
2991    buffer.reserve(header.len() + body.len());
2992    buffer.extend_from_slice(&header);
2993    buffer.extend_from_slice(body);
2994    let write_started = measure.then(Instant::now);
2995    writer
2996        .write_all(buffer)
2997        .await
2998        .map_err(subc_transport::FrameIoError::Io)?;
2999    Ok(write_started.map(|write_started| FrameWriteTiming {
3000        write_started,
3001        write_finished: Instant::now(),
3002        frame_bytes: buffer.len(),
3003    }))
3004}
3005
3006fn spawn_reader_task<R>(
3007    mut read: R,
3008    tx: mpsc::Sender<Result<DecodedFrame, SubcError>>,
3009) -> JoinHandle<()>
3010where
3011    R: AsyncRead + Unpin + Send + 'static,
3012{
3013    tokio::spawn(async move {
3014        loop {
3015            match read_frame(&mut read).await {
3016                Ok(Some(frame)) => {
3017                    let decoded = DecodedFrame {
3018                        frame,
3019                        phase_trace: PhaseTrace::new(Instant::now()),
3020                    };
3021                    if tx.send(Ok(decoded)).await.is_err() {
3022                        return;
3023                    }
3024                }
3025                Ok(None) => {
3026                    // EOF: let the loop observe channel close as "daemon closed".
3027                    return;
3028                }
3029                Err(error) => {
3030                    // A killed daemon surfaces as ConnectionReset (RST) on
3031                    // Windows where Unix delivers a clean EOF (FIN); a
3032                    // mid-teardown daemon can also abort the socket. Both mean
3033                    // "daemon went away", not a wire fault — normalize them to
3034                    // the clean-close path so module exit behavior matches
3035                    // across platforms (same class subc-core fixed in d33d9a71).
3036                    if let subc_transport::FrameIoError::Io(io_error) = &error {
3037                        if matches!(
3038                            io_error.kind(),
3039                            std::io::ErrorKind::ConnectionReset
3040                                | std::io::ErrorKind::ConnectionAborted
3041                        ) {
3042                            log::info!(
3043                                "subc attach: connection reset by daemon; treating as close"
3044                            );
3045                            return;
3046                        }
3047                    }
3048                    let _ = tx.send(Err(SubcError::FrameIo(error))).await;
3049                    return;
3050                }
3051            }
3052        }
3053    })
3054}
3055
3056async fn finish_writer_task(
3057    mut writer_task: JoinHandle<Result<(), subc_transport::FrameIoError>>,
3058) -> Result<(), SubcError> {
3059    match tokio::time::timeout(Duration::from_millis(100), &mut writer_task).await {
3060        Ok(Ok(Ok(()))) => Ok(()),
3061        Ok(Ok(Err(error))) => Err(SubcError::FrameIo(error)),
3062        Ok(Err(error)) => Err(SubcError::WriterJoin(error)),
3063        Err(_) => {
3064            writer_task.abort();
3065            Ok(())
3066        }
3067    }
3068}
3069
3070fn register_actor_for_bind(
3071    shared_app: &Arc<App>,
3072    executor: &Arc<Executor>,
3073    push_senders: &PushSenders,
3074    bind_root_id: &ProjectRootId,
3075    route_channel: u16,
3076    root_was_live: bool,
3077) -> bool {
3078    if executor.actor_registered(bind_root_id) {
3079        log::debug!(
3080            "subc attach: reusing actor for route {} root {}",
3081            route_channel,
3082            bind_root_id.as_path().display()
3083        );
3084        return false;
3085    }
3086
3087    if root_was_live {
3088        log::warn!(
3089            "subc attach: recreating missing actor for live root {} on route {}",
3090            bind_root_id.as_path().display(),
3091            route_channel
3092        );
3093    }
3094
3095    let actor_ctx = Arc::new(AppContext::from_app(
3096        Arc::clone(shared_app),
3097        Config::default(),
3098    ));
3099    install_bash_compressor(&actor_ctx);
3100    actor_ctx.set_progress_sender(Some(push::progress_sender_for_root(
3101        push_senders.clone(),
3102        bind_root_id.clone(),
3103    )));
3104    let inserted = executor.register_actor(bind_root_id.clone(), Arc::clone(&actor_ctx));
3105    drop(actor_ctx);
3106    if inserted {
3107        // Do not insert into live_roots until configure succeeds: live_roots
3108        // drives maintenance, and a half-configured new actor must not be
3109        // maintenance-eligible before its route/session identity exists.
3110        log::debug!(
3111            "subc attach: registered actor for route {} root {}",
3112            route_channel,
3113            bind_root_id.as_path().display()
3114        );
3115    } else {
3116        log::debug!(
3117            "subc attach: actor appeared while binding route {} root {}; reusing it",
3118            route_channel,
3119            bind_root_id.as_path().display()
3120        );
3121    }
3122    inserted
3123}
3124
3125fn rollback_pending_bind_actor(
3126    executor: &Arc<Executor>,
3127    live_roots: &HashMap<ProjectRootId, RootMeta>,
3128    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3129    root_id: &ProjectRootId,
3130    inserted_new_actor: bool,
3131) {
3132    if !inserted_new_actor || live_roots.contains_key(root_id) {
3133        return;
3134    }
3135
3136    if let Some((route, pending)) = pending_binds
3137        .iter_mut()
3138        .find(|(_, pending)| &pending.bind_root_id == root_id)
3139    {
3140        pending.inserted_new_actor = true;
3141        log::debug!(
3142            "subc attach: transferred rollback ownership for root {} to pending route {}",
3143            root_id.as_path().display(),
3144            route
3145        );
3146        return;
3147    }
3148
3149    executor.remove_actor(root_id);
3150}
3151
3152fn route_bind_error_code_for_configure_response(response: &Response) -> &'static str {
3153    match response.data.get("code").and_then(|code| code.as_str()) {
3154        // Preserve typed configure rejections across the bind boundary: a
3155        // malformed fed fingerprint means a federation-module bug or
3156        // fingerprint-format drift, and the fed side matches on the code rather
3157        // than parsing prose.
3158        Some("bad_harness_fingerprint") => "bad_harness_fingerprint",
3159        // Cache-key probe failures are transient (fd pressure, git spawn
3160        // contention); the client retries the bind rather than treating the
3161        // root as permanently divergent.
3162        Some("cache_key_probe_failed") => "cache_key_probe_failed",
3163        // Actor lifecycle gaps are transient from the daemon/client viewpoint:
3164        // a fresh bind can create or join a healthy actor, so do not classify
3165        // them as permanent config divergence.
3166        Some("actor_not_registered" | "actor_fatal") => "actor_not_ready",
3167        _ => "config_divergence",
3168    }
3169}
3170
3171fn queue_post_bind_configure_and_completion_maintenance(
3172    root_id: &ProjectRootId,
3173    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3174) {
3175    let Some(meta) = live_roots.get_mut(root_id) else {
3176        return;
3177    };
3178    if meta.maintenance_poisoned || meta.maintenance_pending {
3179        return;
3180    }
3181
3182    meta.maintenance_pending = true;
3183    meta.maintenance_queued_kinds
3184        .push_back(MaintenanceDrainKind::ConfigureTail);
3185    meta.maintenance_queued_kinds
3186        .push_back(MaintenanceDrainKind::CompletionDrains);
3187}
3188
3189#[allow(clippy::too_many_arguments)]
3190async fn handle_route_bind_completion(
3191    tx: &WriterSender,
3192    completion: RouteBindCompletion,
3193    routes: &mut HashMap<RouteChannel, RouteIdentity>,
3194    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
3195    session_identity: &mut HashMap<(ProjectRootId, String), RetainedSessionIdentity>,
3196    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
3197    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3198    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3199    installed_route_epochs: &mut HashMap<u16, u32>,
3200    executor: &Arc<Executor>,
3201    shutdown: &Arc<Notify>,
3202    metrics: &Arc<DispatchPathMetrics>,
3203) -> Result<(), SubcError> {
3204    let route_id = completion.route;
3205    let Some(pending) = pending_binds.remove(&route_id) else {
3206        log::warn!(
3207            "subc attach: dropping RouteBind completion for non-pending route {}",
3208            completion.route
3209        );
3210        rollback_pending_bind_actor(
3211            executor,
3212            live_roots,
3213            pending_binds,
3214            &completion.bind_root_id,
3215            completion.inserted_new_actor,
3216        );
3217        let has_pending_bind = pending_binds
3218            .values()
3219            .any(|pending| pending.bind_root_id == completion.bind_root_id);
3220        if !root_channels
3221            .get(&completion.bind_root_id)
3222            .is_some_and(|channels| !channels.is_empty())
3223            && !has_pending_bind
3224        {
3225            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3226        }
3227        remove_installed_route(installed_route_epochs, route_id);
3228        return Ok(());
3229    };
3230
3231    if pending.bind_root_id != completion.bind_root_id {
3232        log::warn!(
3233            "subc attach: pending RouteBind root mismatch for route {} (pending {} completion {})",
3234            completion.route,
3235            pending.bind_root_id.as_path().display(),
3236            completion.bind_root_id.as_path().display()
3237        );
3238    }
3239
3240    let inserted_new_actor = pending.inserted_new_actor || completion.inserted_new_actor;
3241    if pending.cancelled {
3242        rollback_pending_bind_actor(
3243            executor,
3244            live_roots,
3245            pending_binds,
3246            &completion.bind_root_id,
3247            inserted_new_actor,
3248        );
3249        let has_pending_bind = pending_binds
3250            .values()
3251            .any(|pending| pending.bind_root_id == completion.bind_root_id);
3252        if !root_channels
3253            .get(&completion.bind_root_id)
3254            .is_some_and(|channels| !channels.is_empty())
3255            && !has_pending_bind
3256        {
3257            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3258        }
3259        log::debug!(
3260            "subc attach: discarded completed RouteBind for cancelled route {} root {}",
3261            completion.route,
3262            completion.bind_root_id.as_path().display()
3263        );
3264        remove_installed_route(installed_route_epochs, route_id);
3265        return Ok(());
3266    }
3267
3268    let failure = if !completion.configure_response.success {
3269        Some((
3270            &completion.configure_response,
3271            "configure failed during route bind",
3272        ))
3273    } else {
3274        None
3275    };
3276
3277    if let Some((response, fallback)) = failure {
3278        rollback_pending_bind_actor(
3279            executor,
3280            live_roots,
3281            pending_binds,
3282            &completion.bind_root_id,
3283            inserted_new_actor,
3284        );
3285        let has_pending_bind = pending_binds
3286            .values()
3287            .any(|pending| pending.bind_root_id == completion.bind_root_id);
3288        if !root_channels
3289            .get(&completion.bind_root_id)
3290            .is_some_and(|channels| !channels.is_empty())
3291            && !has_pending_bind
3292        {
3293            quiesce_unbound_root(&completion.bind_root_id, live_roots, executor);
3294        }
3295        let message = response_message(response, fallback);
3296        let fatal = response_is_fatal_panic(response);
3297        let error_code = route_bind_error_code_for_configure_response(response);
3298        send_route_bind_error_parts(
3299            tx,
3300            completion.ver,
3301            completion.corr,
3302            completion.flags,
3303            error_code,
3304            &message,
3305            metrics,
3306        )
3307        .await?;
3308        remove_installed_route(installed_route_epochs, route_id);
3309        if fatal {
3310            signal_fatal_teardown(
3311                tx,
3312                Some(completion.route),
3313                completion.ver,
3314                completion.corr,
3315                shutdown,
3316                metrics,
3317            )
3318            .await;
3319        }
3320        return Ok(());
3321    }
3322
3323    remember_session_identity(session_identity, &completion.identity);
3324    let replay_key = push::ReplayKey::from_identity(&completion.identity);
3325    let bind_trust = completion.identity.trust;
3326    insert_route_channel(routes, root_channels, route_id, completion.identity);
3327    let restore_watcher = live_roots
3328        .get(&completion.bind_root_id)
3329        .is_some_and(|meta| meta.idle_artifacts_evicted || meta.unbound_quiesced);
3330    live_roots
3331        .entry(completion.bind_root_id.clone())
3332        .and_modify(|meta| {
3333            meta.reactivate_bound();
3334            meta.diagnostics_on_edit = completion.diagnostics_on_edit;
3335            meta.maintenance_poisoned = false;
3336        })
3337        .or_insert_with(|| RootMeta::new(Instant::now()));
3338    if let Some(meta) = live_roots.get_mut(&completion.bind_root_id) {
3339        meta.diagnostics_on_edit = completion.diagnostics_on_edit;
3340        meta.maintenance_poisoned = false;
3341    }
3342    if let Some(ctx) = executor.actor_context(&completion.bind_root_id) {
3343        ctx.mark_subc_bound();
3344        if restore_watcher {
3345            crate::commands::configure::ensure_project_watcher(&ctx);
3346        }
3347    }
3348
3349    let ack =
3350        serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).map_err(SubcError::Json)?;
3351    let response = Frame::build_with_version(
3352        completion.ver,
3353        FrameType::Response,
3354        control_flags(),
3355        0,
3356        0,
3357        completion.corr,
3358        ack,
3359    )
3360    .map_err(SubcError::FrameBuild)?;
3361    send_reliable_writer_frame(tx, metrics, response, "RouteBindAck").await?;
3362    queue_post_bind_configure_and_completion_maintenance(&completion.bind_root_id, live_roots);
3363    let replayed = push::replay_buffered_push_frames(
3364        tx,
3365        metrics,
3366        route_id,
3367        push_buffer,
3368        &replay_key,
3369        bind_trust,
3370    );
3371    if replayed > 0 {
3372        log::debug!(
3373            "subc attach: replayed {} buffered Push frame(s) to route {} root {} harness {} session {}",
3374            replayed,
3375            completion.route,
3376            replay_key.root.as_path().display(),
3377            replay_key.harness,
3378            replay_key.session
3379        );
3380    }
3381    log::info!(
3382        "subc attach: route {} bound to root {}",
3383        completion.route,
3384        completion.bind_root_id.as_path().display()
3385    );
3386    Ok(())
3387}
3388
3389async fn expire_overdue_route_binds(
3390    tx: &WriterSender,
3391    executor: &Arc<Executor>,
3392    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3393    installed_route_epochs: &mut HashMap<u16, u32>,
3394    metrics: &DispatchPathMetrics,
3395) -> Result<(), SubcError> {
3396    let now = Instant::now();
3397    let expired: Vec<_> = pending_binds
3398        .iter()
3399        .filter_map(|(route, pending)| {
3400            let age = now.saturating_duration_since(pending.started_at);
3401            (!pending.deadline_reported && age >= ROUTE_BIND_DEADLINE).then(|| {
3402                (
3403                    *route,
3404                    pending.corr,
3405                    pending.ver,
3406                    pending.flags,
3407                    pending.bind_root_id.clone(),
3408                    pending.configure_request_id.clone(),
3409                    age,
3410                )
3411            })
3412        })
3413        .collect();
3414
3415    for (route, corr, ver, flags, root_id, configure_request_id, age) in expired {
3416        if let Some(pending) = pending_binds.get_mut(&route) {
3417            pending.cancelled = true;
3418            pending.deadline_reported = true;
3419            let outcome = executor.cancel_job(&pending.bind_root_id, &pending.cancellation);
3420            log::debug!(
3421                "subc attach: cancelled overdue RouteBind configure for route {route} ({outcome:?})"
3422            );
3423        }
3424        remove_installed_route(installed_route_epochs, route);
3425        let age_ms = age.as_millis().min(u128::from(u64::MAX)) as u64;
3426        let deadline_ms = ROUTE_BIND_DEADLINE.as_millis();
3427        send_route_bind_error_parts(
3428            tx,
3429            ver,
3430            corr,
3431            flags,
3432            "actor_not_ready",
3433            &format!("route bind deadline exceeded after {age_ms}ms (deadline {deadline_ms}ms)"),
3434            metrics,
3435        )
3436        .await?;
3437        log::warn!(
3438            "subc attach: route {} bind for root {} exceeded {}ms deadline (configure_request_id={})",
3439            route,
3440            root_id.as_path().display(),
3441            deadline_ms,
3442            configure_request_id
3443        );
3444    }
3445
3446    Ok(())
3447}
3448
3449/// channel-0 control requests: RouteBind plus the cached health probe. RouteBind
3450/// still reconciles the route's RootConfig through the executor's Mutating lane
3451/// and resolves completion on a loop-owned control-completion channel so slow
3452/// configure jobs do not block the transport loop.
3453#[allow(clippy::too_many_arguments)]
3454async fn handle_control_request(
3455    tx: &WriterSender,
3456    frame: &Frame,
3457    shared_app: &Arc<App>,
3458    executor: &Arc<Executor>,
3459    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3460    pending_binds: &mut HashMap<RouteChannel, PendingBind>,
3461    installed_route_epochs: &mut HashMap<u16, u32>,
3462    routes: &mut HashMap<RouteChannel, RouteIdentity>,
3463    root_channels: &mut HashMap<ProjectRootId, HashSet<RouteChannel>>,
3464    bg_subs: &mut HashMap<RouteChannel, BgSub>,
3465    bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
3466    bg_wake_pending: &mut HashSet<RouteChannel>,
3467    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
3468    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
3469    retry_buffer: &mut RetryBuffer,
3470    push_buffer: &mut HashMap<push::ReplayKey, VecDeque<PushFrame>>,
3471    shutdown: &Arc<Notify>,
3472    control_completion_tx: &mpsc::Sender<RouteBindCompletion>,
3473    metrics: &Arc<DispatchPathMetrics>,
3474    push_senders: &PushSenders,
3475    dispatch: DispatchFn,
3476    user_config_path: Option<&Path>,
3477) -> Result<(), SubcError> {
3478    let request =
3479        serde_json::from_slice::<ModuleControlRequest>(&frame.body).map_err(SubcError::Json)?;
3480    match request {
3481        ModuleControlRequest::RouteBind {
3482            route_channel,
3483            epoch,
3484            target: _,
3485            identity,
3486            principal,
3487            consumer_capabilities,
3488        } => {
3489            let route_id = route_key(route_channel, epoch);
3490            if epoch == 0 {
3491                return send_route_bind_error(
3492                    tx,
3493                    frame,
3494                    "config_divergence",
3495                    "route bind uses an invalid channel generation",
3496                    metrics,
3497                )
3498                .await;
3499            }
3500            let mut bind_root_id = None;
3501            if let Some(installed_epoch) = installed_route_epochs.get(&route_channel).copied() {
3502                if installed_epoch >= epoch {
3503                    return send_route_bind_error(
3504                        tx,
3505                        frame,
3506                        "config_divergence",
3507                        "route bind generation is not newer than the installed generation",
3508                        metrics,
3509                    )
3510                    .await;
3511                }
3512
3513                let replacement_root = match ProjectRootId::from_path(&identity.project_root) {
3514                    Ok(root_id) => root_id,
3515                    Err(error) => {
3516                        return send_route_bind_error(
3517                            tx,
3518                            frame,
3519                            "config_divergence",
3520                            &format!("invalid route project root: {error}"),
3521                            metrics,
3522                        )
3523                        .await;
3524                    }
3525                };
3526                teardown_installed_route(
3527                    tx,
3528                    metrics,
3529                    executor,
3530                    route_key(route_channel, installed_epoch),
3531                    "higher-epoch RouteBind",
3532                    Some(&replacement_root),
3533                    installed_route_epochs,
3534                    routes,
3535                    root_channels,
3536                    bg_subs,
3537                    bg_sub_by_session,
3538                    bg_wake_pending,
3539                    pending_bash_asks,
3540                    live_roots,
3541                    route_bash_cancels,
3542                    pending_binds,
3543                    retry_buffer,
3544                    push_buffer,
3545                    shutdown,
3546                )
3547                .await?;
3548                bind_root_id = Some(replacement_root);
3549            }
3550            if pending_binds.contains_key(&route_id) {
3551                return send_route_bind_error(
3552                    tx,
3553                    frame,
3554                    "config_divergence",
3555                    "route bind is already pending for channel",
3556                    metrics,
3557                )
3558                .await;
3559            }
3560            let bind_root_id = match bind_root_id {
3561                Some(root_id) => root_id,
3562                None => match ProjectRootId::from_path(&identity.project_root) {
3563                    Ok(root_id) => root_id,
3564                    Err(error) => {
3565                        return send_route_bind_error(
3566                            tx,
3567                            frame,
3568                            "config_divergence",
3569                            &format!("invalid route project root: {error}"),
3570                            metrics,
3571                        )
3572                        .await;
3573                    }
3574                },
3575            };
3576
3577            // Reconcile RootConfig: build a configure request from the bind
3578            // identity + forwarded config tiers and run it through the executor.
3579            let request_id = format!("subc-bind-{route_channel}");
3580            let bind_project_root = identity.project_root.clone();
3581            let bind_harness = identity.harness.clone();
3582            let bind_session = identity.session.clone();
3583            let bind_trust = trust_for_bind(&bind_harness, &principal);
3584            let bind_principal_id = principal_id(&principal);
3585            // Typed capability declaration from the consumer: the facade stamps it
3586            // from the MCP host's initialize-advertised capabilities. Absent
3587            // means no reverse-request capability — flat deny, fail-closed. A
3588            // consumer over-declaring only earns asks that TTL-deny.
3589            let consumer_elicitation_capable = consumer_capabilities
3590                .as_ref()
3591                .is_some_and(|capabilities| capabilities.iter().any(|c| c == "elicitation"));
3592            log::info!(
3593                "subc attach: route {} harness={} principal={} trust={} elicitation={}",
3594                route_channel,
3595                bind_harness,
3596                principal_label(&principal),
3597                bind_trust.label(),
3598                consumer_elicitation_capable
3599            );
3600
3601            // Config is single-per-project, read by AFT directly from the
3602            // CortexKit config files (user: ~/.config/cortexkit/aft.jsonc,
3603            // project: <root>/.cortexkit/aft.jsonc). Wire-relayed config tiers are
3604            // IGNORED entirely: a front (runner, mcp:*, or fed:*) cannot push config over
3605            // the wire. This is what makes config harness-INDEPENDENT — every
3606            // harness binding a project gets the identical on-disk config, so two
3607            // trust domains sharing the per-root actor can never diverge or
3608            // inherit each other's capabilities (the cross-bind escalation class).
3609            // Wire-relayed config tiers (if the protocol still carries them) are
3610            // ignored entirely; the per-tier trust boundary (user trusted, project
3611            // privileged-dropped) is applied to the FILE tiers in handle_configure.
3612            let local_tiers = crate::subc_config::read_local_cortexkit_config_tiers(
3613                user_config_path,
3614                Path::new(&bind_project_root),
3615            );
3616            let config_tiers: Vec<Value> = local_tiers
3617                .iter()
3618                .map(|t| json!({ "tier": t.tier, "source": t.source, "doc": t.doc }))
3619                .collect();
3620            let diagnostics_on_edit = diagnostics_on_edit_from_tiers(&local_tiers);
3621            let configure_json = json!({
3622                "id": request_id,
3623                "command": "configure",
3624                "project_root": bind_project_root,
3625                "harness": bind_harness,
3626                "session_id": bind_session.clone(),
3627                "config": config_tiers,
3628            });
3629            let configure_req = match serde_json::from_value::<RawRequest>(configure_json) {
3630                Ok(req) => req,
3631                Err(error) => {
3632                    return send_route_bind_error(
3633                        tx,
3634                        frame,
3635                        "config_divergence",
3636                        &format!("failed to build configure request: {error}"),
3637                        metrics,
3638                    )
3639                    .await;
3640                }
3641            };
3642
3643            let route_identity = RouteIdentity(Arc::new(RouteIdentityData {
3644                root: bind_root_id.clone(),
3645                project_root: PathBuf::from(&bind_project_root),
3646                harness: bind_harness.clone(),
3647                session: bind_session.clone(),
3648                trust: bind_trust,
3649                spawn_principal: AuthenticatedPrincipal::RouteBind {
3650                    trust: bind_trust.sandbox_trust(),
3651                    route_channel,
3652                    route_epoch: epoch,
3653                    project_root: PathBuf::from(&bind_project_root),
3654                    harness: bind_harness.clone(),
3655                    session_id: bind_session.clone(),
3656                    principal_id: bind_principal_id,
3657                },
3658                consumer_elicitation_capable,
3659            }));
3660            let configure_session = route_identity.session.clone();
3661            let root_was_live = live_roots.contains_key(&bind_root_id);
3662            let inserted_new_actor = register_actor_for_bind(
3663                shared_app,
3664                executor,
3665                push_senders,
3666                &bind_root_id,
3667                route_channel,
3668                root_was_live,
3669            );
3670
3671            let configure_request_id = configure_req.id.clone();
3672            installed_route_epochs.insert(route_channel, epoch);
3673            if let Some(meta) = live_roots.get_mut(&bind_root_id) {
3674                meta.maintenance_queued_kinds.clear();
3675                meta.maintenance_pending = meta.maintenance_jobs_in_flight > 0;
3676            }
3677            let (configure_rx, configure_cancellation) = executor.submit_cancellable_async(
3678                bind_root_id.clone(),
3679                Lane::Mutating,
3680                configure_request_id.clone(),
3681                Box::new(move |ctx| {
3682                    log_ctx::with_session(Some(configure_session.clone()), || {
3683                        dispatch(configure_req, ctx)
3684                    })
3685                }),
3686            );
3687            pending_binds.insert(
3688                route_id,
3689                PendingBind {
3690                    bind_root_id: bind_root_id.clone(),
3691                    inserted_new_actor,
3692                    cancelled: false,
3693                    configure_request_id: configure_request_id.clone(),
3694                    started_at: Instant::now(),
3695                    warned_half_deadline: false,
3696                    deadline_reported: false,
3697                    corr: frame.header.corr,
3698                    ver: frame.header.ver,
3699                    flags: frame.header.flags,
3700                    cancellation: configure_cancellation,
3701                },
3702            );
3703
3704            let completion_tx = control_completion_tx.clone();
3705            let completion_identity = route_identity;
3706            let completion_root = bind_root_id.clone();
3707            let completion_route_channel = route_channel;
3708            let completion_ver = frame.header.ver;
3709            let completion_corr = frame.header.corr;
3710            let completion_flags = frame.header.flags;
3711            let completion_metrics = Arc::clone(metrics);
3712            tokio::spawn(async move {
3713                let _response_task = ResponseTaskGuard::new(&completion_metrics);
3714                let configure_response =
3715                    await_executor_response(configure_rx, configure_request_id.clone()).await;
3716                // Send the route-bind acknowledgment as soon as configure succeeds.
3717                // Installing completed search or callgraph builds only refreshes cached
3718                // read data, so a later maintenance pass can do it without delaying the
3719                // daemon's confirmation that the route is usable.
3720                let completion = RouteBindCompletion {
3721                    route: route_key(completion_route_channel, epoch),
3722                    identity: completion_identity,
3723                    bind_root_id: completion_root,
3724                    inserted_new_actor,
3725                    configure_response,
3726                    diagnostics_on_edit,
3727                    ver: completion_ver,
3728                    corr: completion_corr,
3729                    flags: completion_flags,
3730                };
3731                if send_counted_channel(
3732                    &completion_tx,
3733                    &completion_metrics.control_completion_queued,
3734                    completion,
3735                )
3736                .await
3737                .is_err()
3738                {
3739                    log::debug!(
3740                        "subc attach: dropped RouteBind completion for route {} after loop exit",
3741                        completion_route_channel
3742                    );
3743                }
3744            });
3745
3746            Ok(())
3747        }
3748        ModuleControlRequest::HealthCheck {} => {
3749            let report = build_health_report(executor, pending_binds, metrics, shared_app);
3750            let body = serde_json::to_vec(&ModuleControlResponse::from(report))
3751                .map_err(SubcError::Json)?;
3752            let response = Frame::build_with_version(
3753                frame.header.ver,
3754                FrameType::Response,
3755                frame.header.flags,
3756                0,
3757                0,
3758                frame.header.corr,
3759                body,
3760            )
3761            .map_err(SubcError::FrameBuild)?;
3762            send_frame(tx, metrics, response).await
3763        }
3764    }
3765}
3766
3767fn install_bash_compressor(ctx: &AppContext) {
3768    // Mirrors main.rs per-actor compressor installation for subc-created actors.
3769    let filter_registry_handle = ctx.shared_filter_registry();
3770    let compress_flag = ctx.bash_compress_flag();
3771    ctx.bash_background().set_compressor_with_exit_code(
3772        move |command: &str, output: String, exit_code: Option<i32>| {
3773            if !compress_flag.load(std::sync::atomic::Ordering::Relaxed) {
3774                return crate::compress::CompressionResult::new(output);
3775            }
3776            let registry_guard = match filter_registry_handle.read() {
3777                Ok(g) => g,
3778                Err(poisoned) => poisoned.into_inner(),
3779            };
3780            crate::compress::compress_with_registry_exit_code(
3781                command,
3782                &output,
3783                exit_code,
3784                &registry_guard,
3785            )
3786        },
3787    );
3788}
3789
3790fn diagnostics_on_edit_from_tiers(tiers: &[ConfigTier]) -> bool {
3791    let mut diagnostics_on_edit = false;
3792    for tier in tiers {
3793        if let Some(value) = diagnostics_on_edit_from_doc(&tier.doc) {
3794            diagnostics_on_edit = value;
3795        }
3796    }
3797    diagnostics_on_edit
3798}
3799
3800fn diagnostics_on_edit_from_doc(doc: &str) -> Option<bool> {
3801    let stripped = strip_jsonc(doc);
3802    let value = serde_json::from_str::<Value>(&stripped).ok()?;
3803    value
3804        .get("lsp")
3805        .and_then(Value::as_object)?
3806        .get("diagnostics_on_edit")
3807        .and_then(Value::as_bool)
3808}
3809
3810async fn send_route_bind_error(
3811    tx: &WriterSender,
3812    frame: &Frame,
3813    code: &str,
3814    message: &str,
3815    metrics: &DispatchPathMetrics,
3816) -> Result<(), SubcError> {
3817    send_route_bind_error_parts(
3818        tx,
3819        frame.header.ver,
3820        frame.header.corr,
3821        frame.header.flags,
3822        code,
3823        message,
3824        metrics,
3825    )
3826    .await
3827}
3828
3829async fn send_route_bind_error_parts(
3830    tx: &WriterSender,
3831    ver: u8,
3832    corr: u64,
3833    flags: Flags,
3834    code: &str,
3835    message: &str,
3836    metrics: &DispatchPathMetrics,
3837) -> Result<(), SubcError> {
3838    let response = build_error_frame(ver, 0, 0, corr, flags, code, message)?;
3839    send_reliable_writer_frame(tx, metrics, response, "RouteBind error").await?;
3840    log_route_bind_rejection(code, message);
3841    Ok(())
3842}
3843
3844/// Per-message rate limit for the bind-rejection warn line. A caller that
3845/// re-attaches a dead root forever turns this line into the entire readable
3846/// tail of the SHARED daemon log (measured 2026-08-09: 2.08M copies, ~40/sec,
3847/// 936MB log — other modules' incident lines pushed out of the tail).
3848/// The line itself stays byte-identical so external counters keep matching;
3849/// repeats inside the window are summarized with a suppressed count on the
3850/// next emission (volume stays diagnosable, per the log-diet convention).
3851fn log_route_bind_rejection(code: &str, message: &str) {
3852    const WINDOW: Duration = Duration::from_secs(60);
3853    static SUPPRESSED: OnceLock<StdMutex<HashMap<String, (Instant, u64)>>> = OnceLock::new();
3854    let map = SUPPRESSED.get_or_init(|| StdMutex::new(HashMap::new()));
3855    let mut map = match map.try_lock() {
3856        Ok(map) => map,
3857        // Contended: log unsuppressed rather than blocking or dropping.
3858        Err(_) => {
3859            log::warn!("subc attach: route bind rejected ({code}): {message}");
3860            return;
3861        }
3862    };
3863    let now = Instant::now();
3864    // Bound the map: dead roots churn, and an unbounded suppression map is
3865    // its own leak. Sweep expired entries once it grows past a fleet-sized
3866    // number of distinct rejection messages.
3867    if map.len() > 512 {
3868        map.retain(|_, (start, _)| now.duration_since(*start) < WINDOW);
3869    }
3870    match map.get_mut(message) {
3871        Some((window_start, suppressed)) if now.duration_since(*window_start) < WINDOW => {
3872            *suppressed += 1;
3873        }
3874        Some((window_start, suppressed)) => {
3875            if *suppressed > 0 {
3876                log::warn!(
3877                    "subc attach: route bind rejected ({code}): {message} (repeated {}x in last 60s)",
3878                    *suppressed
3879                );
3880            } else {
3881                log::warn!("subc attach: route bind rejected ({code}): {message}");
3882            }
3883            *window_start = now;
3884            *suppressed = 0;
3885        }
3886        None => {
3887            log::warn!("subc attach: route bind rejected ({code}): {message}");
3888            map.insert(message.to_string(), (now, 0));
3889        }
3890    }
3891}
3892
3893/// Route-channel tool call: `{name, arguments}` → executor lane → dispatch to
3894/// the sync command core → wrap the structured Response in a CallToolResult
3895/// `{content, isError}`. Tool-result mapping: the whole `{success, ...}` Response
3896/// serialized into ONE text block; `isError` carries `success == false`.
3897async fn handle_tool_call(
3898    tx: &WriterSender,
3899    frame: &Frame,
3900    mut phase_trace: PhaseTrace,
3901    routes: &HashMap<RouteChannel, RouteIdentity>,
3902    pending_binds: &HashMap<RouteChannel, PendingBind>,
3903    live_roots: &mut HashMap<ProjectRootId, RootMeta>,
3904    executor: &Arc<Executor>,
3905    shutdown: &Arc<Notify>,
3906    connection_cancel: &PersistentCancelSignal,
3907    bash_deferred_tx: &mpsc::Sender<bash::BashDeferredCompletion>,
3908    bash_poll_touch_tx: &mpsc::Sender<ProjectRootId>,
3909    metrics: &Arc<DispatchPathMetrics>,
3910    route_bash_cancels: &mut HashMap<RouteChannel, bash::RouteBashCancel>,
3911    pending_bash_asks: &mut HashMap<ReverseCorrKey, PendingBashAsk>,
3912    next_bash_ask_corr: &mut u64,
3913    bg_subs: &mut HashMap<RouteChannel, BgSub>,
3914    bg_sub_by_session: &mut HashMap<(ProjectRootId, String), RouteChannel>,
3915    bg_wake_pending: &mut HashSet<RouteChannel>,
3916    bg_wake_epoch: &mut HashMap<(ProjectRootId, String), u64>,
3917    dispatch: DispatchFn,
3918    allow_native_passthrough: bool,
3919) -> Result<(), SubcError> {
3920    let route_id = route_key(frame.header.channel, frame.header.epoch);
3921    if pending_binds.contains_key(&route_id) {
3922        let error = build_error_frame(
3923            frame.header.ver,
3924            frame.header.channel,
3925            frame.header.epoch,
3926            frame.header.corr,
3927            frame.header.flags,
3928            "route_not_bound",
3929            "route is not bound before tool call",
3930        )?;
3931        return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
3932    }
3933
3934    let Some(identity) = routes.get(&route_id).cloned() else {
3935        let error = build_error_frame(
3936            frame.header.ver,
3937            frame.header.channel,
3938            frame.header.epoch,
3939            frame.header.corr,
3940            frame.header.flags,
3941            "route_not_bound",
3942            "route is not bound before tool call",
3943        )?;
3944        return send_reliable_writer_frame(tx, metrics, error, "route_not_bound error").await;
3945    };
3946    let restore_watcher = live_roots
3947        .get(&identity.root)
3948        .is_some_and(|meta| meta.idle_artifacts_evicted);
3949    if let Some(meta) = live_roots.get_mut(&identity.root) {
3950        meta.reactivate_bound();
3951    }
3952    if restore_watcher {
3953        if let Some(ctx) = executor.actor_context(&identity.root) {
3954            crate::commands::configure::ensure_project_watcher(&ctx);
3955        }
3956    }
3957
3958    let route_request =
3959        serde_json::from_slice::<RouteRequest>(&frame.body).map_err(SubcError::Json)?;
3960    if matches!(
3961        route_request,
3962        RouteRequest::BgEvents(BgEventsRequest {
3963            op: BgEventsOp::BgEvents
3964        })
3965    ) {
3966        if let Some(old_sub) = bg_subs.get(&route_id).copied() {
3967            push::send_reliable_bg_stream_end(tx, metrics, route_id, &old_sub).await?;
3968        }
3969        if !identity.trust.allows_bash_observation() {
3970            bg_subs.remove(&route_id);
3971            bg_wake_pending.remove(&route_id);
3972            remove_bg_subscription_index(bg_sub_by_session, route_id, Some(&identity));
3973            let denied_sub = BgSub {
3974                corr: frame.header.corr,
3975                ver: frame.header.ver,
3976                flags: frame.header.flags,
3977            };
3978            push::send_reliable_bg_stream_end(tx, metrics, route_id, &denied_sub).await?;
3979            return Ok(());
3980        }
3981        bg_subs.insert(
3982            route_id,
3983            BgSub {
3984                corr: frame.header.corr,
3985                ver: frame.header.ver,
3986                flags: frame.header.flags,
3987            },
3988        );
3989        bg_sub_by_session.insert((identity.root.clone(), identity.session.clone()), route_id);
3990        push::arm_bg_wake(
3991            identity.root.clone(),
3992            identity.session.clone(),
3993            route_id,
3994            bg_wake_pending,
3995            bg_wake_epoch,
3996        );
3997        return Ok(());
3998    }
3999
4000    let RouteRequest::ToolCall(call) = route_request else {
4001        unreachable!("background event subscription returned above")
4002    };
4003    let bare_name = call.name;
4004    let arguments = strip_agent_preview_arg_owned(call.arguments);
4005    let format_context = crate::subc_format::FormatContext::from_tool_call(
4006        &bare_name,
4007        &arguments,
4008        identity.project_root.as_path(),
4009    );
4010
4011    let request_id = format!("subc-{}-{}", frame.header.channel, frame.header.corr);
4012    let bind_trust = identity.trust;
4013    let diagnostics_on_edit = live_roots
4014        .get(&identity.root)
4015        .map(|meta| meta.diagnostics_on_edit)
4016        .unwrap_or(false);
4017
4018    let requests_host = bare_name == "bash"
4019        && arguments
4020            .get("sandbox")
4021            .or_else(|| {
4022                arguments
4023                    .get("params")
4024                    .and_then(|params| params.get("sandbox"))
4025            })
4026            .and_then(Value::as_str)
4027            == Some("host");
4028    if matches!(bind_trust, BindTrust::Untrusted) && requests_host {
4029        let response = Response::error(
4030            request_id.clone(),
4031            "sandbox_escalation_denied",
4032            "sandbox host escalation is unavailable to untrusted principals",
4033        );
4034        let text = crate::subc_format::format_response_with_context(
4035            &bare_name,
4036            &response,
4037            &format_context,
4038        );
4039        let result = ToolCallResult { text, response };
4040        let response_frame = build_tool_response_frame(
4041            frame.header.ver,
4042            route_id,
4043            frame.header.corr,
4044            frame.header.flags,
4045            &result,
4046            bind_trust,
4047        )?;
4048        return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4049    }
4050
4051    if matches!(bind_trust, BindTrust::Untrusted)
4052        && is_bash_family_tool(&bare_name)
4053        && (bare_name != "bash" || !identity.consumer_elicitation_capable)
4054    {
4055        let response = bash::bash_denied_untrusted_response(request_id.clone());
4056        let text = crate::subc_format::format_response_with_context(
4057            &bare_name,
4058            &response,
4059            &format_context,
4060        );
4061        let result = ToolCallResult { text, response };
4062        let response_frame = build_tool_response_frame(
4063            frame.header.ver,
4064            route_id,
4065            frame.header.corr,
4066            frame.header.flags,
4067            &result,
4068            bind_trust,
4069        )?;
4070        return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4071    }
4072
4073    // A non-core name is NOT in the tool manifest. AFT fails closed and
4074    // does not trust subc to enforce the manifest: rejecting here is the
4075    // defense-in-depth backstop that prevents a forwarded native command
4076    // (e.g. `configure`, which would reach handle_configure and bypass
4077    // the RouteBind config-trust cap) from ever reaching dispatch. Only
4078    // the integration-test harness (run_subc_mode_for_test) opens this to
4079    // drive synthetic native commands through the executor.
4080    if !is_subc_agent_core_tool(&bare_name)
4081        && !is_subc_native_plumbing_tool(&bare_name)
4082        && !allow_native_passthrough
4083    {
4084        log::warn!(
4085            "subc tool call: rejecting non-manifest tool name {:?} on route {} (fail-closed)",
4086            bare_name,
4087            frame.header.channel
4088        );
4089        let response = Response::error(
4090            request_id.clone(),
4091            "unknown_tool",
4092            format!("tool {:?} is not in the AFT tool manifest", bare_name),
4093        );
4094        let text = crate::subc_format::format_response_with_context(
4095            &bare_name,
4096            &response,
4097            &format_context,
4098        );
4099        let result = ToolCallResult { text, response };
4100        let response_frame = build_tool_response_frame(
4101            frame.header.ver,
4102            route_id,
4103            frame.header.corr,
4104            frame.header.flags,
4105            &result,
4106            bind_trust,
4107        )?;
4108        return send_reliable_writer_frame(tx, metrics, response_frame, "tool response").await;
4109    }
4110
4111    if bare_name == "bash" {
4112        if matches!(bind_trust, BindTrust::Untrusted) {
4113            let plan = match bash::prepare_bash_elicitation_plan(
4114                &arguments,
4115                identity.project_root.as_path(),
4116            ) {
4117                Ok(plan) => plan,
4118                Err(error) => {
4119                    let response = Response::error(request_id.clone(), error.code, error.message);
4120                    let text = crate::subc_format::format_response_with_context(
4121                        &bare_name,
4122                        &response,
4123                        &format_context,
4124                    );
4125                    let result = ToolCallResult { text, response };
4126                    let response_frame = build_tool_response_frame(
4127                        frame.header.ver,
4128                        route_id,
4129                        frame.header.corr,
4130                        frame.header.flags,
4131                        &result,
4132                        bind_trust,
4133                    )?;
4134                    return send_reliable_writer_frame(
4135                        tx,
4136                        metrics,
4137                        response_frame,
4138                        "tool response",
4139                    )
4140                    .await;
4141                }
4142            };
4143
4144            let reverse_corr =
4145                allocate_reverse_corr(pending_bash_asks, route_id, next_bash_ask_corr);
4146            let ask_frame = build_bash_elicitation_request_frame(
4147                frame.header.ver,
4148                route_id,
4149                reverse_corr,
4150                frame.header.flags,
4151                &plan.command,
4152                &plan.asks,
4153            )?;
4154
4155            let meta = live_roots
4156                .entry(identity.root.clone())
4157                .or_insert_with(|| RootMeta::new(Instant::now()));
4158            meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
4159            meta.reactivate_bound();
4160
4161            let route_cancel =
4162                route_bash_cancels
4163                    .entry(route_id)
4164                    .or_insert_with(|| bash::RouteBashCancel {
4165                        token: PersistentCancelSignal::new(),
4166                        active_waits: 0,
4167                    });
4168            route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
4169            let cancel = bash::BashWaitCancel {
4170                connection: connection_cancel.clone(),
4171                route: route_cancel.token.clone(),
4172            };
4173            pending_bash_asks.insert(
4174                ReverseCorrKey {
4175                    route: route_id,
4176                    corr: reverse_corr,
4177                },
4178                PendingBashAsk {
4179                    route: route_id,
4180                    tool_corr: frame.header.corr,
4181                    tool_flags: frame.header.flags,
4182                    tool_ver: frame.header.ver,
4183                    root: identity.root.clone(),
4184                    project_root: identity.project_root.clone(),
4185                    session_id: identity.session.clone(),
4186                    spawn_principal: identity.spawn_principal.clone(),
4187                    request_id,
4188                    arguments,
4189                    format_context,
4190                    cancel,
4191                    grants: plan.grants,
4192                    expires_at: Instant::now() + bash_elicitation_timeout(),
4193                },
4194            );
4195            return send_reliable_writer_frame(tx, metrics, ask_frame, "bash elicitation request")
4196                .await;
4197        }
4198
4199        let meta = live_roots
4200            .entry(identity.root.clone())
4201            .or_insert_with(|| RootMeta::new(Instant::now()));
4202        meta.active_bash_waits = meta.active_bash_waits.saturating_add(1);
4203        meta.reactivate_bound();
4204
4205        let route_cancel =
4206            route_bash_cancels
4207                .entry(route_id)
4208                .or_insert_with(|| bash::RouteBashCancel {
4209                    token: PersistentCancelSignal::new(),
4210                    active_waits: 0,
4211                });
4212        route_cancel.active_waits = route_cancel.active_waits.saturating_add(1);
4213        let cancel = bash::BashWaitCancel {
4214            connection: connection_cancel.clone(),
4215            route: route_cancel.token.clone(),
4216        };
4217
4218        bash::submit_deferred_bash(
4219            executor,
4220            bash_deferred_tx,
4221            bash_poll_touch_tx,
4222            metrics,
4223            dispatch,
4224            identity.root.clone(),
4225            identity.project_root.clone(),
4226            identity.session.clone(),
4227            request_id,
4228            route_id,
4229            frame.header.corr,
4230            frame.header.flags,
4231            frame.header.ver,
4232            arguments,
4233            format_context,
4234            cancel,
4235            bind_trust,
4236            identity.spawn_principal.clone(),
4237            None,
4238        );
4239        return Ok(());
4240    }
4241
4242    let lane = command_lane(&bare_name);
4243    let tool_call_context = ToolCallContext {
4244        project_root: identity.project_root.clone(),
4245        session_id: Some(identity.session.clone()),
4246        request_id: request_id.clone(),
4247        diagnostics_on_edit,
4248        preview: call.preview,
4249    };
4250    let bare_name_for_frame = bare_name.clone();
4251    let identity_for_run = identity.clone();
4252    let completion_session = identity.session.clone();
4253    let completion_root = identity.project_root.clone();
4254    let request_id_for_force = request_id.clone();
4255    let format_context_for_frame = format_context.clone();
4256    let (tool_call_tx, tool_call_rx) = oneshot::channel::<ToolCallCompletion>();
4257    phase_trace.mark_executor_submitted();
4258    let rx = executor.submit_async(
4259        identity.root.clone(),
4260        lane,
4261        request_id.clone(),
4262        Box::new(move |ctx| {
4263            phase_trace.mark_job_admitted();
4264            log_ctx::with_session(Some(identity_for_run.session.clone()), || {
4265                let run = || {
4266                    let finalizer = |response: &mut Response| {
4267                        crate::response_finalize::finalize_response_with_bg_completions(
4268                            response,
4269                            ctx,
4270                            &identity_for_run.session,
4271                            &bare_name,
4272                            bind_trust.allows_bash_observation(),
4273                        );
4274                    };
4275                    match run_tool_call(
4276                        &bare_name,
4277                        arguments,
4278                        &format_context,
4279                        &tool_call_context,
4280                        ctx,
4281                        &dispatch,
4282                        Some(&finalizer),
4283                        Some(&mut phase_trace),
4284                    ) {
4285                        ToolCallOutcome::Unary(result) => {
4286                            let response = result.response;
4287                            let _ = tool_call_tx.send(ToolCallCompletion {
4288                                text: result.text,
4289                                phase_trace,
4290                            });
4291                            response
4292                        }
4293                    }
4294                };
4295                if matches!(bind_trust, BindTrust::Untrusted) {
4296                    ctx.with_force_restrict(&request_id_for_force, run)
4297                } else {
4298                    run()
4299                }
4300            })
4301        }),
4302    );
4303    let completion_tx = tx.clone();
4304    let completion_shutdown = Arc::clone(shutdown);
4305    let route = route_id;
4306    let corr = frame.header.corr;
4307    let flags = frame.header.flags;
4308    let ver = frame.header.ver;
4309    let completion_metrics = Arc::clone(metrics);
4310    tokio::spawn(async move {
4311        let _response_task = ResponseTaskGuard::new(&completion_metrics);
4312        let response = await_executor_response(rx, request_id.clone()).await;
4313        let (text, phase_trace) = match tool_call_rx.await {
4314            Ok(completion) => (completion.text, Some(completion.phase_trace)),
4315            Err(_) => (
4316                crate::subc_format::format_response_with_context(
4317                    &bare_name_for_frame,
4318                    &response,
4319                    &format_context_for_frame,
4320                ),
4321                None,
4322            ),
4323        };
4324        let result = ToolCallResult { text, response };
4325        let fatal = response_is_fatal_panic(&result.response);
4326        match build_tool_response_frame(ver, route, corr, flags, &result, bind_trust) {
4327            Ok(response_frame) => {
4328                let send_result = if let Some(phase_trace) = phase_trace {
4329                    let trace = ToolResponseWriteTrace::new(
4330                        phase_trace,
4331                        bare_name_for_frame,
4332                        completion_root,
4333                        completion_session,
4334                        route.channel,
4335                        corr,
4336                    );
4337                    send_traced_tool_response_frame(
4338                        &completion_tx,
4339                        &completion_metrics,
4340                        response_frame,
4341                        trace,
4342                    )
4343                    .await
4344                } else {
4345                    send_reliable_writer_frame(
4346                        &completion_tx,
4347                        &completion_metrics,
4348                        response_frame,
4349                        "tool response",
4350                    )
4351                    .await
4352                };
4353                if let Err(error) = send_result {
4354                    log::warn!("subc attach: failed to queue tool response frame: {error}");
4355                }
4356            }
4357            Err(error) => {
4358                log::error!("subc attach: failed to build tool response frame: {error}");
4359            }
4360        }
4361        if fatal {
4362            signal_fatal_teardown(
4363                &completion_tx,
4364                Some(route),
4365                ver,
4366                corr,
4367                &completion_shutdown,
4368                &completion_metrics,
4369            )
4370            .await;
4371        }
4372    });
4373    Ok(())
4374}
4375
4376fn submit_maintenance_job(
4377    executor: &Arc<Executor>,
4378    root_id: ProjectRootId,
4379    kind: MaintenanceDrainKind,
4380    bg_sessions_to_check: Vec<(String, u64)>,
4381    completion_tx: &mpsc::Sender<MaintenanceCompletion>,
4382    metrics: &Arc<DispatchPathMetrics>,
4383) {
4384    let request_id = format!(
4385        "subc-maintenance-drain-{}-{}",
4386        kind.label(),
4387        root_id.as_path().to_string_lossy()
4388    );
4389    let response_id = request_id.clone();
4390    let completion_root_id = root_id.clone();
4391    let maintenance_generation = executor
4392        .actor_context(&root_id)
4393        .map(|ctx| ctx.configure_generation())
4394        .unwrap_or(0);
4395    let (outcome_tx, outcome_rx) = oneshot::channel::<MaintenanceJobOutcome>();
4396    // ConfigureTail runs deferred configure mutations and needs the actor
4397    // epoch write gate. The other drain kinds only mutate subsystem state
4398    // behind that subsystem's own lock, so they run on MaintenanceCommit and
4399    // overlap interactive reads instead of excluding them.
4400    let lane = match kind {
4401        MaintenanceDrainKind::ConfigureTail => Lane::Mutating,
4402        MaintenanceDrainKind::Watcher
4403        | MaintenanceDrainKind::Lsp
4404        | MaintenanceDrainKind::CompletionDrains => Lane::MaintenanceCommit,
4405    };
4406    let rx = executor.submit_maintenance_async(
4407        root_id,
4408        lane,
4409        request_id.clone(),
4410        Box::new(move |ctx| {
4411            let outcome = match kind {
4412                MaintenanceDrainKind::Watcher => {
4413                    let drained = runtime_drain::drain_watcher_events_bounded(
4414                        ctx,
4415                        runtime_drain::WATCHER_PATH_DRAIN_BATCH_CAP,
4416                    );
4417                    MaintenanceJobOutcome {
4418                        empty_bg_sessions: Vec::new(),
4419                        requeue_kind: drained.has_more.then_some(kind),
4420                    }
4421                }
4422                MaintenanceDrainKind::Lsp => {
4423                    let drained = runtime_drain::drain_lsp_events_bounded(
4424                        ctx,
4425                        runtime_drain::LSP_EVENT_DRAIN_BATCH_CAP,
4426                    );
4427                    MaintenanceJobOutcome {
4428                        empty_bg_sessions: Vec::new(),
4429                        requeue_kind: drained.has_more.then_some(kind),
4430                    }
4431                }
4432                MaintenanceDrainKind::ConfigureTail => {
4433                    runtime_drain::drain_deferred_configure_maintenance(ctx);
4434                    runtime_drain::drain_configure_warning_events(ctx);
4435                    MaintenanceJobOutcome::default()
4436                }
4437                MaintenanceDrainKind::CompletionDrains => {
4438                    runtime_drain::drain_search_index_events(ctx);
4439                    runtime_drain::drain_callgraph_store_events(ctx);
4440                    runtime_drain::drain_semantic_index_events(ctx);
4441                    runtime_drain::drain_semantic_refresh_events(ctx);
4442                    runtime_drain::drain_inspect_events_for_generation(ctx, maintenance_generation);
4443                    let empty_bg_sessions = bg_sessions_to_check
4444                        .into_iter()
4445                        .filter(|(session, _)| {
4446                            !ctx.bash_background()
4447                                .has_completions_for_session(Some(session.as_str()))
4448                        })
4449                        .collect();
4450                    MaintenanceJobOutcome {
4451                        empty_bg_sessions,
4452                        requeue_kind: None,
4453                    }
4454                }
4455            };
4456            let requeued = outcome.requeue_kind.is_some();
4457            let _ = outcome_tx.send(outcome);
4458            Response::success(
4459                response_id,
4460                json!({ "drained": true, "kind": kind.label(), "requeued": requeued }),
4461            )
4462        }),
4463    );
4464    let completion_tx = completion_tx.clone();
4465    let completion_metrics = Arc::clone(metrics);
4466    tokio::spawn(async move {
4467        let _response_task = ResponseTaskGuard::new(&completion_metrics);
4468        let response = await_executor_response(rx, request_id).await;
4469        let outcome = outcome_rx.await.unwrap_or_default();
4470        let _ = send_counted_channel(
4471            &completion_tx,
4472            &completion_metrics.maintenance_queued,
4473            MaintenanceCompletion {
4474                root_id: completion_root_id,
4475                kind,
4476                response,
4477                empty_bg_sessions: outcome.empty_bg_sessions,
4478                requeue_kind: outcome.requeue_kind,
4479            },
4480        )
4481        .await;
4482    });
4483}
4484
4485async fn await_executor_response(rx: oneshot::Receiver<Response>, request_id: String) -> Response {
4486    rx.await
4487        .unwrap_or_else(|_| Response::error(request_id, "internal_error", "executor dropped"))
4488}
4489async fn signal_fatal_teardown(
4490    tx: &WriterSender,
4491    route: Option<RouteChannel>,
4492    ver: u8,
4493    corr: u64,
4494    shutdown: &Arc<Notify>,
4495    metrics: &DispatchPathMetrics,
4496) {
4497    if let Some(route) = route {
4498        if let Ok(frame) = build_goodbye_frame(ver, route.channel, route.epoch, corr) {
4499            if let Err(error) = send_frame(tx, metrics, frame).await {
4500                log::warn!(
4501                    "subc attach: failed to queue fatal route Goodbye for route {route}: {error}"
4502                );
4503            }
4504        }
4505    }
4506    if let Ok(frame) = build_goodbye_frame(ver, 0, 0, 0) {
4507        if let Err(error) = send_frame(tx, metrics, frame).await {
4508            log::warn!("subc attach: failed to queue fatal channel-0 Goodbye: {error}");
4509        }
4510    }
4511    shutdown.notify_one();
4512}
4513#[derive(Debug, Deserialize)]
4514#[serde(untagged)]
4515enum RouteRequest {
4516    BgEvents(BgEventsRequest),
4517    ToolCall(ToolCallRequest),
4518}
4519
4520#[derive(Debug, Deserialize)]
4521struct BgEventsRequest {
4522    op: BgEventsOp,
4523}
4524
4525#[derive(Debug, Deserialize)]
4526#[serde(rename_all = "snake_case")]
4527enum BgEventsOp {
4528    BgEvents,
4529}
4530
4531#[derive(Debug, Deserialize)]
4532struct ToolCallRequest {
4533    name: String,
4534    #[serde(default)]
4535    arguments: Value,
4536    /// Server-owned preview control (B1c-0): the plugin's mutation flow is
4537    /// preview -> permission ask -> apply. Dropping this field made "preview"
4538    /// calls mutate disk before the permission prompt and the subsequent
4539    /// apply fail with not-found.
4540    #[serde(default)]
4541    preview: bool,
4542}
4543
4544#[cfg(test)]
4545pub(crate) mod test_support {
4546    use super::*;
4547    use crate::bash_background::BgTaskStatus;
4548    use crate::protocol::{
4549        BashCompletedFrame, BashLongRunningFrame, BashPatternMatchFrame, ConfigureWarningsFrame,
4550        ProgressFrame, StatusChangedFrame,
4551    };
4552    use serde_json::json;
4553
4554    pub(super) fn test_root(name: &str) -> (tempfile::TempDir, ProjectRootId) {
4555        let dir = tempfile::Builder::new()
4556            .prefix(name)
4557            .tempdir()
4558            .expect("temp root");
4559        let root = ProjectRootId::from_path(dir.path()).expect("project root id");
4560        (dir, root)
4561    }
4562
4563    pub(super) fn test_ctx() -> Arc<AppContext> {
4564        Arc::new(AppContext::new(
4565            Box::new(crate::parser::TreeSitterProvider::new()),
4566            crate::config::Config::default(),
4567        ))
4568    }
4569
4570    pub(super) fn wait_for_watcher_count(ctx: &AppContext, expected: usize) {
4571        let deadline = Instant::now() + Duration::from_secs(30);
4572        loop {
4573            let observed = ctx.watcher_registry_count();
4574            if observed == expected {
4575                return;
4576            }
4577            assert!(
4578                Instant::now() < deadline,
4579                "watcher count did not settle before deadline: expected={expected}, observed={observed}"
4580            );
4581            std::thread::sleep(Duration::from_millis(50));
4582        }
4583    }
4584
4585    /// Sweep until `root` is forgotten, mirroring how production reaps.
4586    ///
4587    /// `reap_idle_roots` probes actor idleness with a try-lock and retains the
4588    /// root when the scheduler holds that lock — correct behavior, since the
4589    /// real caller sweeps on a timer and simply catches the root next tick. A
4590    /// test that asserts a single sweep succeeds is therefore asserting it wins
4591    /// a lock race that nothing in production depends on: `register_actor` wakes
4592    /// the scheduler, which grabs the same lock, and on a loaded runner that
4593    /// window is wide enough to lose. Sweep to the outcome instead.
4594    pub(super) fn reap_until_forgotten(
4595        root: &ProjectRootId,
4596        live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4597        pending_binds: &HashMap<RouteChannel, PendingBind>,
4598        root_channels: &HashMap<ProjectRootId, HashSet<RouteChannel>>,
4599        executor: &Arc<Executor>,
4600        metrics: &DispatchPathMetrics,
4601    ) -> IdleReapOutcome {
4602        let deadline = Instant::now() + Duration::from_secs(30);
4603        loop {
4604            let outcome = reap_idle_roots(
4605                Instant::now(),
4606                live_roots,
4607                pending_binds,
4608                root_channels,
4609                executor,
4610                metrics,
4611            );
4612            if outcome.forgotten_deleted_roots.contains(root) {
4613                return outcome;
4614            }
4615            assert!(
4616                Instant::now() < deadline,
4617                "deleted root was never forgotten: {root:?}"
4618            );
4619            std::thread::sleep(Duration::from_millis(10));
4620        }
4621    }
4622
4623    pub(super) fn wait_for_actor_root_count(app: &App, expected: usize) {
4624        let deadline = Instant::now() + Duration::from_secs(30);
4625        loop {
4626            let observed = app.actor_root_count();
4627            if observed == expected {
4628                return;
4629            }
4630            assert!(
4631                Instant::now() < deadline,
4632                "actor root count did not settle before deadline: expected={expected}, observed={observed}"
4633            );
4634            std::thread::sleep(Duration::from_millis(50));
4635        }
4636    }
4637
4638    pub(super) fn status_frame(seq: u64) -> PushFrame {
4639        status_frame_with_session(seq, None)
4640    }
4641
4642    pub(super) fn status_frame_with_session(seq: u64, session_id: Option<&str>) -> PushFrame {
4643        PushFrame::StatusChanged(StatusChangedFrame {
4644            frame_type: "status_changed",
4645            session_id: session_id.map(str::to_string),
4646            snapshot: json!({ "seq": seq }),
4647        })
4648    }
4649
4650    pub(super) fn completion_frame(task_id: &str) -> PushFrame {
4651        completion_frame_with_session(task_id, "session-1")
4652    }
4653
4654    pub(super) fn completion_frame_with_session(task_id: &str, session_id: &str) -> PushFrame {
4655        PushFrame::BashCompleted(BashCompletedFrame {
4656            frame_type: "bash_completed",
4657            task_id: task_id.to_string(),
4658            session_id: session_id.to_string(),
4659            status: BgTaskStatus::Completed,
4660            exit_code: Some(0),
4661            command: format!("echo {task_id}"),
4662            output_preview: String::new(),
4663            output_truncated: false,
4664            original_tokens: None,
4665            compressed_tokens: None,
4666            tokens_skipped: false,
4667            status_reason: None,
4668        })
4669    }
4670
4671    pub(super) fn long_running_frame(task_id: &str, elapsed_ms: u64) -> PushFrame {
4672        long_running_frame_with_session(task_id, "session-1", elapsed_ms)
4673    }
4674
4675    pub(super) fn long_running_frame_with_session(
4676        task_id: &str,
4677        session_id: &str,
4678        elapsed_ms: u64,
4679    ) -> PushFrame {
4680        PushFrame::BashLongRunning(BashLongRunningFrame {
4681            frame_type: "bash_long_running",
4682            task_id: task_id.to_string(),
4683            session_id: session_id.to_string(),
4684            command: format!("sleep {elapsed_ms}"),
4685            elapsed_ms,
4686        })
4687    }
4688
4689    pub(super) fn pattern_match_frame(session_id: &str) -> PushFrame {
4690        PushFrame::BashPatternMatch(BashPatternMatchFrame {
4691            frame_type: "bash_pattern_match",
4692            task_id: "task-pattern".to_string(),
4693            session_id: session_id.to_string(),
4694            watch_id: "watch-1".to_string(),
4695            match_text: "needle".to_string(),
4696            match_offset: 7,
4697            context: "haystack needle".to_string(),
4698            once: true,
4699            reason: "pattern_match",
4700        })
4701    }
4702
4703    pub(super) fn configure_warnings_frame(session_id: Option<&str>) -> PushFrame {
4704        PushFrame::ConfigureWarnings(ConfigureWarningsFrame {
4705            frame_type: "configure_warnings",
4706            session_id: session_id.map(str::to_string),
4707            project_root: "/tmp/subc-test".to_string(),
4708            warnings: Vec::new(),
4709        })
4710    }
4711
4712    pub(super) fn route_identity(root: &ProjectRootId, session_id: &str) -> RouteIdentity {
4713        route_identity_with_trust(root, session_id, BindTrust::FirstParty)
4714    }
4715
4716    pub(super) fn route_identity_with_trust(
4717        root: &ProjectRootId,
4718        session_id: &str,
4719        trust: BindTrust,
4720    ) -> RouteIdentity {
4721        RouteIdentity(Arc::new(RouteIdentityData {
4722            root: root.clone(),
4723            project_root: root.as_path().to_path_buf(),
4724            harness: "opencode".to_string(),
4725            session: session_id.to_string(),
4726            trust,
4727            spawn_principal: AuthenticatedPrincipal::RouteBind {
4728                trust: trust.sandbox_trust(),
4729                route_channel: 0,
4730                route_epoch: 0,
4731                project_root: root.as_path().to_path_buf(),
4732                harness: "opencode".to_string(),
4733                session_id: session_id.to_string(),
4734                principal_id: Some(match trust {
4735                    BindTrust::FirstParty => "direct".to_string(),
4736                    BindTrust::Untrusted => "unverified".to_string(),
4737                }),
4738            },
4739            consumer_elicitation_capable: false,
4740        }))
4741    }
4742
4743    pub(super) fn progress_frame(request_id: &str, kind: ProgressKind, chunk: &str) -> PushFrame {
4744        PushFrame::Progress(ProgressFrame::new(request_id, kind, chunk))
4745    }
4746
4747    pub(super) fn status_seq(frame: &PushFrame) -> Option<u64> {
4748        match frame {
4749            PushFrame::StatusChanged(status) => status.snapshot.get("seq").and_then(|v| v.as_u64()),
4750            _ => None,
4751        }
4752    }
4753
4754    pub(super) fn completion_task(frame: &PushFrame) -> Option<&str> {
4755        match frame {
4756            PushFrame::BashCompleted(completion) => Some(completion.task_id.as_str()),
4757            _ => None,
4758        }
4759    }
4760
4761    pub(super) fn push_frame_task_id(frame: &Frame) -> Option<String> {
4762        let body: serde_json::Value = serde_json::from_slice(&frame.body).expect("push body");
4763        body.get("task_id")
4764            .and_then(serde_json::Value::as_str)
4765            .map(str::to_string)
4766    }
4767}
4768
4769#[cfg(test)]
4770mod tests {
4771    use super::test_support::{
4772        completion_frame, reap_until_forgotten, route_identity, test_ctx, test_root,
4773        wait_for_actor_root_count, wait_for_watcher_count,
4774    };
4775    use super::*;
4776    use crate::bash_background::BgTaskStatus;
4777
4778    fn attach_error(kind: io::ErrorKind) -> SubcError {
4779        SubcError::Connect {
4780            endpoint: "127.0.0.1:1".to_string(),
4781            source: io::Error::new(kind, "constructed attach failure"),
4782        }
4783    }
4784
4785    fn auth_io_error(kind: io::ErrorKind) -> SubcError {
4786        SubcError::Auth {
4787            endpoint: "127.0.0.1:1".to_string(),
4788            source: subc_transport::AuthError::Io {
4789                stage: subc_transport::AuthStage::ServerProof,
4790                source: io::Error::new(kind, "constructed auth failure"),
4791            },
4792        }
4793    }
4794
4795    #[test]
4796    fn initial_attach_error_classifier_distinguishes_transient_and_permanent_failures() {
4797        let transient_errors = vec![
4798            attach_error(io::ErrorKind::ConnectionRefused),
4799            attach_error(io::ErrorKind::TimedOut),
4800            attach_error(io::ErrorKind::ConnectionReset),
4801            auth_io_error(io::ErrorKind::ConnectionAborted),
4802            auth_io_error(io::ErrorKind::BrokenPipe),
4803            SubcError::Auth {
4804                endpoint: "127.0.0.1:1".to_string(),
4805                source: subc_transport::AuthError::UnexpectedEof {
4806                    stage: subc_transport::AuthStage::ServerProof,
4807                    expected: 4,
4808                    actual: 0,
4809                },
4810            },
4811            SubcError::Auth {
4812                endpoint: "127.0.0.1:1".to_string(),
4813                source: subc_transport::AuthError::Timeout {
4814                    stage: subc_transport::AuthStage::ServerProof,
4815                    deadline: AUTH_DEADLINE,
4816                },
4817            },
4818        ];
4819        for error in &transient_errors {
4820            assert_eq!(
4821                classify_attach_error(error),
4822                AttachErrorClass::Transient,
4823                "expected transient: {error}"
4824            );
4825        }
4826
4827        let permanent_errors = vec![
4828            attach_error(io::ErrorKind::PermissionDenied),
4829            auth_io_error(io::ErrorKind::InvalidData),
4830            SubcError::Auth {
4831                endpoint: "127.0.0.1:1".to_string(),
4832                source: subc_transport::AuthError::InvalidServerProof,
4833            },
4834            SubcError::Auth {
4835                endpoint: "127.0.0.1:1".to_string(),
4836                source: subc_transport::AuthError::DaemonIdMismatch,
4837            },
4838            SubcError::ConnectionFile {
4839                path: PathBuf::from("subc-connection.json"),
4840                source: subc_transport::ConnectionFileError::Invalid {
4841                    reason: "constructed invalid file".to_string(),
4842                },
4843            },
4844            SubcError::NoEndpoint {
4845                path: PathBuf::from("subc-connection.json"),
4846            },
4847            SubcError::InvalidEndpoint {
4848                path: PathBuf::from("subc-connection.json"),
4849                endpoint: "not-an-ip:1234".to_string(),
4850            },
4851        ];
4852        for error in &permanent_errors {
4853            assert_eq!(
4854                classify_attach_error(error),
4855                AttachErrorClass::Permanent,
4856                "expected permanent: {error}"
4857            );
4858        }
4859    }
4860
4861    #[test]
4862    fn incompatible_wire_version_is_rejected_before_tcp_connect() {
4863        let conn_dir = tempfile::tempdir().expect("connection tempdir");
4864        let conn_path = conn_dir.path().join("subc-connection.json");
4865        let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind listener");
4866        listener
4867            .set_nonblocking(true)
4868            .expect("set listener nonblocking");
4869        let port = listener.local_addr().expect("listener addr").port();
4870        connection_file::write_atomic(
4871            &conn_path,
4872            &connection_file::ConnectionInfo {
4873                schema: connection_file::SCHEMA_VERSION,
4874                wire_version: Some(PROTOCOL_VERSION.wrapping_add(1)),
4875                endpoints: vec![connection_file::Endpoint {
4876                    host: "127.0.0.1".to_string(),
4877                    port,
4878                }],
4879                key: vec![0x42; subc_transport::KEY_LEN],
4880                daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
4881                pid: std::process::id(),
4882                daemon_ver: "subc-test".to_string(),
4883            },
4884        )
4885        .expect("write connection file");
4886
4887        let runtime = tokio::runtime::Builder::new_current_thread()
4888            .enable_all()
4889            .build()
4890            .expect("test runtime");
4891        let result = runtime.block_on(connect_and_authenticate_with_policy(
4892            &conn_path,
4893            AttachRetryPolicy {
4894                budget: Duration::from_secs(1),
4895                initial_backoff: Duration::from_millis(5),
4896                max_backoff: Duration::from_millis(10),
4897                jitter_percent: 0,
4898            },
4899        ));
4900        assert!(matches!(
4901            result,
4902            Err(SubcError::ConnectionFile {
4903                source: connection_file::ConnectionFileError::WireVersionMismatch { .. },
4904                ..
4905            })
4906        ));
4907        assert!(matches!(
4908            listener.accept(),
4909            Err(error) if error.kind() == std::io::ErrorKind::WouldBlock
4910        ));
4911    }
4912
4913    #[test]
4914    fn initial_attach_unreachable_endpoint_retries_until_budget_then_fails_loud() {
4915        let conn_dir = tempfile::tempdir().expect("connection tempdir");
4916        let conn_path = conn_dir.path().join("subc-connection.json");
4917        let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve port");
4918        let port = listener.local_addr().expect("reserved addr").port();
4919        drop(listener);
4920        connection_file::write_atomic(
4921            &conn_path,
4922            &connection_file::ConnectionInfo {
4923                schema: connection_file::SCHEMA_VERSION,
4924                wire_version: Some(PROTOCOL_VERSION),
4925                endpoints: vec![connection_file::Endpoint {
4926                    host: "127.0.0.1".to_string(),
4927                    port,
4928                }],
4929                key: vec![0x42; subc_transport::KEY_LEN],
4930                daemon_id: [0x24; subc_transport::DAEMON_ID_LEN],
4931                pid: std::process::id(),
4932                daemon_ver: "subc-test".to_string(),
4933            },
4934        )
4935        .expect("write connection file");
4936
4937        let policy = AttachRetryPolicy {
4938            budget: Duration::from_millis(40),
4939            initial_backoff: Duration::from_millis(5),
4940            max_backoff: Duration::from_millis(10),
4941            jitter_percent: 0,
4942        };
4943        let runtime = tokio::runtime::Builder::new_current_thread()
4944            .enable_all()
4945            .build()
4946            .expect("test runtime");
4947        let started_at = Instant::now();
4948        let result = runtime.block_on(connect_and_authenticate_with_policy(&conn_path, policy));
4949        let elapsed = started_at.elapsed();
4950        let error = match result {
4951            Ok(_) => panic!("unreachable endpoint unexpectedly attached"),
4952            Err(error) => error,
4953        };
4954
4955        assert!(matches!(error, SubcError::Connect { .. }), "{error}");
4956        assert!(
4957            elapsed >= Duration::from_millis(35),
4958            "retry budget ended too early: {elapsed:?}"
4959        );
4960        assert!(
4961            elapsed < Duration::from_secs(1),
4962            "retry budget was not bounded: {elapsed:?}"
4963        );
4964    }
4965
4966    fn due_maintenance_jobs_without_actor_context(
4967        live_roots: &mut HashMap<ProjectRootId, RootMeta>,
4968        budget: usize,
4969        pending_bind_roots: &HashSet<ProjectRootId>,
4970    ) -> (Vec<(ProjectRootId, MaintenanceDrainKind)>, bool) {
4971        due_maintenance_jobs(
4972            live_roots,
4973            None,
4974            &HashMap::new(),
4975            &HashSet::new(),
4976            budget,
4977            pending_bind_roots,
4978        )
4979    }
4980
4981    fn actor_ctx_with_dirty_search_index(
4982        root: &Path,
4983        storage: &Path,
4984        file_name: &str,
4985        old_contents: &str,
4986        new_contents: &str,
4987    ) -> (Arc<AppContext>, PathBuf, PathBuf) {
4988        let file = root.join(file_name);
4989        std::fs::write(&file, old_contents).expect("write source");
4990        let canonical_root = std::fs::canonicalize(root).expect("canonical root");
4991        let ctx = Arc::new(AppContext::new(
4992            Box::new(crate::parser::TreeSitterProvider::new()),
4993            Config {
4994                project_root: Some(root.to_path_buf()),
4995                storage_dir: Some(storage.to_path_buf()),
4996                ..Config::default()
4997            },
4998        ));
4999        ctx.set_canonical_cache_root(canonical_root.clone());
5000
5001        let cache_dir = crate::search_index::resolve_cache_dir(&canonical_root, Some(storage));
5002        let mut index = crate::search_index::SearchIndex::build(&canonical_root);
5003        let git_head = index.stored_git_head().map(str::to_owned);
5004        index.write_to_disk(&cache_dir, git_head.as_deref());
5005
5006        std::fs::write(&file, new_contents).expect("edit source");
5007        index.update_file(&file);
5008        *ctx.search_index()
5009            .write()
5010            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
5011        (ctx, canonical_root, cache_dir)
5012    }
5013
5014    #[test]
5015    fn graceful_shutdown_flushes_every_actor_search_index() {
5016        let storage = tempfile::tempdir().expect("storage tempdir");
5017        let (root1_dir, root1) = test_root("shutdown-flush-root-1");
5018        let (root2_dir, root2) = test_root("shutdown-flush-root-2");
5019        let (ctx1, canonical_root1, cache_dir1) = actor_ctx_with_dirty_search_index(
5020            root1_dir.path(),
5021            storage.path(),
5022            "alpha.txt",
5023            "old actor one token\n",
5024            "new actor one token\n",
5025        );
5026        let (ctx2, canonical_root2, cache_dir2) = actor_ctx_with_dirty_search_index(
5027            root2_dir.path(),
5028            storage.path(),
5029            "beta.txt",
5030            "old actor two token\n",
5031            "new actor two token\n",
5032        );
5033
5034        let executor = Executor::new();
5035        assert!(executor.register_actor(root1.clone(), Arc::clone(&ctx1)));
5036        assert!(executor.register_actor(root2.clone(), Arc::clone(&ctx2)));
5037
5038        flush_actor_indexes_on_graceful_shutdown(&executor.actor_contexts());
5039
5040        let mut restored1 =
5041            crate::search_index::SearchIndex::read_from_disk(&cache_dir1, &canonical_root1)
5042                .expect("load flushed root one index");
5043        restored1.ready = true;
5044        assert_eq!(
5045            restored1
5046                .grep("new actor one token", true, &[], &[], &canonical_root1, 10)
5047                .matches
5048                .len(),
5049            1,
5050            "graceful subc shutdown should flush the first root's trigram delta"
5051        );
5052
5053        let mut restored2 =
5054            crate::search_index::SearchIndex::read_from_disk(&cache_dir2, &canonical_root2)
5055                .expect("load flushed root two index");
5056        restored2.ready = true;
5057        assert_eq!(
5058            restored2
5059                .grep("new actor two token", true, &[], &[], &canonical_root2, 10)
5060                .matches
5061                .len(),
5062            1,
5063            "graceful subc shutdown should flush every registered root"
5064        );
5065    }
5066
5067    #[test]
5068    fn idle_root_reaper_closes_artifacts_and_stops_watcher() {
5069        let _ = env_logger::builder().is_test(true).try_init();
5070        let (root_dir, root) = test_root("idle-root-reaper");
5071        let storage = tempfile::tempdir().expect("storage tempdir");
5072        std::fs::write(
5073            root_dir.path().join("main.rs"),
5074            "fn entry() { leaf(); }\nfn leaf() {}\n",
5075        )
5076        .expect("source file");
5077        let canonical_root = std::fs::canonicalize(root_dir.path()).expect("canonical root");
5078        let app = App::default_shared();
5079        let ctx = Arc::new(AppContext::from_app(
5080            Arc::clone(&app),
5081            Config {
5082                project_root: Some(canonical_root.clone()),
5083                storage_dir: Some(storage.path().to_path_buf()),
5084                callgraph_store: true,
5085                search_index: true,
5086                ..Config::default()
5087            },
5088        ));
5089        ctx.set_canonical_cache_root(canonical_root.clone());
5090        let project_key = crate::search_index::artifact_cache_key(&canonical_root);
5091        crate::root_cache::configure_artifact_access(&canonical_root, &project_key, false);
5092        assert!(ctx
5093            .ensure_callgraph_store()
5094            .expect("build callgraph store")
5095            .is_some());
5096
5097        let cache_dir =
5098            crate::search_index::resolve_cache_dir(&canonical_root, Some(storage.path()));
5099        let mut index = crate::search_index::SearchIndex::build(&canonical_root);
5100        let git_head = index.stored_git_head().map(str::to_owned);
5101        index.write_to_disk(&cache_dir, git_head.as_deref());
5102        *ctx.search_index()
5103            .write()
5104            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(index);
5105        // Seed a completed warm verification so the test can prove eviction
5106        // downgrades it to Strict rather than passing vacuously.
5107        let seeded_generation =
5108            crate::cache_freshness::artifact_generation(&cache_dir.join("cache.bin"))
5109                .expect("seeded artifact generation");
5110        crate::cache_freshness::record_verify_completed(
5111            &canonical_root,
5112            crate::cache_freshness::VerifyArtifact::Search,
5113            Some(seeded_generation),
5114        );
5115        assert!(
5116            matches!(
5117                crate::cache_freshness::warm_verify_plan(
5118                    canonical_root.as_path(),
5119                    crate::cache_freshness::VerifyArtifact::Search,
5120                    Some(seeded_generation),
5121                ),
5122                crate::cache_freshness::WarmVerifyPlan::Skip
5123            ),
5124            "memo must be warm before eviction for the downgrade assertion to bite"
5125        );
5126
5127        let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
5128        let _dispatch_tx = dispatch_tx;
5129        let shutdown = Arc::new(AtomicBool::new(false));
5130        let thread_shutdown = Arc::clone(&shutdown);
5131        let join = std::thread::spawn(move || {
5132            while !thread_shutdown.load(Ordering::SeqCst) {
5133                std::thread::yield_now();
5134            }
5135        });
5136        ctx.install_watcher_runtime(
5137            dispatch_rx,
5138            crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
5139        );
5140        wait_for_watcher_count(&ctx, 1);
5141
5142        let executor = Arc::new(Executor::new());
5143        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5144        ctx.mark_subc_unbound();
5145        let mut live_roots = HashMap::new();
5146        let mut meta = RootMeta::new(Instant::now());
5147        meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
5148        meta.unbound_quiesced = true;
5149        live_roots.insert(root.clone(), meta);
5150
5151        let message = idle_root_eviction_message(&root, &ctx.memory_root_snapshot(), None);
5152        assert!(message.contains("evicted idle root"));
5153        assert!(message.contains("freed ~"));
5154        assert!(message.contains("semantic"));
5155        assert!(!message.contains("semantic not estimated retained"));
5156        assert!(message.contains("trigram"));
5157        assert!(message.contains("retained: bash"));
5158        assert!(message.contains("parser_pool"));
5159
5160        assert_eq!(
5161            reap_idle_roots(
5162                Instant::now(),
5163                &mut live_roots,
5164                &HashMap::new(),
5165                &HashMap::new(),
5166                &executor,
5167                &DispatchPathMetrics::new(),
5168            )
5169            .evicted,
5170            1
5171        );
5172        assert!(ctx.search_index().read().unwrap().is_none());
5173        wait_for_watcher_count(&ctx, 0);
5174        // The watcher stopped with the eviction, so the idle interval is
5175        // unobserved: the pre-seeded warm-verify memo (Skip) must fall back to
5176        // strict content verification (stat-first would miss same-size,
5177        // preserved-mtime edits made while nobody was watching).
5178        assert!(
5179            matches!(
5180                crate::cache_freshness::warm_verify_plan(
5181                    canonical_root.as_path(),
5182                    crate::cache_freshness::VerifyArtifact::Search,
5183                    Some(seeded_generation),
5184                ),
5185                crate::cache_freshness::WarmVerifyPlan::Strict
5186            ),
5187            "idle eviction must force strict re-verification"
5188        );
5189        assert!(
5190            crate::search_index::SearchIndex::read_from_disk(&cache_dir, &canonical_root).is_some()
5191        );
5192        ctx.mark_subc_bound();
5193        assert!(ctx
5194            .ensure_callgraph_store()
5195            .expect("reopen callgraph store")
5196            .is_some());
5197        assert!(live_roots[&root].idle_artifacts_evicted);
5198    }
5199
5200    #[test]
5201    fn idle_root_reaper_applies_ttl_to_unbound_roots() {
5202        let (_root_dir, root) = test_root("idle-root-ttl-gate");
5203        let ctx = test_ctx();
5204        let executor = Arc::new(Executor::new());
5205        assert!(executor.register_actor(root.clone(), ctx));
5206        let ctx = executor.actor_context(&root).expect("actor context");
5207        ctx.mark_subc_unbound();
5208        let now = Instant::now();
5209        let mut meta = RootMeta::new(now);
5210        meta.unbound_quiesced = true;
5211        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5212
5213        // A recently-unbound root stays warm: a transient unbind (host
5214        // restart) must not pay the strict-verify + forced-rebuild teardown
5215        // on the next maintenance sweep.
5216        assert_eq!(
5217            reap_idle_roots(
5218                now,
5219                &mut live_roots,
5220                &HashMap::new(),
5221                &HashMap::new(),
5222                &executor,
5223                &DispatchPathMetrics::new(),
5224            )
5225            .evicted,
5226            0
5227        );
5228        assert!(!live_roots[&root].idle_artifacts_evicted);
5229
5230        // Past the TTL the same unbound root pays the full teardown. The
5231        // pending reconciliation paths retained across the transient-unbind
5232        // window would block eviction forever through
5233        // `artifact_eviction_blocked`; the reaper disposes them because the
5234        // strict gap invalidation subsumes their purpose.
5235        ctx.add_pending_search_index_paths([root.as_path().join("retained.rs")]);
5236        assert_eq!(
5237            reap_idle_roots(
5238                now + IDLE_ROOT_TTL,
5239                &mut live_roots,
5240                &HashMap::new(),
5241                &HashMap::new(),
5242                &executor,
5243                &DispatchPathMetrics::new(),
5244            )
5245            .evicted,
5246            1
5247        );
5248        assert!(live_roots[&root].idle_artifacts_evicted);
5249        assert!(
5250            ctx.take_pending_search_index_paths().is_empty(),
5251            "TTL eviction must dispose retained pending reconciliation paths"
5252        );
5253    }
5254
5255    #[test]
5256    fn blocked_ttl_eviction_restores_taken_pending_reconciliation_state() {
5257        let (_root_dir, root) = test_root("ttl-eviction-blocked-restore");
5258        let ctx = test_ctx();
5259        let executor = Arc::new(Executor::new());
5260        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5261        ctx.mark_subc_unbound();
5262
5263        // Retained pending path from the transient-unbind window plus a
5264        // SECONDARY eviction blocker (a non-ready resident search index, the
5265        // dirty-index blocker in artifact_eviction_blocked). Disposal must be
5266        // transactional: the blocked eviction may be followed by a rebind, and
5267        // the path is the only repair record for its consumed watcher event.
5268        let pending = root.as_path().join("edited-while-unbound.rs");
5269        ctx.add_pending_search_index_paths([pending.clone()]);
5270        let dirty_source = root.as_path().join("dirty.rs");
5271        std::fs::write(&dirty_source, "fn dirty() {}\n").expect("dirty source");
5272        let mut dirty = crate::search_index::SearchIndex::new();
5273        dirty.ready = true;
5274        dirty.update_file(&dirty_source);
5275        *ctx.search_index()
5276            .write()
5277            .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(dirty);
5278        assert!(ctx.artifact_eviction_blocked());
5279
5280        let mut live_roots = HashMap::new();
5281        let mut meta = RootMeta::new(Instant::now());
5282        meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
5283        meta.unbound_quiesced = true;
5284        live_roots.insert(root.clone(), meta);
5285
5286        assert_eq!(
5287            reap_idle_roots(
5288                Instant::now(),
5289                &mut live_roots,
5290                &HashMap::new(),
5291                &HashMap::new(),
5292                &executor,
5293                &DispatchPathMetrics::new(),
5294            )
5295            .evicted,
5296            0,
5297            "the dirty index must still block this eviction"
5298        );
5299        assert_eq!(
5300            ctx.take_pending_search_index_paths(),
5301            vec![pending],
5302            "a blocked eviction must restore the taken pending paths"
5303        );
5304    }
5305
5306    #[test]
5307    fn idle_reap_with_bound_route_keeps_watcher_running() {
5308        let (_root_dir, root) = test_root("bound-root-reap-gate");
5309        let ctx = test_ctx();
5310        let executor = Arc::new(Executor::new());
5311        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5312
5313        let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
5314        let _dispatch_tx = dispatch_tx;
5315        let shutdown = Arc::new(AtomicBool::new(false));
5316        let thread_shutdown = Arc::clone(&shutdown);
5317        let join = std::thread::spawn(move || {
5318            while !thread_shutdown.load(Ordering::SeqCst) {
5319                std::thread::yield_now();
5320            }
5321        });
5322        ctx.install_watcher_runtime(
5323            dispatch_rx,
5324            crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
5325        );
5326
5327        let mut meta = RootMeta::new(Instant::now());
5328        meta.last_touched = Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1);
5329        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5330        let bound = HashMap::from([(root, HashSet::from([route_key(7, 1)]))]);
5331        assert_eq!(
5332            reap_idle_roots(
5333                Instant::now(),
5334                &mut live_roots,
5335                &HashMap::new(),
5336                &bound,
5337                &executor,
5338                &DispatchPathMetrics::new(),
5339            )
5340            .evicted,
5341            0
5342        );
5343        wait_for_watcher_count(&ctx, 1);
5344        ctx.stop_watcher_runtime_in_background();
5345        wait_for_watcher_count(&ctx, 0);
5346    }
5347
5348    #[test]
5349    fn deleted_root_with_bound_route_is_reclaimed_after_confirmation_and_routes_are_purged() {
5350        let (root_dir, root) = test_root("deleted-bound-root-reap");
5351        let executor = Arc::new(Executor::new());
5352        assert!(executor.register_actor(root.clone(), test_ctx()));
5353        root_dir.close().expect("delete project root");
5354
5355        let route = route_key(19, 3);
5356        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
5357        let cancel_signal = PersistentCancelSignal::new();
5358        let mut routes = HashMap::from([(route, route_identity(&root, "deleted-route"))]);
5359        let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
5360        let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
5361        let mut route_bash_cancels = HashMap::from([(
5362            route,
5363            bash::RouteBashCancel {
5364                token: cancel_signal.clone(),
5365                active_waits: 0,
5366            },
5367        )]);
5368        let metrics = DispatchPathMetrics::new();
5369
5370        let first = reap_idle_roots(
5371            Instant::now(),
5372            &mut live_roots,
5373            &HashMap::new(),
5374            &root_channels,
5375            &executor,
5376            &metrics,
5377        );
5378        assert!(first.forgotten_deleted_roots.is_empty());
5379        assert!(executor.actor_registered(&root));
5380
5381        let mut forgotten = Vec::new();
5382        for _ in 0..100 {
5383            let outcome = reap_idle_roots(
5384                Instant::now(),
5385                &mut live_roots,
5386                &HashMap::new(),
5387                &root_channels,
5388                &executor,
5389                &metrics,
5390            );
5391            if !outcome.forgotten_deleted_roots.is_empty() {
5392                forgotten = outcome.forgotten_deleted_roots;
5393                break;
5394            }
5395            std::thread::sleep(Duration::from_millis(10));
5396        }
5397        assert_eq!(forgotten, vec![root.clone()]);
5398        assert!(!executor.actor_registered(&root));
5399
5400        let mut retry_buffer = HashMap::new();
5401        let mut reclaimed_routes = ReclaimedRoutes::default();
5402        let mut session_identity = HashMap::new();
5403        let mut push_buffer = HashMap::new();
5404        let mut bg_subs = HashMap::new();
5405        let mut bg_sub_by_session = HashMap::new();
5406        let mut bg_wake_pending = HashSet::new();
5407        let mut bg_wake_epoch = HashMap::new();
5408        let mut pending_bash_asks = HashMap::new();
5409        purge_deleted_root_residents(
5410            &root,
5411            &mut routes,
5412            &mut root_channels,
5413            &mut installed_route_epochs,
5414            &mut route_bash_cancels,
5415            &mut retry_buffer,
5416            &mut reclaimed_routes,
5417            &mut session_identity,
5418            &mut push_buffer,
5419            &mut bg_subs,
5420            &mut bg_sub_by_session,
5421            &mut bg_wake_pending,
5422            &mut bg_wake_epoch,
5423            &mut pending_bash_asks,
5424        );
5425
5426        assert!(routes.is_empty());
5427        assert!(root_channels.is_empty());
5428        assert!(installed_route_epochs.is_empty());
5429        assert!(route_bash_cancels.is_empty());
5430        assert!(reclaimed_routes.contains(route));
5431        assert!(cancel_signal.is_cancelled());
5432    }
5433
5434    /// The control for the deleted-root reclamation above: a root whose
5435    /// directory still EXISTS must stay retained while it holds a bound route,
5436    /// even with every other reap precondition satisfied. Without this, the
5437    /// suite cannot tell "reclaim roots that are provably gone" apart from
5438    /// "reap any root that looks idle" — the second would tear down live
5439    /// sessions, and both satisfy the deleted-root tests.
5440    #[test]
5441    fn live_root_with_bound_route_is_never_reclaimed() {
5442        let (_root_dir, root) = test_root("live-bound-root-retained");
5443        let ctx = test_ctx();
5444        ctx.mark_subc_unbound();
5445        let executor = Arc::new(Executor::new());
5446        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5447
5448        let route = RouteChannel {
5449            channel: 7,
5450            epoch: 1,
5451        };
5452        let mut meta = RootMeta::new(Instant::now() - IDLE_ROOT_TTL - Duration::from_secs(1));
5453        meta.unbound_quiesced = true;
5454        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5455        let root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
5456
5457        // Sweep three times because reclamation requires the root path to be
5458        // observed missing on two CONSECUTIVE sweeps. A single sweep would pass
5459        // here even if reclamation were wrongly unconditional, since the first
5460        // absence never reclaims on its own.
5461        for _ in 0..3 {
5462            let outcome = reap_idle_roots(
5463                Instant::now(),
5464                &mut live_roots,
5465                &HashMap::new(),
5466                &root_channels,
5467                &executor,
5468                &DispatchPathMetrics::new(),
5469            );
5470            assert!(
5471                outcome.forgotten_deleted_roots.is_empty(),
5472                "a root whose directory exists must never be forgotten"
5473            );
5474        }
5475
5476        assert!(live_roots.contains_key(&root), "live root must be retained");
5477        assert!(
5478            executor.actor_registered(&root),
5479            "live root's actor must survive"
5480        );
5481        assert!(
5482            root.as_path().exists(),
5483            "test vehicle must keep the directory alive; otherwise this control proves nothing"
5484        );
5485    }
5486
5487    #[test]
5488    fn deleted_root_is_not_reclaimed_on_first_absence_observation() {
5489        let (root_dir, root) = test_root("deleted-root-first-observation");
5490        let ctx = test_ctx();
5491        ctx.mark_subc_unbound();
5492        let executor = Arc::new(Executor::new());
5493        assert!(executor.register_actor(root.clone(), ctx));
5494        root_dir.close().expect("delete project root");
5495
5496        let mut meta = RootMeta::new(Instant::now());
5497        meta.unbound_quiesced = true;
5498        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5499        let outcome = reap_idle_roots(
5500            Instant::now(),
5501            &mut live_roots,
5502            &HashMap::new(),
5503            &HashMap::new(),
5504            &executor,
5505            &DispatchPathMetrics::new(),
5506        );
5507
5508        assert!(outcome.forgotten_deleted_roots.is_empty());
5509        assert!(live_roots.contains_key(&root));
5510        assert!(executor.actor_registered(&root));
5511    }
5512
5513    fn spawn_background_for_root(
5514        ctx: &AppContext,
5515        root: &ProjectRootId,
5516        storage: &tempfile::TempDir,
5517        session_id: &str,
5518    ) -> (String, u32) {
5519        // Windows refuses to delete a directory that is a running process's
5520        // cwd (ERROR_SHARING_VIOLATION), so the task must not live inside the
5521        // project root these tests delete. The kill path matches on the task's
5522        // registered project_root, not its cwd, so pointing the cwd at task
5523        // storage keeps the association under test intact.
5524        let command = if cfg!(windows) {
5525            // timeout.exe requires a console; ping is the standard sleep shim.
5526            "ping -n 31 127.0.0.1 > nul"
5527        } else {
5528            "sleep 30"
5529        };
5530        let task_id = ctx
5531            .bash_background()
5532            .spawn(
5533                crate::sandbox_spawn::SpawnPlan::Unsandboxed,
5534                command,
5535                session_id.to_string(),
5536                storage.path().to_path_buf(),
5537                HashMap::new(),
5538                Some(Duration::from_secs(60)),
5539                storage.path().to_path_buf(),
5540                8,
5541                true,
5542                false,
5543                Some(root.as_path().to_path_buf()),
5544            )
5545            .expect("spawn background task");
5546        let snapshot = ctx
5547            .bash_background()
5548            .status(
5549                &task_id,
5550                session_id,
5551                Some(root.as_path()),
5552                Some(storage.path()),
5553                0,
5554            )
5555            .expect("background task status");
5556        (task_id, snapshot.child_pid.expect("background child pid"))
5557    }
5558
5559    fn wait_for_background_exit(pid: u32) {
5560        let deadline = Instant::now() + Duration::from_secs(5);
5561        while crate::bash_background::process::is_process_alive(pid) {
5562            assert!(
5563                Instant::now() < deadline,
5564                "background task process survived kill"
5565            );
5566            std::thread::sleep(Duration::from_millis(20));
5567        }
5568    }
5569
5570    #[test]
5571    fn deleted_root_reclaims_background_task_after_two_absence_sweeps() {
5572        let (root_dir, root) = test_root("deleted-root-background-task");
5573        let storage = tempfile::tempdir().expect("task storage");
5574        let ctx = test_ctx();
5575        let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "reclaim-session");
5576        assert!(crate::bash_background::process::is_process_alive(pid));
5577
5578        let executor = Arc::new(Executor::new());
5579        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5580        assert!(executor.actor_is_idle(&root));
5581        root_dir.close().expect("delete project root");
5582        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
5583        let pending_binds = HashMap::new();
5584        let root_channels = HashMap::new();
5585        let metrics = DispatchPathMetrics::new();
5586
5587        let first = reap_idle_roots(
5588            Instant::now(),
5589            &mut live_roots,
5590            &pending_binds,
5591            &root_channels,
5592            &executor,
5593            &metrics,
5594        );
5595        assert!(first.forgotten_deleted_roots.is_empty());
5596        assert!(crate::bash_background::process::is_process_alive(pid));
5597
5598        let outcome = reap_until_forgotten(
5599            &root,
5600            &mut live_roots,
5601            &pending_binds,
5602            &root_channels,
5603            &executor,
5604            &metrics,
5605        );
5606        assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
5607        wait_for_background_exit(pid);
5608
5609        let snapshot = ctx
5610            .bash_background()
5611            .status(
5612                &task_id,
5613                "reclaim-session",
5614                Some(root.as_path()),
5615                Some(storage.path()),
5616                0,
5617            )
5618            .expect("reclaimed task status");
5619        assert_eq!(snapshot.info.status, BgTaskStatus::Killed);
5620        assert_eq!(
5621            snapshot.info.status_reason.as_deref(),
5622            Some(crate::bash_background::registry::ROOT_RECLAIMED_REASON)
5623        );
5624        assert_eq!(
5625            serde_json::to_value(&snapshot).expect("serialize bash status")["status_reason"],
5626            crate::bash_background::registry::ROOT_RECLAIMED_REASON
5627        );
5628        let completion = ctx
5629            .bash_background()
5630            .drain_completions_for_session(Some("reclaim-session"))
5631            .pop()
5632            .expect("reclaimed task completion");
5633        assert_eq!(
5634            completion.status_reason.as_deref(),
5635            Some(crate::bash_background::registry::ROOT_RECLAIMED_REASON)
5636        );
5637    }
5638
5639    #[test]
5640    fn existing_unbound_root_keeps_background_task_alive_across_sweeps() {
5641        let (root_dir, root) = test_root("existing-root-background-task");
5642        let storage = tempfile::tempdir().expect("task storage");
5643        let ctx = test_ctx();
5644        ctx.mark_subc_unbound();
5645        let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "existing-session");
5646
5647        let executor = Arc::new(Executor::new());
5648        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5649        let mut meta = RootMeta::new(
5650            Instant::now()
5651                .checked_sub(IDLE_ROOT_TTL + Duration::from_secs(1))
5652                .expect("old root timestamp"),
5653        );
5654        meta.unbound_quiesced = true;
5655        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5656        let pending_binds = HashMap::new();
5657        let root_channels = HashMap::new();
5658        let metrics = DispatchPathMetrics::new();
5659
5660        for _ in 0..8 {
5661            reap_idle_roots(
5662                Instant::now(),
5663                &mut live_roots,
5664                &pending_binds,
5665                &root_channels,
5666                &executor,
5667                &metrics,
5668            );
5669            std::thread::sleep(Duration::from_millis(10));
5670        }
5671        assert!(root_dir.path().exists());
5672        assert!(crate::bash_background::process::is_process_alive(pid));
5673        let snapshot = ctx
5674            .bash_background()
5675            .status(
5676                &task_id,
5677                "existing-session",
5678                Some(root.as_path()),
5679                Some(storage.path()),
5680                0,
5681            )
5682            .expect("existing task status");
5683        assert_eq!(snapshot.info.status, BgTaskStatus::Running);
5684        let _ = ctx.bash_background().kill(&task_id, "existing-session");
5685        wait_for_background_exit(pid);
5686    }
5687
5688    #[test]
5689    fn restored_root_between_absence_sweeps_keeps_background_task_alive() {
5690        let (root_dir, root) = test_root("restored-root-background-task");
5691        let storage = tempfile::tempdir().expect("task storage");
5692        let ctx = test_ctx();
5693        let (task_id, pid) = spawn_background_for_root(&ctx, &root, &storage, "restored-session");
5694
5695        let executor = Arc::new(Executor::new());
5696        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5697        root_dir.close().expect("delete project root");
5698        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
5699        let pending_binds = HashMap::new();
5700        let root_channels = HashMap::new();
5701        let metrics = DispatchPathMetrics::new();
5702
5703        let first = reap_idle_roots(
5704            Instant::now(),
5705            &mut live_roots,
5706            &pending_binds,
5707            &root_channels,
5708            &executor,
5709            &metrics,
5710        );
5711        assert!(first.forgotten_deleted_roots.is_empty());
5712        std::fs::create_dir_all(root.as_path()).expect("restore project root");
5713        let second = reap_idle_roots(
5714            Instant::now(),
5715            &mut live_roots,
5716            &pending_binds,
5717            &root_channels,
5718            &executor,
5719            &metrics,
5720        );
5721        assert!(second.forgotten_deleted_roots.is_empty());
5722        assert!(crate::bash_background::process::is_process_alive(pid));
5723        let _ = ctx.bash_background().kill(&task_id, "restored-session");
5724        wait_for_background_exit(pid);
5725    }
5726
5727    #[test]
5728    fn observing_root_again_resets_deleted_sweep_confirmation() {
5729        let (root_dir, root) = test_root("deleted-root-observation-reset");
5730        let ctx = test_ctx();
5731        ctx.mark_subc_unbound();
5732        let executor = Arc::new(Executor::new());
5733        assert!(executor.register_actor(root.clone(), ctx));
5734        root_dir.close().expect("delete project root");
5735
5736        let mut meta = RootMeta::new(Instant::now());
5737        meta.unbound_quiesced = true;
5738        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5739        let pending_binds = HashMap::new();
5740        let root_channels = HashMap::new();
5741        let metrics = DispatchPathMetrics::new();
5742
5743        let first = reap_idle_roots(
5744            Instant::now(),
5745            &mut live_roots,
5746            &pending_binds,
5747            &root_channels,
5748            &executor,
5749            &metrics,
5750        );
5751        assert!(first.forgotten_deleted_roots.is_empty());
5752
5753        std::fs::create_dir_all(root.as_path()).expect("restore project root");
5754        reap_idle_roots(
5755            Instant::now(),
5756            &mut live_roots,
5757            &pending_binds,
5758            &root_channels,
5759            &executor,
5760            &metrics,
5761        );
5762        std::fs::remove_dir_all(root.as_path()).expect("delete project root again");
5763
5764        let after_reset = reap_idle_roots(
5765            Instant::now(),
5766            &mut live_roots,
5767            &pending_binds,
5768            &root_channels,
5769            &executor,
5770            &metrics,
5771        );
5772        assert!(after_reset.forgotten_deleted_roots.is_empty());
5773        assert!(live_roots.contains_key(&root));
5774        assert!(executor.actor_registered(&root));
5775    }
5776
5777    #[test]
5778    fn deleted_idle_root_is_fully_forgotten_and_status_counts_drop() {
5779        let (root_dir, root) = test_root("deleted-root-reap");
5780        let app = App::default_shared();
5781        let ctx = Arc::new(AppContext::from_app(
5782            Arc::clone(&app),
5783            Config {
5784                project_root: Some(root.as_path().to_path_buf()),
5785                ..Config::default()
5786            },
5787        ));
5788        ctx.set_canonical_cache_root(root.as_path().to_path_buf());
5789        ctx.mark_subc_unbound();
5790        let executor = Arc::new(Executor::new());
5791        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5792        assert_eq!(app.actor_root_count(), 1);
5793        drop(ctx);
5794        root_dir.close().expect("delete project root");
5795
5796        let mut meta = RootMeta::new(Instant::now());
5797        meta.unbound_quiesced = true;
5798        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5799        let outcome = reap_until_forgotten(
5800            &root,
5801            &mut live_roots,
5802            &HashMap::new(),
5803            &HashMap::new(),
5804            &executor,
5805            &DispatchPathMetrics::new(),
5806        );
5807        assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
5808        assert!(!executor.actor_registered(&root));
5809        assert!(!live_roots.contains_key(&root));
5810        wait_for_actor_root_count(&app, 0);
5811
5812        let status_ctx = AppContext::from_app(app, Config::default());
5813        let status = status_ctx.build_status_snapshot();
5814        assert_eq!(status["runtime"]["live_actor_roots"], 0);
5815        assert_eq!(status["runtime"]["open_routes"], 0);
5816    }
5817
5818    #[test]
5819    fn deleted_root_reap_blocker_census_is_exposed_in_health_metrics() {
5820        let (root_dir, root) = test_root("deleted-root-reap-census");
5821        let executor = Arc::new(Executor::new());
5822        assert!(executor.register_actor(root.clone(), test_ctx()));
5823        root_dir.close().expect("delete project root");
5824
5825        let mut live_roots = HashMap::from([(root, RootMeta::new(Instant::now()))]);
5826        let metrics = DispatchPathMetrics::new();
5827        let outcome = reap_idle_roots(
5828            Instant::now(),
5829            &mut live_roots,
5830            &HashMap::new(),
5831            &HashMap::new(),
5832            &executor,
5833            &metrics,
5834        );
5835        assert_eq!(outcome.evicted, 0);
5836
5837        let report = build_health_report(
5838            &executor,
5839            &HashMap::new(),
5840            &metrics,
5841            &crate::context::App::default_shared(),
5842        );
5843        let reap = report
5844            .metrics
5845            .as_ref()
5846            .and_then(|metrics| metrics.get("reap"))
5847            .expect("reap health metrics");
5848        assert_eq!(reap["deleted_retained"].as_u64(), Some(1));
5849        assert_eq!(reap["blockers"]["absence_unconfirmed"].as_u64(), Some(1));
5850        assert_eq!(reap["blockers"]["unbound_quiesced"].as_u64(), Some(0));
5851        assert_eq!(reap["blockers"]["actor_busy"].as_u64(), Some(0));
5852    }
5853
5854    #[test]
5855    fn connection_exit_quiesces_queued_maintenance_and_deleted_root_is_purged() {
5856        let (root_dir, root) = test_root("connection-exit-deleted-root");
5857        let executor = Arc::new(Executor::new());
5858        assert!(executor.register_actor(root.clone(), test_ctx()));
5859
5860        let route = route_key(11, 1);
5861        let mut meta = RootMeta::new(Instant::now());
5862        meta.maintenance_pending = true;
5863        meta.maintenance_queued_kinds
5864            .push_back(MaintenanceDrainKind::CompletionDrains);
5865        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5866        let mut pending_binds = HashMap::new();
5867        let mut routes = HashMap::from([(route, route_identity(&root, "abandoned"))]);
5868        let mut root_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
5869        let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
5870        let mut route_bash_cancels = HashMap::new();
5871
5872        quiesce_connection_roots(
5873            &mut live_roots,
5874            &mut pending_binds,
5875            &mut routes,
5876            &mut root_channels,
5877            &mut installed_route_epochs,
5878            &mut route_bash_cancels,
5879            &executor,
5880        );
5881        assert!(live_roots[&root].unbound_quiesced);
5882        assert!(!live_roots[&root].maintenance_pending);
5883        assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
5884        assert!(routes.is_empty());
5885        assert!(root_channels.is_empty());
5886
5887        root_dir.close().expect("delete project root");
5888        let metrics = DispatchPathMetrics::new();
5889        let outcome = reap_until_forgotten(
5890            &root,
5891            &mut live_roots,
5892            &pending_binds,
5893            &root_channels,
5894            &executor,
5895            &metrics,
5896        );
5897        let mut session_identity = HashMap::new();
5898        let mut push_buffer = HashMap::new();
5899        let mut bg_subs = HashMap::new();
5900        let mut bg_sub_by_session = HashMap::new();
5901        let mut bg_wake_pending = HashSet::new();
5902        let mut bg_wake_epoch = HashMap::new();
5903        let mut pending_bash_asks = HashMap::new();
5904        let mut retry_buffer = HashMap::new();
5905        let mut reclaimed_routes = ReclaimedRoutes::default();
5906        for forgotten in &outcome.forgotten_deleted_roots {
5907            purge_deleted_root_residents(
5908                forgotten,
5909                &mut routes,
5910                &mut root_channels,
5911                &mut installed_route_epochs,
5912                &mut route_bash_cancels,
5913                &mut retry_buffer,
5914                &mut reclaimed_routes,
5915                &mut session_identity,
5916                &mut push_buffer,
5917                &mut bg_subs,
5918                &mut bg_sub_by_session,
5919                &mut bg_wake_pending,
5920                &mut bg_wake_epoch,
5921                &mut pending_bash_asks,
5922            );
5923        }
5924
5925        assert_eq!(outcome.forgotten_deleted_roots, vec![root.clone()]);
5926        assert!(!executor.actor_registered(&root));
5927        assert!(!live_roots.contains_key(&root));
5928    }
5929
5930    #[test]
5931    fn unbound_root_quiesces_maintenance_without_removing_actor() {
5932        let (_root_dir, root) = test_root("unbound-root-quiesce");
5933        let ctx = test_ctx();
5934        let executor = Arc::new(Executor::new());
5935        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
5936        let mut meta = RootMeta::new(Instant::now());
5937        meta.maintenance_pending = true;
5938        meta.maintenance_jobs_in_flight = 1;
5939        meta.maintenance_queued_kinds
5940            .push_back(MaintenanceDrainKind::ConfigureTail);
5941        let mut live_roots = HashMap::from([(root.clone(), meta)]);
5942        // Warm state planted before the unbind: quiesce must keep it.
5943        *ctx.search_index()
5944            .write()
5945            .unwrap_or_else(std::sync::PoisonError::into_inner) =
5946            Some(crate::search_index::SearchIndex::new());
5947        ctx.set_cache_writer_capabilities(true, true);
5948        let pending = root.as_path().join("pending.rs");
5949        ctx.add_pending_search_index_paths([pending.clone()]);
5950        // A warm verify memo must survive the transient unbind: the watcher
5951        // keeps running, so no unobserved window exists and the next warm
5952        // reload must not pay a strict full-corpus re-hash.
5953        let canonical_root = root.as_path().to_path_buf();
5954        let artifact = canonical_root.join("cache.bin");
5955        std::fs::write(&artifact, b"warm-artifact").expect("write artifact");
5956        let seeded_generation = crate::cache_freshness::artifact_generation(&artifact);
5957        crate::cache_freshness::record_verify_completed(
5958            &canonical_root,
5959            crate::cache_freshness::VerifyArtifact::Search,
5960            seeded_generation,
5961        );
5962        assert!(matches!(
5963            crate::cache_freshness::warm_verify_plan(
5964                &canonical_root,
5965                crate::cache_freshness::VerifyArtifact::Search,
5966                seeded_generation,
5967            ),
5968            crate::cache_freshness::WarmVerifyPlan::Skip
5969        ));
5970        // A live watcher runtime must survive quiesce (its events accumulate
5971        // for the rebind replay).
5972        let (dispatch_tx, dispatch_rx) = crate::watcher_filter::watcher_dispatch_channel();
5973        let _dispatch_tx = dispatch_tx;
5974        let shutdown = Arc::new(AtomicBool::new(false));
5975        let thread_shutdown = Arc::clone(&shutdown);
5976        let join = std::thread::spawn(move || {
5977            while !thread_shutdown.load(Ordering::SeqCst) {
5978                std::thread::yield_now();
5979            }
5980        });
5981        ctx.install_watcher_runtime(
5982            dispatch_rx,
5983            crate::watcher_filter::WatcherThreadHandle::new(shutdown, join),
5984        );
5985        assert!(ctx.watcher_runtime_active());
5986
5987        quiesce_unbound_root(&root, &mut live_roots, &executor);
5988        let meta = &live_roots[&root];
5989        assert!(meta.unbound_quiesced);
5990        assert!(ctx.subc_unbound_quiesced());
5991        assert!(meta.maintenance_pending);
5992        assert!(meta.maintenance_queued_kinds.is_empty());
5993        assert!(executor.actor_registered(&root));
5994        // Transient unbind keeps the root warm: resident artifacts stay
5995        // resident, no forced callgraph rebuild is planted, and pending
5996        // reconciliation paths survive for the rebind replay.
5997        assert!(
5998            ctx.search_index()
5999                .read()
6000                .unwrap_or_else(std::sync::PoisonError::into_inner)
6001                .is_some(),
6002            "quiesce must not evict resident artifacts"
6003        );
6004        assert_eq!(
6005            ctx.pending_callgraph_store_force_token(),
6006            None,
6007            "quiesce must not force a callgraph rebuild"
6008        );
6009        assert_eq!(
6010            ctx.take_pending_search_index_paths(),
6011            vec![pending],
6012            "quiesce must retain pending watcher-derived paths"
6013        );
6014        assert!(
6015            matches!(
6016                crate::cache_freshness::warm_verify_plan(
6017                    &canonical_root,
6018                    crate::cache_freshness::VerifyArtifact::Search,
6019                    seeded_generation,
6020                ),
6021                crate::cache_freshness::WarmVerifyPlan::Skip
6022            ),
6023            "quiesce must not invalidate the warm verify memo"
6024        );
6025        assert!(
6026            ctx.watcher_runtime_active(),
6027            "quiesce must not stop a running watcher"
6028        );
6029        ctx.stop_watcher_runtime();
6030
6031        let meta = live_roots.get_mut(&root).expect("root metadata");
6032        note_maintenance_completion(
6033            meta,
6034            Some(MaintenanceDrainKind::ConfigureTail),
6035            false,
6036            meta.unbound_quiesced,
6037        );
6038        assert!(!meta.maintenance_pending);
6039        assert!(meta.maintenance_queued_kinds.is_empty());
6040    }
6041
6042    #[test]
6043    fn same_root_higher_epoch_replacement_does_not_quiesce_between_generations() {
6044        let (_dir, root) = test_root("same-root-replacement");
6045        let route = route_key(7, 1);
6046        let installed_channels = HashMap::from([(root.clone(), HashSet::from([route]))]);
6047        let root_channels = HashMap::new();
6048
6049        assert!(!route_removal_will_quiesce_root(
6050            &root,
6051            route,
6052            &installed_channels,
6053            false,
6054            Some(&root),
6055        ));
6056        assert!(route_removal_will_quiesce_root(
6057            &root,
6058            route,
6059            &installed_channels,
6060            false,
6061            None,
6062        ));
6063        assert!(!should_quiesce_removed_root(
6064            &root,
6065            &root_channels,
6066            false,
6067            Some(&root),
6068        ));
6069        assert!(should_quiesce_removed_root(
6070            &root,
6071            &root_channels,
6072            false,
6073            None,
6074        ));
6075        assert!(!should_quiesce_removed_root(
6076            &root,
6077            &root_channels,
6078            true,
6079            None,
6080        ));
6081    }
6082
6083    #[test]
6084    fn root_quiesces_only_after_its_last_route_is_removed_and_reactivates_on_bind() {
6085        let (_root_dir, root) = test_root("unbound-root-route-count");
6086        let executor = Arc::new(Executor::new());
6087        assert!(executor.register_actor(root.clone(), test_ctx()));
6088        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
6089        let mut root_channels = HashMap::from([(
6090            root.clone(),
6091            HashSet::from([route_key(7, 1), route_key(8, 1)]),
6092        )]);
6093
6094        remove_root_channel(&mut root_channels, &root, route_key(7, 1));
6095        if !root_channels.contains_key(&root) {
6096            quiesce_unbound_root(&root, &mut live_roots, &executor);
6097        }
6098        assert!(!live_roots[&root].unbound_quiesced);
6099
6100        remove_root_channel(&mut root_channels, &root, route_key(8, 1));
6101        if !root_channels.contains_key(&root) {
6102            quiesce_unbound_root(&root, &mut live_roots, &executor);
6103        }
6104        assert!(live_roots[&root].unbound_quiesced);
6105
6106        live_roots
6107            .get_mut(&root)
6108            .expect("root metadata")
6109            .note_activity();
6110        assert!(
6111            live_roots[&root].unbound_quiesced,
6112            "late asynchronous activity must not reactivate an unbound root"
6113        );
6114
6115        live_roots
6116            .get_mut(&root)
6117            .expect("root metadata")
6118            .reactivate_bound();
6119        assert!(!live_roots[&root].unbound_quiesced);
6120    }
6121
6122    #[test]
6123    fn allocator_pressure_relief_requires_every_root_to_be_idle() {
6124        let (_idle_dir, idle_root) = test_root("allocator-relief-idle");
6125        let (_active_dir, active_root) = test_root("allocator-relief-active");
6126        let now = Instant::now();
6127        let mut live_roots = HashMap::new();
6128        let mut idle = RootMeta::new(now);
6129        idle.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
6130        live_roots.insert(idle_root, idle);
6131        assert!(process_has_been_idle(now, &live_roots));
6132
6133        live_roots.insert(active_root.clone(), RootMeta::new(now));
6134        assert!(!process_has_been_idle(now, &live_roots));
6135
6136        let active = live_roots
6137            .get_mut(&active_root)
6138            .expect("active root metadata");
6139        active.last_touched = now - IDLE_ROOT_TTL - Duration::from_secs(1);
6140        active.active_bash_waits = 1;
6141        assert!(!process_has_been_idle(now, &live_roots));
6142    }
6143
6144    #[test]
6145    fn pressure_relief_log_reports_before_and_after_measurements() {
6146        let allocator = crate::memory::AllocatorMemorySnapshot {
6147            status: "measured",
6148            bytes_in_use: Some(8 * 1024 * 1024),
6149            size_allocated: Some(12 * 1024 * 1024),
6150            retained_slack_bytes: Some(4 * 1024 * 1024),
6151            not_estimated: None,
6152        };
6153        let relief = crate::memory::AllocatorPressureRelief {
6154            bytes_released: 3 * 1024 * 1024,
6155            rss_before_bytes: Some(20 * 1024 * 1024),
6156            rss_after_bytes: Some(17 * 1024 * 1024),
6157            allocator_before: allocator.clone(),
6158            allocator_after: crate::memory::AllocatorMemorySnapshot {
6159                size_allocated: Some(9 * 1024 * 1024),
6160                retained_slack_bytes: Some(1024 * 1024),
6161                ..allocator
6162            },
6163        };
6164        let message = pressure_relief_label(&relief);
6165        assert!(message.contains("RSS 20.0 MB -> 17.0 MB"));
6166        assert!(message.contains("allocated 12.0 MB -> 9.0 MB"));
6167        assert!(message.contains("slack 4.0 MB -> 1.0 MB"));
6168        assert!(message.contains("reported 3.0 MB released"));
6169    }
6170
6171    #[test]
6172    fn due_maintenance_jobs_skip_poisoned_roots() {
6173        let (_healthy_dir, healthy_root) = test_root("maintenance-healthy");
6174        let (_poisoned_dir, poisoned_root) = test_root("maintenance-poisoned");
6175        let mut live_roots = HashMap::new();
6176        live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
6177        let mut poisoned_meta = RootMeta::new(Instant::now());
6178        poisoned_meta.maintenance_poisoned = true;
6179        live_roots.insert(poisoned_root.clone(), poisoned_meta);
6180
6181        let (due, deferred) = due_maintenance_jobs_without_actor_context(
6182            &mut live_roots,
6183            MAINTENANCE_SUBMIT_BUDGET,
6184            &HashSet::new(),
6185        );
6186
6187        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6188        assert!(due.iter().all(|(root, _)| root == &healthy_root));
6189        assert!(!deferred);
6190        assert!(live_roots[&healthy_root].maintenance_pending);
6191        assert_eq!(
6192            live_roots[&healthy_root].maintenance_jobs_in_flight,
6193            INITIAL_MAINTENANCE_JOB_COUNT
6194        );
6195        assert!(!live_roots[&poisoned_root].maintenance_pending);
6196    }
6197
6198    #[test]
6199    fn due_maintenance_jobs_do_not_restart_quiesced_root_work() {
6200        let (_dir, root) = test_root("maintenance-unbound");
6201        let mut meta = RootMeta::new(Instant::now());
6202        meta.unbound_quiesced = true;
6203        let mut live_roots = HashMap::from([(root.clone(), meta)]);
6204
6205        let (due, deferred) = due_maintenance_jobs_without_actor_context(
6206            &mut live_roots,
6207            MAINTENANCE_SUBMIT_BUDGET,
6208            &HashSet::new(),
6209        );
6210
6211        assert!(due.is_empty());
6212        assert!(!deferred);
6213        assert!(!live_roots[&root].maintenance_pending);
6214    }
6215
6216    #[test]
6217    fn idle_bg_subscription_queues_no_jobs_until_a_wake_arrives() {
6218        let (_dir, root) = test_root("maintenance-idle-bg-subscription");
6219        let ctx = test_ctx();
6220        assert!(!ctx.completion_drains_have_work());
6221
6222        let executor = Executor::new();
6223        assert!(executor.register_actor(root.clone(), ctx));
6224        let mut live_roots = HashMap::from([(root.clone(), RootMeta::new(Instant::now()))]);
6225        let session = "idle-session".to_string();
6226        let channel = route_key(17, 1);
6227        let bg_sub_by_session = HashMap::from([((root.clone(), session.clone()), channel)]);
6228        let mut bg_wake_pending = HashSet::new();
6229
6230        let (idle_tick_jobs, deferred) = due_maintenance_jobs(
6231            &mut live_roots,
6232            Some(&executor),
6233            &bg_sub_by_session,
6234            &bg_wake_pending,
6235            MAINTENANCE_SUBMIT_BUDGET,
6236            &HashSet::new(),
6237        );
6238        assert!(idle_tick_jobs.is_empty());
6239        assert!(!deferred);
6240        assert!(!live_roots[&root].maintenance_pending);
6241
6242        // A completion can arm its wake after the idle tick's probes. The wake
6243        // remains loop-owned state, so the following tick must observe it.
6244        let mut bg_wake_epoch = HashMap::new();
6245        push::arm_bg_wake(
6246            root.clone(),
6247            session,
6248            channel,
6249            &mut bg_wake_pending,
6250            &mut bg_wake_epoch,
6251        );
6252        let (next_tick_jobs, deferred) = due_maintenance_jobs(
6253            &mut live_roots,
6254            Some(&executor),
6255            &bg_sub_by_session,
6256            &bg_wake_pending,
6257            MAINTENANCE_SUBMIT_BUDGET,
6258            &HashSet::new(),
6259        );
6260        assert_eq!(
6261            next_tick_jobs,
6262            vec![(root, MaintenanceDrainKind::CompletionDrains)]
6263        );
6264        assert!(!deferred);
6265    }
6266
6267    #[tokio::test]
6268    async fn subc_configure_tail_precedes_completed_search_install() {
6269        let root_dir = tempfile::tempdir().unwrap();
6270        let storage = tempfile::tempdir().unwrap();
6271        let root = ProjectRootId::from_path(root_dir.path()).unwrap();
6272        let (ctx, ignored_path) =
6273            runtime_drain::configure_search_order_context_for_test(root_dir.path(), storage.path());
6274        let ctx = Arc::new(ctx);
6275        assert!(!runtime_drain::watcher_path_is_ignored_by_current_matcher(
6276            &ctx,
6277            &ignored_path
6278        ));
6279
6280        let executor = Arc::new(Executor::new());
6281        assert!(executor.register_actor(root.clone(), Arc::clone(&ctx)));
6282        let metrics = Arc::new(DispatchPathMetrics::new());
6283        let (completion_tx, mut completion_rx) = mpsc::channel(4);
6284        submit_maintenance_job(
6285            &executor,
6286            root.clone(),
6287            MaintenanceDrainKind::ConfigureTail,
6288            Vec::new(),
6289            &completion_tx,
6290            &metrics,
6291        );
6292        submit_maintenance_job(
6293            &executor,
6294            root,
6295            MaintenanceDrainKind::CompletionDrains,
6296            Vec::new(),
6297            &completion_tx,
6298            &metrics,
6299        );
6300
6301        let first = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
6302            .await
6303            .expect("configure-tail completion timed out")
6304            .expect("configure-tail completion channel closed");
6305        let second = tokio::time::timeout(Duration::from_secs(5), completion_rx.recv())
6306            .await
6307            .expect("completion-drains completion timed out")
6308            .expect("completion-drains completion channel closed");
6309        assert!(first.response.id.contains("configure-tail"));
6310        assert!(second.response.id.contains("completion-drains"));
6311        assert!(runtime_drain::watcher_path_is_ignored_by_current_matcher(
6312            &ctx,
6313            &ignored_path
6314        ));
6315        assert_eq!(
6316            ctx.search_index()
6317                .read()
6318                .unwrap_or_else(std::sync::PoisonError::into_inner)
6319                .as_ref()
6320                .expect("completed search index installed")
6321                .file_count(),
6322            0,
6323            "configure must install the ignore matcher before pending paths replay"
6324        );
6325        ctx.stop_watcher_runtime();
6326    }
6327
6328    #[test]
6329    fn post_bind_configure_and_completion_jobs_are_queued_in_order() {
6330        let (_dir, root) = test_root("maintenance-post-bind");
6331        let mut live_roots = HashMap::new();
6332        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6333
6334        queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
6335        queue_post_bind_configure_and_completion_maintenance(&root, &mut live_roots);
6336
6337        let meta = live_roots.get(&root).expect("root metadata");
6338        assert!(meta.maintenance_pending);
6339        assert_eq!(meta.maintenance_jobs_in_flight, 0);
6340        assert_eq!(
6341            meta.maintenance_queued_kinds
6342                .iter()
6343                .copied()
6344                .collect::<Vec<_>>(),
6345            vec![
6346                MaintenanceDrainKind::ConfigureTail,
6347                MaintenanceDrainKind::CompletionDrains,
6348            ]
6349        );
6350
6351        let (due, deferred) = due_maintenance_jobs_without_actor_context(
6352            &mut live_roots,
6353            MAINTENANCE_SUBMIT_BUDGET,
6354            &HashSet::new(),
6355        );
6356
6357        assert_eq!(
6358            due,
6359            vec![
6360                (root.clone(), MaintenanceDrainKind::ConfigureTail),
6361                (root.clone(), MaintenanceDrainKind::CompletionDrains),
6362            ]
6363        );
6364        assert!(!deferred);
6365        assert_eq!(live_roots[&root].maintenance_jobs_in_flight, 2);
6366        assert!(live_roots[&root].maintenance_queued_kinds.is_empty());
6367    }
6368
6369    #[test]
6370    fn due_maintenance_jobs_defers_unsubmitted_roots_without_marking_pending() {
6371        let mut live_roots = HashMap::new();
6372        let mut root_ids = Vec::new();
6373        let mut _dirs = Vec::new();
6374        for index in 0..4 {
6375            let (dir, root_id) = test_root(&format!("maintenance-budget-{index}"));
6376            live_roots.insert(root_id.clone(), RootMeta::new(Instant::now()));
6377            root_ids.push(root_id);
6378            _dirs.push(dir);
6379        }
6380
6381        let small_budget = INITIAL_MAINTENANCE_JOB_COUNT + 1;
6382        let (first_due, first_deferred) = due_maintenance_jobs_without_actor_context(
6383            &mut live_roots,
6384            small_budget,
6385            &HashSet::new(),
6386        );
6387
6388        assert_eq!(first_due.len(), small_budget);
6389        assert!(first_deferred);
6390        let first_due_set: HashSet<_> = first_due.into_iter().map(|(root, _)| root).collect();
6391        assert!(first_due_set
6392            .iter()
6393            .all(|root| live_roots[root].maintenance_pending));
6394        assert!(first_due_set
6395            .iter()
6396            .any(|root| !live_roots[root].maintenance_queued_kinds.is_empty()));
6397
6398        let all_roots: HashSet<_> = root_ids.into_iter().collect();
6399        let deferred_roots: HashSet<_> = all_roots.difference(&first_due_set).cloned().collect();
6400        assert!(deferred_roots
6401            .iter()
6402            .all(|root| !live_roots[root].maintenance_pending));
6403    }
6404
6405    #[test]
6406    fn due_maintenance_jobs_defers_pending_bind_roots() {
6407        let (_bind_dir, bind_root) = test_root("maintenance-pending-bind");
6408        let (_healthy_dir, healthy_root) = test_root("maintenance-no-bind");
6409        let mut live_roots = HashMap::new();
6410        live_roots.insert(bind_root.clone(), RootMeta::new(Instant::now()));
6411        live_roots.insert(healthy_root.clone(), RootMeta::new(Instant::now()));
6412        let pending_bind_roots = HashSet::from([bind_root.clone()]);
6413
6414        let (due, deferred) = due_maintenance_jobs_without_actor_context(
6415            &mut live_roots,
6416            usize::MAX,
6417            &pending_bind_roots,
6418        );
6419
6420        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6421        assert!(due.iter().all(|(root, _)| root == &healthy_root));
6422        assert!(!deferred);
6423        assert!(!live_roots[&bind_root].maintenance_pending);
6424        assert!(live_roots[&bind_root].maintenance_queued_kinds.is_empty());
6425    }
6426
6427    #[test]
6428    fn maintenance_pending_survives_requeue_and_clears_after_final_batch() {
6429        let (_dir, root) = test_root("maintenance-requeue");
6430        let mut live_roots = HashMap::new();
6431        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6432        let (due, deferred) = due_maintenance_jobs_without_actor_context(
6433            &mut live_roots,
6434            usize::MAX,
6435            &HashSet::new(),
6436        );
6437        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6438        assert!(due.iter().all(|(due_root, _)| due_root == &root));
6439        assert!(!deferred);
6440
6441        let meta = live_roots.get_mut(&root).unwrap();
6442        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, false);
6443        assert!(meta.maintenance_pending);
6444        assert_eq!(
6445            meta.maintenance_jobs_in_flight,
6446            INITIAL_MAINTENANCE_JOB_COUNT - 1
6447        );
6448        assert_eq!(meta.maintenance_queued_kinds.len(), 1);
6449
6450        let (requeued, deferred) =
6451            due_maintenance_jobs_without_actor_context(&mut live_roots, 1, &HashSet::new());
6452        assert_eq!(
6453            requeued,
6454            vec![(root.clone(), MaintenanceDrainKind::Watcher)]
6455        );
6456        assert!(!deferred);
6457        let meta = live_roots.get_mut(&root).unwrap();
6458        assert_eq!(
6459            meta.maintenance_jobs_in_flight,
6460            INITIAL_MAINTENANCE_JOB_COUNT
6461        );
6462        assert!(meta.maintenance_queued_kinds.is_empty());
6463
6464        for _ in 0..INITIAL_MAINTENANCE_JOB_COUNT {
6465            note_maintenance_completion(meta, None, false, false);
6466        }
6467        assert!(!meta.maintenance_pending);
6468        assert_eq!(meta.maintenance_jobs_in_flight, 0);
6469    }
6470
6471    #[test]
6472    fn maintenance_requeue_drops_while_bind_is_pending() {
6473        let (_dir, root) = test_root("maintenance-bind-requeue");
6474        let mut live_roots = HashMap::new();
6475        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6476        let (due, _) = due_maintenance_jobs_without_actor_context(
6477            &mut live_roots,
6478            usize::MAX,
6479            &HashSet::new(),
6480        );
6481        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6482
6483        let meta = live_roots.get_mut(&root).unwrap();
6484        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), false, true);
6485
6486        assert_eq!(
6487            meta.maintenance_jobs_in_flight,
6488            INITIAL_MAINTENANCE_JOB_COUNT - 1
6489        );
6490        assert!(meta.maintenance_queued_kinds.is_empty());
6491        assert!(meta.maintenance_pending);
6492    }
6493
6494    #[test]
6495    fn parked_lsp_completion_never_requiesces_or_cancels_a_pending_bind() {
6496        let mut meta = RootMeta::new(Instant::now());
6497        meta.unbound_quiesced = true;
6498
6499        assert!(!should_requiesce_after_maintenance(
6500            &meta,
6501            MaintenanceDrainKind::Lsp,
6502            false,
6503        ));
6504        assert!(!should_requiesce_after_maintenance(
6505            &meta,
6506            MaintenanceDrainKind::ConfigureTail,
6507            true,
6508        ));
6509        assert!(should_requiesce_after_maintenance(
6510            &meta,
6511            MaintenanceDrainKind::ConfigureTail,
6512            false,
6513        ));
6514    }
6515
6516    #[test]
6517    fn maintenance_pending_clears_and_poison_stops_requeue_after_fatal() {
6518        let (_dir, root) = test_root("maintenance-fatal");
6519        let mut live_roots = HashMap::new();
6520        live_roots.insert(root.clone(), RootMeta::new(Instant::now()));
6521        let (due, _) = due_maintenance_jobs_without_actor_context(
6522            &mut live_roots,
6523            usize::MAX,
6524            &HashSet::new(),
6525        );
6526        assert_eq!(due.len(), INITIAL_MAINTENANCE_JOB_COUNT);
6527
6528        let meta = live_roots.get_mut(&root).unwrap();
6529        note_maintenance_completion(meta, Some(MaintenanceDrainKind::Watcher), true, false);
6530        assert!(meta.maintenance_poisoned);
6531        assert!(meta.maintenance_queued_kinds.is_empty());
6532
6533        for _ in 1..INITIAL_MAINTENANCE_JOB_COUNT {
6534            note_maintenance_completion(meta, None, false, false);
6535        }
6536        assert!(!meta.maintenance_pending);
6537        assert_eq!(meta.maintenance_jobs_in_flight, 0);
6538    }
6539
6540    #[test]
6541    fn trust_for_principal_matrix() {
6542        assert_eq!(
6543            trust_for_principal(&Some(Principal::Direct)),
6544            BindTrust::FirstParty
6545        );
6546        // Every first-party reserved id is asserted BY NAME, in one loop over
6547        // the full set, so two failure classes stay distinguishable: an empty
6548        // or broken allowlist reddens every name at once, while a dropped
6549        // single entry (the rename hazard) reddens exactly the missing name.
6550        // Both halves of each transitional rename pair stay listed until the
6551        // flip settles (see the allowlist comment).
6552        for module_id in [
6553            "llm-runner",
6554            "aft",
6555            "broca",
6556            "alfonso-core",
6557            "prefrontal",
6558            "prefrontal-core",
6559        ] {
6560            assert_eq!(
6561                trust_for_principal(&Some(Principal::Reserved {
6562                    module_id: module_id.to_string(),
6563                })),
6564                BindTrust::FirstParty,
6565                "reserved module id '{module_id}' must resolve to first-party trust"
6566            );
6567        }
6568        assert_eq!(
6569            trust_for_principal(&Some(Principal::Reserved {
6570                module_id: "subc-mcp".to_string(),
6571            })),
6572            BindTrust::Untrusted
6573        );
6574        assert_eq!(
6575            trust_for_principal(&Some(Principal::Reserved {
6576                module_id: "anything-unknown".to_string(),
6577            })),
6578            BindTrust::Untrusted
6579        );
6580        assert_eq!(
6581            trust_for_principal(&Some(Principal::Unverified)),
6582            BindTrust::Untrusted
6583        );
6584        assert_eq!(trust_for_principal(&None), BindTrust::Untrusted);
6585    }
6586
6587    #[test]
6588    fn fed_harness_class_maps_to_untrusted_regardless_of_fingerprint_value() {
6589        let principal = Some(Principal::Direct);
6590        let fingerprint_a = "fed:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
6591        let fingerprint_b = "fed:0123456789abcdef111111111111111111111111111111111111111111111111";
6592
6593        assert_eq!(
6594            trust_for_bind(fingerprint_a, &principal),
6595            BindTrust::Untrusted
6596        );
6597        assert_eq!(
6598            trust_for_bind(fingerprint_b, &principal),
6599            BindTrust::Untrusted
6600        );
6601    }
6602
6603    /// The table above proves `trust_for_principal` maps correctly, and the
6604    /// test above proves the `fed:` harness override wins — but neither
6605    /// exercises the ordinary path, so an implementation that ignored the
6606    /// principal entirely for non-fed harnesses would satisfy both. Pin the
6607    /// delegation itself: on a normal harness the verdict must still come from
6608    /// the principal, in both directions.
6609    #[test]
6610    fn trust_for_bind_delegates_to_the_principal_on_ordinary_harnesses() {
6611        for harness in ["opencode", "pi", "runner", "mcp:claude"] {
6612            assert_eq!(
6613                trust_for_bind(harness, &Some(Principal::Direct)),
6614                BindTrust::FirstParty,
6615                "a direct principal must stay first-party on {harness}"
6616            );
6617            assert_eq!(
6618                trust_for_bind(harness, &Some(Principal::Unverified)),
6619                BindTrust::Untrusted,
6620                "an unverified principal must stay untrusted on {harness}"
6621            );
6622            assert_eq!(
6623                trust_for_bind(harness, &None),
6624                BindTrust::Untrusted,
6625                "an absent principal must fail closed on {harness}"
6626            );
6627            assert_eq!(
6628                trust_for_bind(
6629                    harness,
6630                    &Some(Principal::Reserved {
6631                        module_id: "subc-mcp".to_string(),
6632                    })
6633                ),
6634                BindTrust::Untrusted,
6635                "a non-allowlisted reserved module must stay untrusted on {harness}"
6636            );
6637        }
6638    }
6639
6640    #[tokio::test]
6641    async fn persistent_cancel_resolves_when_fired_before_await() {
6642        // The lost-wakeup guard: cancel() fires exactly once via notify_waiters()
6643        // (no stored permit). A waiter that registers AFTER the cancel must still
6644        // observe it via the flag; a waiter racing the cancel must still be woken.
6645        let signal = PersistentCancelSignal::new();
6646        signal.cancel();
6647        // Fired before we ever call cancelled() — must return immediately, not park.
6648        tokio::time::timeout(Duration::from_secs(1), signal.cancelled())
6649            .await
6650            .expect("cancelled() must resolve when cancel fired beforehand");
6651
6652        // A fresh signal cancelled concurrently with an in-flight cancelled().
6653        let racing = PersistentCancelSignal::new();
6654        let racing_for_task = racing.clone();
6655        let waiter = tokio::spawn(async move { racing_for_task.cancelled().await });
6656        racing.cancel();
6657        tokio::time::timeout(Duration::from_secs(1), waiter)
6658            .await
6659            .expect("cancelled() must resolve when cancel races the await")
6660            .expect("waiter task panicked");
6661    }
6662
6663    #[test]
6664    fn ingress_epoch_validation_rejects_reclaimed_requests_and_drops_other_stale_epochs() {
6665        let installed = HashMap::from([(7, 9)]);
6666        let mut reclaimed = ReclaimedRoutes::default();
6667        reclaimed.insert(route_key(8, 1));
6668        for ty in [
6669            FrameType::Request,
6670            FrameType::Response,
6671            FrameType::Error,
6672            FrameType::Push,
6673            FrameType::Cancel,
6674            FrameType::Goodbye,
6675        ] {
6676            let body = if ty.is_pure_header() {
6677                Vec::new()
6678            } else {
6679                br#"{}"#.to_vec()
6680            };
6681            let stale = Frame::build(ty, control_flags(), 7, 8, 41, body).unwrap();
6682            assert!(
6683                !ingress_route_should_be_processed(&installed, &reclaimed, &stale),
6684                "{ty:?}"
6685            );
6686        }
6687
6688        let reclaimed_request = Frame::build(
6689            FrameType::Request,
6690            control_flags(),
6691            8,
6692            1,
6693            42,
6694            br#"{}"#.to_vec(),
6695        )
6696        .unwrap();
6697        assert!(ingress_route_should_be_processed(
6698            &installed,
6699            &reclaimed,
6700            &reclaimed_request
6701        ));
6702
6703        let never_installed = Frame::build(
6704            FrameType::Request,
6705            control_flags(),
6706            9,
6707            1,
6708            43,
6709            br#"{}"#.to_vec(),
6710        )
6711        .unwrap();
6712        assert!(!ingress_route_should_be_processed(
6713            &installed,
6714            &reclaimed,
6715            &never_installed
6716        ));
6717
6718        let current = Frame::build(
6719            FrameType::Request,
6720            control_flags(),
6721            7,
6722            9,
6723            43,
6724            br#"{}"#.to_vec(),
6725        )
6726        .unwrap();
6727        let control = Frame::build(FrameType::Ping, control_flags(), 0, 0, 44, Vec::new()).unwrap();
6728        assert!(ingress_route_should_be_processed(
6729            &installed, &reclaimed, &current
6730        ));
6731        assert!(ingress_route_should_be_processed(
6732            &installed, &reclaimed, &control
6733        ));
6734        assert_eq!(installed, HashMap::from([(7, 9)]));
6735    }
6736
6737    #[tokio::test]
6738    async fn route_bind_ack_precedes_route_egress_in_writer_queue() {
6739        let (_dir, root) = test_root("route-bind-b2-ordering");
6740        let route = route_key(7, 3);
6741        let identity = RouteIdentity(Arc::new(RouteIdentityData {
6742            root: root.clone(),
6743            project_root: root.as_path().to_path_buf(),
6744            harness: "opencode".to_string(),
6745            session: "b2-session".to_string(),
6746            trust: BindTrust::FirstParty,
6747            spawn_principal: AuthenticatedPrincipal::FirstParty,
6748            consumer_elicitation_capable: false,
6749        }));
6750        let replay_key = push::ReplayKey::from_identity(&identity);
6751        let completion = RouteBindCompletion {
6752            route,
6753            identity,
6754            bind_root_id: root.clone(),
6755            inserted_new_actor: false,
6756            configure_response: Response::success("subc-bind-7", json!({})),
6757            diagnostics_on_edit: false,
6758            ver: PROTOCOL_VERSION,
6759            corr: 91,
6760            flags: control_flags(),
6761        };
6762        let mut pending_binds = HashMap::from([(
6763            route,
6764            PendingBind {
6765                bind_root_id: root,
6766                inserted_new_actor: false,
6767                cancelled: false,
6768                configure_request_id: "subc-bind-7".to_string(),
6769                started_at: Instant::now(),
6770                warned_half_deadline: false,
6771                deadline_reported: false,
6772                corr: 91,
6773                ver: PROTOCOL_VERSION,
6774                flags: control_flags(),
6775                cancellation: crate::executor::JobCancellation::new(),
6776            },
6777        )]);
6778        let mut installed_route_epochs = HashMap::from([(route.channel, route.epoch)]);
6779        let mut push_buffer =
6780            HashMap::from([(replay_key, VecDeque::from([completion_frame("b2-replay")]))]);
6781        let (writer_tx, mut writer_rx) = mpsc::channel(8);
6782        let metrics = Arc::new(DispatchPathMetrics::new());
6783
6784        handle_route_bind_completion(
6785            &writer_tx,
6786            completion,
6787            &mut HashMap::new(),
6788            &mut HashMap::new(),
6789            &mut HashMap::new(),
6790            &mut push_buffer,
6791            &mut HashMap::new(),
6792            &mut pending_binds,
6793            &mut installed_route_epochs,
6794            &Arc::new(Executor::new()),
6795            &Arc::new(Notify::new()),
6796            &metrics,
6797        )
6798        .await
6799        .unwrap();
6800
6801        let ack = writer_rx.try_recv().expect("RouteBindAck");
6802        assert_eq!(ack.header.ty, FrameType::Response);
6803        assert_eq!((ack.header.channel, ack.header.epoch), (0, 0));
6804        let route_frame = writer_rx.try_recv().expect("post-ack route frame");
6805        assert_eq!(route_frame.header.ty, FrameType::Push);
6806        assert_eq!(
6807            (route_frame.header.channel, route_frame.header.epoch),
6808            (route.channel, route.epoch)
6809        );
6810    }
6811}