Skip to main content

subc_daemon/
control.rs

1use std::{
2    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3    fmt,
4    path::{Path, PathBuf},
5    sync::{Arc, Mutex},
6    time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14    ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20    error_codes,
21    manifest::{
22        validate_hello_capability_grammar, validate_hello_self_signal_declarations,
23        CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
24        ProviderRole,
25    },
26    session::{
27        HealthReport, ModuleControlPush, ModuleControlRequest, ModuleControlRequestFromModule,
28        ModuleControlResponse, ModuleControlResponseToModule, MODULE_CONTROL_OP_HEALTH_CHECK,
29        MODULE_TO_SUBC_OP_CATALOG_UPDATE,
30    },
31    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
32    Priority, RouteTarget, PROTOCOL_VERSION,
33};
34use tokio::time::{timeout_at, Instant};
35use tracing::{debug, info, warn};
36
37use crate::{
38    capability_requirements::{
39        log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
40        CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
41        RuntimeModule,
42    },
43    daemon_config::RestartRequiredSection,
44    forwarding::{
45        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
46        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
47        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
48    },
49    observability::{
50        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
51    },
52    provenance::{
53        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
54    },
55    registry::{ChannelState, ConnectionId, Registry, RegistryError},
56    router::{RouteCtx, RouterError},
57    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
58    stderr_tail::{CaptureState, TailEntry},
59    supervise::{
60        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
61        SupervisorHandle, SwapHelloAdmission,
62    },
63    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
64};
65
66/// Lowest envelope version this subc build will negotiate.
67///
68/// Module HELLO negotiation is exact: peers must use the daemon's locked
69/// protocol version. Older and newer peers receive `version_unsupported` and
70/// are not registered.
71pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
72
73const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
74const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
75const CAP_PING_PONG: &str = "ping_pong_v1";
76const CAP_SESSION_ATTACH: &str = "session_attach_v1";
77const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
78
79const SUBC_CONTROL_OPS: &[&str] = &[
80    ops::SERVER_DESCRIBE,
81    ops::CATALOG_LIST,
82    ops::ROUTE_OPEN,
83    ops::ROUTE_POLL,
84    ops::ROUTE_CLOSING,
85    ops::ROUTE_CLOSED,
86    ops::SUPERVISOR_LIST,
87    ops::SUPERVISOR_RESTART,
88    ops::SUPERVISOR_SWAP,
89    ops::SUPERVISOR_RELOAD,
90    ops::SUPERVISOR_RESCAN,
91    ops::SUPERVISOR_RELEASE_RESERVED,
92    ops::SUPERVISOR_SET_ENABLED,
93    ops::SUPERVISOR_HEALTH_PROBE,
94    ops::SUPERVISOR_HEALTH,
95    ops::SUPERVISOR_STDERR_TAIL,
96    ops::SUPERVISOR_TERMINALS,
97    ops::SUPERVISOR_ROUTES,
98    ops::SUPERVISOR_PROVENANCE,
99    ops::SUPERVISOR_SPAWN_SNAPSHOT,
100    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
101];
102
103const MODULE_TO_SUBC_CONTROL_OPS: &[&str] =
104    &[MODULE_TO_SUBC_OP_CATALOG_UPDATE, "supervisor.live_roots"];
105
106const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
107
108/// How long subc waits for a module to ack a relayed route.bind before returning
109/// `module_timeout`. The ack waits on the module's own configure, which for AFT
110/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
111/// and DB-open work — on a cold page cache or a large repo that legitimately
112/// exceeds a couple of seconds. The default is generous because rejecting a VALID
113/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
114/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
115pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
116
117/// How many CONSECUTIVE full-budget relay timeouts against one target module
118/// open that module's bind-relay breaker.
119///
120/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
121/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
122/// so three consecutive full-budget timeouts take ~36s to observe: every client
123/// whose open contributed to opening the breaker had already given up on its
124/// own. That is what makes opening the breaker unable to turn a call that would
125/// have succeeded into a refusal — it can only make an already-failing module
126/// fail faster.
127///
128/// Two would be reachable inside one default deadline. One would convict a
129/// module on a single cold-cache bind, which is exactly the valid-but-slow case
130/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
131pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
132
133/// How long a module's bind-relay breaker stays open before exactly one
134/// `route.open` is let through as a probe.
135///
136/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
137/// re-pays a full-budget stall almost continuously, and the breaker stops being
138/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
139/// deadline: a client that starts retrying after the module recovers has to get
140/// a probe opportunity inside its own deadline, or the breaker converts a
141/// recovered module into a failed call — the failure it exists to prevent,
142/// pointed the other way.
143///
144/// 20s sits between those with room on both sides, and it caps what a wedged
145/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
146/// rather than one per `route.open` per connection. The stall that motivated
147/// this, with its measurements, is written up in
148/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
149/// waited the whole budget out.
150pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
151
152const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
153const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
154
155fn reload_verdict(
156    configured: &Path,
157    spawned_from: Option<&Path>,
158    image: subc_control::RunningImageAgreement,
159) -> PendingReloadVerdict {
160    let path = match spawned_from {
161        Some(spawned_from) if configured == spawned_from => ReloadPathAgreement::Match,
162        Some(spawned_from) => ReloadPathAgreement::Mismatch {
163            configured: configured.to_path_buf(),
164            spawned_from: spawned_from.to_path_buf(),
165        },
166        None => ReloadPathAgreement::Unavailable {
167            reason: if matches!(
168                image,
169                subc_control::RunningImageAgreement::Unavailable {
170                    reason: subc_control::RunningImageUnavailableReason::NotRunning
171                }
172            ) {
173                ReloadPathUnavailableReason::NotRunning
174            } else {
175                ReloadPathUnavailableReason::SpawnedPathUnavailable
176            },
177        },
178    };
179    PendingReloadVerdict { path, image }
180}
181
182#[derive(Clone)]
183struct DaemonProvenanceFacts {
184    build: DaemonBuildProvenance,
185    pid: Option<u32>,
186    started_at_ms: Option<u64>,
187    start_clock: Option<crate::clock::StartClock>,
188    executable_path: Option<PathBuf>,
189    executable_identity: Option<SpawnedFileIdentity>,
190    process_start_time: Option<u64>,
191    probe: ExecutableIdentityProbe,
192}
193
194impl Default for DaemonProvenanceFacts {
195    fn default() -> Self {
196        Self {
197            build: DaemonBuildProvenance {
198                build_git_sha: None,
199                build_lock_digest: None,
200            },
201            pid: None,
202            started_at_ms: None,
203            start_clock: None,
204            executable_path: None,
205            executable_identity: None,
206            process_start_time: None,
207            probe: ExecutableIdentityProbe::default(),
208        }
209    }
210}
211
212#[derive(Debug, Clone)]
213struct SupervisorRescanContext {
214    supervisor: Supervisor,
215    config_path: PathBuf,
216    configured_port: Option<u16>,
217    storage_config: Option<crate::daemon_config::StorageConfig>,
218    admission_facts_carrier_module_id: Option<String>,
219    admission_facts_targets: Option<Vec<String>>,
220}
221
222/// Real channel-0 control handler for subc itself.
223#[derive(Clone)]
224pub struct ControlHandler {
225    registry: Arc<Registry>,
226    forwarding: Arc<ForwardingTable>,
227    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
228    supervisor: SupervisorHandle,
229    subc_capabilities: Arc<[String]>,
230    /// Daemon-wide route.bind relay budget. Used as the fallback when the
231    /// target module has no per-module override in
232    /// `route_bind_relay_timeouts`.
233    route_bind_relay_timeout: Duration,
234    /// Per-module route.bind relay budget overrides, keyed by module id. When
235    /// `handle_route_open` resolves the deadline for a target module, a
236    /// per-module entry wins over the daemon-wide value above.
237    route_bind_relay_timeouts: BTreeMap<String, Duration>,
238    /// Per-target-module bind-relay breaker state. Shared with the forwarding
239    /// table, which is where a new module connection resets it.
240    route_bind_breakers: RouteBindBreakers,
241    /// Live relay admissions keyed by target module. Shared through the
242    /// forwarding table so cloned or separately built handlers enforce one cap.
243    route_bind_concurrency: RouteBindConcurrency,
244    /// Consecutive relay timeouts that open a module's breaker.
245    route_bind_breaker_threshold: u32,
246    /// How long a breaker stays open before one probe is admitted.
247    route_bind_breaker_cooldown: Duration,
248    health_probe_timeout: Duration,
249    /// Central storage policy. When set, each registering module receives its
250    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
251    storage_config: Option<crate::daemon_config::StorageConfig>,
252    /// The machine id established at boot, served on every HELLO_ACK and on
253    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
254    /// changes the file, never this value. `None` serves no id.
255    machine_id: Option<crate::machine_id::MachineId>,
256    admission_facts_carrier_module_id: Option<String>,
257    admission_facts_targets: Option<Vec<String>>,
258    rescan: Option<SupervisorRescanContext>,
259    connected_clients: ConnectedClients,
260    counters: DaemonCounters,
261    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
262    daemon_provenance: DaemonProvenanceFacts,
263    #[cfg(test)]
264    control_dispatch_delay: Option<Duration>,
265    #[cfg(test)]
266    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
267}
268
269impl fmt::Debug for ControlHandler {
270    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
271        f.debug_struct("ControlHandler")
272            .field("registry", &self.registry)
273            .field("forwarding", &self.forwarding)
274            .field("process_liveness", &self.process_liveness.is_some())
275            .field("supervisor", &self.supervisor)
276            .field("subc_capabilities", &self.subc_capabilities)
277            .finish()
278    }
279}
280
281struct RouteOpenRequest {
282    target: RouteTarget,
283    identity: BindIdentity,
284    consumer_identity: Option<ConsumerIdentity>,
285    consumer_capabilities: Option<Vec<String>>,
286    admission_facts: Option<serde_json::Value>,
287}
288
289struct RouteBindReservationGuard {
290    forwarding: Arc<ForwardingTable>,
291    endpoint: ModuleEndpointId,
292    relay_corr: u64,
293    armed: bool,
294}
295
296struct ModuleControlRpcGuard {
297    forwarding: Arc<ForwardingTable>,
298    endpoint: ModuleEndpointId,
299    corr: u64,
300    armed: bool,
301}
302
303impl ModuleControlRpcGuard {
304    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
305        Self {
306            forwarding,
307            endpoint,
308            corr,
309            armed: true,
310        }
311    }
312
313    fn disarm(&mut self) {
314        self.armed = false;
315    }
316}
317
318impl Drop for ModuleControlRpcGuard {
319    fn drop(&mut self) {
320        if self.armed {
321            let _ = self
322                .forwarding
323                .cancel_module_control_rpc(self.endpoint, self.corr);
324        }
325    }
326}
327
328impl RouteBindReservationGuard {
329    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
330        Self {
331            forwarding,
332            endpoint,
333            relay_corr,
334            armed: true,
335        }
336    }
337
338    fn release_and_disarm(&mut self) {
339        if !self.armed {
340            return;
341        }
342        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
343            self.endpoint,
344            self.relay_corr,
345            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
346        ) {
347            send_goodbye_target_best_effort(
348                &self.forwarding.counters(),
349                &target,
350                "canceled route.bind",
351            );
352        }
353        self.armed = false;
354    }
355
356    fn disarm(&mut self) {
357        self.armed = false;
358    }
359}
360
361impl Drop for RouteBindReservationGuard {
362    fn drop(&mut self) {
363        self.release_and_disarm();
364    }
365}
366
367/// Per-target-module circuit breaker around the `route.bind` relay.
368///
369/// The connection reader is serial per connection, so a module whose `on_bind`
370/// sits on the ack blocks every LATER frame on the connections that call it,
371/// including calls to unrelated modules. This does not make any module's bind
372/// fast; it stops the daemon paying the full budget again and again for a
373/// condition it has already observed.
374///
375/// State is keyed by TARGET MODULE and shared by every connection: a wedged
376/// module wedges everyone, so what one connection learned should protect the
377/// rest.
378///
379/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
380/// relay to that module has actually timed out, and is removed again when a
381/// relay is accepted or the module reconnects, so it cannot grow with traffic
382/// or with modules that behave.
383///
384/// # Why a `std` mutex here is not the head-of-line defect again
385///
386/// Acquisition never awaits. The critical section is a hash lookup plus a few
387/// integer updates, with no I/O and no `.await` inside it, so a reader task
388/// cannot be descheduled behind it the way it can behind
389/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
390/// primitive, held for the same kind of work, as the refusal counter this very
391/// path already increments.
392///
393/// It is also NOT on the data-plane splice path: only `route.open` and module
394/// registration touch it, so bound-route frames gain no state check and no
395/// contention.
396#[derive(Debug, Clone, Default)]
397pub(crate) struct RouteBindBreakers {
398    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
399}
400
401#[derive(Debug, Clone, Default)]
402pub(crate) struct RouteBindConcurrency {
403    modules: Arc<Mutex<HashMap<String, usize>>>,
404}
405
406struct RouteBindConcurrencyGuard {
407    concurrency: RouteBindConcurrency,
408    module_id: String,
409}
410
411impl RouteBindConcurrency {
412    /// Admit without waiting. Waiting here would move the bind stall from the
413    /// module reply to a semaphore and restore reader head-of-line blocking.
414    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
415        let mut modules = self
416            .modules
417            .lock()
418            .expect("route.bind concurrency mutex poisoned");
419        let in_flight = modules.entry(module_id.to_string()).or_default();
420        if *in_flight >= limit {
421            return Err(*in_flight);
422        }
423        *in_flight += 1;
424        Ok(RouteBindConcurrencyGuard {
425            concurrency: self.clone(),
426            module_id: module_id.to_string(),
427        })
428    }
429}
430
431impl Drop for RouteBindConcurrencyGuard {
432    fn drop(&mut self) {
433        let mut modules = self
434            .concurrency
435            .modules
436            .lock()
437            .expect("route.bind concurrency mutex poisoned");
438        let remove = {
439            let in_flight = modules
440                .get_mut(&self.module_id)
441                .expect("admitted route.bind has a concurrency entry");
442            *in_flight -= 1;
443            *in_flight == 0
444        };
445        if remove {
446            modules.remove(&self.module_id);
447        }
448    }
449}
450
451#[derive(Debug, Default)]
452struct ModuleBreakerState {
453    /// Relay timeouts observed with no accepted relay in between.
454    consecutive_timeouts: u32,
455    /// `Some` while the breaker is open: the instant the cooldown expires and
456    /// the next arrival may probe. `None` means closed.
457    cooldown_until: Option<Instant>,
458    /// A half-open probe has been admitted and has not settled yet. This is
459    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
460    /// that read the cooldown, so concurrent opens arriving at the moment the
461    /// cooldown expires cannot all decide that they are the probe.
462    probe_in_flight: bool,
463}
464
465/// What the breaker decided for one `route.open`, before any relay work.
466enum RouteBindAdmission<'a> {
467    Admitted {
468        guard: RouteBindBreakerGuard<'a>,
469        /// This open is the single half-open probe, so the transition is worth
470        /// one log line.
471        probe: bool,
472    },
473    Refused {
474        consecutive_timeouts: u32,
475        /// What is left of the cooldown. Zero when the refusal is because the
476        /// one probe is already in flight rather than because the cooldown has
477        /// not elapsed.
478        retry_in: Duration,
479        probe_in_flight: bool,
480    },
481}
482
483/// An outstanding admission, which must be told how its relay settled.
484///
485/// `Drop` settles it as inconclusive, so an early return between admission and
486/// the relay -- or the whole handler being cancelled when the client
487/// disconnects -- releases a half-open probe slot instead of leaving the
488/// breaker wedged half-open with no further probes.
489struct RouteBindBreakerGuard<'a> {
490    breakers: RouteBindBreakers,
491    module_id: &'a str,
492    settled: bool,
493}
494
495impl RouteBindBreakerGuard<'_> {
496    /// The module answered within the budget and took the bind. THE ONLY
497    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
498    /// breaker, which is a transition worth logging.
499    fn record_accepted(&mut self) -> bool {
500        self.settled = true;
501        self.breakers.record_accepted(self.module_id)
502    }
503
504    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
505    /// COUNTS TOWARD OPENING.
506    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
507        self.settled = true;
508        self.breakers
509            .record_timeout(self.module_id, threshold, cooldown)
510    }
511
512    /// Everything else: the module REJECTED the bind, its connection went away
513    /// mid-relay, or the waiter was cancelled.
514    ///
515    /// None of these is evidence that a module is slow, and each already has
516    /// its own refusal with its own code. A module that rejects a bind in
517    /// microseconds is healthy and must never be convicted for it; a module
518    /// that died has said nothing about the module that replaces it. So these
519    /// neither increment nor reset the count -- they only release a probe slot.
520    fn record_inconclusive(&mut self) {
521        self.settled = true;
522        self.breakers.record_inconclusive(self.module_id);
523    }
524}
525
526impl Drop for RouteBindBreakerGuard<'_> {
527    fn drop(&mut self) {
528        if !self.settled {
529            self.breakers.record_inconclusive(self.module_id);
530        }
531    }
532}
533
534/// The breaker moved to open, reported so the caller can log it outside the
535/// lock. Opening is rare and load-bearing; the refusals that follow are
536/// frequent and are counted rather than logged.
537struct BreakerOpened {
538    consecutive_timeouts: u32,
539    /// True when a failed probe re-opened an already-open breaker, which reads
540    /// very differently in a log from a first opening.
541    reopened_after_probe: bool,
542}
543
544impl RouteBindBreakers {
545    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
546        self.modules
547            .lock()
548            .expect("route.bind breaker mutex poisoned")
549    }
550
551    /// Decide whether this `route.open` may attempt its relay. Takes the map
552    /// lock and nothing else, and never awaits.
553    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
554        let admitted = |probe| RouteBindAdmission::Admitted {
555            guard: RouteBindBreakerGuard {
556                breakers: self.clone(),
557                module_id,
558                settled: false,
559            },
560            probe,
561        };
562
563        let mut modules = self.lock();
564        let Some(state) = modules.get_mut(module_id) else {
565            return admitted(false);
566        };
567        let Some(cooldown_until) = state.cooldown_until else {
568            return admitted(false);
569        };
570        if state.probe_in_flight {
571            return RouteBindAdmission::Refused {
572                consecutive_timeouts: state.consecutive_timeouts,
573                retry_in: Duration::ZERO,
574                probe_in_flight: true,
575            };
576        }
577        let now = Instant::now();
578        if now < cooldown_until {
579            return RouteBindAdmission::Refused {
580                consecutive_timeouts: state.consecutive_timeouts,
581                retry_in: cooldown_until - now,
582                probe_in_flight: false,
583            };
584        }
585        state.probe_in_flight = true;
586        admitted(true)
587    }
588
589    fn record_accepted(&self, module_id: &str) -> bool {
590        self.lock()
591            .remove(module_id)
592            .is_some_and(|state| state.cooldown_until.is_some())
593    }
594
595    fn record_timeout(
596        &self,
597        module_id: &str,
598        threshold: u32,
599        cooldown: Duration,
600    ) -> Option<BreakerOpened> {
601        let mut modules = self.lock();
602        let state = modules.entry(module_id.to_string()).or_default();
603        let was_open = state.cooldown_until.is_some();
604        let was_probe = state.probe_in_flight;
605        state.probe_in_flight = false;
606        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
607        if state.consecutive_timeouts < threshold {
608            return None;
609        }
610        state.cooldown_until = Some(Instant::now() + cooldown);
611        Some(BreakerOpened {
612            consecutive_timeouts: state.consecutive_timeouts,
613            reopened_after_probe: was_open && was_probe,
614        })
615    }
616
617    fn record_inconclusive(&self, module_id: &str) {
618        if let Some(state) = self.lock().get_mut(module_id) {
619            state.probe_in_flight = false;
620        }
621    }
622
623    /// Discard what was learned about a module, because the process it was
624    /// learned about is gone. Returns the discarded count when it was non-zero.
625    ///
626    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
627    /// `module_id` is a configuration identity that outlives any particular
628    /// child; what the breaker observed was the process behind the module
629    /// connection of the moment. When a new connection registers under that id
630    /// the verdict's subject no longer exists, so the verdict is stale by
631    /// construction rather than merely likely to be wrong. Keeping it would
632    /// apply a dead process's record to a live one, which is the same defect
633    /// class this breaker exists to stop the daemon committing.
634    ///
635    /// A half-open probe in flight is discarded with the rest: it was a
636    /// question about the old process.
637    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
638        self.lock()
639            .remove(module_id)
640            .map(|state| state.consecutive_timeouts)
641            .filter(|discarded| *discarded > 0)
642    }
643
644    /// Open breakers, for the `server.describe` counters object. `None` when
645    /// none is open, so the key stays absent rather than present-and-empty.
646    ///
647    /// This is the operator's answer to "is this module refusing instantly or
648    /// is it fine?", which look identical from a client that retries and then
649    /// succeeds.
650    fn open_snapshot(&self) -> Option<serde_json::Value> {
651        let now = Instant::now();
652        let modules = self.lock();
653        let open = modules
654            .iter()
655            .filter_map(|(module_id, state)| {
656                let cooldown_until = state.cooldown_until?;
657                Some((
658                    module_id.clone(),
659                    serde_json::json!({
660                        "consecutive_timeouts": state.consecutive_timeouts,
661                        "cooldown_remaining_ms":
662                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
663                        "probe_in_flight": state.probe_in_flight,
664                    }),
665                ))
666            })
667            .collect::<serde_json::Map<String, serde_json::Value>>();
668        (!open.is_empty()).then_some(serde_json::Value::Object(open))
669    }
670}
671
672impl ControlHandler {
673    pub fn new(registry: Arc<Registry>) -> Self {
674        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
675    }
676
677    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
678        let counters = forwarding.counters();
679        // Taken from the forwarding table rather than created here, so that the
680        // breaker a `route.open` consults is the same one a module's
681        // registration resets, however many handlers are built over one table.
682        let route_bind_breakers = forwarding.route_bind_breakers();
683        let route_bind_concurrency = forwarding.route_bind_concurrency();
684        Self {
685            registry,
686            forwarding,
687            process_liveness: None,
688            supervisor: SupervisorHandle::new(),
689            subc_capabilities: Arc::from([
690                CAP_MANIFEST_REGISTRATION.to_string(),
691                CAP_CHANNEL_LIFECYCLE.to_string(),
692                CAP_PING_PONG.to_string(),
693                CAP_SESSION_ATTACH.to_string(),
694                CAP_ADMISSION_FACTS_RELAY.to_string(),
695            ]),
696            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
697            route_bind_relay_timeouts: BTreeMap::new(),
698            route_bind_breakers,
699            route_bind_concurrency,
700            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
701            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
702            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
703            storage_config: None,
704            machine_id: None,
705            admission_facts_carrier_module_id: None,
706            admission_facts_targets: None,
707            rescan: None,
708            connected_clients: ConnectedClients::new(),
709            counters,
710            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
711            daemon_provenance: DaemonProvenanceFacts::default(),
712            #[cfg(test)]
713            control_dispatch_delay: None,
714            #[cfg(test)]
715            provenance_probe_override: None,
716        }
717    }
718
719    /// Set the central storage policy: registering modules then receive their
720    /// resolved storage descriptor in HELLO_ACK.
721    pub fn with_storage_config(
722        mut self,
723        storage_config: Option<crate::daemon_config::StorageConfig>,
724    ) -> Self {
725        self.storage_config = storage_config;
726        self
727    }
728
729    /// Set the machine id served to every registering module (HELLO_ACK) and on
730    /// `server.describe`.
731    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
732        self.machine_id = machine_id;
733        self
734    }
735
736    /// Configure the exact reserved module and target ids permitted to relay
737    /// opaque admission facts. Config-file loading validates this authority;
738    /// this builder keeps the same policy available to embedded test daemons.
739    pub fn with_admission_facts_config(
740        mut self,
741        carrier_module_id: Option<String>,
742        targets: Option<Vec<String>>,
743    ) -> Self {
744        self.admission_facts_carrier_module_id = carrier_module_id;
745        self.admission_facts_targets = targets;
746        self
747    }
748
749    /// Override the route.bind relay timeout. Used by tests that assert the
750    /// timeout path so they don't block on the production-safe default.
751    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
752        self.route_bind_relay_timeout = timeout;
753        self
754    }
755
756    /// Install per-module route.bind relay budget overrides. A module id
757    /// listed here wins over the daemon-wide default set via
758    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
759    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
760    /// the same `Duration` the bind path will use.
761    pub fn with_route_bind_relay_timeouts(
762        mut self,
763        timeouts: impl IntoIterator<Item = (String, Duration)>,
764    ) -> Self {
765        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
766        self
767    }
768
769    /// Resolve the route.bind relay budget for a specific target module id.
770    /// Per-module overrides win; the daemon-wide value (set via
771    /// `with_route_bind_relay_timeout` or the built-in default) is the
772    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
773    /// the same resolution `handle_route_open` will use.
774    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
775        self.route_bind_relay_timeouts
776            .get(module_id)
777            .copied()
778            .unwrap_or(self.route_bind_relay_timeout)
779    }
780
781    /// Override the per-module bind-relay breaker policy.
782    ///
783    /// Used by tests, which cannot spend three production budgets opening a
784    /// breaker or twenty seconds waiting for its cooldown. The production
785    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
786    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
787    /// reasoning for the numbers.
788    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
789        self.route_bind_breaker_threshold = threshold.max(1);
790        self.route_bind_breaker_cooldown = cooldown;
791        self
792    }
793
794    #[cfg(test)]
795    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
796        self.health_probe_timeout = timeout;
797        self
798    }
799
800    #[cfg(test)]
801    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
802        self.control_dispatch_delay = Some(delay);
803        self
804    }
805
806    pub fn with_process_liveness(
807        mut self,
808        process_liveness: Arc<dyn ModuleProcessLiveness>,
809    ) -> Self {
810        self.process_liveness = Some(process_liveness);
811        self
812    }
813
814    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
815        self.supervisor = supervisor;
816        self
817    }
818
819    pub fn with_daemon_provenance(
820        mut self,
821        pid: u32,
822        started_at_ms: u64,
823        executable_path: Option<PathBuf>,
824        build_git_sha: Option<String>,
825        build_lock_digest: Option<String>,
826    ) -> Self {
827        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
828        let process_start_time = process_start_time(pid);
829        self.daemon_provenance = DaemonProvenanceFacts {
830            build: DaemonBuildProvenance {
831                build_git_sha,
832                build_lock_digest,
833            },
834            pid: Some(pid),
835            started_at_ms: Some(started_at_ms),
836            start_clock: None,
837            executable_path,
838            executable_identity,
839            process_start_time,
840            probe: ExecutableIdentityProbe::default(),
841        };
842        self
843    }
844
845    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
846        self.daemon_provenance.start_clock = Some(clock);
847        self
848    }
849
850    #[cfg(test)]
851    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
852        self.provenance_probe_override = Some(result);
853        self
854    }
855
856    /// Install the configured module set and its reserved capability bindings.
857    /// Bindings are configuration-scoped and may point at a provider that has not
858    /// been installed yet, so this does not require the bound module to exist.
859    pub fn with_capability_config(
860        self,
861        modules: impl IntoIterator<Item = (String, bool)>,
862        reserved_capabilities: BTreeMap<String, String>,
863    ) -> Self {
864        self.capability_evaluator
865            .configure(modules, reserved_capabilities);
866        self
867    }
868
869    pub fn with_supervisor_rescan(
870        mut self,
871        supervisor: Supervisor,
872        config_path: impl Into<PathBuf>,
873        configured_port: Option<u16>,
874    ) -> Self {
875        self.rescan = Some(SupervisorRescanContext {
876            supervisor,
877            config_path: config_path.into(),
878            configured_port,
879            storage_config: self.storage_config.clone(),
880            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
881            admission_facts_targets: self.admission_facts_targets.clone(),
882        });
883        self
884    }
885
886    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
887        self.connected_clients = connected_clients;
888        self
889    }
890
891    pub fn forwarding(&self) -> Arc<ForwardingTable> {
892        Arc::clone(&self.forwarding)
893    }
894
895    pub(crate) fn counters(&self) -> DaemonCounters {
896        self.counters.clone()
897    }
898
899    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
900    /// requirement event without depending on an operator polling a status command.
901    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
902        tokio::spawn(async move {
903            loop {
904                self.capability_evaluator
905                    .wait_for_change_or_deadline()
906                    .await;
907                self.refresh_capability_requirements();
908            }
909        });
910    }
911
912    fn runtime_capability_snapshot(
913        &self,
914    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
915        let runtime = self
916            .supervisor
917            .list()
918            .into_iter()
919            .map(|module| {
920                let status = module.status().map_err(|err| {
921                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
922                })?;
923                Ok(RuntimeModule {
924                    module_id: status.module_id,
925                    state: status.state,
926                    enabled: status.enabled,
927                })
928            })
929            .collect::<Result<Vec<_>, RouterError>>()?;
930        let (_, registrations) = self.registry.list_modules().map_err(|err| {
931            RouterError::backend(
932                0,
933                0,
934                format!("failed to list capability registrations: {err}"),
935            )
936        })?;
937        let registrations = registrations
938            .into_iter()
939            .map(|registration| RegisteredModule {
940                module_id: registration.manifest.module_id,
941                module_version: registration.manifest.module_version,
942                capabilities: registration.manifest.capabilities,
943            })
944            .collect();
945        Ok((runtime, registrations))
946    }
947
948    /// The capability side effects of a module becoming the active registration
949    /// for its id: cache its manifest (warning if its claims drifted), run the
950    /// deny census when its declarations call for one, and recompute the
951    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
952    /// candidate's does not, and the supervisor does it at promotion instead,
953    /// through [`crate::supervise::SwapPromotionObserver`].
954    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
955        let cached_registration = RegisteredModule {
956            module_id: registration.manifest.module_id.clone(),
957            module_version: registration.manifest.module_version.clone(),
958            capabilities: registration.manifest.capabilities.clone(),
959        };
960        if self.capability_evaluator.record_hello(&cached_registration) {
961            warn!(
962                module_id = %cached_registration.module_id,
963                "capability claims drifted from the cached manifest"
964            );
965        }
966        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
967            self.enforce_capability_denies();
968        }
969        self.refresh_capability_requirements();
970    }
971
972    /// Point the shared supervisor handle at this handler for swap promotions.
973    /// Called wherever a handler is put behind the `Arc` the router serves, so
974    /// it can be held weakly.
975    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
976        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
977            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
978        self.supervisor.set_swap_promotion_observer(observer);
979    }
980
981    pub fn refresh_capability_requirements(&self) {
982        match self.runtime_capability_snapshot() {
983            Ok((runtime, registrations)) => {
984                log_requirement_events(
985                    self.capability_evaluator
986                        .evaluate_now(&runtime, &registrations),
987                );
988            }
989            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
990        }
991    }
992
993    /// Reconcile only live, attested route bindings after a capability deny edge
994    /// or target claim was added. This is deliberately a control-plane census:
995    /// the opaque forwarding hot path must not grow a per-frame capability check.
996    fn enforce_capability_denies(&self) {
997        let (_, registrations) = match self.registry.list_modules() {
998            Ok(snapshot) => snapshot,
999            Err(err) => {
1000                warn!(error = %err, "failed to read registrations for capability deny census");
1001                return;
1002            }
1003        };
1004        let manifests = registrations
1005            .into_iter()
1006            .map(|registration| {
1007                (
1008                    registration.manifest.module_id.clone(),
1009                    registration.manifest,
1010                )
1011            })
1012            .collect::<BTreeMap<_, _>>();
1013        let census = match self.forwarding.route_census(None) {
1014            Ok(census) => census,
1015            Err(err) => {
1016                warn!(error = %err, "failed to read route census for capability deny enforcement");
1017                return;
1018            }
1019        };
1020
1021        for (target_module_id, routes) in census {
1022            let Some(target_manifest) = manifests.get(&target_module_id) else {
1023                continue;
1024            };
1025            let mut closed_routes = Vec::new();
1026            let mut module_goodbyes = Vec::new();
1027            for route in routes {
1028                let Principal::Reserved {
1029                    module_id: opening_module_id,
1030                } = &route.principal
1031                else {
1032                    continue;
1033                };
1034                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1035                    continue;
1036                };
1037                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1038                    continue;
1039                };
1040
1041                match self.forwarding.release_client_route(
1042                    route.goodbye_target.connection_id,
1043                    route.goodbye_target.channel,
1044                    route.goodbye_target.epoch,
1045                ) {
1046                    Ok(RouteRelease::Removed(module_goodbye)) => {
1047                        warn!(
1048                            opening_module_id,
1049                            target_module_id,
1050                            capability,
1051                            "force-closing route because an attested capability deny edge now matches"
1052                        );
1053                        closed_routes.push(route);
1054                        module_goodbyes.push(module_goodbye);
1055                    }
1056                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1057                    Err(err) => warn!(
1058                        opening_module_id,
1059                        target_module_id,
1060                        capability,
1061                        error = %err,
1062                        "failed to force-close capability-denied route"
1063                    ),
1064                }
1065            }
1066
1067            if closed_routes.is_empty() {
1068                continue;
1069            }
1070            send_route_control_pushes(
1071                &self.forwarding,
1072                closed_routes,
1073                ClientControlPush::RouteClosed {
1074                    module_id: target_module_id,
1075                    reason: RouteCloseReason::CapabilityDenied,
1076                    drained: false,
1077                    abandoned: 0,
1078                    excluded_subscriptions: 0,
1079                    terminal: Some(false),
1080                },
1081            );
1082            self.emit_route_goodbyes(module_goodbyes);
1083        }
1084    }
1085
1086    /// Why a registered module is not accepting new route binds, or `None` when
1087    /// it is. This is the module's effective readiness: its declared readiness
1088    /// first, then every `need: required` capability it declares evaluating to
1089    /// `provided`. `route.open` and `catalog.list` both read it here so the
1090    /// catalog never reports a module routable that `route.open` would refuse.
1091    fn not_ready_reason(
1092        &self,
1093        registration: &crate::registry::ModuleRegistration,
1094    ) -> Option<NotReadyReason> {
1095        if !registration.ready {
1096            return Some(NotReadyReason {
1097                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1098                capability: None,
1099            });
1100        }
1101        self.first_unprovided_required_capability(registration)
1102            .map(|capability| NotReadyReason {
1103                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1104                capability: Some(capability),
1105            })
1106    }
1107
1108    /// The lexicographically first capability this registration declares
1109    /// `need: required` whose evaluator verdict is not `provided`.
1110    ///
1111    /// The verdicts are the capability evaluator's own; nothing here decides
1112    /// what "provided" means. The evaluator counts a capability provided as
1113    /// soon as a module claiming it has REGISTERED, not once that module is
1114    /// ready. That distinction is what keeps two modules that require each
1115    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1116    /// is ready", each would wait for the other to become ready first and
1117    /// neither ever would. Do not tighten it to readiness.
1118    ///
1119    /// A required capability with no verdict at all means this registration's
1120    /// HELLO or catalog.update landed after the last recompute; recompute once
1121    /// rather than let a missing verdict read as either answer. If it is still
1122    /// missing (the recompute itself failed) the capability counts as
1123    /// unprovided: the refusal is retryable, and routing a module whose
1124    /// required provider is unknown is the outcome this check exists to stop.
1125    fn first_unprovided_required_capability(
1126        &self,
1127        registration: &crate::registry::ModuleRegistration,
1128    ) -> Option<String> {
1129        let required = registration
1130            .manifest
1131            .capabilities
1132            .iter()
1133            .flat_map(|declarations| declarations.requires.iter())
1134            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1135            .map(|requirement| requirement.capability.as_str())
1136            .collect::<BTreeSet<_>>();
1137        if required.is_empty() {
1138            return None;
1139        }
1140        let module_id = registration.manifest.module_id.as_str();
1141        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1142        if required
1143            .iter()
1144            .any(|capability| verdict(capability).is_none())
1145        {
1146            self.refresh_capability_requirements();
1147        }
1148        required
1149            .into_iter()
1150            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1151            .map(str::to_string)
1152    }
1153
1154    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1155        self.capability_evaluator
1156            .statuses()
1157            .into_iter()
1158            .map(capability_requirement_status)
1159            .collect()
1160    }
1161
1162    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1163    /// registration-release watch. The signal is what the supervisor waits on
1164    /// before spawning a replacement, so it must only fire once forwarding
1165    /// teardown is also done (see [`Self::cleanup_connection`] /
1166    /// [`Self::handle_goodbye`]). Used directly only where there is no forwarding
1167    /// state to tear down (a HELLO that failed before module registration).
1168    fn deregister_connection(
1169        &self,
1170        connection_id: ConnectionId,
1171    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1172        self.registry.deregister_connection(connection_id)
1173    }
1174
1175    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1176        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1177            return None;
1178        }
1179        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1180            parse_client_control_request(&frame.body)
1181        else {
1182            return None;
1183        };
1184        Some(target_module_id(&target).to_string())
1185    }
1186
1187    pub(crate) fn route_open_capacity_refusal(
1188        &self,
1189        ctx: &RouteCtx,
1190        frame: &Frame,
1191        target_module_id: &str,
1192        limit: usize,
1193    ) -> Result<Frame, RouterError> {
1194        self.route_open_admission_refusal_frame(
1195            ctx,
1196            frame,
1197            target_module_id,
1198            format!(
1199                "connection already has {limit} route.open binds in flight; retry after one settles"
1200            ),
1201        )
1202    }
1203
1204    /// Admission pressure clears as existing binds settle, so its refusal must
1205    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
1206    /// `module_reloading`, `module_warming`, `target_unavailable`, or
1207    /// `module_timeout`. `target_unavailable` is honest for an attempt that
1208    /// cannot currently reach its target; `module_timeout` would falsely claim
1209    /// that a wait expired. A new, cleaner code would be terminal to deployed
1210    /// clients, so it requires a client-tolerance rollout before daemon emission.
1211    fn route_open_admission_refusal_frame(
1212        &self,
1213        ctx: &RouteCtx,
1214        frame: &Frame,
1215        target_module_id: &str,
1216        message: impl Into<String>,
1217    ) -> Result<Frame, RouterError> {
1218        self.route_open_refusal_frame(
1219            ctx,
1220            frame,
1221            target_module_id,
1222            "open_admission_full",
1223            error_codes::TARGET_UNAVAILABLE,
1224            message,
1225        )
1226    }
1227
1228    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
1229    ///
1230    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
1231    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
1232    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
1233    #[cfg(test)]
1234    pub fn handle_control(
1235        &self,
1236        connection_id: ConnectionId,
1237        frame: Frame,
1238    ) -> Result<Vec<Frame>, RouterError> {
1239        match frame.header.ty {
1240            FrameType::Ping => Ok(vec![pong(&frame)?]),
1241            FrameType::Hello => self.handle_hello(connection_id, None, frame),
1242            FrameType::Goodbye => self.handle_goodbye(connection_id),
1243            ty => Ok(vec![control_error_frame(
1244                &frame,
1245                "unsupported_control_frame",
1246                format!("unsupported channel-0 frame {ty:?}"),
1247            )?]),
1248        }
1249    }
1250
1251    pub async fn handle_control_frame(
1252        &self,
1253        ctx: &RouteCtx,
1254        frame: Frame,
1255    ) -> Result<Vec<Frame>, RouterError> {
1256        self.handle_control_frame_timed(ctx, frame, None).await
1257    }
1258
1259    pub(crate) async fn handle_control_frame_timed(
1260        &self,
1261        ctx: &RouteCtx,
1262        frame: Frame,
1263        dispatch_started_at: Option<StdInstant>,
1264    ) -> Result<Vec<Frame>, RouterError> {
1265        match frame.header.ty {
1266            FrameType::Ping => Ok(vec![pong(&frame)?]),
1267            FrameType::Hello => {
1268                self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1269            }
1270            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1271            FrameType::Cancel => {
1272                if self
1273                    .supervisor
1274                    .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1275                {
1276                    Ok(Vec::new())
1277                } else {
1278                    Ok(vec![control_error_frame(
1279                        &frame,
1280                        "unknown_subscription",
1281                        "no supervisor spawn subscription has this correlation id",
1282                    )?])
1283                }
1284            }
1285            FrameType::Request => {
1286                if self
1287                    .forwarding
1288                    .module_endpoint_for_connection(ctx.connection_id)
1289                    .map_err(RouterError::Forwarding)?
1290                    .is_some()
1291                {
1292                    if !is_known_module_request_op(&frame.body) {
1293                        return Ok(vec![control_error_frame(
1294                            &frame,
1295                            "unsupported_control_frame",
1296                            "module-originated channel-0 REQUEST is not supported",
1297                        )?]);
1298                    }
1299                    let request = match parse_module_control_request_from_module(&frame.body) {
1300                        Ok(request) => request,
1301                        Err((err, ControlRequestBodyError::UnknownOp)) => {
1302                            return Ok(vec![control_error_frame(
1303                                &frame,
1304                                "unsupported_control_frame",
1305                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
1306                            )?])
1307                        }
1308                        Err((err, ControlRequestBodyError::InvalidBody)) => {
1309                            return Ok(vec![control_error_frame(
1310                                &frame,
1311                                "invalid_control_body",
1312                                format!("malformed module control body: {err}"),
1313                            )?])
1314                        }
1315                    };
1316                    let op = module_control_request_op(&request);
1317                    let corr = frame.header.corr;
1318                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
1319                    let result =
1320                        self.handle_module_control_request(ctx.connection_id, frame, request);
1321                    log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1322                    return result;
1323                }
1324
1325                if is_known_module_request_op(&frame.body) {
1326                    return Ok(vec![control_error_frame(
1327                        &frame,
1328                        "not_registered",
1329                        "catalog.update requires an active module registration owned by this connection",
1330                    )?]);
1331                }
1332
1333                let request = match parse_client_control_request(&frame.body) {
1334                    Ok(request) => request,
1335                    Err((err, ControlRequestBodyError::UnknownOp)) => {
1336                        return Ok(vec![control_error_frame(
1337                            &frame,
1338                            "unknown_control_op",
1339                            format!("unknown client control op: {err}"),
1340                        )?])
1341                    }
1342                    Err((err, ControlRequestBodyError::InvalidBody)) => {
1343                        return Ok(vec![control_error_frame(
1344                            &frame,
1345                            "invalid_control_body",
1346                            format!("malformed client control body: {err}"),
1347                        )?])
1348                    }
1349                };
1350                let op = client_control_request_op(&request);
1351                let corr = frame.header.corr;
1352                log_control_dispatch_arrival(op, ctx.connection_id, corr);
1353                #[cfg(test)]
1354                if let Some(delay) = self.control_dispatch_delay {
1355                    tokio::time::sleep(delay).await;
1356                }
1357                let result = self
1358                    .handle_client_control_request(ctx, frame, request)
1359                    .await;
1360                log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1361                result
1362            }
1363            FrameType::Push => {
1364                let Some(endpoint) = self
1365                    .forwarding
1366                    .module_endpoint_for_connection(ctx.connection_id)
1367                    .map_err(RouterError::Forwarding)?
1368                else {
1369                    return Ok(vec![control_error_frame(
1370                        &frame,
1371                        "unsupported_control_frame",
1372                        "client-originated channel-0 PUSH is not supported",
1373                    )?]);
1374                };
1375                self.handle_status_update(endpoint, frame)
1376            }
1377            FrameType::Response | FrameType::Error
1378                if self
1379                    .forwarding
1380                    .module_endpoint_for_connection(ctx.connection_id)
1381                    .map_err(RouterError::Forwarding)?
1382                    .is_some() =>
1383            {
1384                self.handle_module_relay_response(ctx.connection_id, frame)
1385            }
1386            ty => Ok(vec![control_error_frame(
1387                &frame,
1388                "unsupported_control_frame",
1389                format!("unsupported channel-0 frame {ty:?}"),
1390            )?]),
1391        }
1392    }
1393
1394    pub fn cleanup_connection(
1395        &self,
1396        connection_id: ConnectionId,
1397    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1398        let crash_closed = self
1399            .registry
1400            .get_module_by_connection(connection_id)?
1401            .and_then(|registration| {
1402                self.forwarding
1403                    .module_endpoint_for_connection(connection_id)
1404                    .ok()
1405                    .flatten()
1406                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1407                    .map(|routes| (registration.manifest.module_id, routes))
1408            });
1409        if let Some((module_id, routes)) = crash_closed {
1410            let terminal = match self.supervisor.get(&module_id) {
1411                None => false,
1412                Some(module) => match module.will_recover_after_connection_loss() {
1413                    Ok(will_recover) => !will_recover,
1414                    Err(err) => {
1415                        warn!(
1416                            %module_id,
1417                            error = %err,
1418                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
1419                        );
1420                        false
1421                    }
1422                },
1423            };
1424            send_route_control_pushes(
1425                &self.forwarding,
1426                routes,
1427                ClientControlPush::RouteClosed {
1428                    module_id,
1429                    reason: RouteCloseReason::Crash,
1430                    drained: false,
1431                    abandoned: 0,
1432                    excluded_subscriptions: 0,
1433                    terminal: Some(terminal),
1434                },
1435            );
1436        }
1437        let registrations = self.deregister_connection(connection_id);
1438        if let Ok(released_routes) = self.forwarding.cleanup_connection(connection_id) {
1439            self.emit_route_goodbyes(released_routes);
1440        }
1441        // Signal the registration-release watch only now that BOTH registry and
1442        // forwarding teardown are done, so a supervisor waiting to spawn a
1443        // replacement never observes release while old routes still exist.
1444        if matches!(&registrations, Ok(r) if !r.is_empty()) {
1445            crate::supervise::notify_registration_release();
1446            self.capability_evaluator.wake_deadline_loop();
1447            self.refresh_capability_requirements();
1448        }
1449        self.supervisor.remove_spawn_subscribers(connection_id);
1450        registrations
1451    }
1452
1453    pub(crate) fn handle_route_goodbye(
1454        &self,
1455        connection_id: ConnectionId,
1456        route_channel: u16,
1457        route_epoch: u32,
1458    ) -> Result<bool, RouterError> {
1459        debug!(
1460            connection_id = connection_id.get(),
1461            route_channel, route_epoch, "handling route GOODBYE"
1462        );
1463        let RouteRelease::Removed(released_route) = self
1464            .forwarding
1465            .release_client_route(connection_id, route_channel, route_epoch)
1466            .map_err(RouterError::Forwarding)?
1467        else {
1468            return Ok(false);
1469        };
1470        self.emit_route_goodbyes(vec![released_route]);
1471        Ok(true)
1472    }
1473
1474    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1475        for released in released_routes {
1476            let frame = match Frame::build_with_version(
1477                released.negotiated_ver,
1478                FrameType::Goodbye,
1479                control_flags(),
1480                released.channel,
1481                released.epoch,
1482                0,
1483                Vec::new(),
1484            ) {
1485                Ok(frame) => frame,
1486                Err(err) => {
1487                    warn!(
1488                        route_channel = released.channel,
1489                        error = %err,
1490                        "failed to build route GOODBYE frame"
1491                    );
1492                    continue;
1493                }
1494            };
1495            if !released.close_on_delivery_failure() {
1496                crate::forwarding::send_module_route_goodbye(
1497                    &self.counters,
1498                    &released.sink,
1499                    frame,
1500                    released.module_id.as_deref(),
1501                    "client route released",
1502                );
1503                continue;
1504            }
1505            if let Err(err) = released.sink.try_send(frame) {
1506                warn!(
1507                    target_connection_id = released.connection_id.get(),
1508                    route_channel = released.channel,
1509                    error = %err,
1510                    "route GOODBYE was not delivered to client; closing target connection"
1511                );
1512                if self
1513                    .forwarding
1514                    .escalate_client_delivery_failure(
1515                        released.connection_id,
1516                        released.channel,
1517                        released.epoch,
1518                        CloseReason::new(
1519                            "route_goodbye_delivery_failed",
1520                            format!(
1521                                "failed to enqueue route GOODBYE for channel {}: {err}",
1522                                released.channel
1523                            ),
1524                        ),
1525                        crate::forwarding::UndeliveredFrame {
1526                            module_id: released.module_id.as_deref(),
1527                            sink: &released.sink,
1528                        },
1529                    )
1530                    .unwrap_or(false)
1531                {
1532                    self.counters.increment_goodbye_relay_client_failed();
1533                }
1534            }
1535        }
1536    }
1537
1538    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
1539    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
1540    /// own commit failed after the module had already accepted). Without this, a
1541    /// module that accepts late keeps a binding subc has torn down, so a later
1542    /// frame on that module channel could misdeliver if the channel is reused.
1543    ///
1544    /// Never closes the shared module connection on failure: a dropped notification
1545    /// only wastes a bounded amount of warm module-side state, which the module's
1546    /// own idle reaper reclaims. Only call this once the route.bind relay was
1547    /// actually enqueued to the module — if the relay send itself failed, the
1548    /// module never created a binding and there is nothing to tear down.
1549    fn send_abandoned_route_bind_goodbye(
1550        &self,
1551        module_sink: &crate::FrameSink,
1552        negotiated_ver: u8,
1553        module_channel: u16,
1554        module_epoch: u32,
1555    ) {
1556        let frame = match Frame::build_with_version(
1557            negotiated_ver,
1558            FrameType::Goodbye,
1559            control_flags(),
1560            module_channel,
1561            module_epoch,
1562            0,
1563            Vec::new(),
1564        ) {
1565            Ok(frame) => frame,
1566            Err(err) => {
1567                warn!(
1568                    route_channel = module_channel,
1569                    error = %err,
1570                    "failed to build GOODBYE for abandoned route.bind"
1571                );
1572                return;
1573            }
1574        };
1575        crate::forwarding::send_module_route_goodbye(
1576            &self.counters,
1577            module_sink,
1578            frame,
1579            None,
1580            "abandoned route.bind",
1581        );
1582    }
1583
1584    fn handle_hello(
1585        &self,
1586        connection_id: ConnectionId,
1587        sink: Option<crate::FrameSink>,
1588        frame: Frame,
1589    ) -> Result<Vec<Frame>, RouterError> {
1590        debug!(
1591            connection_id = connection_id.get(),
1592            corr = frame.header.corr,
1593            "handling HELLO"
1594        );
1595        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1596            Ok(value) => value,
1597            Err(err) => {
1598                return Ok(vec![control_error_frame(
1599                    &frame,
1600                    "invalid_hello",
1601                    format!("malformed HELLO body: {err}"),
1602                )?])
1603            }
1604        };
1605        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1606            return Ok(vec![control_error_frame(
1607                &frame,
1608                "invalid_capability_grammar",
1609                err.to_string(),
1610            )?]);
1611        }
1612        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
1613            return Ok(vec![control_error_frame(
1614                &frame,
1615                "invalid_manifest",
1616                err.to_string(),
1617            )?]);
1618        }
1619        if let Some(provenance) = hello_value
1620            .get("manifest")
1621            .and_then(|manifest| manifest.get("provenance"))
1622        {
1623            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
1624                return Ok(vec![control_error_frame(
1625                    &frame,
1626                    "invalid_manifest",
1627                    format!("malformed manifest provenance: {err}"),
1628                )?]);
1629            }
1630        }
1631        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
1632            Ok(hello) => hello,
1633            Err(err) => {
1634                return Ok(vec![control_error_frame(
1635                    &frame,
1636                    "invalid_hello",
1637                    format!("malformed HELLO body: {err}"),
1638                )?])
1639            }
1640        };
1641
1642        if hello.protocol_ver != hello.manifest.protocol_ver {
1643            return Ok(vec![control_error_frame(
1644                &frame,
1645                "invalid_manifest",
1646                format!(
1647                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
1648                    hello.protocol_ver, hello.manifest.protocol_ver
1649                ),
1650            )?]);
1651        }
1652
1653        if hello.manifest.module_id.trim().is_empty() {
1654            return Ok(vec![control_error_frame(
1655                &frame,
1656                "invalid_manifest",
1657                "manifest module_id must not be empty",
1658            )?]);
1659        }
1660
1661        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
1662            Ok(negotiated_ver) => negotiated_ver,
1663            Err(message) => {
1664                return Ok(vec![control_error_frame(
1665                    &frame,
1666                    "version_unsupported",
1667                    message,
1668                )?])
1669            }
1670        };
1671
1672        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
1673        // swap is open for this id, the only HELLO admitted as a second process
1674        // is the one carrying the candidate's launch nonce (the swap token), and
1675        // it registers into the candidate slot rather than being refused as a
1676        // duplicate. Run after the reserved gate, a reserved module's candidate
1677        // would be refused `reserved_module` for presenting a nonce that gate
1678        // does not know. See `SupervisorHandle::swap_hello_admission`.
1679        let swap_admission = self
1680            .supervisor
1681            .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
1682        if swap_admission == SwapHelloAdmission::Refused {
1683            warn!(
1684                module_id = %hello.manifest.module_id,
1685                connection_id = connection_id.get(),
1686                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
1687            );
1688            return Ok(vec![control_error_frame(
1689                &frame,
1690                "swap_token_invalid",
1691                format!(
1692                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
1693                    hello.manifest.module_id
1694                ),
1695            )?]);
1696        }
1697        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
1698
1699        // Reserved-module identity gate: a module_id configured `reserved` may be
1700        // registered ONLY by the process subc spawned for it, proven by echoing the
1701        // one-time launch nonce subc injected. A non-reserved id has no recorded
1702        // nonce and always passes. This blocks a key-holder from impersonating a
1703        // security-boundary module (e.g. the credential vault) while the real one is
1704        // down/restarting and its registration slot is momentarily free. A swap
1705        // candidate has already proven the same thing with its own nonce above.
1706        if let Some(rejection) = (!swap_candidate)
1707            .then(|| {
1708                self.supervisor.reserved_hello_rejection(
1709                    &hello.manifest.module_id,
1710                    hello.launch_nonce.as_deref(),
1711                )
1712            })
1713            .flatten()
1714        {
1715            let message = match rejection {
1716                ReservedHelloRejection::Exact { module_id } => format!(
1717                    "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
1718                ),
1719                ReservedHelloRejection::Prefix {
1720                    prefix,
1721                    owner_module_id,
1722                } => format!(
1723                    "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
1724                    hello.manifest.module_id
1725                ),
1726            };
1727            return Ok(vec![control_error_frame(
1728                &frame,
1729                "reserved_module",
1730                message,
1731            )?]);
1732        }
1733
1734        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
1735            &hello.manifest.module_id,
1736            hello.manifest.capabilities.as_ref(),
1737        );
1738        if let Some(refusal) = reserved_capability_refusals.first() {
1739            let capability = refusal.capability.clone();
1740            let bound_module = refusal.claimants[0].clone();
1741            log_duplicate_claim_events(reserved_capability_refusals);
1742            return Ok(vec![control_error_frame(
1743                &frame,
1744                "reserved_capability",
1745                format!(
1746                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
1747                    capability, bound_module, hello.manifest.module_id
1748                ),
1749            )?]);
1750        }
1751
1752        // A connection that already opened client routes must not also register as
1753        // a module: cleanup would then release only one side and leak the other.
1754        if self
1755            .forwarding
1756            .connection_has_client_routes(connection_id)
1757            .map_err(RouterError::Forwarding)?
1758        {
1759            return Ok(vec![control_error_frame(
1760                &frame,
1761                "invalid_hello",
1762                "connection has open client routes and cannot also register as a module",
1763            )?]);
1764        }
1765
1766        let control_ops = effective_module_control_ops(hello.control_ops);
1767        // Built before anything is registered so an encoding failure leaves no
1768        // registry or forwarding state behind.
1769        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
1770        if swap_candidate {
1771            return self.register_swap_candidate(
1772                connection_id,
1773                sink,
1774                &frame,
1775                hello.manifest,
1776                negotiated_ver,
1777                control_ops,
1778                hello_ack,
1779            );
1780        }
1781        let registration = match self.registry.register_with_control_ops(
1782            hello.manifest,
1783            negotiated_ver,
1784            connection_id,
1785            control_ops,
1786        ) {
1787            Ok(registration) => registration,
1788            Err(RegistryError::DuplicateModuleId { module_id }) => {
1789                return Ok(vec![control_error_frame(
1790                    &frame,
1791                    "duplicate_module_id",
1792                    format!(
1793                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
1794                    ),
1795                )?])
1796            }
1797            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
1798                return Ok(vec![control_error_frame(
1799                    &frame,
1800                    "invalid_module_id",
1801                    err.to_string(),
1802                )?])
1803            }
1804            Err(err) => {
1805                return Ok(vec![control_error_frame(
1806                    &frame,
1807                    "registry_error",
1808                    err.to_string(),
1809                )?])
1810            }
1811        };
1812
1813        let reply = if let Some(sink) = sink {
1814            // The forwarding table's module store is also the daemon-to-module
1815            // control-RPC lane, so every HELLO gets a live endpoint even when the
1816            // manifest has no routable provider role. Non-routable modules still
1817            // cannot receive route.bind in production: `handle_route_open` checks
1818            // the registry manifest with `target_has_required_role` before the
1819            // only production call to `begin_route_bind_relay_for` below that
1820            // route.open path. The remaining direct relay callers are unit tests
1821            // and benchmark harnesses that construct forwarding state explicitly.
1822            //
1823            // The HELLO_ACK is queued by the forwarding table itself, before the
1824            // endpoint becomes visible, and is NOT returned as a reply. A module
1825            // reads HELLO_ACK first and exits on anything else; a reply is only
1826            // written after this handler returns, by which time a route.open on
1827            // another connection could already have queued a route.bind request
1828            // for this module ahead of it.
1829            let concurrency = manifest_concurrency(&registration.manifest);
1830            if let Err(err) = self.forwarding.register_module_connection_acked(
1831                connection_id,
1832                registration.manifest.module_id.clone(),
1833                negotiated_ver,
1834                concurrency,
1835                sink,
1836                hello_ack,
1837            ) {
1838                // Forwarding registration failed, so there is no forwarding
1839                // state to tear down. Remove the registry entry and signal the
1840                // release watch directly.
1841                if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
1842                    crate::supervise::notify_registration_release();
1843                }
1844                return Ok(vec![control_error_frame(
1845                    &frame,
1846                    forwarding_error_code(&err),
1847                    err.to_string(),
1848                )?]);
1849            }
1850            Vec::new()
1851        } else {
1852            // No sink means no forwarding endpoint, so nothing can be routed
1853            // ahead of the ack; it goes out as the reply.
1854            vec![hello_ack]
1855        };
1856
1857        // Exposure over assumption: Concurrency's serde default is pinned to the
1858        // pre-field behavior (ModuleManaged), so a management surface that is
1859        // genuinely Serial and just never declared it inherits concurrent
1860        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
1861        // turns "no module has been bitten yet" into the checkable claim "no
1862        // module is exposed" -- one read of the boot log instead of a fleet
1863        // audit. Detected from the raw HELLO bytes because the serde default
1864        // deliberately erases the absent/declared distinction from the type.
1865        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
1866            info!(
1867                module_id = %registration.manifest.module_id,
1868                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
1869            );
1870        }
1871
1872        self.apply_registration_capabilities(&registration);
1873
1874        info!(
1875            module_id = %registration.manifest.module_id,
1876            module_version = %registration.manifest.module_version,
1877            negotiated_ver,
1878            routable_provider = manifest_provides_routable_role(&registration.manifest),
1879            connection_id = connection_id.get(),
1880            "module registered"
1881        );
1882
1883        Ok(reply)
1884    }
1885
1886    /// Register a HELLO the swap gate admitted into the candidate slot of the
1887    /// registry and of forwarding, where it is reachable over its own
1888    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
1889    /// nothing routes to it until the supervisor cuts over.
1890    ///
1891    /// Registry first, then forwarding, the same order as an ordinary HELLO;
1892    /// a forwarding failure removes the registry entry again. The capability
1893    /// census is not run: it describes routable modules, and this one is not
1894    /// routable until promotion.
1895    #[allow(clippy::too_many_arguments)]
1896    fn register_swap_candidate(
1897        &self,
1898        connection_id: ConnectionId,
1899        sink: Option<crate::FrameSink>,
1900        frame: &Frame,
1901        manifest: ModuleManifest,
1902        negotiated_ver: u8,
1903        control_ops: Vec<String>,
1904        hello_ack: Frame,
1905    ) -> Result<Vec<Frame>, RouterError> {
1906        let module_id = manifest.module_id.clone();
1907        let registration = match self.registry.register_candidate_with_control_ops(
1908            manifest,
1909            negotiated_ver,
1910            connection_id,
1911            control_ops,
1912        ) {
1913            Ok(registration) => registration,
1914            Err(RegistryError::DuplicateModuleId { module_id }) => {
1915                return Ok(vec![control_error_frame(
1916                    frame,
1917                    "duplicate_module_id",
1918                    format!(
1919                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
1920                    ),
1921                )?])
1922            }
1923            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
1924                return Ok(vec![control_error_frame(
1925                    frame,
1926                    "invalid_module_id",
1927                    err.to_string(),
1928                )?])
1929            }
1930            Err(err) => {
1931                return Ok(vec![control_error_frame(
1932                    frame,
1933                    "registry_error",
1934                    err.to_string(),
1935                )?])
1936            }
1937        };
1938        let reply = if let Some(sink) = sink {
1939            // Same ordering as an ordinary HELLO: the forwarding table queues
1940            // the HELLO_ACK before the candidate endpoint is inserted, because
1941            // a module exits if its first frame after HELLO is anything else.
1942            let concurrency = manifest_concurrency(&registration.manifest);
1943            if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
1944                connection_id,
1945                module_id.clone(),
1946                negotiated_ver,
1947                concurrency,
1948                sink,
1949                hello_ack,
1950            ) {
1951                if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
1952                    crate::supervise::notify_registration_release();
1953                }
1954                return Ok(vec![control_error_frame(
1955                    frame,
1956                    forwarding_error_code(&err),
1957                    err.to_string(),
1958                )?]);
1959            }
1960            Vec::new()
1961        } else {
1962            vec![hello_ack]
1963        };
1964        self.supervisor.mark_swap_candidate_admitted(&module_id);
1965        info!(
1966            module_id = %module_id,
1967            module_version = %registration.manifest.module_version,
1968            negotiated_ver,
1969            ready = registration.ready,
1970            connection_id = connection_id.get(),
1971            "swap candidate registered; not routable until cutover"
1972        );
1973        Ok(reply)
1974    }
1975
1976    fn build_hello_ack(
1977        &self,
1978        frame: &Frame,
1979        negotiated_ver: u8,
1980        module_id: &str,
1981    ) -> Result<Frame, RouterError> {
1982        let ack = ModuleHelloAckBody {
1983            negotiated_ver,
1984            subc_ops: module_subc_ops(),
1985            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
1986            storage: self
1987                .storage_config
1988                .as_ref()
1989                .map(|cfg| cfg.descriptor_for(module_id)),
1990            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
1991        };
1992        let body = serde_json::to_vec(&ack).map_err(|err| {
1993            RouterError::backend(
1994                0,
1995                frame.header.corr,
1996                format!("failed to encode HELLO_ACK: {err}"),
1997            )
1998        })?;
1999
2000        Frame::build_with_version(
2001            negotiated_ver,
2002            FrameType::HelloAck,
2003            control_flags(),
2004            0,
2005            0,
2006            frame.header.corr,
2007            body,
2008        )
2009        .map_err(RouterError::FrameBuild)
2010    }
2011
2012    async fn handle_client_control_request(
2013        &self,
2014        ctx: &RouteCtx,
2015        frame: Frame,
2016        request: ClientControlRequest,
2017    ) -> Result<Vec<Frame>, RouterError> {
2018        match request {
2019            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2020            ClientControlRequest::CatalogList { module_id } => {
2021                self.handle_catalog_list(frame, module_id)
2022            }
2023            ClientControlRequest::RouteOpen {
2024                target,
2025                identity,
2026                consumer_identity,
2027                consumer_capabilities,
2028                admission_facts,
2029            } => {
2030                self.handle_route_open(
2031                    ctx,
2032                    frame,
2033                    RouteOpenRequest {
2034                        target,
2035                        identity,
2036                        consumer_identity,
2037                        consumer_capabilities,
2038                        admission_facts,
2039                    },
2040                )
2041                .await
2042            }
2043            ClientControlRequest::RoutePoll {
2044                route_channel,
2045                route_epoch,
2046                kind,
2047            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2048            ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2049            ClientControlRequest::SupervisorSpawnSnapshot {} => {
2050                self.handle_supervisor_spawn_snapshot(frame)
2051            }
2052            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2053                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2054            }
2055            ClientControlRequest::SupervisorRestart {
2056                module_id,
2057                drain_timeout_ms,
2058            } => {
2059                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2060                    .await
2061            }
2062            ClientControlRequest::SupervisorSwap {
2063                module_id,
2064                ready_timeout_ms,
2065            } => {
2066                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2067                    .await
2068            }
2069            ClientControlRequest::SupervisorReload { module_id } => {
2070                self.handle_supervisor_reload(frame, module_id).await
2071            }
2072            ClientControlRequest::SupervisorRescan { preview } => {
2073                self.handle_supervisor_rescan(frame, preview).await
2074            }
2075            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2076                self.handle_supervisor_release_reserved(frame, module_id)
2077                    .await
2078            }
2079            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2080                self.handle_supervisor_set_enabled(frame, module_id, enabled)
2081                    .await
2082            }
2083            ClientControlRequest::SupervisorHealthProbe { module_id } => {
2084                self.handle_supervisor_health_probe(frame, module_id).await
2085            }
2086            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2087            ClientControlRequest::SupervisorRoutes { module_id } => {
2088                self.handle_supervisor_routes(frame, module_id)
2089            }
2090            ClientControlRequest::SupervisorProvenance { module_id } => {
2091                self.handle_supervisor_provenance(frame, module_id).await
2092            }
2093            ClientControlRequest::SupervisorStderrTail {
2094                module_id,
2095                max_lines,
2096                max_bytes,
2097            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2098            ClientControlRequest::SupervisorTerminals { module_id } => {
2099                self.handle_supervisor_terminals(frame, module_id).await
2100            }
2101        }
2102    }
2103
2104    fn handle_module_control_request(
2105        &self,
2106        connection_id: ConnectionId,
2107        frame: Frame,
2108        request: ModuleControlRequestFromModule,
2109    ) -> Result<Vec<Frame>, RouterError> {
2110        match request {
2111            ModuleControlRequestFromModule::CatalogUpdate {
2112                provides,
2113                capabilities,
2114                ready,
2115            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2116            ModuleControlRequestFromModule::LiveRoots {} => {
2117                let registered = self
2118                    .registry
2119                    .get_module_by_connection(connection_id)
2120                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2121                let Some(registration) = registered else {
2122                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2123                };
2124                let response = self
2125                    .forwarding
2126                    .live_roots(&registration.manifest.module_id)
2127                    .map_err(RouterError::Forwarding)?;
2128                Ok(vec![control_response_body_frame(
2129                    &frame,
2130                    &response,
2131                    "ModuleControlResponseToModule::LiveRoots",
2132                )?])
2133            }
2134        }
2135    }
2136
2137    fn handle_catalog_update(
2138        &self,
2139        connection_id: ConnectionId,
2140        frame: Frame,
2141        provides: Vec<ProviderRole>,
2142        capabilities: Option<CapabilityDeclarations>,
2143        ready: Option<bool>,
2144    ) -> Result<Vec<Frame>, RouterError> {
2145        self.refresh_capability_requirements();
2146        let Some(registration) = self
2147            .registry
2148            .get_module_by_connection(connection_id)
2149            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2150        else {
2151            return Ok(vec![control_error_frame(
2152                &frame,
2153                "not_registered",
2154                "catalog.update requires an active module registration owned by this connection",
2155            )?]);
2156        };
2157
2158        if let Some(message) =
2159            catalog_update_frozen_field_message(&registration.manifest, &provides)
2160        {
2161            return Ok(vec![control_error_frame(
2162                &frame,
2163                "catalog_update_frozen_field",
2164                message,
2165            )?]);
2166        }
2167
2168        let mut candidate = registration.manifest.clone();
2169        candidate.provides = provides.clone();
2170        candidate.capabilities = capabilities
2171            .clone()
2172            .or_else(|| registration.manifest.capabilities.clone());
2173        if let Err(err) = candidate.validate_capability_grammar() {
2174            return Ok(vec![control_error_frame(
2175                &frame,
2176                "invalid_capability_grammar",
2177                err.to_string(),
2178            )?]);
2179        }
2180
2181        let updated = self
2182            .registry
2183            .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
2184            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2185        if updated.is_none() {
2186            return Ok(vec![control_error_frame(
2187                &frame,
2188                "not_registered",
2189                "catalog.update requires an active module registration owned by this connection",
2190            )?]);
2191        }
2192        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
2193            log_duplicate_claim_events(
2194                self.capability_evaluator
2195                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
2196            );
2197        }
2198        if capability_census_trigger(
2199            registration.manifest.capabilities.as_ref(),
2200            updated
2201                .as_ref()
2202                .and_then(|entry| entry.manifest.capabilities.as_ref()),
2203        ) {
2204            self.enforce_capability_denies();
2205        }
2206        self.refresh_capability_requirements();
2207
2208        let response = ModuleControlResponseToModule::CatalogUpdate {};
2209        control_response_body_frame(
2210            &frame,
2211            &response,
2212            "ModuleControlResponseToModule::CatalogUpdate",
2213        )
2214        .map(|frame| vec![frame])
2215    }
2216
2217    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2218        self.refresh_capability_requirements();
2219        // A bare connection count is ambiguous between many clients holding a
2220        // route each and one client accumulating hundreds, so publish the
2221        // concentration alongside it. Route state is best-effort here: a
2222        // diagnostic endpoint must still answer if the forwarding lock is
2223        // contended.
2224        let mut counters = self.counters.snapshot();
2225        if let (Ok((connections_with_routes, max)), Some(obj)) = (
2226            self.forwarding.client_route_concentration(),
2227            counters.as_object_mut(),
2228        ) {
2229            obj.insert(
2230                "client_connections_with_routes".into(),
2231                connections_with_routes.into(),
2232            );
2233            obj.insert("max_routes_on_one_connection".into(), max.into());
2234        }
2235        // A module that is being fast-refused and a module that is fine look
2236        // identical from a client that retries and succeeds, so name the open
2237        // breakers here. This rides the existing free-form counters object
2238        // rather than a new wire field, so no sibling that deserializes
2239        // `ServerDescribe` has to be rebuilt to keep reading it.
2240        if let (Some(open_breakers), Some(obj)) = (
2241            self.route_bind_breakers.open_snapshot(),
2242            counters.as_object_mut(),
2243        ) {
2244            obj.insert("route_bind_breakers_open".into(), open_breakers);
2245        }
2246        let response = ClientControlResponse::ServerDescribe {
2247            protocol_ver: PROTOCOL_VERSION,
2248            subc_ops: subc_ops(),
2249            capabilities: self.subc_capabilities.as_ref().to_vec(),
2250            connected_clients: self.connected_clients.count(),
2251            counters: Some(counters),
2252            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
2253            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
2254            capability_requirements: self.capability_requirement_statuses(),
2255            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2256        };
2257        Ok(vec![control_response_body_frame(
2258            &frame,
2259            &response,
2260            "ClientControlResponse::ServerDescribe",
2261        )?])
2262    }
2263
2264    fn handle_catalog_list(
2265        &self,
2266        frame: Frame,
2267        module_id: Option<String>,
2268    ) -> Result<Vec<Frame>, RouterError> {
2269        let (generation, modules) = self.registry.list_modules().map_err(|err| {
2270            RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
2271        })?;
2272        let entries = modules
2273            .into_iter()
2274            .filter(|registration| {
2275                module_id
2276                    .as_deref()
2277                    .map(|wanted| registration.manifest.module_id == wanted)
2278                    .unwrap_or(true)
2279            })
2280            .map(|registration| {
2281                let not_ready = self.not_ready_reason(&registration);
2282                let roles = registration.manifest.provides;
2283                CatalogEntry {
2284                    module_id: registration.manifest.module_id,
2285                    ready: not_ready.is_none(),
2286                    not_ready,
2287                    module_version: Some(registration.manifest.module_version),
2288                    roles,
2289                    control_ops: registration.control_ops,
2290                    capabilities: registration.manifest.capabilities,
2291                    self_signals: registration.manifest.self_signals,
2292                }
2293            })
2294            .collect();
2295        let response = ClientControlResponse::CatalogList {
2296            generation,
2297            modules: entries,
2298            subc_ops: subc_ops(),
2299        };
2300        Ok(vec![control_response_body_frame(
2301            &frame,
2302            &response,
2303            "ClientControlResponse::CatalogList",
2304        )?])
2305    }
2306
2307    fn route_open_principal(
2308        &self,
2309        frame: &Frame,
2310        consumer_identity: Option<ConsumerIdentity>,
2311    ) -> Result<Result<Principal, Frame>, RouterError> {
2312        let Some(consumer_identity) = consumer_identity else {
2313            return Ok(Ok(Principal::Direct));
2314        };
2315
2316        if self.supervisor.spawned_consumer_authorized(
2317            &consumer_identity.module_id,
2318            &consumer_identity.launch_nonce,
2319        ) {
2320            return Ok(Ok(Principal::Reserved {
2321                module_id: consumer_identity.module_id,
2322            }));
2323        }
2324
2325        Ok(Err(control_error_frame(
2326            frame,
2327            "bad_consumer_identity",
2328            format!(
2329                "consumer_identity for module_id '{}' did not match a supervised launch nonce",
2330                consumer_identity.module_id
2331            ),
2332        )?))
2333    }
2334
2335    /// Every refusal of a `route.open` goes through here so the daemon can
2336    /// attest which code it sent: without the event, a client's "the daemon
2337    /// refused me" and the daemon's own view could only be reconciled by
2338    /// argument. Malformed input (`invalid_project_root`) does not come here;
2339    /// rejecting a request that was never a valid open is not a refusal of one.
2340    fn route_open_refusal_frame(
2341        &self,
2342        ctx: &RouteCtx,
2343        frame: &Frame,
2344        module_id: &str,
2345        reason: &'static str,
2346        code: &'static str,
2347        message: impl Into<String>,
2348    ) -> Result<Frame, RouterError> {
2349        self.observe_route_open_refusal(ctx, module_id, reason, code);
2350        control_error_frame(frame, code, message.into())
2351    }
2352
2353    /// Refuse a `route.open` because the target module's bind-relay breaker is
2354    /// open, without attempting the relay.
2355    ///
2356    /// The wire code is `module_timeout`, which is the truth (the module has
2357    /// not been answering binds) and which both SDKs already classify as
2358    /// retryable with capped backoff. Reusing it is what keeps this change out
2359    /// of both SDKs; the daemon-side distinction lives in the counter key
2360    /// instead.
2361    ///
2362    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
2363    /// While a breaker is open this fires on every open to that module, and the
2364    /// stall written up in `docs/designs/route-open-head-of-line.md` already
2365    /// produced 261 lines about a single module inside 3000 lines of daemon
2366    /// log. The rare transitions are logged at warn/info instead and the volume
2367    /// is carried by the counter, so the evidence survives without the flood.
2368    /// The debug line keeps a per-refusal record reachable for whoever turns
2369    /// the level up.
2370    fn route_open_breaker_refusal_frame(
2371        &self,
2372        ctx: &RouteCtx,
2373        frame: &Frame,
2374        module_id: &str,
2375        consecutive_timeouts: u32,
2376        retry_in: Duration,
2377        probe_in_flight: bool,
2378    ) -> Result<Frame, RouterError> {
2379        self.counters
2380            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
2381        debug!(
2382            target: "control",
2383            code = "module_timeout",
2384            module_id = ?module_id,
2385            connection_id = ctx.connection_id.get(),
2386            consecutive_timeouts,
2387            retry_in_ms = retry_in.as_millis() as u64,
2388            probe_in_flight,
2389            "route.open refused by open bind-relay breaker"
2390        );
2391        let detail = if probe_in_flight {
2392            "one probe bind is already in flight; retry once it settles".to_string()
2393        } else {
2394            format!("not relaying for another {retry_in:?}")
2395        };
2396        control_error_frame(
2397            frame,
2398            "module_timeout",
2399            format!(
2400                "module_id '{module_id}' failed {consecutive_timeouts} consecutive route.bind \
2401                 relays; {detail}"
2402            ),
2403        )
2404    }
2405
2406    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
2407    /// requester's bytes (an unknown target is whatever the client sent) and
2408    /// is Debug-formatted so control characters land in the log escaped
2409    /// rather than as terminal sequences for whoever tails it.
2410    ///
2411    /// `reason` names the check that refused, because one wire code has
2412    /// several senders: after a module registers, `target_unavailable` can
2413    /// come from a missing role, an inactive registration, a supervisor that
2414    /// has not marked the process live, a missing forwarding connection, or a
2415    /// failed relay, and a log that records only the code cannot say which of
2416    /// them fired. It is a static, daemon-chosen label per branch, so it is
2417    /// safe to print plainly and stays a closed set.
2418    fn observe_route_open_refusal(
2419        &self,
2420        ctx: &RouteCtx,
2421        module_id: &str,
2422        reason: &'static str,
2423        code: &'static str,
2424    ) {
2425        self.counters.increment_route_open_refused(code);
2426        info!(
2427            target: "control",
2428            code,
2429            reason,
2430            module_id = ?module_id,
2431            connection_id = ctx.connection_id.get(),
2432            "route.open refused"
2433        );
2434    }
2435
2436    /// Record an ACCEPTED route.open.
2437    ///
2438    /// Refusals have been logged and counted since the attestation work; accepts
2439    /// were invisible, so the daemon knew every principal it stamped and wrote
2440    /// none of them down. The party that attests the identity was the only party
2441    /// not recording it, which left a credential vault unable to name the sender
2442    /// of a call that reached it (claustrum #43) and left the launch-nonce
2443    /// concurrency question unanswerable from the outside.
2444    ///
2445    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
2446    /// `code`/`module_id`/`connection_id` returns both directions of the same
2447    /// decision rather than two shapes a reader has to join by hand.
2448    ///
2449    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
2450    /// and the difference carries information rather than being an
2451    /// inconsistency. This line is only reachable after a successful bind to a
2452    /// REGISTERED module, so the value has already passed HELLO validation
2453    /// including the path-hazard refusal and cannot contain control bytes. A
2454    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
2455    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
2456    ///
2457    /// Bare is also what every other daemon line already emits (`module
2458    /// registered`, `configured module supervised`). Shipping `?module_id` here
2459    /// made this instrument the only one in the file whose ids did not answer
2460    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
2461    /// line whose whole purpose is being grepped beside its sibling.
2462    ///
2463    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
2464    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
2465    /// forwards every field type through it and a bare `&str` and a `?`-escaped
2466    /// one are recorded identically. A test written against that harness passes
2467    /// either way -- I wrote one, measured it, and deleted it rather than ship a
2468    /// green assertion that cannot fail. The same limit applies to the escaping
2469    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
2470    /// it reads as a guard on the Debug escaping and cannot detect its removal.
2471    /// Fencing either needs the real formatter, not the capture layer.
2472    ///
2473    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
2474    /// unix-socket options and subc is loopback TCP, so there is no peer identity
2475    /// to record. The ephemeral port would decay within minutes and answer only a
2476    /// live question. The identity question is instead answered by counting
2477    /// distinct live connections presenting one module's `consumer_identity` --
2478    /// "is anyone else holding this secret" rather than "is this the right
2479    /// process".
2480    fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
2481        self.counters.increment_route_open_accepted(principal);
2482        info!(
2483            target: "control",
2484            principal,
2485            module_id,
2486            connection_id = ctx.connection_id.get(),
2487            "route.open accepted"
2488        );
2489    }
2490
2491    fn supervised_absent_route_open_refusal_frame(
2492        &self,
2493        ctx: &RouteCtx,
2494        frame: &Frame,
2495        module_id: &str,
2496        code: &'static str,
2497        status: &crate::supervise::ModuleStatus,
2498    ) -> Result<Frame, RouterError> {
2499        self.counters.increment_route_open_refused(code);
2500        info!(
2501            target: "control",
2502            code,
2503            reason = "supervised_not_registered",
2504            module_id = ?module_id,
2505            connection_id = ctx.connection_id.get(),
2506            state = %status.state,
2507            enabled = status.enabled,
2508            live = status.live,
2509            "route.open refused"
2510        );
2511        control_error_frame(
2512            frame,
2513            code,
2514            format!(
2515                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
2516                status.state, status.enabled, status.live
2517            ),
2518        )
2519    }
2520
2521    async fn handle_route_open(
2522        &self,
2523        ctx: &RouteCtx,
2524        frame: Frame,
2525        request: RouteOpenRequest,
2526    ) -> Result<Vec<Frame>, RouterError> {
2527        let RouteOpenRequest {
2528            target,
2529            mut identity,
2530            consumer_identity,
2531            consumer_capabilities,
2532            admission_facts,
2533        } = request;
2534        let target_module_id = target_module_id(&target).to_string();
2535        debug!(
2536            connection_id = ctx.connection_id.get(),
2537            corr = frame.header.corr,
2538            module_id = %target_module_id,
2539            "handling route.open"
2540        );
2541
2542        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
2543        // opposite. Below, a caller learns whether a module is unregistered,
2544        // supervised-but-down (with state/enabled/live), or registered without the
2545        // requested role. Elsewhere that is an enumeration leak: a probe learning
2546        // the shape of a fleet it cannot otherwise see.
2547        //
2548        // It is not one here, and the reason is the ACCESS MODEL rather than
2549        // anything about these errors. Reaching route.open requires the
2550        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
2551        // connection file, so any caller who completes it already runs as this
2552        // user -- and can read subc.jsonc for the module list and `ck module
2553        // status` for live state. The reply discloses nothing the caller cannot
2554        // read more easily from disk, while the precision is load-bearing:
2555        // `unknown_module` is retryable and a missing role is not.
2556        //
2557        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
2558        // remote transport, a sandboxed caller, a shared-host mode -- THAT
2559        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
2560        let Some(registration) = self
2561            .registry
2562            .get_module(&target_module_id)
2563            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2564        else {
2565            if let Some((status, warming)) =
2566                self.supervisor_status(&target_module_id, frame.header.corr)?
2567            {
2568                // BEFORE the two availability codes below, because for a module
2569                // that speaks no subc wire both of them are false comfort: they
2570                // say "not right now" and are retried, and this module will
2571                // never register no matter how long the caller waits. The
2572                // absence here is the declaration being honoured, not a module
2573                // that is late.
2574                if status.protocol == ModuleProtocol::None {
2575                    return Ok(vec![self.route_open_refusal_frame(
2576                        ctx,
2577                        &frame,
2578                        &target_module_id,
2579                        "protocol_none",
2580                        error_codes::MODULE_NO_PROTOCOL,
2581                        format!(
2582                            "module_id '{target_module_id}' is declared protocol: none; \
2583                             it speaks no subc wire and serves no routes"
2584                        ),
2585                    )?]);
2586                }
2587                let code = if warming {
2588                    "module_warming"
2589                } else {
2590                    "target_unavailable"
2591                };
2592                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
2593                    ctx,
2594                    &frame,
2595                    &target_module_id,
2596                    code,
2597                    &status,
2598                )?]);
2599            }
2600            if let Some(removed_ago_ms) =
2601                self.supervisor.removal_tombstone_age_ms(&target_module_id)
2602            {
2603                return Ok(vec![self.route_open_refusal_frame(
2604                    ctx,
2605                    &frame,
2606                    &target_module_id,
2607                    "removed",
2608                    error_codes::MODULE_REMOVED,
2609                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
2610                )?]);
2611            }
2612            return Ok(vec![self.route_open_refusal_frame(
2613                ctx,
2614                &frame,
2615                &target_module_id,
2616                "not_registered",
2617                error_codes::UNKNOWN_MODULE,
2618                format!("module_id '{target_module_id}' is not registered"),
2619            )?]);
2620        };
2621
2622        // Best-effort only: registry readiness and forwarding reservation use
2623        // different locks, so a module can flip readiness between this read and
2624        // the relay. Modules must still tolerate an `on_bind` while not ready.
2625        if !registration.ready {
2626            self.counters
2627                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
2628            info!(
2629                target: "control",
2630                code = error_codes::MODULE_WARMING,
2631                module_id = ?target_module_id,
2632                connection_id = ctx.connection_id.get(),
2633                reason = "declared_not_ready",
2634                "route.open refused"
2635            );
2636            return Ok(vec![control_error_body_frame(
2637                &frame,
2638                ErrorBody {
2639                    code: error_codes::MODULE_WARMING.to_string(),
2640                    message: format!(
2641                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
2642                    ),
2643                    detail: Some(serde_json::json!({
2644                        "reason": "declared_not_ready"
2645                    })),
2646                },
2647            )?]);
2648        }
2649
2650        // Effective readiness, second half: a module that declares a capability
2651        // `need: required` is not routable while that capability has no
2652        // registered provider. It is enforced HERE, as a retryable routing
2653        // refusal, and deliberately not as spawn ordering or a boot block. The
2654        // module is still started and registered and can make its own calls;
2655        // spawn ordering is a promise that cannot be kept once a provider
2656        // crashes at runtime, and refusing to boot would stop the whole
2657        // machine, including the tools needed to fix its configuration.
2658        //
2659        // "Provided" is the evaluator's verdict, which counts a provider as
2660        // soon as it has REGISTERED, not once it is ready. Two modules that
2661        // require each other's capabilities are therefore both routable once
2662        // both register; counting readiness instead would deadlock them.
2663        //
2664        // Only new opens are refused. Routes already bound when a provider
2665        // goes away stay bound: nothing here tears them down, and the module
2666        // answers them as it can. Like the readiness read above this is
2667        // best-effort against a provider registering or leaving concurrently.
2668        if let Some(capability) = self.first_unprovided_required_capability(&registration) {
2669            self.counters
2670                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
2671            info!(
2672                target: "control",
2673                code = error_codes::MODULE_WARMING,
2674                module_id = ?target_module_id,
2675                connection_id = ctx.connection_id.get(),
2676                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
2677                capability = %capability,
2678                "route.open refused"
2679            );
2680            return Ok(vec![control_error_body_frame(
2681                &frame,
2682                ErrorBody {
2683                    code: error_codes::MODULE_WARMING.to_string(),
2684                    message: format!(
2685                        "module_id '{target_module_id}' requires capability '{capability}', \
2686                         which no registered module provides; retry"
2687                    ),
2688                    detail: Some(serde_json::json!({
2689                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
2690                        "capability": capability,
2691                    })),
2692                },
2693            )?]);
2694        }
2695
2696        if !target_has_required_role(&target, &registration.manifest.provides) {
2697            return Ok(vec![self.route_open_refusal_frame(
2698                ctx,
2699                &frame,
2700                &target_module_id,
2701                "role_not_provided",
2702                "target_unavailable",
2703                format!("module_id '{target_module_id}' does not provide the requested target"),
2704            )?]);
2705        }
2706
2707        if registration.state != ChannelState::Active {
2708            return Ok(vec![self.route_open_refusal_frame(
2709                ctx,
2710                &frame,
2711                &target_module_id,
2712                "registration_not_active",
2713                "target_unavailable",
2714                format!("module_id '{target_module_id}' is not active"),
2715            )?]);
2716        }
2717
2718        if self
2719            .forwarding
2720            .module_is_draining(&target_module_id)
2721            .map_err(RouterError::Forwarding)?
2722        {
2723            return Ok(vec![self.route_open_refusal_frame(
2724                ctx,
2725                &frame,
2726                &target_module_id,
2727                "reloading",
2728                "module_reloading",
2729                format!("module_id '{target_module_id}' is reloading"),
2730            )?]);
2731        }
2732
2733        if self
2734            .process_liveness
2735            .as_ref()
2736            .and_then(|process_liveness| process_liveness.process_live(&target_module_id))
2737            == Some(false)
2738        {
2739            return Ok(vec![self.route_open_refusal_frame(
2740                ctx,
2741                &frame,
2742                &target_module_id,
2743                "supervisor_not_live",
2744                "target_unavailable",
2745                format!("module_id '{target_module_id}' is not live"),
2746            )?]);
2747        }
2748
2749        if !self
2750            .forwarding
2751            .has_live_module_connection(&target_module_id)
2752            .map_err(RouterError::Forwarding)?
2753        {
2754            return Ok(vec![self.route_open_refusal_frame(
2755                ctx,
2756                &frame,
2757                &target_module_id,
2758                "no_forwarding_connection",
2759                "target_unavailable",
2760                format!("module_id '{target_module_id}' has no live forwarding connection"),
2761            )?]);
2762        }
2763
2764        if let Some(error) =
2765            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
2766        {
2767            self.observe_route_open_refusal(
2768                ctx,
2769                &target_module_id,
2770                "op_not_allowed",
2771                "op_not_allowed",
2772            );
2773            return Ok(vec![error]);
2774        }
2775
2776        let principal = match self.route_open_principal(&frame, consumer_identity)? {
2777            Ok(principal) => principal,
2778            Err(error) => {
2779                self.observe_route_open_refusal(
2780                    ctx,
2781                    &target_module_id,
2782                    "bad_consumer_identity",
2783                    "bad_consumer_identity",
2784                );
2785                return Ok(vec![error]);
2786            }
2787        };
2788
2789        // This is attested, control-plane policy for supervised module origins.
2790        // Keep it before route reservation and out of the opaque forwarding hot
2791        // path: data frames must never acquire a per-frame capability check.
2792        if let Principal::Reserved {
2793            module_id: opening_module_id,
2794        } = &principal
2795        {
2796            if let Some(opening_registration) = self
2797                .registry
2798                .get_module(opening_module_id)
2799                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2800            {
2801                if let Some(capability) =
2802                    denied_capability(&opening_registration.manifest, &registration.manifest)
2803                {
2804                    warn!(
2805                        opening_module_id,
2806                        target_module_id,
2807                        capability,
2808                        "refusing route.open because an attested capability deny edge matches"
2809                    );
2810                    return Ok(vec![self.route_open_refusal_frame(
2811                        ctx,
2812                        &frame,
2813                        &target_module_id,
2814                        "capability_deny_edge",
2815                        "capability_forbidden",
2816                        format!(
2817                            "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
2818                        ),
2819                    )?]);
2820                }
2821            }
2822        }
2823
2824        if admission_facts.is_some() {
2825            let carrier_matches = matches!(
2826                &principal,
2827                Principal::Reserved { module_id }
2828                    if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
2829            );
2830            if !carrier_matches {
2831                return Ok(vec![self.route_open_refusal_frame(
2832                    ctx,
2833                    &frame,
2834                    &target_module_id,
2835                    "admission_facts_carrier_not_permitted",
2836                    "admission_facts_not_permitted",
2837                    "admission facts may only be carried by the configured reserved module",
2838                )?]);
2839            }
2840
2841            let target_allowed = self
2842                .admission_facts_targets
2843                .as_ref()
2844                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
2845            if !target_allowed {
2846                return Ok(vec![self.route_open_refusal_frame(
2847                    ctx,
2848                    &frame,
2849                    &target_module_id,
2850                    "admission_facts_target_not_listed",
2851                    "admission_facts_target_not_allowed",
2852                    format!(
2853                        "admission facts are not permitted for target module_id '{target_module_id}'"
2854                    ),
2855                )?]);
2856            }
2857
2858            // Keep the value opaque to subc. The downstream admission validator owns
2859            // schema and semantic checks; this daemon only enforces carrier authority
2860            // and the configured destination allowlist.
2861        }
2862
2863        // Bind admits a root that no longer exists on disk, because refusing here
2864        // closes the only exit from a paused run: cancel needs a bound route, and a
2865        // renamed or reclaimed directory makes that route unopenable forever. The
2866        // run itself is intact and still addressable by its recorded identity.
2867        //
2868        // This does NOT relax the rule the strict constructor protects. That rule is
2869        // that no root is ever aliased into NEW durable state -- a missing component
2870        // can reappear as a symlink elsewhere, which would move the identity and
2871        // split a session's history across two of them. The engine now refuses the
2872        // two operations that create such state (send and import) at admission,
2873        // which is a narrower way to hold the same invariant: reads and terminations
2874        // are admitted, writes are not. That refusal had to ship before this line
2875        // changed, or there is an interval where a send commits under a provisional
2876        // identity -- the exact failure the original policy existed to prevent.
2877        //
2878        // Resolution follows realpath rather than lexical cleanup: the longest
2879        // existing ancestor is canonicalized and the missing tail re-appended, so a
2880        // live root is unchanged and a vanished leaf keeps the identity it was
2881        // admitted under. Lexical cleanup would mint a DIFFERENT identity for the
2882        // same caller the moment the directory vanished, which strands the run more
2883        // quietly than refusing it.
2884        let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
2885            Ok(project_root) => project_root,
2886            Err(err) => {
2887                return Ok(vec![control_error_frame(
2888                    &frame,
2889                    "invalid_project_root",
2890                    err.to_string(),
2891                )?])
2892            }
2893        };
2894        identity.project_root = project_root.as_path().to_path_buf();
2895
2896        // Last gate before any relay work, and deliberately after the cheap
2897        // registry and availability checks above: those name a more precise
2898        // condition (unknown, removed, reloading) and a caller is better served
2899        // by the precise code than by this one.
2900        //
2901        // Everything below this point costs an egress permit, a reserved handle
2902        // pair and, if the module does not answer, the whole relay budget. The
2903        // reader no longer waits for that budget, so cap each target explicitly;
2904        // serial dispatch used to provide the accidental cap of one relay per
2905        // connection. Admission is a mutex-protected count and never waits.
2906        let _concurrency_guard = match self
2907            .route_bind_concurrency
2908            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
2909        {
2910            Ok(guard) => guard,
2911            Err(in_flight) => {
2912                return Ok(vec![self.route_open_admission_refusal_frame(
2913                    ctx,
2914                    &frame,
2915                    &target_module_id,
2916                    format!(
2917                        "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
2918                    ),
2919                )?]);
2920            }
2921        };
2922
2923        // A module that has already burned the whole budget `threshold` times
2924        // in a row does not get to charge it again until a probe says it recovered.
2925        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
2926            RouteBindAdmission::Admitted { guard, probe } => {
2927                if probe {
2928                    info!(
2929                        module_id = %target_module_id,
2930                        connection_id = ctx.connection_id.get(),
2931                        "route.bind breaker half-open: admitting one probe"
2932                    );
2933                }
2934                guard
2935            }
2936            RouteBindAdmission::Refused {
2937                consecutive_timeouts,
2938                retry_in,
2939                probe_in_flight,
2940            } => {
2941                return Ok(vec![self.route_open_breaker_refusal_frame(
2942                    ctx,
2943                    &frame,
2944                    &target_module_id,
2945                    consecutive_timeouts,
2946                    retry_in,
2947                    probe_in_flight,
2948                )?]);
2949            }
2950        };
2951
2952        // Resolve the per-module budget here so the wait matches the operator's
2953        // intent for this specific target. A per-module override in
2954        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
2955        // daemons) wins over the daemon-wide default.
2956        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
2957        let relay_deadline = Instant::now() + route_bind_relay_timeout;
2958        let pending = match self
2959            .forwarding
2960            .begin_route_bind_relay_for(
2961                ctx.connection_id,
2962                ctx.egress.clone(),
2963                response_version(&frame),
2964                frame.header.corr,
2965                &target_module_id,
2966                principal.clone(),
2967                Some(project_root),
2968                relay_deadline,
2969            )
2970            .await
2971        {
2972            Ok(pending) => pending,
2973            Err(err) => {
2974                return Ok(vec![self.route_open_refusal_frame(
2975                    ctx,
2976                    &frame,
2977                    &target_module_id,
2978                    "relay_reservation_failed",
2979                    forwarding_error_code(&err),
2980                    err.to_string(),
2981                )?])
2982            }
2983        };
2984        let crate::forwarding::PendingRouteBindRelay {
2985            endpoint,
2986            module_sink,
2987            negotiated_ver,
2988            client_channel,
2989            client_epoch,
2990            module_channel,
2991            module_epoch,
2992            corr: relay_corr,
2993            receiver,
2994        } = pending;
2995        let mut reservation =
2996            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
2997
2998        debug!(
2999            connection_id = ctx.connection_id.get(),
3000            client_channel,
3001            client_epoch,
3002            module_channel,
3003            module_epoch,
3004            "reserved route handle pair"
3005        );
3006        // Rendered BEFORE the move into the relay, because the accept arm below
3007        // is where it is logged and the principal is gone by then.
3008        let principal_label = match &principal {
3009            Principal::Reserved { module_id } => format!("reserved:{module_id}"),
3010            Principal::Direct => "direct".to_string(),
3011            other => format!("{other:?}"),
3012        };
3013        let relay = ModuleControlRequest::RouteBind {
3014            route_channel: module_channel,
3015            epoch: module_epoch,
3016            target,
3017            identity,
3018            principal: Some(principal),
3019            consumer_capabilities,
3020            admission_facts,
3021        };
3022        let relay_body = serde_json::to_vec(&relay).map_err(|err| {
3023            RouterError::backend(
3024                0,
3025                frame.header.corr,
3026                format!("failed to encode route.bind request: {err}"),
3027            )
3028        })?;
3029        let relay_frame = Frame::build_with_version(
3030            negotiated_ver,
3031            FrameType::Request,
3032            control_flags(),
3033            0,
3034            0,
3035            relay_corr,
3036            relay_body,
3037        )
3038        .map_err(RouterError::FrameBuild)?;
3039
3040        if let Err(err) = module_sink.send(relay_frame).await {
3041            reservation.release_and_disarm();
3042            return Ok(vec![self.route_open_refusal_frame(
3043                ctx,
3044                &frame,
3045                &target_module_id,
3046                "relay_send_failed",
3047                "target_unavailable",
3048                err.to_string(),
3049            )?]);
3050        }
3051
3052        if !self
3053            .forwarding
3054            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
3055            .map_err(RouterError::Forwarding)?
3056        {
3057            self.send_abandoned_route_bind_goodbye(
3058                &module_sink,
3059                negotiated_ver,
3060                module_channel,
3061                module_epoch,
3062            );
3063        }
3064
3065        match timeout_at(relay_deadline, receiver).await {
3066            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
3067                reservation.disarm();
3068                if breaker.record_accepted() {
3069                    info!(
3070                        module_id = %target_module_id,
3071                        "route.bind breaker closed: the probe was accepted"
3072                    );
3073                }
3074                self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
3075                Ok(Vec::new())
3076            }
3077            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
3078                reservation.release_and_disarm();
3079                // A module that says no in microseconds is healthy. Rejection
3080                // is a different condition with its own refusal and must not
3081                // move the breaker.
3082                breaker.record_inconclusive();
3083                self.counters
3084                    .increment_route_open_refused("module_rejected");
3085                info!(
3086                    target: "control",
3087                    code = "module_rejected",
3088                    module_code = ?body.code,
3089                    module_id = ?target_module_id,
3090                    connection_id = ctx.connection_id.get(),
3091                    "route.open refused"
3092                );
3093                Ok(vec![control_error_body_frame(&frame, body)?])
3094            }
3095            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
3096                reservation.release_and_disarm();
3097                breaker.record_inconclusive();
3098                // Fires when the module's connection closes while a relayed
3099                // bind is pending -- typically a caller racing a module restart
3100                // whose bind was relayed BEFORE the drain mark went up. Logged
3101                // because the caller sees only its own error and the fleet has
3102                // already spent one diagnosis round unable to tell this arm
3103                // from a relay timeout without daemon-side evidence.
3104                tracing::warn!(
3105                    module_id = %target_module_id,
3106                    "route.bind relay abandoned: {message}"
3107                );
3108                Ok(vec![self.route_open_refusal_frame(
3109                    ctx,
3110                    &frame,
3111                    &target_module_id,
3112                    "relay_abandoned",
3113                    "target_unavailable",
3114                    message,
3115                )?])
3116            }
3117            Ok(Err(_)) => {
3118                reservation.release_and_disarm();
3119                breaker.record_inconclusive();
3120                Ok(vec![self.route_open_refusal_frame(
3121                    ctx,
3122                    &frame,
3123                    &target_module_id,
3124                    "relay_waiter_canceled",
3125                    "target_unavailable",
3126                    "route.bind relay waiter was canceled before the module responded",
3127                )?])
3128            }
3129            Err(_) => {
3130                reservation.release_and_disarm();
3131                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
3132                // answer at all is the one condition a fast refusal can
3133                // usefully stand in for; every other arm already answered.
3134                if let Some(opened) = breaker.record_timeout(
3135                    self.route_bind_breaker_threshold,
3136                    self.route_bind_breaker_cooldown,
3137                ) {
3138                    warn!(
3139                        module_id = %target_module_id,
3140                        consecutive_timeouts = opened.consecutive_timeouts,
3141                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
3142                        reopened_after_probe = opened.reopened_after_probe,
3143                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
3144                    );
3145                }
3146                // The generous budget just burned to no answer: the module is
3147                // registered and its connection is up, but its bind handler sat
3148                // on the ack for the full budget (warm-on-bind, cold configure,
3149                // or a wedged handler). Every earlier unavailability shape
3150                // fast-refuses BEFORE the relay, so this arm firing means the
3151                // slowness is module-side -- log it so the per-module timeline
3152                // is reconstructable without client audit rows.
3153                tracing::warn!(
3154                    module_id = %target_module_id,
3155                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
3156                    "route.bind relay timed out: module did not ack within budget"
3157                );
3158                Ok(vec![self.route_open_refusal_frame(
3159                    ctx,
3160                    &frame,
3161                    &target_module_id,
3162                    "relay_timed_out",
3163                    "module_timeout",
3164                    format!(
3165                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
3166                        route_bind_relay_timeout
3167                    ),
3168                )?])
3169            }
3170        }
3171    }
3172
3173    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3174        let response = ClientControlResponse::SupervisorSpawnSnapshot {
3175            snapshot: self.supervisor.spawn_snapshot(),
3176        };
3177        Ok(vec![control_response_body_frame(
3178            &frame,
3179            &response,
3180            "ClientControlResponse::SupervisorSpawnSnapshot",
3181        )?])
3182    }
3183
3184    fn handle_supervisor_spawn_subscribe(
3185        &self,
3186        ctx: &RouteCtx,
3187        frame: Frame,
3188        since: Option<SpawnCursor>,
3189    ) -> Result<Vec<Frame>, RouterError> {
3190        match self.supervisor.subscribe_spawns(
3191            ctx.connection_id,
3192            frame.header.corr,
3193            response_version(&frame),
3194            since,
3195            ctx.egress.clone(),
3196        ) {
3197            Ok(()) => Ok(Vec::new()),
3198            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
3199                Ok(vec![control_error_body_frame(
3200                    &frame,
3201                    ErrorBody {
3202                        code: "spawn_cursor_incarnation_mismatch".to_string(),
3203                        message: "spawn cursor belongs to a different daemon incarnation"
3204                            .to_string(),
3205                        detail: Some(serde_json::json!({
3206                            "current_daemon_incarnation": current
3207                        })),
3208                    },
3209                )?])
3210            }
3211            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
3212                &frame,
3213                ErrorBody {
3214                    code: "spawn_cursor_too_old".to_string(),
3215                    message: "spawn cursor predates the retained event ring".to_string(),
3216                    detail: Some(serde_json::json!({
3217                        "oldest_retained_cursor": oldest
3218                    })),
3219                },
3220            )?]),
3221            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
3222                0,
3223                frame.header.corr,
3224                format!("failed to open supervisor spawn subscription: {error}"),
3225            )),
3226        }
3227    }
3228
3229    async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3230        let generation = self
3231            .registry
3232            .generation()
3233            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3234        let mut modules = Vec::new();
3235        for module in self.supervisor.list() {
3236            let status = module.status_for_control("list").map_err(|err| {
3237                RouterError::backend(
3238                    0,
3239                    frame.header.corr,
3240                    format!("failed to read supervisor status: {err}"),
3241                )
3242            })?;
3243            let (configured, _) = module.configuration().map_err(|err| {
3244                RouterError::backend(
3245                    0,
3246                    frame.header.corr,
3247                    format!("failed to read module configuration: {err}"),
3248                )
3249            })?;
3250            // Status and configuration snapshots release their locks before the image probe awaits.
3251            let image = module.running_image_agreement().await;
3252            let pending_reload = Some(reload_verdict(
3253                &configured.program,
3254                status.spawned_from.as_deref(),
3255                image,
3256            ));
3257            modules.push(SupervisorEntry {
3258                module_id: status.module_id,
3259                state: status.state.to_string(),
3260                enabled: status.enabled,
3261                live: status.live,
3262                protocol: status.protocol,
3263                health: status.health.status,
3264                pending_reload,
3265                last_probe_ms: status.health.last_probe_ms,
3266                last_exit_code: status.last_exit.as_ref().and_then(|e| e.code),
3267                last_exit_signal: status.last_exit.as_ref().and_then(|e| e.signal),
3268                last_exit_ms: status.last_exit.as_ref().map(|e| e.at_ms),
3269                last_exit_kind: status.last_exit.as_ref().map(|e| e.kind.into()),
3270                restart_count: Some(status.restart_count),
3271                max_restarts: Some(status.max_restarts),
3272                lifetime_restarts: Some(status.lifetime_restarts),
3273                spawn_generation: Some(status.spawn_generation),
3274                restart_window_secs: Some(status.restart_window.as_secs()),
3275                drain_timeout_ms: Some(status.drain_timeout.as_millis() as u64),
3276                restart_backoff_ms: Some(status.restart_backoff.as_millis() as u64),
3277                restart_max_backoff_ms: Some(status.restart_max_backoff.as_millis() as u64),
3278            });
3279        }
3280        let response = ClientControlResponse::SupervisorList {
3281            generation,
3282            modules,
3283        };
3284        Ok(vec![control_response_body_frame(
3285            &frame,
3286            &response,
3287            "ClientControlResponse::SupervisorList",
3288        )?])
3289    }
3290
3291    fn handle_supervisor_stderr_tail(
3292        &self,
3293        frame: Frame,
3294        module_id: String,
3295        max_lines: Option<u32>,
3296        max_bytes: Option<u32>,
3297    ) -> Result<Vec<Frame>, RouterError> {
3298        let Some(module) = self.supervisor.get(&module_id) else {
3299            return Ok(vec![control_error_frame(
3300                &frame,
3301                "unknown_module",
3302                format!("module_id '{module_id}' is not supervised"),
3303            )?]);
3304        };
3305
3306        let snapshot = module.stderr_tail(
3307            max_lines.map(|value| value as usize),
3308            max_bytes.map(|value| value as usize),
3309        );
3310
3311        let response = ClientControlResponse::SupervisorStderrTail {
3312            module_id,
3313            tail: StderrTail {
3314                capture: match snapshot.capture {
3315                    CaptureState::Captured => StderrCaptureState::Captured,
3316                    CaptureState::Incomplete { reason } => {
3317                        StderrCaptureState::Incomplete { reason }
3318                    }
3319                    CaptureState::NotCaptured { reason } => {
3320                        StderrCaptureState::NotCaptured { reason }
3321                    }
3322                },
3323                entries: snapshot
3324                    .entries
3325                    .into_iter()
3326                    .map(|entry| match entry {
3327                        TailEntry::Line { text, truncated } => {
3328                            StderrTailEntry::Line { text, truncated }
3329                        }
3330                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
3331                    })
3332                    .collect(),
3333                dropped_lines: snapshot.dropped_lines,
3334            },
3335        };
3336        Ok(vec![control_response_body_frame(
3337            &frame,
3338            &response,
3339            "ClientControlResponse::SupervisorStderrTail",
3340        )?])
3341    }
3342
3343    async fn handle_supervisor_terminals(
3344        &self,
3345        frame: Frame,
3346        module_id: String,
3347    ) -> Result<Vec<Frame>, RouterError> {
3348        let Some(module) = self.supervisor.get(&module_id) else {
3349            return Ok(vec![control_error_frame(
3350                &frame,
3351                "unknown_module",
3352                format!("module_id '{module_id}' is not supervised"),
3353            )?]);
3354        };
3355
3356        // The journal read runs on a blocking thread: it can be megabytes of
3357        // file I/O and must not occupy a runtime worker.
3358        let terminals = module
3359            .read_durable_terminal_history()
3360            .await
3361            .map_err(|error| {
3362                RouterError::backend(
3363                    0,
3364                    frame.header.corr,
3365                    format!("failed to read terminal history: {error}"),
3366                )
3367            })?;
3368        let response = ClientControlResponse::SupervisorTerminals {
3369            module_id,
3370            terminals,
3371        };
3372        Ok(vec![control_response_body_frame(
3373            &frame,
3374            &response,
3375            "ClientControlResponse::SupervisorTerminals",
3376        )?])
3377    }
3378
3379    fn handle_supervisor_routes(
3380        &self,
3381        frame: Frame,
3382        module_id: Option<String>,
3383    ) -> Result<Vec<Frame>, RouterError> {
3384        let modules = self
3385            .forwarding
3386            .route_census(module_id.as_deref())
3387            .map_err(RouterError::Forwarding)?
3388            .into_iter()
3389            .map(|(module_id, routes)| SupervisorRouteModule {
3390                module_id,
3391                routes: routes
3392                    .into_iter()
3393                    .map(|route| SupervisorRoute {
3394                        consumer: match route.principal {
3395                            Principal::Reserved { module_id } => {
3396                                SupervisorRouteConsumer::Reserved { module_id }
3397                            }
3398                            Principal::Direct | Principal::Unverified => {
3399                                SupervisorRouteConsumer::Direct {
3400                                    connection_id: route.goodbye_target.connection_id.get(),
3401                                }
3402                            }
3403                        },
3404                        age_ms: Instant::now()
3405                            .saturating_duration_since(route.bound_at)
3406                            .as_millis()
3407                            .try_into()
3408                            .unwrap_or(u64::MAX),
3409                        draining: route.draining,
3410                        drain_reason: route.drain_reason,
3411                    })
3412                    .collect(),
3413            })
3414            .collect();
3415        let response = ClientControlResponse::SupervisorRoutes { modules };
3416        Ok(vec![control_response_body_frame(
3417            &frame,
3418            &response,
3419            "ClientControlResponse::SupervisorRoutes",
3420        )?])
3421    }
3422
3423    async fn handle_supervisor_provenance(
3424        &self,
3425        frame: Frame,
3426        module_id: Option<String>,
3427    ) -> Result<Vec<Frame>, RouterError> {
3428        let mut selected = if let Some(module_id) = module_id {
3429            let Some(module) = self.supervisor.get(&module_id) else {
3430                return Ok(vec![control_error_frame(
3431                    &frame,
3432                    "unknown_module",
3433                    format!("module_id '{module_id}' is not supervised"),
3434                )?]);
3435            };
3436            vec![module]
3437        } else {
3438            self.supervisor.list()
3439        };
3440
3441        let mut modules = Vec::with_capacity(selected.len());
3442        for module in selected.drain(..) {
3443            let status = module.status().map_err(|err| {
3444                RouterError::backend(
3445                    0,
3446                    frame.header.corr,
3447                    format!("failed to read supervisor status: {err}"),
3448                )
3449            })?;
3450            let module_declared = self
3451                .registry
3452                .get_module(&status.module_id)
3453                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3454                .and_then(|registration| registration.manifest.provenance)
3455                .map(|build| ModuleDeclaredProvenance::Reported { build })
3456                .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
3457            #[cfg(test)]
3458            let running_image = match &self.provenance_probe_override {
3459                Some(result) => result.clone(),
3460                None => module.running_image_agreement().await,
3461            };
3462            #[cfg(not(test))]
3463            let running_image = module.running_image_agreement().await;
3464            modules.push(SupervisorModuleProvenance {
3465                module_id: status.module_id,
3466                module_declared,
3467                daemon_observed: SupervisorObservedProcess {
3468                    pid: status.pid,
3469                    spawned_at_ms: status.spawned_at_ms,
3470                    spawned_from: status.spawned_from,
3471                    running_image,
3472                },
3473            });
3474        }
3475        let daemon = SupervisorDaemonProvenance {
3476            daemon_build: self.daemon_provenance.build.clone(),
3477            daemon_observed: DaemonObservedProcess {
3478                pid: self.daemon_provenance.pid,
3479                started_at_ms: self
3480                    .daemon_provenance
3481                    .start_clock
3482                    .map(|clock| clock.started_at_ms())
3483                    .or(self.daemon_provenance.started_at_ms),
3484                running_image: self
3485                    .daemon_provenance
3486                    .probe
3487                    .observe(
3488                        self.daemon_provenance.pid,
3489                        self.daemon_provenance.executable_path.as_deref(),
3490                        self.daemon_provenance.executable_identity,
3491                        self.daemon_provenance.process_start_time,
3492                    )
3493                    .await,
3494            },
3495        };
3496        let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
3497        Ok(vec![control_response_body_frame(
3498            &frame,
3499            &response,
3500            "ClientControlResponse::SupervisorProvenance",
3501        )?])
3502    }
3503
3504    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3505        self.refresh_capability_requirements();
3506        let generation = self
3507            .registry
3508            .generation()
3509            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3510        let modules = self
3511            .supervisor
3512            .list()
3513            .into_iter()
3514            .map(|module| {
3515                let status = module.status_for_control("health").map_err(|err| {
3516                    RouterError::backend(
3517                        0,
3518                        frame.header.corr,
3519                        format!("failed to read supervisor health: {err}"),
3520                    )
3521                })?;
3522                let module_id = status.module_id;
3523                let capability_detail = self
3524                    .capability_evaluator
3525                    .required_problem_detail(&module_id);
3526                Ok(SupervisorHealthEntry {
3527                    module_id,
3528                    status: status.health.status,
3529                    detail: append_capability_problem_detail(
3530                        status.health.detail,
3531                        capability_detail,
3532                    ),
3533                    metrics: status.health.metrics,
3534                    consecutive_failures: status.health.consecutive_failures,
3535                    late_answer_count: status.health.late_answer_count,
3536                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
3537                    last_action: status.health.last_action,
3538                    last_action_ms: status.health.last_action_ms,
3539                    last_probe_ms: status.health.last_probe_ms,
3540                })
3541            })
3542            .collect::<Result<Vec<_>, RouterError>>()?;
3543        let response = ClientControlResponse::SupervisorHealth {
3544            generation,
3545            modules,
3546        };
3547        Ok(vec![control_response_body_frame(
3548            &frame,
3549            &response,
3550            "ClientControlResponse::SupervisorHealth",
3551        )?])
3552    }
3553
3554    async fn handle_supervisor_restart(
3555        &self,
3556        frame: Frame,
3557        module_id: String,
3558        drain_timeout_ms: Option<u64>,
3559    ) -> Result<Vec<Frame>, RouterError> {
3560        let operation_lock = self.supervisor.operation_lock();
3561        let _operation_guard = operation_lock.lock().await;
3562        let Some(module) = self.supervisor.get(&module_id) else {
3563            return Ok(vec![control_error_frame(
3564                &frame,
3565                "unknown_module",
3566                format!("module_id '{module_id}' is not supervised"),
3567            )?]);
3568        };
3569
3570        if let Err(err) = module.restart(drain_timeout_ms).await {
3571            let (code, message) = match err {
3572                crate::supervise::SuperviseError::Disabled { .. } => {
3573                    ("module_disabled", err.to_string())
3574                }
3575                crate::supervise::SuperviseError::SwapInProgress { .. } => {
3576                    ("swap_in_progress", err.to_string())
3577                }
3578                _ => (
3579                    "target_unavailable",
3580                    format!("failed to restart module_id '{module_id}': {err}"),
3581                ),
3582            };
3583            return Ok(vec![control_error_frame(&frame, code, message)?]);
3584        }
3585
3586        let response = ClientControlResponse::SupervisorAck {
3587            module_id,
3588            applied: true,
3589        };
3590        Ok(vec![control_response_body_frame(
3591            &frame,
3592            &response,
3593            "ClientControlResponse::SupervisorAck",
3594        )?])
3595    }
3596
3597    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
3598    /// when the old process has finished draining: a caller whose own lane
3599    /// rides the old process must get its reply before that drain waits on it.
3600    async fn handle_supervisor_swap(
3601        &self,
3602        frame: Frame,
3603        module_id: String,
3604        ready_timeout_ms: Option<u64>,
3605    ) -> Result<Vec<Frame>, RouterError> {
3606        // The daemon-wide operation lock is held only to resolve the handle,
3607        // not across the swap. The swap can take its whole readiness budget,
3608        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
3609        // holding it here would park an operator's stop behind the swap it is
3610        // meant to abort. A rescan or stop that reaches the module during the
3611        // swap is served by the swap itself (see `supervise_swap`).
3612        let module = {
3613            let operation_lock = self.supervisor.operation_lock();
3614            let _operation_guard = operation_lock.lock().await;
3615            self.supervisor.get(&module_id)
3616        };
3617        let Some(module) = module else {
3618            return Ok(vec![control_error_frame(
3619                &frame,
3620                "unknown_module",
3621                format!("module_id '{module_id}' is not supervised"),
3622            )?]);
3623        };
3624
3625        if let Err(err) = module
3626            .swap(ready_timeout_ms.map(Duration::from_millis))
3627            .await
3628        {
3629            use crate::supervise::SuperviseError;
3630            let message = err.to_string();
3631            let error = match err {
3632                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
3633                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
3634                    code: "swap_refused".to_string(),
3635                    message,
3636                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
3637                },
3638                SuperviseError::SwapFailed {
3639                    arm,
3640                    candidate_exit,
3641                    ..
3642                } => ErrorBody {
3643                    code: "swap_failed".to_string(),
3644                    message,
3645                    detail: Some(serde_json::json!({
3646                        "arm": arm.as_str(),
3647                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
3648                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
3649                    })),
3650                },
3651                _ => ErrorBody::new(
3652                    "target_unavailable",
3653                    format!("failed to swap module_id '{module_id}': {message}"),
3654                ),
3655            };
3656            return Ok(vec![control_error_body_frame(&frame, error)?]);
3657        }
3658
3659        let response = ClientControlResponse::SupervisorAck {
3660            module_id,
3661            applied: true,
3662        };
3663        Ok(vec![control_response_body_frame(
3664            &frame,
3665            &response,
3666            "ClientControlResponse::SupervisorAck",
3667        )?])
3668    }
3669
3670    async fn handle_supervisor_reload(
3671        &self,
3672        frame: Frame,
3673        module_id: String,
3674    ) -> Result<Vec<Frame>, RouterError> {
3675        let operation_lock = self.supervisor.operation_lock();
3676        let _operation_guard = operation_lock.lock().await;
3677        let Some(module) = self.supervisor.get(&module_id) else {
3678            return Ok(vec![control_error_frame(
3679                &frame,
3680                "unknown_module",
3681                format!("module_id '{module_id}' is not supervised"),
3682            )?]);
3683        };
3684
3685        if let Err(err) = module.reload().await {
3686            let (code, message) = match err {
3687                crate::supervise::SuperviseError::Disabled { .. } => {
3688                    ("module_disabled", err.to_string())
3689                }
3690                crate::supervise::SuperviseError::SwapInProgress { .. } => {
3691                    ("swap_in_progress", err.to_string())
3692                }
3693                _ => (
3694                    "reload_failed",
3695                    format!("failed to reload module_id '{module_id}': {err}"),
3696                ),
3697            };
3698            return Ok(vec![control_error_frame(&frame, code, message)?]);
3699        }
3700
3701        let response = ClientControlResponse::SupervisorAck {
3702            module_id,
3703            applied: true,
3704        };
3705        Ok(vec![control_response_body_frame(
3706            &frame,
3707            &response,
3708            "ClientControlResponse::SupervisorAck",
3709        )?])
3710    }
3711
3712    async fn handle_supervisor_rescan(
3713        &self,
3714        frame: Frame,
3715        preview: bool,
3716    ) -> Result<Vec<Frame>, RouterError> {
3717        let Some(context) = self.rescan.clone() else {
3718            return Ok(vec![control_error_frame(
3719                &frame,
3720                "rescan_unavailable",
3721                "the daemon was not started with a reloadable config path".to_string(),
3722            )?]);
3723        };
3724
3725        let operation_lock = self.supervisor.operation_lock();
3726        let _operation_guard = operation_lock.lock().await;
3727        let loaded = match crate::daemon_config::load(&context.config_path) {
3728            Ok(config) => config,
3729            Err(err) => {
3730                return Ok(vec![control_error_frame(
3731                    &frame,
3732                    "invalid_daemon_config",
3733                    format!("supervisor rescan rejected daemon config: {err}"),
3734                )?])
3735            }
3736        };
3737        // `load` reports a missing file as Ok(None), which is correct at boot
3738        // (no config, nothing to supervise) and catastrophic here: rescan treats
3739        // "not in the config" as "remove it", so an absent file would read as an
3740        // empty module list and retire the entire running fleet. An editor
3741        // writing via write-new-then-rename, or a half-finished edit, is enough
3742        // to open that window. Refuse instead: a config that cannot be read
3743        // carries no instruction to remove anything.
3744        let Some(config) = loaded else {
3745            return Ok(vec![control_error_frame(
3746                &frame,
3747                "invalid_daemon_config",
3748                format!(
3749                    "daemon config not found at {}; refusing to rescan (an absent config would \
3750                     retire every supervised module)",
3751                    context.config_path.display()
3752                ),
3753            )?]);
3754        };
3755        let (
3756            configured_port,
3757            storage_config,
3758            admission_facts_carrier_module_id,
3759            admission_facts_targets,
3760            modules,
3761            reserved_capabilities,
3762        ) = (
3763            config.port,
3764            config.storage,
3765            config.admission_facts_carrier_module_id,
3766            config.admission_facts_targets,
3767            config.modules,
3768            config.reserved_capabilities,
3769        );
3770
3771        // Collect the sections rescan cannot apply, so the REPLY carries them.
3772        //
3773        // The warning below has always been correct and has always gone only to
3774        // the journal -- addressed to whoever reads logs, while the person who
3775        // just edited the config is looking at the CLI. Naming each section
3776        // individually rather than setting a flag: "something outside modules
3777        // changed" sends the operator back to diffing their own file, which is
3778        // the work this is meant to save.
3779        let mut restart_required = Vec::new();
3780        for section in RestartRequiredSection::ALL {
3781            let changed = match section {
3782                RestartRequiredSection::Port => configured_port != context.configured_port,
3783                RestartRequiredSection::Storage => storage_config != context.storage_config,
3784                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
3785                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
3786                }
3787                RestartRequiredSection::AdmissionFactsTargets => {
3788                    admission_facts_targets != context.admission_facts_targets
3789                }
3790            };
3791            if changed {
3792                restart_required.push(section.label().to_string());
3793            }
3794        }
3795        if !restart_required.is_empty() {
3796            warn!(
3797                config_path = %context.config_path.display(),
3798                sections = %restart_required.join(", "),
3799                "daemon config changed outside the modules section; restart the daemon to apply those changes"
3800            );
3801        }
3802
3803        for configured in &modules {
3804            if let Err(err) = validate_spec(&configured.module_spec()) {
3805                return Ok(vec![control_error_frame(
3806                    &frame,
3807                    "invalid_daemon_config",
3808                    format!("supervisor rescan rejected daemon config: {err}"),
3809                )?]);
3810            }
3811        }
3812
3813        let configured_capabilities = modules
3814            .iter()
3815            .map(|module| (module.module_id.clone(), module.enabled))
3816            .collect::<Vec<_>>();
3817        let preview_capability_warnings = if preview {
3818            let (_, registrations) = self.runtime_capability_snapshot()?;
3819            let current_modules = self
3820                .supervisor
3821                .list()
3822                .into_iter()
3823                .map(|module| module.module_id().to_string())
3824                .collect::<BTreeSet<_>>();
3825            let resulting_modules = configured_capabilities.clone();
3826            let removed = current_modules
3827                .into_iter()
3828                .filter(|module_id| {
3829                    !resulting_modules
3830                        .iter()
3831                        .any(|(configured_id, _)| configured_id == module_id)
3832                })
3833                .collect::<Vec<_>>();
3834            self.capability_evaluator.preview_removal_warnings(
3835                resulting_modules,
3836                &removed,
3837                &registrations,
3838            )
3839        } else {
3840            Vec::new()
3841        };
3842        let result = match self
3843            .reconcile_supervised_modules(&context.supervisor, modules, preview)
3844            .await
3845        {
3846            Ok(result) => result,
3847            Err(message) => {
3848                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
3849            }
3850        };
3851        if !preview {
3852            self.capability_evaluator
3853                .configure(configured_capabilities, reserved_capabilities);
3854            self.capability_evaluator.wake_deadline_loop();
3855            self.refresh_capability_requirements();
3856        }
3857        let mut result = result;
3858        result.restart_required = restart_required;
3859        result.capability_warnings = preview_capability_warnings;
3860        let response = ClientControlResponse::SupervisorRescan { result };
3861        Ok(vec![control_response_body_frame(
3862            &frame,
3863            &response,
3864            "ClientControlResponse::SupervisorRescan",
3865        )?])
3866    }
3867
3868    async fn handle_supervisor_release_reserved(
3869        &self,
3870        frame: Frame,
3871        module_id: String,
3872    ) -> Result<Vec<Frame>, RouterError> {
3873        let Some(context) = self.rescan.clone() else {
3874            return Ok(vec![control_error_frame(
3875                &frame,
3876                "release_unavailable",
3877                "reserved-id release requires a daemon started with a reloadable config path",
3878            )?]);
3879        };
3880        let operation_lock = self.supervisor.operation_lock();
3881        let _operation_guard = operation_lock.lock().await;
3882        let loaded = match crate::daemon_config::load(&context.config_path) {
3883            Ok(Some(config)) => config,
3884            Ok(None) => {
3885                return Ok(vec![control_error_frame(
3886                    &frame,
3887                    "invalid_daemon_config",
3888                    format!(
3889                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
3890                        context.config_path.display()
3891                    ),
3892                )?])
3893            }
3894            Err(err) => {
3895                return Ok(vec![control_error_frame(
3896                    &frame,
3897                    "invalid_daemon_config",
3898                    format!("unable to verify reserved-id release against daemon config: {err}"),
3899                )?])
3900            }
3901        };
3902        if loaded
3903            .modules
3904            .iter()
3905            .any(|configured| configured.module_id == module_id)
3906        {
3907            return Ok(vec![control_error_frame(
3908                &frame,
3909                "reserved_module_configured",
3910                format!(
3911                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
3912                ),
3913            )?]);
3914        }
3915        if !self.supervisor.release_retained_reserved_gate(&module_id) {
3916            return Ok(vec![control_error_frame(
3917                &frame,
3918                "reserved_gate_not_retained",
3919                format!(
3920                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
3921                ),
3922            )?]);
3923        }
3924
3925        let response = ClientControlResponse::SupervisorAck {
3926            module_id,
3927            applied: true,
3928        };
3929        Ok(vec![control_response_body_frame(
3930            &frame,
3931            &response,
3932            "ClientControlResponse::SupervisorAck",
3933        )?])
3934    }
3935
3936    /// Reconcile the running module set against the configured one.
3937    ///
3938    /// With `preview` set, the diff is computed and returned WITHOUT applying any
3939    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
3940    /// deliberately shares this function with the executing path rather than
3941    /// computing the same diff somewhere else -- two implementations of one
3942    /// decision agree until they do not, and the whole value of a preview is that
3943    /// it describes the operation that will actually run.
3944    async fn reconcile_supervised_modules(
3945        &self,
3946        supervisor: &Supervisor,
3947        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
3948        preview: bool,
3949    ) -> Result<SupervisorRescanResult, String> {
3950        let mut current = BTreeMap::new();
3951        for module in self.supervisor.list() {
3952            let (spec, health) = module.configuration().map_err(|err| {
3953                format!(
3954                    "failed to read configuration for module_id '{}': {err}",
3955                    module.module_id()
3956                )
3957            })?;
3958            let enabled = module
3959                .status()
3960                .map_err(|err| {
3961                    format!(
3962                        "failed to read status for module_id '{}': {err}",
3963                        module.module_id()
3964                    )
3965                })?
3966                .enabled;
3967            current.insert(
3968                module.module_id().to_string(),
3969                (module, spec, health, enabled),
3970            );
3971        }
3972        let configured = configured_modules
3973            .into_iter()
3974            .map(|module| (module.module_id.clone(), module))
3975            .collect::<BTreeMap<_, _>>();
3976
3977        let added = configured
3978            .keys()
3979            .filter(|module_id| !current.contains_key(*module_id))
3980            .cloned()
3981            .collect::<Vec<_>>();
3982        let removed = current
3983            .keys()
3984            .filter(|module_id| !configured.contains_key(*module_id))
3985            .cloned()
3986            .collect::<Vec<_>>();
3987        let mut changed_pending_reload = Vec::new();
3988        let mut configuration_changes = BTreeSet::new();
3989        let mut enabled_changes = BTreeSet::new();
3990        let mut unchanged = 0_u32;
3991
3992        for (module_id, configured_module) in &configured {
3993            let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
3994            else {
3995                continue;
3996            };
3997            let configuration_changed = *current_spec != configured_module.module_spec()
3998                || *current_health != configured_module.health;
3999            let enabled_changed = *current_enabled != configured_module.enabled;
4000            if configuration_changed {
4001                configuration_changes.insert(module_id.clone());
4002                changed_pending_reload.push(module_id.clone());
4003            }
4004            if enabled_changed {
4005                enabled_changes.insert(module_id.clone());
4006            }
4007            if !configuration_changed && !enabled_changed {
4008                unchanged = unchanged.saturating_add(1);
4009            }
4010        }
4011
4012        // Everything above this point is pure computation over two snapshots.
4013        // Everything below MUTATES. The preview returns here so the boundary is a
4014        // single early return rather than a condition repeated at each mutation
4015        // site, where one missed guard would apply part of a change the caller was
4016        // told would not happen.
4017        if preview {
4018            return Ok(SupervisorRescanResult {
4019                added,
4020                removed,
4021                changed_pending_reload,
4022                enabled_changes: enabled_changes.iter().cloned().collect(),
4023                unchanged,
4024                preview: true,
4025                // Filled by the caller on both paths, so the preview reports
4026                // restart-required sections identically to an executed rescan --
4027                // the preview is where an operator is most likely to be looking.
4028                restart_required: Vec::new(),
4029                capability_warnings: Vec::new(),
4030            });
4031        }
4032
4033        for module_id in &removed {
4034            let module = &current
4035                .get(module_id)
4036                .expect("removed module came from current supervisor state")
4037                .0;
4038            module.retire().await.map_err(|err| {
4039                format!("failed to retire module_id '{module_id}' during rescan: {err}")
4040            })?;
4041            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
4042            //
4043            // `handle_route_open` resolves an absent module in three steps:
4044            // registry, then supervisor status, then tombstone. Retiring first
4045            // opens a window where ALL THREE ARE ABSENT -- the registry entry
4046            // went with the teardown above, the supervisor entry went with
4047            // `retire`, and the tombstone does not exist yet -- so a route.open
4048            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
4049            // it") for a module that was deliberately removed and whose caller
4050            // should get `module_removed` (TERMINAL, carrying a removal age).
4051            //
4052            // Writing the tombstone first closes it: during the window the
4053            // supervisor entry still answers, so the caller gets
4054            // `target_unavailable` -- retryable, and TRUE, because the module
4055            // is mid-teardown. After both statements it is `module_removed`.
4056            // No instant remains where a removed module reads as one that
4057            // never existed.
4058            //
4059            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
4060            // the absence of a test beside a fix invites deletion: these are
4061            // two sync statements with no await between them, so reaching the
4062            // window needs a second worker thread to land exactly between them
4063            // and there is no hook to force it. MEASURED: the 25 daemon_config
4064            // tests pass identically with the old order and the new one, so
4065            // the existing suite cannot see this and a green run is not
4066            // evidence either way. What the suite does hold is the
4067            // post-condition -- a removed module answers `module_removed` --
4068            // which this preserves.
4069            //
4070            // Found by an Athena panel reading the shipped tree against a
4071            // design note (2026-09-19), as the one concrete instance of that
4072            // note's class that survived contact with source. Direction is
4073            // benign: retryable where terminal was intended, never the reverse.
4074            self.supervisor.record_rescan_removal(module_id);
4075            self.supervisor.retire(module_id);
4076        }
4077
4078        for module_id in configured.keys() {
4079            let Some((module, _, _, _)) = current.get(module_id) else {
4080                continue;
4081            };
4082            let configured_module = configured
4083                .get(module_id)
4084                .expect("configured module id came from configured map");
4085            if configuration_changes.contains(module_id) {
4086                module
4087                    .update_configuration(
4088                        configured_module.module_spec(),
4089                        configured_module.health,
4090                        configured_module.drain_timeout_ms,
4091                    )
4092                    .await
4093                    .map_err(|err| {
4094                        format!(
4095                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
4096                        )
4097                    })?;
4098            }
4099            if enabled_changes.contains(module_id) {
4100                module
4101                    .set_enabled(configured_module.enabled)
4102                    .await
4103                    .map_err(|err| {
4104                        format!(
4105                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
4106                            configured_module.enabled
4107                        )
4108                    })?;
4109            }
4110        }
4111
4112        for module_id in &added {
4113            let configured_module = configured
4114                .get(module_id)
4115                .expect("added module id came from configured map");
4116            supervisor
4117                .supervise_configured_with_health(
4118                    configured_module.module_spec(),
4119                    configured_module.enabled,
4120                    configured_module.health,
4121                    configured_module.drain_timeout_ms,
4122                    configured_module.restart,
4123                )
4124                .map_err(|err| {
4125                    format!("failed to add module_id '{module_id}' during rescan: {err}")
4126                })?;
4127        }
4128
4129        Ok(SupervisorRescanResult {
4130            added,
4131            removed,
4132            changed_pending_reload,
4133            enabled_changes: enabled_changes.iter().cloned().collect(),
4134            unchanged,
4135            preview: false,
4136            // Filled by the caller, which is the only layer that can see the
4137            // previous config to diff against.
4138            restart_required: Vec::new(),
4139            capability_warnings: Vec::new(),
4140        })
4141    }
4142
4143    async fn handle_supervisor_set_enabled(
4144        &self,
4145        frame: Frame,
4146        module_id: String,
4147        enabled: bool,
4148    ) -> Result<Vec<Frame>, RouterError> {
4149        let operation_lock = self.supervisor.operation_lock();
4150        let _operation_guard = operation_lock.lock().await;
4151        let Some(module) = self.supervisor.get(&module_id) else {
4152            return Ok(vec![control_error_frame(
4153                &frame,
4154                "unknown_module",
4155                format!("module_id '{module_id}' is not supervised"),
4156            )?]);
4157        };
4158
4159        let applied = match module.set_enabled(enabled).await {
4160            Ok(applied) => applied,
4161            Err(err) => {
4162                return Ok(vec![control_error_frame(
4163                    &frame,
4164                    "target_unavailable",
4165                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
4166                )?])
4167            }
4168        };
4169
4170        self.capability_evaluator.wake_deadline_loop();
4171        self.refresh_capability_requirements();
4172        let response = ClientControlResponse::SupervisorAck { module_id, applied };
4173        Ok(vec![control_response_body_frame(
4174            &frame,
4175            &response,
4176            "ClientControlResponse::SupervisorAck",
4177        )?])
4178    }
4179
4180    async fn handle_supervisor_health_probe(
4181        &self,
4182        frame: Frame,
4183        module_id: String,
4184    ) -> Result<Vec<Frame>, RouterError> {
4185        self.refresh_capability_requirements();
4186        let Some(registration) = self
4187            .registry
4188            .get_module(&module_id)
4189            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4190        else {
4191            return Ok(vec![control_error_frame(
4192                &frame,
4193                "unknown_module",
4194                format!("module_id '{module_id}' is not registered"),
4195            )?]);
4196        };
4197
4198        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
4199        // named for it. Making `module_registration_grants_op` return false
4200        // unconditionally reddens five tests, and every one is named for something
4201        // else -- capability relay, probe/bind demultiplexing, supervision-only
4202        // probing. They exercise a successful advertisement check on the way to their
4203        // own subject.
4204        //
4205        // Real protection, fragile in a specific way: narrowing any of those tests to
4206        // focus on its stated subject would silently remove coverage nobody knows
4207        // they are carrying. Recorded here rather than as a sixth test, because the
4208        // useful fact is WHICH tests hold the guard up -- a new test would add
4209        // coverage without telling the next person what the existing ones quietly do.
4210        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
4211        {
4212            return Ok(vec![control_error_frame(
4213                &frame,
4214                "health_not_advertised",
4215                format!("module_id '{module_id}' did not advertise health.check"),
4216            )?]);
4217        }
4218
4219        let deadline = Instant::now() + self.health_probe_timeout;
4220        let pending = match self.forwarding.begin_module_control_rpc_for(
4221            &module_id,
4222            MODULE_CONTROL_OP_HEALTH_CHECK,
4223            deadline,
4224        ) {
4225            Ok(pending) => pending,
4226            Err(err) => {
4227                return Ok(vec![control_error_frame(
4228                    &frame,
4229                    forwarding_error_code(&err),
4230                    err.to_string(),
4231                )?])
4232            }
4233        };
4234
4235        let PendingModuleControlRpc {
4236            endpoint,
4237            module_sink,
4238            negotiated_ver,
4239            corr: probe_corr,
4240            receiver,
4241        } = pending;
4242        let mut guard =
4243            ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
4244        let probe_body =
4245            serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
4246                RouterError::backend(
4247                    0,
4248                    frame.header.corr,
4249                    format!("failed to encode health.check request: {err}"),
4250                )
4251            })?;
4252        let probe_frame = Frame::build_with_version(
4253            negotiated_ver,
4254            FrameType::Request,
4255            control_flags(),
4256            0,
4257            0,
4258            probe_corr,
4259            probe_body,
4260        )
4261        .map_err(RouterError::FrameBuild)?;
4262
4263        if let Err(err) = module_sink.send(probe_frame).await {
4264            return Ok(vec![control_error_frame(
4265                &frame,
4266                "target_unavailable",
4267                err.to_string(),
4268            )?]);
4269        }
4270
4271        match timeout_at(deadline, receiver).await {
4272            Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
4273                guard.disarm();
4274                let Some(report) = response.health_report() else {
4275                    return Ok(vec![control_error_frame(
4276                        &frame,
4277                        "invalid_control_body",
4278                        "health.check RPC returned a non-health response",
4279                    )?]);
4280                };
4281                // Metrics go out whole here. The supervisor's cached snapshot
4282                // caps this blob (see truncate_health_metrics), and this path
4283                // exists precisely to answer without that cap -- so applying it
4284                // here would leave no way to see what the cached view drops.
4285                let HealthReport {
4286                    status,
4287                    detail,
4288                    metrics,
4289                } = report;
4290                let capability_detail = self
4291                    .capability_evaluator
4292                    .required_problem_detail(&module_id);
4293                let response = ClientControlResponse::SupervisorHealthProbe {
4294                    module_id,
4295                    status,
4296                    detail: append_capability_problem_detail(detail, capability_detail),
4297                    metrics,
4298                };
4299                Ok(vec![control_response_body_frame(
4300                    &frame,
4301                    &response,
4302                    "ClientControlResponse::SupervisorHealthProbe",
4303                )?])
4304            }
4305            Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
4306                guard.disarm();
4307                Ok(vec![control_error_body_frame(&frame, body)?])
4308            }
4309            Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
4310                guard.disarm();
4311                Ok(vec![control_error_frame(
4312                    &frame,
4313                    "target_unavailable",
4314                    message,
4315                )?])
4316            }
4317            Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
4318                guard.disarm();
4319                Ok(vec![control_error_frame(
4320                    &frame,
4321                    "invalid_control_body",
4322                    message,
4323                )?])
4324            }
4325            Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
4326                guard.disarm();
4327                Ok(vec![control_error_frame(
4328                    &frame,
4329                    "invalid_control_body",
4330                    format!("expected module-control op '{expected}', got '{actual}'"),
4331                )?])
4332            }
4333            Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
4334                guard.disarm();
4335                Ok(vec![control_error_frame(
4336                    &frame,
4337                    "module_timeout",
4338                    format!(
4339                        "module_id '{module_id}' answered health.check after {:?}",
4340                        self.health_probe_timeout
4341                    ),
4342                )?])
4343            }
4344            Ok(Err(_)) => Ok(vec![control_error_frame(
4345                &frame,
4346                "target_unavailable",
4347                "health.check waiter was canceled before the module responded",
4348            )?]),
4349            Err(_) => Ok(vec![control_error_frame(
4350                &frame,
4351                "module_timeout",
4352                format!(
4353                    "module_id '{module_id}' did not answer health.check within {:?}",
4354                    self.health_probe_timeout
4355                ),
4356            )?]),
4357        }
4358    }
4359
4360    fn supervisor_status(
4361        &self,
4362        module_id: &str,
4363        corr: u64,
4364    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
4365        self.supervisor
4366            .get(module_id)
4367            .map(|module| {
4368                let warming = module.is_warming_for_control("status").map_err(|err| {
4369                    RouterError::backend(
4370                        0,
4371                        corr,
4372                        format!(
4373                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
4374                        ),
4375                    )
4376                })?;
4377                module.status_for_control("status").map_err(|err| {
4378                    RouterError::backend(
4379                        0,
4380                        corr,
4381                        format!(
4382                            "failed to read supervisor status for module_id '{module_id}': {err}"
4383                        ),
4384                    )
4385                }).map(|status| (status, warming))
4386            })
4387            .transpose()
4388    }
4389
4390    fn guard_module_control_op(
4391        &self,
4392        frame: &Frame,
4393        module_id: &str,
4394        op: &str,
4395    ) -> Result<Option<Frame>, RouterError> {
4396        if self.module_grants_op(module_id, op, frame.header.corr)? {
4397            return Ok(None);
4398        }
4399
4400        Ok(Some(control_error_frame(
4401            frame,
4402            "op_not_allowed",
4403            format!("module_id '{module_id}' did not grant control op '{op}'"),
4404        )?))
4405    }
4406
4407    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
4408        let Some(registration) = self
4409            .registry
4410            .get_module(module_id)
4411            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
4412        else {
4413            return Ok(false);
4414        };
4415        Ok(module_registration_grants_op(&registration.control_ops, op))
4416    }
4417
4418    fn handle_status_update(
4419        &self,
4420        endpoint: ModuleEndpointId,
4421        frame: Frame,
4422    ) -> Result<Vec<Frame>, RouterError> {
4423        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
4424            Ok(update) => update,
4425            Err(err) => {
4426                // Forward-compat: a newer module may push a channel-0 op this subc
4427                // version doesn't know. The control contract says unknown push ops
4428                // are IGNORED, never answered with an error. Only a malformed body
4429                // for an op we DO know is a real error worth surfacing.
4430                if is_known_module_push_op(&frame.body) {
4431                    return Ok(vec![control_error_frame(
4432                        &frame,
4433                        "invalid_control_body",
4434                        format!("malformed module control push body: {err}"),
4435                    )?]);
4436                }
4437                return Ok(Vec::new());
4438            }
4439        };
4440
4441        match update {
4442            ModuleControlPush::RouteStatus {
4443                route_channel,
4444                route_epoch,
4445                status,
4446            } => {
4447                self.forwarding
4448                    .cache_status(endpoint, route_channel, route_epoch, status)
4449                    .map_err(RouterError::Forwarding)?;
4450            }
4451        }
4452        Ok(Vec::new())
4453    }
4454
4455    fn handle_route_poll(
4456        &self,
4457        ctx: &RouteCtx,
4458        frame: Frame,
4459        route_channel: u16,
4460        route_epoch: u32,
4461        kind: PollKind,
4462    ) -> Result<Vec<Frame>, RouterError> {
4463        let snapshot = self
4464            .forwarding
4465            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
4466            .map_err(RouterError::Forwarding)?;
4467        let response = match (kind, snapshot) {
4468            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
4469                ClientControlResponse::RoutePoll {
4470                    route_channel,
4471                    route_epoch,
4472                    status,
4473                    live: None,
4474                }
4475            }
4476            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
4477                route_channel,
4478                route_epoch,
4479                status: None,
4480                live: None,
4481            },
4482            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
4483                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
4484                // is what makes reporting `true` correct rather than a
4485                // confident guess. `process_live` returns None only when the
4486                // module id has no supervisor snapshot at all -- an
4487                // externally-started module the daemon did not spawn -- and
4488                // for those the supervisor has no opinion to offer, ever. It
4489                // is never None for a supervised module in an unknown state:
4490                // a supervised module always has a snapshot, and the answer
4491                // comes from `state == Running && process_alive`.
4492                //
4493                // The route is Bound, so the module completed a HELLO on a
4494                // live connection; "the process this route points at is
4495                // running" is therefore attested by the binding rather than
4496                // assumed. Reporting `false` for an unsupervised module would
4497                // be the actual lie -- it would tell a client its healthy
4498                // route is dead because the daemon does not manage the
4499                // process.
4500                //
4501                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
4502                // module whose liveness is genuinely unknown, e.g. a snapshot
4503                // that has not been populated yet -- THIS DEFAULT BECOMES
4504                // WRONG and must split: unsupervised stays true, unknown
4505                // becomes null so the client can tell the two apart. The
4506                // response field is already `Option<bool>`, so the wire can
4507                // carry that distinction today.
4508                let live = self
4509                    .process_liveness
4510                    .as_ref()
4511                    .and_then(|source| source.process_live(&module_id))
4512                    .unwrap_or(true);
4513                ClientControlResponse::RoutePoll {
4514                    route_channel,
4515                    route_epoch,
4516                    status: None,
4517                    live: Some(live),
4518                }
4519            }
4520            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
4521                route_channel,
4522                route_epoch,
4523                status: None,
4524                live: Some(false),
4525            },
4526        };
4527
4528        Ok(vec![control_response_body_frame(
4529            &frame,
4530            &response,
4531            "ClientControlResponse::RoutePoll",
4532        )?])
4533    }
4534
4535    pub(crate) fn observe_module_control_completion(
4536        &self,
4537        completion: ModuleControlRpcCompletion,
4538    ) -> bool {
4539        match completion {
4540            ModuleControlRpcCompletion::Unknown => false,
4541            ModuleControlRpcCompletion::Settled => true,
4542            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
4543                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
4544                info!(
4545                    module_id = %module_id,
4546                    latency_ms,
4547                    "late health.check answer proves the module is alive"
4548                );
4549                match self
4550                    .supervisor
4551                    .record_late_health_answer(&module_id, latency_ms)
4552                {
4553                    Ok(true) => {}
4554                    Ok(false) => debug!(
4555                        module_id = %module_id,
4556                        latency_ms,
4557                        "late health.check answer has no active supervisor snapshot"
4558                    ),
4559                    Err(err) => warn!(
4560                        module_id = %module_id,
4561                        latency_ms,
4562                        error = %err,
4563                        "failed to record late health.check answer"
4564                    ),
4565                }
4566                true
4567            }
4568        }
4569    }
4570
4571    /// Decide whether a failure while settling a relayed `route.bind` belongs to
4572    /// the module connection whose frame is being handled, or to the client that
4573    /// relay was opened for.
4574    ///
4575    /// This runs on the MODULE connection's frame handler, where returning `Err`
4576    /// ends that connection -- and a module connection carries every client's
4577    /// routes to that module, so ending it costs the whole fleet its tools.
4578    /// `ConnectionClosing` carries the id of the connection that is closing, and
4579    /// when that id is a CLIENT's, the condition is entirely about that one
4580    /// client's route.open. A client-scoped condition has no authority over a
4581    /// shared module connection, so it is logged and the single relay is dropped:
4582    /// the client is going away, and `complete_pending_relay` already removed the
4583    /// relay before failing, so there is nothing left to settle. Anything that
4584    /// relay still reserved is released by that client's own connection teardown,
4585    /// which is already under way -- that is what "closing" means.
4586    ///
4587    /// Every other failure is a statement about THIS connection and stays fatal:
4588    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
4589    /// id in `ConnectionClosing` all mean this connection cannot keep serving
4590    /// frames correctly.
4591    fn refuse_to_end_module_connection_for_a_client(
4592        &self,
4593        module_connection_id: ConnectionId,
4594        corr: u64,
4595        err: ForwardingError,
4596    ) -> Result<(), RouterError> {
4597        if let ForwardingError::ConnectionClosing { connection_id } = err {
4598            if connection_id != module_connection_id {
4599                warn!(
4600                    module_connection_id = module_connection_id.get(),
4601                    client_connection_id = connection_id.get(),
4602                    corr,
4603                    "dropping a route.bind response for a closing client; the module connection keeps serving"
4604                );
4605                return Ok(());
4606            }
4607        }
4608        Err(RouterError::Forwarding(err))
4609    }
4610
4611    fn handle_module_relay_response(
4612        &self,
4613        connection_id: ConnectionId,
4614        frame: Frame,
4615    ) -> Result<Vec<Frame>, RouterError> {
4616        let mut secondary_error = None;
4617        let outcome = match frame.header.ty {
4618            FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
4619                Ok(probe) if probe.op == "route.bind" => {
4620                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
4621                        Ok(ModuleControlResponse::RouteBindAck {}) => {
4622                            RouteBindRelayOutcome::Accepted
4623                        }
4624                        Ok(other) => {
4625                            let message =
4626                                format!("route.bind response carried unexpected body: {other:?}");
4627                            secondary_error = Some(control_error_frame(
4628                                &frame,
4629                                "invalid_control_body",
4630                                message.clone(),
4631                            )?);
4632                            RouteBindRelayOutcome::ModuleGone(message)
4633                        }
4634                        Err(err) => {
4635                            let message = format!("malformed route.bind response body: {err}");
4636                            secondary_error = Some(control_error_frame(
4637                                &frame,
4638                                "invalid_control_body",
4639                                message.clone(),
4640                            )?);
4641                            RouteBindRelayOutcome::ModuleGone(message)
4642                        }
4643                    }
4644                }
4645                Ok(probe) => {
4646                    let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
4647                    {
4648                        Ok(response) => ModuleControlRpcOutcome::Response(response),
4649                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
4650                            "malformed {} response body: {err}",
4651                            probe.op
4652                        )),
4653                    };
4654                    let completion = self
4655                        .forwarding
4656                        .complete_module_control_rpc(
4657                            connection_id,
4658                            frame.header.corr,
4659                            Some(&probe.op),
4660                            outcome,
4661                        )
4662                        .map_err(RouterError::Forwarding)?;
4663                    if !self.observe_module_control_completion(completion) {
4664                        debug!(
4665                            connection_id = connection_id.get(),
4666                            corr = frame.header.corr,
4667                            op = %probe.op,
4668                            "dropping late or unknown module-control RPC response"
4669                        );
4670                    }
4671                    return Ok(Vec::new());
4672                }
4673                Err(err) => {
4674                    if let Some(expected_op) = self
4675                        .forwarding
4676                        .pending_module_control_op(connection_id, frame.header.corr)
4677                        .map_err(RouterError::Forwarding)?
4678                    {
4679                        let completion = self
4680                            .forwarding
4681                            .complete_module_control_rpc(
4682                                connection_id,
4683                                frame.header.corr,
4684                                None,
4685                                ModuleControlRpcOutcome::MalformedResponse(format!(
4686                                    "malformed {expected_op} response body: {err}"
4687                                )),
4688                            )
4689                            .map_err(RouterError::Forwarding)?;
4690                        if !self.observe_module_control_completion(completion) {
4691                            debug!(
4692                                connection_id = connection_id.get(),
4693                                corr = frame.header.corr,
4694                                "dropping late malformed module-control RPC response"
4695                            );
4696                        }
4697                        return Ok(Vec::new());
4698                    }
4699                    let message = format!("malformed route.bind response body: {err}");
4700                    secondary_error = Some(control_error_frame(
4701                        &frame,
4702                        "invalid_control_body",
4703                        message.clone(),
4704                    )?);
4705                    RouteBindRelayOutcome::ModuleGone(message)
4706                }
4707            },
4708            FrameType::Error => {
4709                if self
4710                    .forwarding
4711                    .pending_module_control_op(connection_id, frame.header.corr)
4712                    .map_err(RouterError::Forwarding)?
4713                    .is_some()
4714                {
4715                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
4716                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
4717                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
4718                            "malformed module-control ERROR body: {err}"
4719                        )),
4720                    };
4721                    let completion = self
4722                        .forwarding
4723                        .complete_module_control_rpc(
4724                            connection_id,
4725                            frame.header.corr,
4726                            None,
4727                            outcome,
4728                        )
4729                        .map_err(RouterError::Forwarding)?;
4730                    if !self.observe_module_control_completion(completion) {
4731                        debug!(
4732                            connection_id = connection_id.get(),
4733                            corr = frame.header.corr,
4734                            "dropping late or unknown module-control RPC error"
4735                        );
4736                    }
4737                    return Ok(Vec::new());
4738                }
4739                match serde_json::from_slice::<ErrorBody>(&frame.body) {
4740                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
4741                    Err(err) => {
4742                        let message = format!("malformed route.bind ERROR body: {err}");
4743                        secondary_error = Some(control_error_frame(
4744                            &frame,
4745                            "invalid_control_body",
4746                            message.clone(),
4747                        )?);
4748                        RouteBindRelayOutcome::ModuleGone(message)
4749                    }
4750                }
4751            }
4752            ty => {
4753                return Ok(vec![control_error_frame(
4754                    &frame,
4755                    "unsupported_control_frame",
4756                    format!("unsupported module channel-0 frame {ty:?}"),
4757                )?])
4758            }
4759        };
4760
4761        let settled =
4762            self.forwarding
4763                .complete_pending_relay(connection_id, frame.header.corr, outcome);
4764        let completion = match settled {
4765            Ok(completion) => completion,
4766            Err(err) => {
4767                self.refuse_to_end_module_connection_for_a_client(
4768                    connection_id,
4769                    frame.header.corr,
4770                    err,
4771                )?;
4772                return Ok(secondary_error.into_iter().collect());
4773            }
4774        };
4775        if let Some(target) = completion.abandoned.as_ref() {
4776            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
4777        }
4778        if !completion.settled {
4779            debug!(
4780                connection_id = connection_id.get(),
4781                corr = frame.header.corr,
4782                frame_type = ?frame.header.ty,
4783                "dropping late or unknown route.bind relay response"
4784            );
4785        }
4786        Ok(secondary_error.into_iter().collect())
4787    }
4788
4789    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
4790        debug!(connection_id = connection_id.get(), "handling GOODBYE");
4791        let registrations = self
4792            .deregister_connection(connection_id)
4793            .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
4794        let released_routes = self
4795            .forwarding
4796            .cleanup_connection(connection_id)
4797            .map_err(RouterError::Forwarding)?;
4798        self.emit_route_goodbyes(released_routes);
4799        // Notify only after forwarding teardown completes (see cleanup_connection).
4800        if !registrations.is_empty() {
4801            crate::supervise::notify_registration_release();
4802        }
4803        Ok(Vec::new())
4804    }
4805}
4806
4807impl Default for ControlHandler {
4808    fn default() -> Self {
4809        Self::new(Arc::new(Registry::default()))
4810    }
4811}
4812
4813impl crate::supervise::SwapPromotionObserver for ControlHandler {
4814    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
4815        self.apply_registration_capabilities(registration);
4816    }
4817}
4818
4819fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
4820    CapabilityRequirementStatus {
4821        consumer: status.consumer,
4822        capability: status.capability,
4823        need: match status.need {
4824            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
4825            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
4826        },
4827        verdict: status.verdict.as_str().to_string(),
4828        episode_seq: status.episode_seq,
4829        config_satisfiable: status.config_satisfiable,
4830        runtime_available: status.runtime_available,
4831        detail: status.detail,
4832    }
4833}
4834
4835fn append_capability_problem_detail(
4836    detail: Option<String>,
4837    capability_detail: Option<String>,
4838) -> Option<String> {
4839    match (detail, capability_detail) {
4840        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
4841        (Some(detail), None) => Some(detail),
4842        (None, Some(capability_detail)) => Some(capability_detail),
4843        (None, None) => None,
4844    }
4845}
4846
4847fn subc_ops() -> Vec<String> {
4848    SUBC_CONTROL_OPS
4849        .iter()
4850        .map(|op| (*op).to_string())
4851        .collect()
4852}
4853
4854fn module_subc_ops() -> Vec<String> {
4855    SUBC_CONTROL_OPS
4856        .iter()
4857        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
4858        .map(|op| (*op).to_string())
4859        .collect()
4860}
4861
4862#[cfg(test)]
4863fn module_baseline_control_ops() -> Vec<String> {
4864    MODULE_BASELINE_CONTROL_OPS
4865        .iter()
4866        .map(|op| (*op).to_string())
4867        .collect()
4868}
4869
4870fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
4871    let mut seen = HashSet::new();
4872    let mut effective = Vec::new();
4873    for op in MODULE_BASELINE_CONTROL_OPS {
4874        if seen.insert((*op).to_string()) {
4875            effective.push((*op).to_string());
4876        }
4877    }
4878    for op in declared.unwrap_or_default() {
4879        if seen.insert(op.clone()) {
4880            effective.push(op);
4881        }
4882    }
4883    effective
4884}
4885
4886fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
4887    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
4888}
4889
4890fn target_module_id(target: &RouteTarget) -> &str {
4891    match target {
4892        RouteTarget::ToolProvider { module_id }
4893        | RouteTarget::ManagementSurface { module_id }
4894        | RouteTarget::InternalService { module_id, .. } => module_id,
4895    }
4896}
4897
4898fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
4899    roles.iter().any(|role| match (target, role) {
4900        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
4901        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
4902        (
4903            RouteTarget::InternalService { service_id, .. },
4904            ProviderRole::InternalService {
4905                service_id: provided,
4906                ..
4907            },
4908        ) => service_id == provided,
4909        _ => false,
4910    })
4911}
4912
4913fn is_routable_role(role: &ProviderRole) -> bool {
4914    matches!(
4915        role,
4916        ProviderRole::ToolProvider { .. }
4917            | ProviderRole::ManagementSurface { .. }
4918            | ProviderRole::InternalService { .. }
4919    )
4920}
4921
4922#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4923enum ControlRequestBodyError {
4924    UnknownOp,
4925    InvalidBody,
4926}
4927
4928#[derive(Debug, Deserialize)]
4929struct ControlOpProbe {
4930    op: String,
4931}
4932
4933/// Channel-0 push ops this subc version understands. A push whose `op` is not in
4934/// this set is treated as a forward-compat unknown and ignored rather than errored.
4935const MODULE_PUSH_OPS: &[&str] = &["route.status"];
4936
4937fn is_known_module_push_op(body: &[u8]) -> bool {
4938    serde_json::from_slice::<ControlOpProbe>(body)
4939        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
4940        .unwrap_or(false)
4941}
4942
4943fn is_known_module_request_op(body: &[u8]) -> bool {
4944    serde_json::from_slice::<ControlOpProbe>(body)
4945        .map(|probe| MODULE_TO_SUBC_CONTROL_OPS.contains(&probe.op.as_str()))
4946        .unwrap_or(false)
4947}
4948
4949fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
4950    debug!(
4951        op = %op,
4952        connection_id = connection_id.get(),
4953        corr,
4954        "control dispatch"
4955    );
4956}
4957
4958fn log_slow_control_dispatch(
4959    dispatch_started_at: Option<StdInstant>,
4960    op: &'static str,
4961    connection_id: ConnectionId,
4962    corr: u64,
4963) {
4964    let Some(dispatch_started_at) = dispatch_started_at else {
4965        return;
4966    };
4967    let elapsed = dispatch_started_at.elapsed();
4968    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
4969        warn!(
4970            op = %op,
4971            connection_id = connection_id.get(),
4972            corr,
4973            elapsed_ms = elapsed.as_millis() as u64,
4974            "slow control dispatch"
4975        );
4976    }
4977}
4978
4979fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
4980    match request {
4981        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
4982        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
4983        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
4984        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
4985        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
4986        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
4987        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
4988        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
4989        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
4990        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
4991        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
4992        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
4993        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
4994        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
4995        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
4996        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
4997        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
4998        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
4999        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
5000    }
5001}
5002
5003fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
5004    match request {
5005        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
5006        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
5007    }
5008}
5009
5010fn parse_client_control_request(
5011    body: &[u8],
5012) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
5013    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
5014        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5015            Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5016                ControlRequestBodyError::InvalidBody
5017            }
5018            Ok(_) => ControlRequestBodyError::UnknownOp,
5019            Err(_) => ControlRequestBodyError::InvalidBody,
5020        };
5021        (err, classification)
5022    })
5023}
5024
5025fn parse_module_control_request_from_module(
5026    body: &[u8],
5027) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
5028    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
5029        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5030            Ok(probe) if MODULE_TO_SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5031                ControlRequestBodyError::InvalidBody
5032            }
5033            Ok(_) => ControlRequestBodyError::UnknownOp,
5034            Err(_) => ControlRequestBodyError::InvalidBody,
5035        };
5036        (err, classification)
5037    })
5038}
5039
5040#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
5041enum ProviderRoleKind {
5042    ToolProvider,
5043    PipelineStage,
5044    ManagementSurface,
5045    InternalService,
5046}
5047
5048fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
5049    match role {
5050        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
5051        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
5052        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
5053        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
5054    }
5055}
5056
5057fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
5058    roles.iter().map(provider_role_kind).collect()
5059}
5060
5061/// Return whether a catalog change can create a newly violating live route.
5062/// Removing an attested claim is intentionally excluded: it makes fewer routes
5063/// forbidden and therefore must leave the existing route census untouched.
5064fn capability_census_trigger(
5065    old: Option<&CapabilityDeclarations>,
5066    new: Option<&CapabilityDeclarations>,
5067) -> bool {
5068    let old_provides = old
5069        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
5070        .unwrap_or_default();
5071    let old_denies = old
5072        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
5073        .unwrap_or_default();
5074    let new = new.cloned().unwrap_or(CapabilityDeclarations {
5075        provides: Vec::new(),
5076        requires: Vec::new(),
5077        must_never_reach: Vec::new(),
5078    });
5079
5080    new.provides
5081        .iter()
5082        .any(|capability| !old_provides.contains(capability))
5083        || new
5084            .must_never_reach
5085            .iter()
5086            .any(|capability| !old_denies.contains(capability))
5087}
5088
5089/// Find the first capability an attested opener denies that an attested target
5090/// claims. Both manifests are live registry records, never cached or client data.
5091fn denied_capability<'a>(
5092    opening_manifest: &'a ModuleManifest,
5093    target_manifest: &ModuleManifest,
5094) -> Option<&'a str> {
5095    let opening_capabilities = opening_manifest.capabilities.as_ref()?;
5096    let target_capabilities = target_manifest.capabilities.as_ref()?;
5097    opening_capabilities
5098        .must_never_reach
5099        .iter()
5100        .find(|denied| {
5101            target_capabilities
5102                .provides
5103                .iter()
5104                .any(|provided| provided == *denied)
5105        })
5106        .map(String::as_str)
5107}
5108
5109fn catalog_update_frozen_field_message(
5110    registered: &ModuleManifest,
5111    provides: &[ProviderRole],
5112) -> Option<String> {
5113    let old_has_provides = !registered.provides.is_empty();
5114    let new_has_provides = !provides.is_empty();
5115    if old_has_provides != new_has_provides {
5116        return Some(format!(
5117            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
5118            registered.module_id
5119        ));
5120    }
5121
5122    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
5123        return Some(format!(
5124            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
5125            registered.module_id
5126        ));
5127    }
5128
5129    let registered_concurrency = manifest_concurrency(registered);
5130    let mut candidate = registered.clone();
5131    candidate.provides = provides.to_vec();
5132    let candidate_concurrency = manifest_concurrency(&candidate);
5133    if candidate_concurrency != registered_concurrency {
5134        return Some(format!(
5135            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
5136            registered.module_id, registered_concurrency, candidate_concurrency
5137        ));
5138    }
5139
5140    // control_ops live beside the manifest in the HELLO body, not inside
5141    // ModuleManifest, so a provides-only catalog.update cannot change them.
5142    None
5143}
5144
5145fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
5146    manifest.provides.iter().any(is_routable_role)
5147}
5148
5149/// Returns the routable-provider concurrency subc should enforce for this manifest.
5150///
5151/// ToolProvider and ManagementSurface store their delivery concurrency directly.
5152/// InternalService has no role-specific concurrency field, so it retains the
5153/// existing ModuleManaged default for backward compatibility.
5154fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
5155    manifest
5156        .provides
5157        .iter()
5158        .find_map(|provider| match provider {
5159            ProviderRole::ToolProvider { concurrency, .. }
5160            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
5161            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
5162        })
5163        .unwrap_or(Concurrency::ModuleManaged)
5164}
5165
5166/// True when the manifest carries a ManagementSurface role whose concurrency
5167/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
5168/// bytes because the typed manifest deliberately erases that distinction: the
5169/// default exists for wire compatibility, and this probe exists so the default
5170/// stays observable. Any parse irregularity returns false -- the caller only
5171/// logs, and a malformed body already failed registration upstream.
5172fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
5173    let has_management_surface = manifest
5174        .provides
5175        .iter()
5176        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
5177    if !has_management_surface {
5178        return false;
5179    }
5180    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
5181        return false;
5182    };
5183    let Some(provides) = raw
5184        .get("manifest")
5185        .and_then(|manifest| manifest.get("provides"))
5186        .and_then(serde_json::Value::as_array)
5187    else {
5188        return false;
5189    };
5190    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
5191    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
5192    // against the management_surface_manifest_without_concurrency golden, not
5193    // recalled (the externally-tagged guess was this function's first bug).
5194    provides.iter().any(|role| {
5195        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
5196            && role.get("concurrency").is_none()
5197    })
5198}
5199
5200fn negotiate_version(peer_version: u8) -> Result<u8, String> {
5201    if peer_version != PROTOCOL_VERSION {
5202        return Err(format!(
5203            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
5204        ));
5205    }
5206    Ok(PROTOCOL_VERSION)
5207}
5208
5209fn pong(frame: &Frame) -> Result<Frame, RouterError> {
5210    Frame::build_with_version(
5211        response_version(frame),
5212        FrameType::Pong,
5213        frame.header.flags,
5214        0,
5215        0,
5216        frame.header.corr,
5217        Vec::new(),
5218    )
5219    .map_err(RouterError::FrameBuild)
5220}
5221
5222fn control_error_frame(
5223    frame: &Frame,
5224    code: &'static str,
5225    message: impl Into<String>,
5226) -> Result<Frame, RouterError> {
5227    control_error_body_frame(
5228        frame,
5229        ErrorBody {
5230            code: code.to_string(),
5231            message: message.into(),
5232            detail: None,
5233        },
5234    )
5235}
5236
5237fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
5238    let body = serde_json::to_vec(&error).map_err(|err| {
5239        RouterError::backend(
5240            0,
5241            frame.header.corr,
5242            format!("failed to encode control ERROR: {err}"),
5243        )
5244    })?;
5245
5246    Frame::build_with_version(
5247        response_version(frame),
5248        FrameType::Error,
5249        control_flags(),
5250        0,
5251        0,
5252        frame.header.corr,
5253        body,
5254    )
5255    .map_err(RouterError::FrameBuild)
5256}
5257
5258fn control_response_body_frame<T: Serialize>(
5259    frame: &Frame,
5260    reply: &T,
5261    label: &'static str,
5262) -> Result<Frame, RouterError> {
5263    let body = serde_json::to_vec(reply).map_err(|err| {
5264        RouterError::backend(
5265            0,
5266            frame.header.corr,
5267            format!("failed to encode {label}: {err}"),
5268        )
5269    })?;
5270
5271    Frame::build_with_version(
5272        response_version(frame),
5273        FrameType::Response,
5274        control_flags(),
5275        0,
5276        0,
5277        frame.header.corr,
5278        body,
5279    )
5280    .map_err(RouterError::FrameBuild)
5281}
5282
5283/// Map a forwarding failure to the wire code a client sees.
5284///
5285/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
5286/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
5287/// chosen here decides whether a caller retries or gives up.
5288///
5289/// That makes attribution the load-bearing property, not merely having a code. A
5290/// permanent fault published as a retryable one produces a fleet-wide retry storm
5291/// against something that can never recover; a transient fault published as
5292/// permanent gives up on work that would have succeeded. Both look correct in a
5293/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
5294/// the mapping per variant rather than merely asserting that some code exists.
5295///
5296/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
5297/// two codes on the same side of the boundary passes it. Measured rather than
5298/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
5299/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
5300/// a test named for something else that happens to assert the string.
5301///
5302/// That accidental coverage is deliberately left alone rather than promoted to a
5303/// named test, because it guards a property this function does not promise.
5304/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
5305/// specific code within a class, so identity is free to change and only the
5306/// partition is a contract. Splitting it out would assert a guarantee nothing
5307/// depends on — and a suite that promises more than the code does is the harder
5308/// thing to correct later, because the next reader cannot tell which assertions
5309/// are load-bearing.
5310///
5311/// Pin identity here the moment a consumer branches on a specific code.
5312fn forwarding_error_code(err: &ForwardingError) -> &'static str {
5313    match err {
5314        ForwardingError::NoModuleConnection => "target_unavailable",
5315        ForwardingError::ModuleReloading { .. } => "module_reloading",
5316        ForwardingError::ClientRouteChannelExhausted { .. }
5317        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
5318        ForwardingError::StaleModuleEndpoint
5319        | ForwardingError::UnknownReservation { .. }
5320        | ForwardingError::ConnectionClosing { .. }
5321        | ForwardingError::ClientEgressClosed { .. }
5322        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
5323        // Only a swap candidate's registration can produce this, and it means
5324        // exactly what a second active HELLO for a live id means.
5325        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
5326        ForwardingError::RelayCorrelationExhausted
5327        | ForwardingError::RouteOpenBuild(_)
5328        | ForwardingError::Poisoned => "forwarding_error",
5329    }
5330}
5331
5332fn response_version(frame: &Frame) -> u8 {
5333    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
5334        frame.header.ver
5335    } else {
5336        PROTOCOL_VERSION
5337    }
5338}
5339
5340fn control_flags() -> Flags {
5341    Flags::new(false, Priority::Passive, false)
5342}
5343
5344/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
5345/// channel. The target is the module (a client never saw the route), so this
5346/// takes the module path: delivered late rather than dropped when the module's
5347/// queue is momentarily full, and never closing its connection.
5348fn send_goodbye_target_best_effort(
5349    counters: &DaemonCounters,
5350    target: &GoodbyeTarget,
5351    context: &'static str,
5352) {
5353    let Ok(frame) = Frame::build_with_version(
5354        target.negotiated_ver,
5355        FrameType::Goodbye,
5356        control_flags(),
5357        target.channel,
5358        target.epoch,
5359        0,
5360        Vec::new(),
5361    ) else {
5362        return;
5363    };
5364    crate::forwarding::send_module_route_goodbye(
5365        counters,
5366        &target.sink,
5367        frame,
5368        target.module_id.as_deref(),
5369        context,
5370    );
5371}
5372
5373pub(crate) fn send_route_control_pushes(
5374    forwarding: &ForwardingTable,
5375    routes: Vec<EndpointRoute>,
5376    push: ClientControlPush,
5377) {
5378    let body = match serde_json::to_vec(&push) {
5379        Ok(body) => body,
5380        Err(err) => {
5381            warn!(error = %err, "failed to serialize route lifecycle control PUSH");
5382            return;
5383        }
5384    };
5385    let mut targets = Vec::new();
5386    for route in routes {
5387        let target = route.goodbye_target;
5388        if let Some(existing) = targets
5389            .iter()
5390            .find(|existing: &&GoodbyeTarget| existing.connection_id == target.connection_id)
5391        {
5392            debug_assert_eq!(
5393                existing.negotiated_ver, target.negotiated_ver,
5394                "one connection cannot negotiate multiple frame versions"
5395            );
5396            continue;
5397        }
5398        targets.push(target);
5399    }
5400    for target in targets {
5401        let frame = match Frame::build_with_version(
5402            target.negotiated_ver,
5403            FrameType::Push,
5404            control_flags(),
5405            0,
5406            0,
5407            0,
5408            body.clone(),
5409        ) {
5410            Ok(frame) => frame,
5411            Err(err) => {
5412                warn!(
5413                    route_channel = target.channel,
5414                    error = %err,
5415                    "failed to build route lifecycle control PUSH frame"
5416                );
5417                continue;
5418            }
5419        };
5420        if let Err(err) = target.sink.try_send(frame) {
5421            if target.close_on_delivery_failure() {
5422                warn!(
5423                    target_connection_id = target.connection_id.get(),
5424                    route_channel = target.channel,
5425                    error = %err,
5426                    "route lifecycle control PUSH was not delivered to client; closing target connection"
5427                );
5428                let _ = forwarding.escalate_client_delivery_failure(
5429                    target.connection_id,
5430                    target.channel,
5431                    target.epoch,
5432                    CloseReason::new(
5433                        "route_lifecycle_push_delivery_failed",
5434                        format!(
5435                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
5436                            target.channel
5437                        ),
5438                    ),
5439                    crate::forwarding::UndeliveredFrame {
5440                        module_id: target.module_id.as_deref(),
5441                        sink: &target.sink,
5442                    },
5443                );
5444            }
5445        }
5446    }
5447}
5448
5449#[cfg(test)]
5450mod tests {
5451    use std::{
5452        collections::BTreeMap,
5453        fmt,
5454        path::PathBuf,
5455        sync::{Arc, Mutex},
5456        time::Duration,
5457    };
5458
5459    use serde_json::{json, Value};
5460    use subc_protocol::{
5461        manifest::{
5462            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
5463            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
5464        },
5465        session::HealthStatus,
5466        FrameType,
5467    };
5468
5469    use super::*;
5470    use crate::{
5471        forwarding::{DataRoute, DataRouteState},
5472        registry::ChannelState,
5473        router::FrameSink,
5474        stderr_tail::DEFAULT_MAX_LINE_BYTES,
5475        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
5476        test_support::TestTempDir,
5477        RouteCtx, Router,
5478    };
5479    use tokio::{
5480        sync::mpsc,
5481        time::{sleep, Instant},
5482    };
5483    use tracing::{
5484        field::{Field, Visit},
5485        Event, Subscriber,
5486    };
5487    use tracing_subscriber::{layer::Context, prelude::*, Layer};
5488
5489    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
5490    ///
5491    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
5492    /// is only populated for `tests/*.rs` integration test binaries -- this file
5493    /// compiles as part of the library target, which gets neither. This test's
5494    /// own executable path is `<target-dir>/<profile>/deps/subc_core-<hash>`,
5495    /// and the sibling binary lives two directories up at
5496    /// `<target-dir>/<profile>/fake-aft-stub`.
5497    ///
5498    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
5499    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
5500    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
5501    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
5502    /// `NotFound`, which reads as a broken test rather than an unbuilt
5503    /// dependency -- so state the cause and the remedy instead. Deliberately a
5504    /// panic and not a silent skip: a test that quietly passes when it could not
5505    /// run is worse than one that fails, because it reports health it never
5506    /// verified.
5507    fn fake_aft_stub_path() -> PathBuf {
5508        let mut path = std::env::current_exe().expect("current_exe available in tests");
5509        path.pop(); // .../deps/
5510        path.pop(); // .../<profile>/
5511        path.push(if cfg!(windows) {
5512            "fake-aft-stub.exe"
5513        } else {
5514            "fake-aft-stub"
5515        });
5516        assert!(
5517            path.exists(),
5518            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
5519             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
5520            path.display()
5521        );
5522        path
5523    }
5524
5525    /// Whether clients retry `code` in place: the predicate itself, never a copy
5526    /// of its set. A copied list breaks silently when a code is added to or
5527    /// removed from the real one, and a stale copy here would let exactly the
5528    /// failure this test exists to catch pass.
5529    fn client_retries(code: &str) -> bool {
5530        subc_protocol::error_codes::is_retryable_route_open(code)
5531    }
5532
5533    /// A code is not a label — clients branch on it, so publishing the wrong KIND
5534    /// of failure is worse than publishing none. A permanent fault dressed as
5535    /// retryable makes every client in the fleet retry forever against something
5536    /// that cannot recover; a transient fault dressed as permanent abandons work
5537    /// that would have succeeded.
5538    ///
5539    /// Asserting "a code exists" cannot catch either, because the string is free
5540    /// to say anything. This enumerates every variant and pins which side of the
5541    /// retry boundary it lands on, so a new variant must be classified here
5542    /// deliberately rather than inheriting whichever arm it was appended to.
5543    #[test]
5544    fn retryability_of_forwarding_codes_matches_the_failure() {
5545        // Transient by nature: the target is booting, reloading, or its endpoint
5546        // was swapped mid-flight. Retrying is how these resolve.
5547        let transient = [
5548            ForwardingError::NoModuleConnection,
5549            ForwardingError::ModuleReloading {
5550                module_id: "m".into(),
5551            },
5552            ForwardingError::StaleModuleEndpoint,
5553            ForwardingError::UnknownReservation {
5554                client_channel: 1,
5555                module_channel: 1,
5556            },
5557            ForwardingError::ConnectionClosing {
5558                connection_id: ConnectionId::new(1),
5559            },
5560            ForwardingError::ClientEgressClosed {
5561                connection_id: ConnectionId::new(1),
5562            },
5563            ForwardingError::ModuleEgressUnavailable {
5564                connection_id: ConnectionId::new(1),
5565            },
5566        ];
5567        for err in transient {
5568            let code = forwarding_error_code(&err);
5569            assert!(
5570                client_retries(code),
5571                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
5572            );
5573        }
5574
5575        // Not fixed by retrying. Channel and correlation exhaustion need the
5576        // caller to close routes, and a poisoned lock is a daemon that cannot
5577        // recover at all — the worst thing to advertise as retryable, since every
5578        // client would storm a daemon that will never answer.
5579        let permanent = [
5580            ForwardingError::ClientRouteChannelExhausted {
5581                connection_id: ConnectionId::new(1),
5582            },
5583            ForwardingError::ModuleRouteChannelExhausted {
5584                endpoint: ModuleEndpointId {
5585                    connection_id: ConnectionId::new(1),
5586                    generation: 1,
5587                },
5588            },
5589            ForwardingError::RelayCorrelationExhausted,
5590            ForwardingError::RouteOpenBuild("x".into()),
5591            ForwardingError::Poisoned,
5592        ];
5593        for err in permanent {
5594            let code = forwarding_error_code(&err);
5595            assert!(
5596                !client_retries(code),
5597                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
5598            );
5599        }
5600    }
5601
5602    /// The principal is the daemon's answer to "who is calling", and modules
5603    /// branch on it: aft gates bash on it, cerebellum gates browser control,
5604    /// plexus gates connector invocation. So a stamp is an authorization input in
5605    /// another process, not a label — and both possible answers SUCCEED, which is
5606    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
5607    /// first-party capability to something that never proved it; a supervised one
5608    /// stamped `Direct` silently strips a module of capability it is entitled to.
5609    ///
5610    /// Neither shows up in a test that only checks the bind succeeded. Before this
5611    /// test the only coverage was accidental —
5612    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
5613    /// stamped principal on its way past, so narrowing that wire-shape test to its
5614    /// stated subject would have deleted the last assertion on this value. It
5615    /// still asserts the stamp, which is now redundancy rather than the only
5616    /// guard: both fail under the same mutation, and this one names the reason.
5617    /// SCOPE: this handler's supervisor has spawned nothing, so
5618    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
5619    /// is unreachable here. Both assertions below are refusals, and a mutant that
5620    /// refuses everything would satisfy them.
5621    ///
5622    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
5623    /// spawns a supervised consumer, reads its live nonce, and asserts the module
5624    /// observed `principal.kind == "reserved"` carrying that module_id — verified
5625    /// at source rather than assumed, since a citation is a claim about another
5626    /// file and ages like one. Recorded because a harness that structurally
5627    /// cannot reach an arm reports "none" for that arm identically to one that
5628    /// covers it and found nothing.
5629    #[tokio::test]
5630    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
5631        let handler = ControlHandler::default();
5632        let frame =
5633            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
5634
5635        // Absent consumer_identity is the ordinary case: a human at a terminal, or
5636        // any process holding the connection file. Nothing was proved, so nothing
5637        // may be granted beyond the unattested floor.
5638        let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
5639        assert_eq!(
5640            stamped,
5641            Principal::Direct,
5642            "a caller that proved nothing must not be stamped as a supervised module"
5643        );
5644
5645        // A claimed module_id with a nonce no supervised child was given is a
5646        // forgery attempt, not a weaker caller: it must be REFUSED rather than
5647        // quietly demoted to Direct, or an impersonation attempt looks identical
5648        // to an ordinary unattested connection.
5649        let forged = handler
5650            .route_open_principal(
5651                &frame,
5652                Some(ConsumerIdentity {
5653                    module_id: "aft".to_string(),
5654                    launch_nonce: "not-a-real-nonce".to_string(),
5655                }),
5656            )
5657            .unwrap();
5658        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
5659        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
5660    }
5661
5662    /// The test above hands `route_open_principal` an identity it built itself,
5663    /// which proves the stamping rule and nothing about where the identity comes
5664    /// from. The real producer is a wire body, and the two are joined by a serde
5665    /// field name that nothing else asserts.
5666    ///
5667    /// That join fails quietly in one specific way: an unrecognised key is simply
5668    /// absent after parsing, so a renamed or misspelled `consumer_identity`
5669    /// yields `None` and every supervised module silently drops to `Direct`.
5670    /// Capability-wise that is the safe direction, but it surfaces far from its
5671    /// cause — as a module mysteriously refused bash — and it would pass every
5672    /// test that builds its own input.
5673    ///
5674    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
5675    /// would break every client the moment the daemon gains a field, trading a
5676    /// quiet demotion for a hard refusal on additive change. Asserting the join
5677    /// instead means a rename breaks a test here rather than the fleet.
5678    #[test]
5679    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
5680        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
5681        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
5682        let ClientControlRequest::RouteOpen {
5683            consumer_identity, ..
5684        } = parsed
5685        else {
5686            panic!("route.open body must parse as RouteOpen");
5687        };
5688        assert_eq!(
5689            consumer_identity,
5690            Some(ConsumerIdentity {
5691                module_id: "aft".to_string(),
5692                launch_nonce: "n".to_string(),
5693            }),
5694            "the wire field name must reach the value route_open_principal reads"
5695        );
5696    }
5697
5698    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
5699        ModuleManifest::builder(module_id, "0.1.0")
5700            .protocol_ver(protocol_ver)
5701            .provides(vec![ProviderRole::ToolProvider {
5702                tools: vec![Tool {
5703                    name: "read".to_string(),
5704                    description: None,
5705                    execution_mode: ExecutionMode::Pure,
5706                    schema: json!({"type": "object"}),
5707                }],
5708                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
5709                concurrency: Concurrency::ModuleManaged,
5710                emits_push: true,
5711                sub_supervises: true,
5712            }])
5713            .build()
5714    }
5715
5716    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
5717        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
5718    }
5719
5720    fn hello_frame_with_control_ops(
5721        module_id: &str,
5722        protocol_ver: u8,
5723        corr: u64,
5724        control_ops: Option<Vec<String>>,
5725    ) -> Frame {
5726        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
5727    }
5728
5729    fn hello_frame_with_nonce(
5730        module_id: &str,
5731        protocol_ver: u8,
5732        corr: u64,
5733        launch_nonce: Option<&str>,
5734    ) -> Frame {
5735        hello_frame_full(
5736            module_id,
5737            protocol_ver,
5738            corr,
5739            None,
5740            launch_nonce.map(ToOwned::to_owned),
5741        )
5742    }
5743
5744    fn hello_frame_full(
5745        module_id: &str,
5746        protocol_ver: u8,
5747        corr: u64,
5748        control_ops: Option<Vec<String>>,
5749        launch_nonce: Option<String>,
5750    ) -> Frame {
5751        let body = serde_json::to_vec(&ModuleHelloBody {
5752            manifest: manifest(module_id, protocol_ver),
5753            protocol_ver,
5754            control_ops,
5755            launch_nonce,
5756        })
5757        .unwrap();
5758        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
5759    }
5760
5761    fn non_routable_hello_frame_with_control_ops(
5762        module_id: &str,
5763        corr: u64,
5764        control_ops: Option<Vec<String>>,
5765    ) -> Frame {
5766        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
5767        manifest.provides.clear();
5768        let body = serde_json::to_vec(&ModuleHelloBody {
5769            manifest,
5770            protocol_ver: PROTOCOL_VERSION,
5771            control_ops,
5772            launch_nonce: None,
5773        })
5774        .unwrap();
5775        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
5776    }
5777
5778    fn capability_grammar_hello_frame(
5779        capabilities: Value,
5780        runtime_computed: Option<Value>,
5781        corr: u64,
5782    ) -> Frame {
5783        let mut body = serde_json::to_value(ModuleHelloBody {
5784            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
5785            protocol_ver: PROTOCOL_VERSION,
5786            control_ops: None,
5787            launch_nonce: None,
5788        })
5789        .expect("HELLO body serializes");
5790        body["manifest"]["capabilities"] = capabilities;
5791        if let Some(runtime_computed) = runtime_computed {
5792            body["runtime_computed"] = runtime_computed;
5793        }
5794        Frame::build(
5795            FrameType::Hello,
5796            control_flags(),
5797            0,
5798            0,
5799            corr,
5800            serde_json::to_vec(&body).expect("HELLO body reserializes"),
5801        )
5802        .expect("HELLO frame builds")
5803    }
5804
5805    fn channel_request(channel: u16, corr: u64) -> Frame {
5806        Frame::build(
5807            FrameType::Request,
5808            Flags::new(true, Priority::Interactive, false),
5809            channel,
5810            0,
5811            corr,
5812            b"opaque".to_vec(),
5813        )
5814        .unwrap()
5815    }
5816
5817    fn route_ctx(
5818        connection_id: ConnectionId,
5819    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
5820        let (tx, rx) = mpsc::channel(8);
5821        (
5822            RouteCtx {
5823                connection_id,
5824                egress: FrameSink::new(tx),
5825            },
5826            rx,
5827        )
5828    }
5829
5830    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
5831        serde_json::from_slice(&frame.body).unwrap()
5832    }
5833
5834    /// Register a module over a connection that has a sink and return the
5835    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
5836    /// module's own sink rather than returning it as a reply, so the ack is
5837    /// taken off `rx` here and whatever the test reads next is what followed it.
5838    async fn hello_via_sink(
5839        handler: &ControlHandler,
5840        ctx: &RouteCtx,
5841        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
5842        hello: Frame,
5843    ) -> Frame {
5844        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
5845        assert!(
5846            replies.is_empty(),
5847            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
5848        );
5849        let ack = rx
5850            .try_recv()
5851            .expect("HELLO_ACK is queued on the module sink")
5852            .frame;
5853        assert_eq!(ack.header.ty, FrameType::HelloAck);
5854        ack
5855    }
5856
5857    fn parse_error(frame: &Frame) -> Value {
5858        serde_json::from_slice(&frame.body).unwrap()
5859    }
5860
5861    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
5862        serde_json::from_slice(&frame.body).unwrap()
5863    }
5864
5865    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
5866        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
5867            route_channel,
5868            route_epoch: 0,
5869            kind,
5870        })
5871        .unwrap();
5872        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5873    }
5874
5875    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
5876        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
5877            module_id: module_id.to_string(),
5878        })
5879        .unwrap();
5880        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5881    }
5882
5883    fn route_open_frame(corr: u64, module_id: &str, project_root: TestTempDir) -> Frame {
5884        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
5885    }
5886
5887    fn route_open_frame_with_consumer_capabilities(
5888        corr: u64,
5889        module_id: &str,
5890        project_root: TestTempDir,
5891        consumer_capabilities: Option<Vec<String>>,
5892    ) -> Frame {
5893        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
5894            target: RouteTarget::ToolProvider {
5895                module_id: module_id.to_string(),
5896            },
5897            identity: BindIdentity::new(
5898                project_root.path().to_path_buf(),
5899                "unit".to_string(),
5900                "session".to_string(),
5901            ),
5902            consumer_identity: None,
5903            consumer_capabilities,
5904            admission_facts: None,
5905        })
5906        .unwrap();
5907        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5908    }
5909
5910    fn route_open_frame_with_admission_facts(
5911        corr: u64,
5912        module_id: &str,
5913        project_root: TestTempDir,
5914        consumer_identity: Option<subc_control::ConsumerIdentity>,
5915        facts: Option<Value>,
5916    ) -> Frame {
5917        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
5918            target: RouteTarget::ToolProvider {
5919                module_id: module_id.to_string(),
5920            },
5921            identity: BindIdentity::new(
5922                project_root.path().to_path_buf(),
5923                "unit".to_string(),
5924                format!("session-{corr}"),
5925            ),
5926            consumer_identity,
5927            consumer_capabilities: None,
5928            admission_facts: facts,
5929        })
5930        .unwrap();
5931        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5932    }
5933
5934    #[derive(Clone, Default)]
5935    struct EventCapture {
5936        events: Arc<Mutex<Vec<CapturedEvent>>>,
5937    }
5938
5939    #[derive(Clone, Debug)]
5940    struct CapturedEvent {
5941        target: String,
5942        fields: BTreeMap<String, String>,
5943    }
5944
5945    impl EventCapture {
5946        fn events(&self) -> Vec<CapturedEvent> {
5947            self.events.lock().unwrap().clone()
5948        }
5949    }
5950
5951    impl<S> Layer<S> for EventCapture
5952    where
5953        S: Subscriber,
5954    {
5955        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
5956            let mut visitor = EventFieldVisitor::default();
5957            event.record(&mut visitor);
5958            self.events.lock().unwrap().push(CapturedEvent {
5959                target: event.metadata().target().to_string(),
5960                fields: visitor.fields,
5961            });
5962        }
5963    }
5964
5965    #[derive(Default)]
5966    struct EventFieldVisitor {
5967        fields: BTreeMap<String, String>,
5968    }
5969
5970    impl Visit for EventFieldVisitor {
5971        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
5972            self.fields
5973                .insert(field.name().to_string(), format!("{value:?}"));
5974        }
5975    }
5976
5977    fn health_response(corr: u64, status: HealthStatus) -> Frame {
5978        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
5979            status,
5980            detail: Some("warming".to_string()),
5981            metrics: Some(json!({"queue_depth": 3})),
5982        })
5983        .unwrap();
5984        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
5985    }
5986
5987    fn route_bind_ack(corr: u64) -> Frame {
5988        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
5989        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
5990    }
5991
5992    fn unique_project_root(label: &str) -> TestTempDir {
5993        TestTempDir::new(label)
5994    }
5995
5996    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
5997        match parse_route_poll(frame) {
5998            ClientControlResponse::RoutePoll {
5999                status: None,
6000                live: Some(live),
6001                ..
6002            } => assert_eq!(live, expected_live),
6003            other => panic!("unexpected route.poll response: {other:?}"),
6004        }
6005    }
6006
6007    fn bind_liveness_route(
6008        registry: &Registry,
6009        forwarding: &ForwardingTable,
6010        module_id: &str,
6011    ) -> (RouteCtx, u16, u32) {
6012        let module_connection = ConnectionId::new(101);
6013        let client_connection = ConnectionId::new(202);
6014        let registration = registry
6015            .register_with_control_ops(
6016                manifest(module_id, PROTOCOL_VERSION),
6017                PROTOCOL_VERSION,
6018                module_connection,
6019                module_baseline_control_ops(),
6020            )
6021            .unwrap();
6022        let (module_tx, _module_rx) = mpsc::channel(8);
6023        let endpoint = forwarding
6024            .register_module_connection(
6025                module_connection,
6026                module_id.to_string(),
6027                PROTOCOL_VERSION,
6028                manifest_concurrency(&registration.manifest),
6029                FrameSink::new(module_tx),
6030            )
6031            .unwrap();
6032        let (client_ctx, _client_rx) = route_ctx(client_connection);
6033        let pending = forwarding
6034            .begin_route_bind_relay_for_test(
6035                client_connection,
6036                client_ctx.egress.clone(),
6037                1,
6038                module_id,
6039            )
6040            .unwrap();
6041        assert_eq!(pending.endpoint, endpoint);
6042        let route_channel = pending.client_channel;
6043        let route_epoch = pending.client_epoch;
6044        forwarding
6045            .complete_pending_relay(
6046                module_connection,
6047                pending.corr,
6048                RouteBindRelayOutcome::Accepted,
6049            )
6050            .unwrap();
6051        (client_ctx, route_channel, route_epoch)
6052    }
6053
6054    struct FakeProcessLiveness {
6055        live: Option<bool>,
6056    }
6057
6058    impl ModuleProcessLiveness for FakeProcessLiveness {
6059        fn process_live(&self, _module_id: &str) -> Option<bool> {
6060            self.live
6061        }
6062    }
6063
6064    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6065    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
6066    {
6067        let registry = Arc::new(Registry::default());
6068        let supervisor_handle = SupervisorHandle::new();
6069        let supervisor = Supervisor::new(
6070            Arc::clone(&registry),
6071            RestartPolicy::new(1, Duration::from_millis(10)),
6072        )
6073        .with_handle(supervisor_handle.clone());
6074        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
6075        let module = supervisor
6076            .spawn(ModuleSpec {
6077                module_id: "stderr-tail-wire".to_string(),
6078                program: fake_aft_stub_path(),
6079                args: Vec::new(),
6080                env: vec![
6081                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
6082                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
6083                ],
6084                reserved: false,
6085                reserved_prefixes: Vec::new(),
6086                protocol: ModuleProtocol::Subc,
6087                overlap: Default::default(),
6088            })
6089            .unwrap();
6090
6091        let deadline = Instant::now() + Duration::from_secs(5);
6092        loop {
6093            let tail = module.stderr_tail(None, None);
6094            if tail
6095                .entries
6096                .iter()
6097                .any(|entry| matches!(entry, TailEntry::ProcessStart))
6098                && tail.entries.iter().any(|entry| {
6099                    matches!(
6100                        entry,
6101                        TailEntry::Line {
6102                            truncated: true,
6103                            ..
6104                        }
6105                    )
6106                })
6107            {
6108                break;
6109            }
6110            assert!(
6111                Instant::now() < deadline,
6112                "module did not produce a truncated line and restart boundary: {tail:?}"
6113            );
6114            sleep(Duration::from_millis(10)).await;
6115        }
6116
6117        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
6118        let request = ClientControlRequest::SupervisorStderrTail {
6119            module_id: "stderr-tail-wire".to_string(),
6120            max_lines: None,
6121            max_bytes: None,
6122        };
6123        let frame = Frame::build(
6124            FrameType::Request,
6125            control_flags(),
6126            0,
6127            0,
6128            1,
6129            serde_json::to_vec(&request).unwrap(),
6130        )
6131        .unwrap();
6132        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6133        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6134        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
6135            serde_json::from_slice(&responses[0].body).unwrap()
6136        else {
6137            panic!("expected supervisor.stderr_tail response");
6138        };
6139
6140        assert!(
6141            tail.entries
6142                .iter()
6143                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
6144            "the control response lost the restart boundary"
6145        );
6146        let Some(StderrTailEntry::Line { text, truncated }) = tail.entries.iter().find(|entry| {
6147            matches!(
6148                entry,
6149                StderrTailEntry::Line {
6150                    truncated: true,
6151                    ..
6152                }
6153            )
6154        }) else {
6155            panic!("the control response lost the truncated line");
6156        };
6157        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
6158        assert!(*truncated);
6159    }
6160
6161    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
6162    /// read done on the worker thread would stall every other task until it
6163    /// finished; the read must run off the worker so this test's own task keeps
6164    /// running while the read is paused.
6165    #[tokio::test(flavor = "current_thread")]
6166    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
6167        let dir = TestTempDir::new("terminals-off-worker");
6168        let journal_path = dir.join("terminals.jsonl");
6169        let registry = Arc::new(Registry::default());
6170        let supervisor_handle = SupervisorHandle::new();
6171        let supervisor =
6172            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
6173                .with_handle(supervisor_handle.clone())
6174                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
6175        let module = supervisor
6176            .spawn(ModuleSpec {
6177                module_id: "terminal-off-worker".to_string(),
6178                program: fake_aft_stub_path(),
6179                args: Vec::new(),
6180                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6181                reserved: false,
6182                reserved_prefixes: Vec::new(),
6183                protocol: ModuleProtocol::Subc,
6184                overlap: Default::default(),
6185            })
6186            .unwrap();
6187        let deadline = Instant::now() + Duration::from_secs(5);
6188        while module.terminal_history().entries.len() != 2 {
6189            assert!(Instant::now() < deadline, "module did not record two exits");
6190            sleep(Duration::from_millis(10)).await;
6191        }
6192
6193        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
6194        let handler =
6195            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
6196        let frame = Frame::build(
6197            FrameType::Request,
6198            control_flags(),
6199            0,
6200            0,
6201            1,
6202            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
6203                module_id: "terminal-off-worker".to_string(),
6204            })
6205            .unwrap(),
6206        )
6207        .unwrap();
6208        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6209        let spawned_at = std::time::Instant::now();
6210        let read = tokio::spawn({
6211            let handler = Arc::clone(&handler);
6212            async move { handler.handle_control_frame(&ctx, frame).await }
6213        });
6214        // Waiting for the pause from a blocking thread keeps this task pending,
6215        // so the runtime's single worker is free to run the read task.
6216        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
6217            .await
6218            .unwrap()
6219            .expect("the history read reached its pause");
6220        let elapsed = spawned_at.elapsed();
6221        assert!(
6222            elapsed < Duration::from_secs(2) && !read.is_finished(),
6223            "this task could not run while the history read was paused \
6224             (resumed after {elapsed:?}, read finished: {})",
6225            read.is_finished()
6226        );
6227
6228        drop(release);
6229        let responses = read.await.unwrap().unwrap();
6230        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6231        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
6232            panic!("expected supervisor.terminals response");
6233        };
6234        assert_eq!(terminals.entries.len(), 2);
6235        assert_eq!(terminals.journal_skipped_lines, 0);
6236        assert_eq!(terminals.journal_read_errors, 0);
6237    }
6238
6239    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6240    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
6241        let registry = Arc::new(Registry::default());
6242        let supervisor_handle = SupervisorHandle::new();
6243        let supervisor =
6244            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
6245                .with_handle(supervisor_handle.clone());
6246        let module = supervisor
6247            .spawn(ModuleSpec {
6248                module_id: "terminal-golden".to_string(),
6249                program: fake_aft_stub_path(),
6250                args: Vec::new(),
6251                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6252                reserved: false,
6253                reserved_prefixes: Vec::new(),
6254                protocol: ModuleProtocol::Subc,
6255                overlap: Default::default(),
6256            })
6257            .unwrap();
6258
6259        let deadline = Instant::now() + Duration::from_secs(5);
6260        while module.terminal_history().entries.len() != 2 {
6261            assert!(
6262                Instant::now() < deadline,
6263                "module did not retain two terminal exits: {:?}",
6264                module.terminal_history()
6265            );
6266            sleep(Duration::from_millis(10)).await;
6267        }
6268
6269        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
6270        let request = ClientControlRequest::SupervisorTerminals {
6271            module_id: "terminal-golden".to_string(),
6272        };
6273        let frame = Frame::build(
6274            FrameType::Request,
6275            control_flags(),
6276            0,
6277            0,
6278            1,
6279            serde_json::to_vec(&request).unwrap(),
6280        )
6281        .unwrap();
6282        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6283        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6284        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6285        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
6286            panic!("expected supervisor.terminals response");
6287        };
6288        assert_eq!(terminals.entries.len(), 2);
6289        assert_eq!(terminals.dropped, 0);
6290
6291        let mut rendered = serde_json::to_value(response).unwrap();
6292        // Wall-clock fields are the observation contract, but not stable fixture
6293        // bytes; normalize only them after the real handler has shaped the response.
6294        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
6295        for (index, entry) in rendered["entries"]
6296            .as_array_mut()
6297            .expect("terminal response entries array")
6298            .iter_mut()
6299            .enumerate()
6300        {
6301            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
6302        }
6303
6304        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
6305            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
6306        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
6307        if std::env::var_os("UPDATE_GOLDEN").is_some() {
6308            std::fs::write(&golden_path, &serialized).unwrap();
6309        }
6310        let expected: Value =
6311            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
6312        assert_eq!(rendered, expected);
6313    }
6314
6315    #[test]
6316    fn hello_registers_manifest_and_returns_ack() {
6317        let registry = Arc::new(Registry::default());
6318        let handler = ControlHandler::new(Arc::clone(&registry));
6319        let conn = ConnectionId::new(1);
6320
6321        let responses = handler
6322            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
6323            .unwrap();
6324
6325        assert_eq!(responses.len(), 1);
6326        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6327        assert_eq!(responses[0].header.channel, 0);
6328        assert_eq!(responses[0].header.corr, 7);
6329        let ack = parse_ack(&responses[0]);
6330        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
6331        assert!(ack
6332            .subc_capabilities
6333            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
6334        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
6335        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
6336        assert!(ack
6337            .subc_ops
6338            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
6339        assert!(ack
6340            .subc_ops
6341            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
6342
6343        let registration = registry.get_module("aft").unwrap().unwrap();
6344        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
6345        assert_eq!(registration.state, ChannelState::Active);
6346        assert_eq!(registration.connection_id, conn);
6347        assert_eq!(registration.control_ops, module_baseline_control_ops());
6348    }
6349
6350    #[test]
6351    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
6352        let invalid_identifiers = [
6353            ("case_change", "credentials-Provider/v1"),
6354            ("leading_zero", "credentials-provider/v01"),
6355            ("trailing_hyphen", "credentials-provider-/v1"),
6356            ("consecutive_hyphens", "credentials--provider/v1"),
6357            ("uppercase", "Credentials-provider/v1"),
6358            ("missing_v", "credentials-provider/1"),
6359            ("whitespace", "credentials provider/v1"),
6360            ("zero_version", "credentials-provider/v0"),
6361            ("out_of_range_version", "credentials-provider/v4294967296"),
6362            (
6363                "overlength_name",
6364                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
6365            ),
6366        ];
6367        let mut cases = invalid_identifiers
6368            .into_iter()
6369            .map(|(name, identifier)| {
6370                (
6371                    format!("identifier_{name}"),
6372                    "capabilities.provides[0]".to_string(),
6373                    identifier.to_string(),
6374                    json!({ "provides": [identifier] }),
6375                    None,
6376                )
6377            })
6378            .collect::<Vec<_>>();
6379        cases.extend([
6380            (
6381                "unknown_need".to_string(),
6382                "capabilities.requires[0].need".to_string(),
6383                "deferred".to_string(),
6384                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
6385                None,
6386            ),
6387            (
6388                "duplicate_provides".to_string(),
6389                "capabilities.provides[1]".to_string(),
6390                "credentials-provider/v1".to_string(),
6391                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
6392                None,
6393            ),
6394            (
6395                "duplicate_must_never_reach".to_string(),
6396                "capabilities.must_never_reach[1]".to_string(),
6397                "credentials-provider/v1".to_string(),
6398                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
6399                None,
6400            ),
6401            (
6402                "duplicate_requires_same_need".to_string(),
6403                "capabilities.requires[1]".to_string(),
6404                "credentials-provider/v1".to_string(),
6405                json!({ "requires": [
6406                    { "capability": "credentials-provider/v1", "need": "required" },
6407                    { "capability": "credentials-provider/v1", "need": "required" }
6408                ] }),
6409                None,
6410            ),
6411            (
6412                "duplicate_requires_conflicting_need".to_string(),
6413                "capabilities.requires[1]".to_string(),
6414                "credentials-provider/v1".to_string(),
6415                json!({ "requires": [
6416                    { "capability": "credentials-provider/v1", "need": "required" },
6417                    { "capability": "credentials-provider/v1", "need": "optional" }
6418                ] }),
6419                None,
6420            ),
6421            (
6422                "capabilities_root_pointer".to_string(),
6423                "runtime_computed[0]".to_string(),
6424                "/capabilities".to_string(),
6425                json!({}),
6426                Some(json!(["/capabilities"])),
6427            ),
6428            (
6429                "capabilities_descendant_pointer".to_string(),
6430                "runtime_computed[0]".to_string(),
6431                "/capabilities/provides".to_string(),
6432                json!({}),
6433                Some(json!(["/capabilities/provides"])),
6434            ),
6435            (
6436                "malformed_pointer_without_leading_slash".to_string(),
6437                "runtime_computed[0]".to_string(),
6438                "capabilities".to_string(),
6439                json!({}),
6440                Some(json!(["capabilities"])),
6441            ),
6442            (
6443                "malformed_pointer_escape".to_string(),
6444                "runtime_computed[0]".to_string(),
6445                "/roles/~2/tools".to_string(),
6446                json!({}),
6447                Some(json!(["/roles/~2/tools"])),
6448            ),
6449            (
6450                "unknown_capabilities_field".to_string(),
6451                "capabilities.future".to_string(),
6452                "<array>".to_string(),
6453                json!({ "future": [] }),
6454                None,
6455            ),
6456        ]);
6457
6458        for (index, (name, field, value, capabilities, runtime_computed)) in
6459            cases.into_iter().enumerate()
6460        {
6461            let registry = Arc::new(Registry::default());
6462            let handler = ControlHandler::new(Arc::clone(&registry));
6463            let response = handler
6464                .handle_control(
6465                    ConnectionId::new((index + 1) as u64),
6466                    capability_grammar_hello_frame(
6467                        capabilities,
6468                        runtime_computed,
6469                        index as u64 + 1,
6470                    ),
6471                )
6472                .expect("invalid HELLO returns a refusal");
6473
6474            assert_eq!(response.len(), 1, "{name} must emit one refusal");
6475            let error = parse_error(&response[0]);
6476            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
6477            let message = error["message"]
6478                .as_str()
6479                .expect("error message is a string");
6480            assert!(
6481                message.contains(&field),
6482                "{name}: field missing from {message}"
6483            );
6484            assert!(
6485                message.contains(&value),
6486                "{name}: value missing from {message}"
6487            );
6488            assert_eq!(
6489                registry
6490                    .active_registration_count()
6491                    .expect("registry reads"),
6492                0,
6493                "{name}: refused HELLO must not create a catalog entry"
6494            );
6495        }
6496    }
6497
6498    #[test]
6499    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
6500        let registry = Arc::new(Registry::default());
6501        let handler = ControlHandler::new(Arc::clone(&registry));
6502        let capabilities = json!({
6503            "provides": ["credentials-provider/v1"],
6504            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
6505            "must_never_reach": ["federation-transport/v1"]
6506        });
6507        let response = handler
6508            .handle_control(
6509                ConnectionId::new(99),
6510                capability_grammar_hello_frame(
6511                    capabilities.clone(),
6512                    Some(json!(["/roles/0/tools"])),
6513                    99,
6514                ),
6515            )
6516            .expect("valid HELLO registers");
6517        assert_eq!(response[0].header.ty, FrameType::HelloAck);
6518
6519        let request = Frame::build(
6520            FrameType::Request,
6521            control_flags(),
6522            0,
6523            0,
6524            100,
6525            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6526                .expect("catalog request serializes"),
6527        )
6528        .expect("catalog request frame builds");
6529        let response = handler
6530            .handle_catalog_list(request, None)
6531            .expect("catalog list succeeds");
6532        let ClientControlResponse::CatalogList { modules, .. } =
6533            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
6534        else {
6535            panic!("catalog request must return catalog.list");
6536        };
6537        assert_eq!(modules.len(), 1);
6538        assert_eq!(
6539            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
6540            capabilities
6541        );
6542    }
6543
6544    #[test]
6545    fn catalog_list_mirrors_management_operation_description() {
6546        let registry = Arc::new(Registry::default());
6547        let handler = ControlHandler::new(Arc::clone(&registry));
6548        let description = "List managed records and return their identifiers and metadata.";
6549        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
6550        manifest.provides = vec![ProviderRole::ManagementSurface {
6551            operations: vec![ManagementOperation {
6552                name: "records.list".to_string(),
6553                kind: ManagementOperationKind::Query,
6554                description: Some(description.to_string()),
6555            }],
6556            config_schema: json!({"type": "object"}),
6557            observability: vec![ObservabilitySurface {
6558                name: "records.stats".to_string(),
6559                kind: ObservabilityKind::Snapshot,
6560            }],
6561            identity_scope: vec![IdentityScope::Project],
6562            concurrency: Concurrency::ModuleManaged,
6563        }];
6564        registry
6565            .register_with_control_ops(
6566                manifest,
6567                PROTOCOL_VERSION,
6568                ConnectionId::new(99),
6569                Vec::new(),
6570            )
6571            .expect("described management manifest registers");
6572
6573        let request = Frame::build(
6574            FrameType::Request,
6575            control_flags(),
6576            0,
6577            0,
6578            100,
6579            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6580                .expect("catalog request serializes"),
6581        )
6582        .expect("catalog request frame builds");
6583        let response = handler
6584            .handle_catalog_list(request, None)
6585            .expect("catalog list succeeds");
6586        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
6587        assert_eq!(
6588            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
6589            "catalog.list must preserve the declared operation description verbatim"
6590        );
6591    }
6592
6593    #[test]
6594    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
6595        let registry = Arc::new(Registry::default());
6596        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
6597            [("vault".to_string(), true), ("squatter".to_string(), true)],
6598            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
6599        );
6600        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
6601        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
6602            provides: vec!["credentials-provider/v1".to_string()],
6603            requires: Vec::new(),
6604            must_never_reach: Vec::new(),
6605        });
6606        let frame = Frame::build(
6607            FrameType::Hello,
6608            control_flags(),
6609            0,
6610            0,
6611            77,
6612            serde_json::to_vec(&ModuleHelloBody {
6613                manifest: squatter,
6614                protocol_ver: PROTOCOL_VERSION,
6615                control_ops: None,
6616                launch_nonce: None,
6617            })
6618            .expect("HELLO serializes"),
6619        )
6620        .expect("HELLO frame builds");
6621        let response = handler
6622            .handle_control(ConnectionId::new(77), frame)
6623            .expect("reserved claim receives a typed refusal");
6624        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
6625        assert_eq!(
6626            registry
6627                .active_registration_count()
6628                .expect("registry reads"),
6629            0,
6630            "a reserved capability refusal must not leave a catalog entry"
6631        );
6632    }
6633
6634    #[test]
6635    fn server_describe_surfaces_required_capability_verdict_fields() {
6636        let registry = Arc::new(Registry::default());
6637        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
6638            [
6639                ("consumer".to_string(), true),
6640                ("provider".to_string(), false),
6641            ],
6642            BTreeMap::new(),
6643        );
6644        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
6645        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
6646            provides: Vec::new(),
6647            requires: vec![subc_protocol::manifest::CapabilityRequirement {
6648                capability: "credentials-provider/v1".to_string(),
6649                need: subc_protocol::manifest::CapabilityNeed::Required,
6650            }],
6651            must_never_reach: Vec::new(),
6652        });
6653        let hello = Frame::build(
6654            FrameType::Hello,
6655            control_flags(),
6656            0,
6657            0,
6658            78,
6659            serde_json::to_vec(&ModuleHelloBody {
6660                manifest: consumer,
6661                protocol_ver: PROTOCOL_VERSION,
6662                control_ops: None,
6663                launch_nonce: None,
6664            })
6665            .expect("HELLO serializes"),
6666        )
6667        .expect("HELLO frame builds");
6668        handler
6669            .handle_control(ConnectionId::new(78), hello)
6670            .expect("consumer registers");
6671        let describe = Frame::build(
6672            FrameType::Request,
6673            control_flags(),
6674            0,
6675            0,
6676            79,
6677            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
6678                .expect("request serializes"),
6679        )
6680        .expect("describe frame builds");
6681        let response = handler
6682            .handle_server_describe(describe)
6683            .expect("server.describe succeeds");
6684        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
6685        let requirement = &rendered["capability_requirements"][0];
6686        assert_eq!(requirement["consumer"], "consumer");
6687        assert_eq!(requirement["verdict"], "never_provided");
6688        assert_eq!(requirement["episode_seq"], 1);
6689        assert_eq!(requirement["config_satisfiable"], false);
6690        assert_eq!(requirement["runtime_available"], false);
6691        assert!(requirement["detail"]
6692            .as_str()
6693            .expect("detail string")
6694            .contains("credentials-provider/v1"));
6695    }
6696
6697    #[test]
6698    fn catalog_list_omits_capabilities_for_legacy_manifest() {
6699        let registry = Arc::new(Registry::default());
6700        let handler = ControlHandler::new(Arc::clone(&registry));
6701        let hello = handler
6702            .handle_control(
6703                ConnectionId::new(101),
6704                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
6705            )
6706            .expect("legacy HELLO registers");
6707        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
6708
6709        let request = Frame::build(
6710            FrameType::Request,
6711            control_flags(),
6712            0,
6713            0,
6714            102,
6715            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6716                .expect("catalog request serializes"),
6717        )
6718        .expect("catalog request frame builds");
6719        let response = handler
6720            .handle_catalog_list(request, None)
6721            .expect("catalog list succeeds");
6722        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
6723        assert!(
6724            body["modules"][0].get("capabilities").is_none(),
6725            "legacy manifest must retain an absent capabilities field on catalog.list"
6726        );
6727    }
6728
6729    #[test]
6730    fn hello_ack_omits_storage_when_no_storage_config() {
6731        let registry = Arc::new(Registry::default());
6732        let handler = ControlHandler::new(Arc::clone(&registry));
6733        let responses = handler
6734            .handle_control(
6735                ConnectionId::new(1),
6736                hello_frame("aft", PROTOCOL_VERSION, 7),
6737            )
6738            .unwrap();
6739        let ack = parse_ack(&responses[0]);
6740        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
6741        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
6742    }
6743
6744    #[tokio::test]
6745    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
6746        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
6747        let registry = Arc::new(Registry::default());
6748        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
6749        let responses = handler
6750            .handle_control(
6751                ConnectionId::new(1),
6752                hello_frame("aft", PROTOCOL_VERSION, 7),
6753            )
6754            .unwrap();
6755        let ack = parse_ack(&responses[0]);
6756        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
6757
6758        let described = handler
6759            .handle_control_frame(
6760                &route_ctx(ConnectionId::new(2)).0,
6761                Frame::build(
6762                    FrameType::Request,
6763                    control_flags(),
6764                    0,
6765                    0,
6766                    9,
6767                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
6768                )
6769                .unwrap(),
6770            )
6771            .await
6772            .unwrap();
6773        let ClientControlResponse::ServerDescribe { machine_id, .. } =
6774            serde_json::from_slice(&described[0].body).unwrap()
6775        else {
6776            panic!("server.describe answered with another shape");
6777        };
6778        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
6779    }
6780
6781    #[test]
6782    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
6783        // With a central sqlite storage policy, each registering module gets its
6784        // own resolved descriptor in HELLO_ACK, keyed by its module id.
6785        let registry = Arc::new(Registry::default());
6786        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
6787            crate::daemon_config::StorageConfig::Sqlite {
6788                data_home: std::path::PathBuf::from("/data"),
6789            },
6790        ));
6791
6792        let responses = handler
6793            .handle_control(
6794                ConnectionId::new(1),
6795                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
6796            )
6797            .unwrap();
6798        let ack = parse_ack(&responses[0]);
6799        assert_eq!(
6800            ack.storage,
6801            Some(serde_json::json!({
6802                "module_id": "alfonso-routing",
6803                "storage_namespace": "default",
6804                "isolation": { "kind": "module" },
6805                "backend": {
6806                    "backend": "sqlite",
6807                    "path": "/data/cortexkit/alfonso-routing/store.db"
6808                }
6809            })),
6810            "the delivered descriptor is the module's own sqlite store path"
6811        );
6812    }
6813
6814    #[test]
6815    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
6816        let registry = Arc::new(Registry::default());
6817        let handler = ControlHandler::new(Arc::clone(&registry));
6818        let conn = ConnectionId::new(1);
6819        let responses = handler
6820            .handle_control(
6821                conn,
6822                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
6823            )
6824            .unwrap();
6825        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6826        let registration = registry.get_module("aft").unwrap().unwrap();
6827        assert_eq!(registration.control_ops, module_baseline_control_ops());
6828
6829        let frame =
6830            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
6831        assert!(handler
6832            .guard_module_control_op(&frame, "aft", "route.bind")
6833            .unwrap()
6834            .is_none());
6835        let error = handler
6836            .guard_module_control_op(&frame, "aft", "test.synthetic")
6837            .unwrap()
6838            .expect("synthetic ungranted op should be rejected");
6839        assert_eq!(error.header.ty, FrameType::Error);
6840        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
6841    }
6842
6843    #[test]
6844    fn hello_control_ops_some_adds_optional_grants() {
6845        let registry = Arc::new(Registry::default());
6846        let handler = ControlHandler::new(Arc::clone(&registry));
6847        handler
6848            .handle_control(
6849                ConnectionId::new(1),
6850                hello_frame_with_control_ops(
6851                    "aft",
6852                    PROTOCOL_VERSION,
6853                    7,
6854                    Some(vec![
6855                        "future.synthetic".to_string(),
6856                        "route.bind".to_string(),
6857                    ]),
6858                ),
6859            )
6860            .unwrap();
6861        let registration = registry.get_module("aft").unwrap().unwrap();
6862        assert_eq!(
6863            registration.control_ops,
6864            vec![
6865                "route.bind".to_string(),
6866                "route.status".to_string(),
6867                "future.synthetic".to_string(),
6868            ]
6869        );
6870        let frame =
6871            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
6872        assert!(handler
6873            .guard_module_control_op(&frame, "aft", "future.synthetic")
6874            .unwrap()
6875            .is_none());
6876    }
6877
6878    #[tokio::test]
6879    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
6880        let registry = Arc::new(Registry::default());
6881        let forwarding = Arc::new(ForwardingTable::default());
6882        let handler =
6883            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
6884        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
6885        hello_via_sink(
6886            &handler,
6887            &module_ctx,
6888            &mut module_rx,
6889            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
6890        )
6891        .await;
6892
6893        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
6894        let responses = handler
6895            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
6896            .await
6897            .unwrap();
6898        assert_eq!(responses.len(), 1);
6899        assert_eq!(responses[0].header.ty, FrameType::Error);
6900        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
6901        assert!(module_rx.try_recv().is_err());
6902    }
6903
6904    #[tokio::test]
6905    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
6906        let registry = Arc::new(Registry::default());
6907        let forwarding = Arc::new(ForwardingTable::default());
6908        let handler =
6909            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
6910        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
6911        hello_via_sink(
6912            &handler,
6913            &module_ctx,
6914            &mut module_rx,
6915            hello_frame_with_control_ops(
6916                "aft",
6917                PROTOCOL_VERSION,
6918                7,
6919                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
6920            ),
6921        )
6922        .await;
6923
6924        let project_root = unique_project_root("demux");
6925        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
6926        let route_handler = handler.clone();
6927        let route_task = tokio::spawn(async move {
6928            route_handler
6929                .handle_control_frame(
6930                    &route_client_ctx,
6931                    route_open_frame(100, "aft", project_root),
6932                )
6933                .await
6934                .unwrap()
6935        });
6936        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
6937            .await
6938            .unwrap()
6939            .unwrap();
6940        assert!(matches!(
6941            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
6942            ModuleControlRequest::RouteBind { .. }
6943        ));
6944
6945        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
6946        let health_handler = handler.clone();
6947        let health_task = tokio::spawn(async move {
6948            health_handler
6949                .handle_control_frame(
6950                    &health_client_ctx,
6951                    supervisor_health_probe_frame(101, "aft"),
6952                )
6953                .await
6954                .unwrap()
6955        });
6956        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
6957            .await
6958            .unwrap()
6959            .unwrap();
6960        assert_eq!(
6961            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
6962            ModuleControlRequest::HealthCheck {}
6963        );
6964
6965        handler
6966            .handle_control_frame(
6967                &module_ctx,
6968                health_response(health_frame.header.corr, HealthStatus::Degraded),
6969            )
6970            .await
6971            .unwrap();
6972        let health_response = health_task.await.unwrap();
6973        assert_eq!(health_response.len(), 1);
6974        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
6975            ClientControlResponse::SupervisorHealthProbe {
6976                module_id,
6977                status,
6978                detail,
6979                metrics,
6980            } => {
6981                assert_eq!(module_id, "aft");
6982                assert_eq!(status, HealthStatus::Degraded);
6983                assert_eq!(detail.as_deref(), Some("warming"));
6984                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
6985            }
6986            other => panic!("unexpected health response: {other:?}"),
6987        }
6988
6989        handler
6990            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
6991            .await
6992            .unwrap();
6993        let route_response = route_task.await.unwrap();
6994        assert!(route_response.is_empty());
6995        let published = route_client_rx.recv().await.unwrap();
6996        assert!(matches!(
6997            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
6998            ClientControlResponse::RouteOpen { .. }
6999        ));
7000    }
7001
7002    /// Start one `route.open` on `client_connection` and return its still-running
7003    /// handler task together with the `route.bind` the module received for it.
7004    /// The handler blocks until the module answers, so it has to run as a task
7005    /// while the test drives the module side.
7006    async fn relay_route_open(
7007        handler: &ControlHandler,
7008        client_connection: ConnectionId,
7009        client_egress: &FrameSink,
7010        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7011        corr: u64,
7012        module_id: &str,
7013        project_root_label: &str,
7014    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
7015        let ctx = RouteCtx {
7016            connection_id: client_connection,
7017            egress: client_egress.clone(),
7018        };
7019        let handler = handler.clone();
7020        let project_root = unique_project_root(project_root_label);
7021        let module_id = module_id.to_string();
7022        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
7023        let task = tokio::spawn(async move {
7024            let _guard = tracing::dispatcher::set_default(&dispatch);
7025            handler
7026                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
7027                .await
7028                .unwrap()
7029        });
7030        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
7031            .await
7032            .expect("module receives the relayed route.bind")
7033            .expect("module egress is open");
7034        (task, bind.frame)
7035    }
7036
7037    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
7038        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
7039            ModuleControlRequest::RouteBind {
7040                route_channel,
7041                epoch,
7042                ..
7043            } => (route_channel, epoch),
7044            other => panic!("expected a route.bind request, got {other:?}"),
7045        }
7046    }
7047
7048    fn published_route(frame: &Frame) -> (u16, u32) {
7049        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
7050            ClientControlResponse::RouteOpen {
7051                route_channel,
7052                route_epoch,
7053            } => (route_channel, route_epoch),
7054            other => panic!("expected a route.open response, got {other:?}"),
7055        }
7056    }
7057
7058    /// Reproduction of a production outage. A client had `route.open`s in
7059    /// flight to a module and was already marked closing -- its egress had refused a
7060    /// module frame, so the daemon asked its connection to end -- while its sink
7061    /// was still open. When the module acked those binds, the daemon refused to
7062    /// commit a route for a closing client, and that refusal was returned from
7063    /// the MODULE connection's frame handler, where a router error that has no
7064    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
7065    /// the supervisor correctly did not respawn a clean exit, and every seat lost
7066    /// its tools for hours -- one client's teardown took down a connection
7067    /// carrying ~170 other routes.
7068    ///
7069    /// The window is opened here by calling the production path that opens it
7070    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
7071    /// state that matters is "in `closing_connections`, sink still open, relay
7072    /// still pending", and it lasts only from the close request until the
7073    /// connection loop reacts to it; a socket-level test can flood a client into
7074    /// that escalation but cannot pin the module's ack inside the window. Closing
7075    /// the socket instead takes the other path entirely -- connection teardown
7076    /// removes the pending relay under the same lock, so the ack finds nothing.
7077    #[tokio::test]
7078    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
7079        let registry = Arc::new(Registry::default());
7080        let forwarding = Arc::new(ForwardingTable::default());
7081        let handler =
7082            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
7083
7084        let module_connection = ConnectionId::new(30);
7085        let (module_ctx, mut module_rx) = route_ctx(module_connection);
7086        hello_via_sink(
7087            &handler,
7088            &module_ctx,
7089            &mut module_rx,
7090            hello_frame("aft", PROTOCOL_VERSION, 7),
7091        )
7092        .await;
7093
7094        let dying_client = ConnectionId::new(31);
7095        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
7096
7097        // A published route on the dying client. The escalation below only marks
7098        // a connection closing for a route it has already published.
7099        let (first_task, first_bind) = relay_route_open(
7100            &handler,
7101            dying_client,
7102            &dying_ctx.egress,
7103            &mut module_rx,
7104            100,
7105            "aft",
7106            "closing-first",
7107        )
7108        .await;
7109        handler
7110            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
7111            .await
7112            .unwrap();
7113        assert!(first_task.await.unwrap().is_empty());
7114        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
7115
7116        // A second route.open from the same client, relayed and awaiting its ack.
7117        let (second_task, second_bind) = relay_route_open(
7118            &handler,
7119            dying_client,
7120            &dying_ctx.egress,
7121            &mut module_rx,
7122            101,
7123            "aft",
7124            "closing-second",
7125        )
7126        .await;
7127        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
7128
7129        // The window: the client is closing, its sink is still open, and its
7130        // second bind is still pending.
7131        assert!(forwarding
7132            .escalate_client_delivery_failure(
7133                dying_client,
7134                first_channel,
7135                first_epoch,
7136                CloseReason::new(
7137                    "module_to_client_delivery_failed",
7138                    "client egress refused a module frame",
7139                ),
7140                crate::forwarding::UndeliveredFrame {
7141                    module_id: None,
7142                    sink: &dying_ctx.egress,
7143                },
7144            )
7145            .unwrap());
7146        assert!(!dying_ctx.egress.is_closed());
7147
7148        // The frame that used to end the module connection.
7149        let ack = handler
7150            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
7151            .await;
7152        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
7153        if module_loop_error.is_some() {
7154            // What the server's connection loop does with a router error that has
7155            // no ERROR-frame translation: end the connection, which releases the
7156            // module's registration and every route on it.
7157            handler.cleanup_connection(module_connection).unwrap();
7158        }
7159        // Read the module's next frame before opening the co-tenant's route, so
7160        // the GOODBYE assertion below is about THIS ack and not about later
7161        // traffic. `None` means the module was told nothing.
7162        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7163            .await
7164            .ok()
7165            .flatten();
7166
7167        // 1. The module connection is still registered.
7168        assert!(
7169            registry
7170                .get_module_by_connection(module_connection)
7171                .unwrap()
7172                .is_some(),
7173            "one client's closing connection ended the shared module connection: \
7174             {module_loop_error:?}"
7175        );
7176        // ...and still serving: another client can open and use a route on it.
7177        let cotenant = ConnectionId::new(32);
7178        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
7179        let (cotenant_task, cotenant_bind) = relay_route_open(
7180            &handler,
7181            cotenant,
7182            &cotenant_ctx.egress,
7183            &mut module_rx,
7184            102,
7185            "aft",
7186            "closing-cotenant",
7187        )
7188        .await;
7189        handler
7190            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
7191            .await
7192            .unwrap();
7193        assert!(cotenant_task.await.unwrap().is_empty());
7194        let (cotenant_channel, cotenant_epoch) =
7195            published_route(&cotenant_rx.recv().await.unwrap());
7196        assert!(matches!(
7197            forwarding
7198                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
7199                .unwrap(),
7200            DataRoute::Client(DataRouteState::Bound(_))
7201        ));
7202
7203        // 2. The module was told to drop the binding it created for the route
7204        //    that will never be published.
7205        let goodbye = post_ack_module_frame
7206            .expect("module receives a GOODBYE for the abandoned route channel");
7207        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
7208        assert_eq!(goodbye.header.channel, abandoned_channel);
7209        assert_eq!(goodbye.header.epoch, abandoned_epoch);
7210
7211        // 3. The dying client received nothing: no route was ever published to
7212        //    it. Its route.open is answered as unavailable, which the connection
7213        //    loop would write to a socket that is already going away.
7214        assert!(dying_rx.try_recv().is_err());
7215        let second_response = second_task.await.unwrap();
7216        assert_eq!(second_response.len(), 1);
7217        assert_eq!(
7218            parse_error(&second_response[0])["code"],
7219            "target_unavailable"
7220        );
7221    }
7222
7223    /// The fence at the module-loop boundary, stated as its own contract: which
7224    /// forwarding failures are allowed to end the module connection that is being
7225    /// served. A `ConnectionClosing` naming some client is about that client, and
7226    /// a module connection is shared; the same error naming the module's own
7227    /// connection is about this connection and must stay fatal, as must failures
7228    /// that are about the forwarding table itself.
7229    #[test]
7230    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
7231        let handler = ControlHandler::default();
7232        let module_connection = ConnectionId::new(30);
7233        let client_connection = ConnectionId::new(31);
7234
7235        handler
7236            .refuse_to_end_module_connection_for_a_client(
7237                module_connection,
7238                77,
7239                ForwardingError::ConnectionClosing {
7240                    connection_id: client_connection,
7241                },
7242            )
7243            .expect("a closing client must never end the module connection");
7244
7245        assert!(matches!(
7246            handler.refuse_to_end_module_connection_for_a_client(
7247                module_connection,
7248                78,
7249                ForwardingError::ConnectionClosing {
7250                    connection_id: module_connection,
7251                },
7252            ),
7253            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
7254                connection_id
7255            })) if connection_id == module_connection
7256        ));
7257        assert!(matches!(
7258            handler.refuse_to_end_module_connection_for_a_client(
7259                module_connection,
7260                79,
7261                ForwardingError::Poisoned,
7262            ),
7263            Err(RouterError::Forwarding(ForwardingError::Poisoned))
7264        ));
7265        assert!(matches!(
7266            handler.refuse_to_end_module_connection_for_a_client(
7267                module_connection,
7268                80,
7269                ForwardingError::StaleModuleEndpoint,
7270            ),
7271            Err(RouterError::Forwarding(
7272                ForwardingError::StaleModuleEndpoint
7273            ))
7274        ));
7275    }
7276
7277    /// The spawn-attestation guard is what stops a connected module from claiming
7278    /// another module's identity and being stamped `Reserved` for it. Every other
7279    /// test that supplies a consumer_identity supplies a CORRECT one, because a
7280    /// correct one is what the rest of the flow needs -- so the guard's rejection
7281    /// branch was never the subject of an assertion, only its acceptance branch.
7282    ///
7283    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
7284    /// whole subc-core library suite green; only the forwarding integration tests
7285    /// notice, and they notice for unrelated reasons. This test exists so the
7286    /// refusal itself is asserted where the guard lives: it fails if the identity
7287    /// check stops refusing, which is the direction that matters, since a guard
7288    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
7289    #[tokio::test]
7290    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
7291        let registry = Arc::new(Registry::default());
7292        let forwarding = Arc::new(ForwardingTable::default());
7293        let supervisor = SupervisorHandle::new();
7294        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7295        let handler =
7296            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
7297                .with_supervisor(supervisor);
7298
7299        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
7300        hello_via_sink(
7301            &handler,
7302            &target_ctx,
7303            &mut target_rx,
7304            hello_frame("target", PROTOCOL_VERSION, 1),
7305        )
7306        .await;
7307
7308        // A real supervised module id presenting the wrong nonce. This is the
7309        // impersonation case: the attacker knows a privileged module_id, which is
7310        // public, and guesses at the nonce, which is not.
7311        let wrong_nonce = handler
7312            .handle_control_frame(
7313                &route_ctx(ConnectionId::new(91)).0,
7314                route_open_frame_with_admission_facts(
7315                    20,
7316                    "target",
7317                    unique_project_root("admission-facts"),
7318                    Some(subc_control::ConsumerIdentity {
7319                        module_id: "fed".to_string(),
7320                        launch_nonce: "not-the-real-nonce".to_string(),
7321                    }),
7322                    None,
7323                ),
7324            )
7325            .await
7326            .unwrap();
7327        assert_eq!(
7328            parse_error(&wrong_nonce[0])["code"],
7329            "bad_consumer_identity",
7330            "a mismatched launch nonce must be refused, not stamped Reserved"
7331        );
7332
7333        // A module id the supervisor never spawned at all, so no nonce exists to
7334        // compare against. An implementation that treats "no record" as "nothing
7335        // to check" fails open here while passing the case above.
7336        let never_spawned = handler
7337            .handle_control_frame(
7338                &route_ctx(ConnectionId::new(92)).0,
7339                route_open_frame_with_admission_facts(
7340                    21,
7341                    "target",
7342                    unique_project_root("admission-facts"),
7343                    Some(subc_control::ConsumerIdentity {
7344                        module_id: "never-spawned".to_string(),
7345                        launch_nonce: "any-nonce".to_string(),
7346                    }),
7347                    None,
7348                ),
7349            )
7350            .await
7351            .unwrap();
7352        assert_eq!(
7353            parse_error(&never_spawned[0])["code"],
7354            "bad_consumer_identity",
7355            "an unspawned module_id must be refused rather than accepted for lack of a record"
7356        );
7357    }
7358
7359    /// The refusal test above proves the guard says NO. Nothing proved it can say
7360    /// YES, and the difference is not academic: replacing the whole authorization
7361    /// with `false` -- admitting no consumer identity at all, revoking Reserved
7362    /// standing for every supervised module in the fleet -- leaves 110 of the 111
7363    /// library tests GREEN. The one that notices does so by HANGING, because it
7364    /// waits for a bind that can no longer happen.
7365    ///
7366    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
7367    /// or flaky test, invites a RETRY rather than an investigation, and the retry
7368    /// hangs too and gets blamed on the runner. So a total revocation of the
7369    /// daemon's trust grant would have shipped behind a symptom nobody attributes
7370    /// to code.
7371    ///
7372    /// The bias is structural rather than accidental. A REFUSAL looks like a
7373    /// failure someone writes a test for; a GRANT looks like the happy path. Every
7374    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
7375    /// suite for that reason, and this one is the purest case in the daemon.
7376    ///
7377    /// This test asserts the EFFECT rather than the absence of an error: the module
7378    /// receives a RouteBind and it carries `Reserved` naming the attested module.
7379    /// A guard that admitted nobody would produce no bind at all; one that admitted
7380    /// everybody would stamp the wrong principal, which the refusal test catches.
7381    #[tokio::test]
7382    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
7383        let registry = Arc::new(Registry::default());
7384        let forwarding = Arc::new(ForwardingTable::default());
7385        let supervisor = SupervisorHandle::new();
7386        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7387        let handler =
7388            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
7389                .with_supervisor(supervisor);
7390
7391        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
7392        hello_via_sink(
7393            &handler,
7394            &target_ctx,
7395            &mut target_rx,
7396            hello_frame("target", PROTOCOL_VERSION, 1),
7397        )
7398        .await;
7399
7400        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
7401        let route_handler = handler.clone();
7402        let route_task = tokio::spawn(async move {
7403            route_handler
7404                .handle_control_frame(
7405                    &client_ctx,
7406                    route_open_frame_with_admission_facts(
7407                        30,
7408                        "target",
7409                        unique_project_root("admission-facts"),
7410                        Some(subc_control::ConsumerIdentity {
7411                            module_id: "fed".to_string(),
7412                            launch_nonce: "fed-nonce".to_string(),
7413                        }),
7414                        None,
7415                    ),
7416                )
7417                .await
7418                .unwrap()
7419        });
7420
7421        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
7422        // the very mutation it exists to catch -- a guard that admits nobody -- no
7423        // bind is ever sent, so it HUNG rather than failing. That reproduces the
7424        // exact defect being fixed: a total revocation detected only as a stalled
7425        // suite, which reads as flakiness and invites a retry. An acceptance test
7426        // that waits for an effect must bound the wait, or a red becomes a hang.
7427        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7428            .await
7429            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
7430            .expect("module control channel closed before route.bind");
7431        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
7432        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
7433            panic!("expected route.bind")
7434        };
7435        assert_eq!(
7436            principal,
7437            Some(Principal::Reserved {
7438                module_id: "fed".to_string()
7439            }),
7440            "a correctly attested consumer must be stamped Reserved for its own id"
7441        );
7442
7443        handler
7444            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
7445            .await
7446            .unwrap();
7447        assert!(route_task.await.unwrap().is_empty());
7448        assert!(
7449            matches!(
7450                serde_json::from_slice::<ClientControlResponse>(
7451                    &client_rx.recv().await.unwrap().body
7452                )
7453                .unwrap(),
7454                ClientControlResponse::RouteOpen { .. }
7455            ),
7456            "the route must actually open, not merely avoid an error"
7457        );
7458    }
7459
7460    #[tokio::test(start_paused = true)]
7461    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
7462        let registry = Arc::new(Registry::default());
7463        let forwarding = Arc::new(ForwardingTable::default());
7464        let supervisor = SupervisorHandle::new();
7465        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7466        let handler =
7467            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
7468                .with_supervisor(supervisor);
7469
7470        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
7471        hello_via_sink(
7472            &handler,
7473            &target_ctx,
7474            &mut target_rx,
7475            hello_frame("target", PROTOCOL_VERSION, 1),
7476        )
7477        .await;
7478
7479        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
7480        let direct_handler = handler.clone();
7481        let direct_open = tokio::spawn(async move {
7482            direct_handler
7483                .handle_control_frame(
7484                    &direct_ctx,
7485                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
7486                )
7487                .await
7488                .unwrap()
7489        });
7490        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7491            .await
7492            .expect("no direct route.bind within 5s")
7493            .expect("target control channel closed before direct route.bind");
7494        handler
7495            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
7496            .await
7497            .unwrap();
7498        assert!(direct_open.await.unwrap().is_empty());
7499        let _ = direct_rx.recv().await.unwrap();
7500
7501        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
7502        let reserved_handler = handler.clone();
7503        let reserved_open = tokio::spawn(async move {
7504            reserved_handler
7505                .handle_control_frame(
7506                    &reserved_ctx,
7507                    route_open_frame_with_admission_facts(
7508                        3,
7509                        "target",
7510                        unique_project_root("admission-facts"),
7511                        Some(ConsumerIdentity {
7512                            module_id: "fed".to_string(),
7513                            launch_nonce: "fed-nonce".to_string(),
7514                        }),
7515                        None,
7516                    ),
7517                )
7518                .await
7519                .unwrap()
7520        });
7521        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7522            .await
7523            .expect("no reserved route.bind within 5s")
7524            .expect("target control channel closed before reserved route.bind");
7525        handler
7526            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
7527            .await
7528            .unwrap();
7529        assert!(reserved_open.await.unwrap().is_empty());
7530        let _ = reserved_rx.recv().await.unwrap();
7531
7532        forwarding
7533            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
7534            .unwrap();
7535        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
7536        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
7537            module_id: Some("target".to_string()),
7538        })
7539        .unwrap();
7540        let census_frame =
7541            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
7542        let response = handler
7543            .handle_control_frame(&census_ctx, census_frame)
7544            .await
7545            .unwrap()
7546            .pop()
7547            .unwrap();
7548        let actual: Value = serde_json::from_slice(&response.body).unwrap();
7549        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
7550        assert!(matches!(
7551            decoded,
7552            ClientControlResponse::SupervisorRoutes { .. }
7553        ));
7554        let routes = actual["modules"][0]["routes"].as_array().unwrap();
7555        assert_eq!(routes.len(), 2);
7556        assert!(routes.iter().all(|route| route["draining"] == true));
7557        // The census carries WHY: the reason the drain was begun with, in the
7558        // route.closing vocabulary, on every draining route this drain marked.
7559        assert!(
7560            routes.iter().all(|route| route["drain_reason"] == "reload"),
7561            "draining routes must name the drain's reason: {routes:?}"
7562        );
7563        assert!(routes.iter().any(|route| {
7564            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
7565        }));
7566        assert!(routes.iter().any(|route| {
7567            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
7568        }));
7569
7570        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7571            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
7572        if std::env::var_os("UPDATE_GOLDEN").is_some() {
7573            std::fs::write(
7574                &golden_path,
7575                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
7576            )
7577            .unwrap();
7578        }
7579        let expected: Value =
7580            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
7581        assert_eq!(actual, expected);
7582    }
7583
7584    async fn query_live_roots(
7585        handler: &ControlHandler,
7586        module_ctx: &RouteCtx,
7587    ) -> ModuleControlResponseToModule {
7588        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
7589        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
7590        let response = handler
7591            .handle_control_frame(module_ctx, frame)
7592            .await
7593            .unwrap()
7594            .pop()
7595            .unwrap();
7596        serde_json::from_slice(&response.body).unwrap()
7597    }
7598
7599    #[tokio::test(start_paused = true)]
7600    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
7601        let registry = Arc::new(Registry::default());
7602        let forwarding = Arc::new(ForwardingTable::default());
7603        let handler = ControlHandler::with_forwarding(registry, forwarding);
7604        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
7605        hello_via_sink(
7606            &handler,
7607            &target_ctx,
7608            &mut target_rx,
7609            hello_frame("target", PROTOCOL_VERSION, 1),
7610        )
7611        .await;
7612        let root = unique_project_root("live-roots-known");
7613        let path = ProjectRootId::from_path_allowing_missing(root.path())
7614            .unwrap()
7615            .as_path()
7616            .to_path_buf();
7617        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
7618        let open_handler = handler.clone();
7619        let opened = tokio::spawn(async move {
7620            open_handler
7621                .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
7622                .await
7623                .unwrap()
7624        });
7625        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7626            .await
7627            .unwrap()
7628            .unwrap();
7629        handler
7630            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
7631            .await
7632            .unwrap();
7633        assert!(opened.await.unwrap().is_empty());
7634        let _ = client_rx.recv().await.unwrap();
7635
7636        let root = unique_project_root("live-roots-pending");
7637        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
7638            .unwrap()
7639            .as_path()
7640            .to_path_buf();
7641        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
7642        let open_handler = handler.clone();
7643        let pending = tokio::spawn(async move {
7644            open_handler
7645                .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
7646                .await
7647                .unwrap()
7648        });
7649        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7650            .await
7651            .unwrap()
7652            .unwrap();
7653        let actual = query_live_roots(&handler, &target_ctx).await;
7654        let ModuleControlResponseToModule::LiveRoots {
7655            roots,
7656            unknown_root_bindings,
7657            total_bindings,
7658        } = actual
7659        else {
7660            panic!("expected live roots")
7661        };
7662        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
7663        assert_eq!(unknown_root_bindings, 0);
7664        assert_eq!(
7665            roots.len(),
7666            2,
7667            "root-known arm must retain each canonical root"
7668        );
7669        assert_eq!(
7670            total_bindings,
7671            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7672        );
7673        let counts = roots
7674            .iter()
7675            .map(|root| (root.project_root.clone(), root.bound, root.pending))
7676            .collect::<Vec<_>>();
7677        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
7678        expected.sort_by(|a, b| a.0.cmp(&b.0));
7679        assert_eq!(
7680            counts, expected,
7681            "roots must sort by path and count pending separately"
7682        );
7683        handler
7684            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
7685            .await
7686            .unwrap();
7687        assert!(pending.await.unwrap().is_empty());
7688    }
7689
7690    #[tokio::test(start_paused = true)]
7691    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
7692        let forwarding = Arc::new(ForwardingTable::default());
7693        let handler =
7694            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
7695        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
7696        hello_via_sink(
7697            &handler,
7698            &target_ctx,
7699            &mut target_rx,
7700            hello_frame("target", PROTOCOL_VERSION, 1),
7701        )
7702        .await;
7703        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
7704        let pending = forwarding
7705            .begin_route_bind_relay_for_test(
7706                client_ctx.connection_id,
7707                client_ctx.egress.clone(),
7708                2,
7709                "target",
7710            )
7711            .unwrap();
7712        forwarding
7713            .complete_pending_relay(
7714                target_ctx.connection_id,
7715                pending.corr,
7716                RouteBindRelayOutcome::Accepted,
7717            )
7718            .unwrap();
7719        let actual = query_live_roots(&handler, &target_ctx).await;
7720        let ModuleControlResponseToModule::LiveRoots {
7721            roots,
7722            unknown_root_bindings,
7723            total_bindings,
7724        } = actual
7725        else {
7726            panic!("expected live roots")
7727        };
7728        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
7729        assert_eq!(
7730            unknown_root_bindings, 1,
7731            "unknown-root arm must not read as no bindings"
7732        );
7733        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
7734        assert_eq!(
7735            total_bindings,
7736            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7737        );
7738    }
7739
7740    /// A module reads HELLO_ACK as its first frame and exits on anything else,
7741    /// so the ack has to be on its outbound queue before the module is
7742    /// routable. The connection loop writes a handler's replies only after the
7743    /// handler returns; this test stops in exactly that gap, runs a real
7744    /// route.open from another connection, and only then writes whatever the
7745    /// HELLO handler returned, the way the loop would. If the ack were still a
7746    /// reply, the route.bind request would reach the module first.
7747    #[tokio::test(start_paused = true)]
7748    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
7749        let forwarding = Arc::new(ForwardingTable::default());
7750        let handler =
7751            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
7752        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
7753        let replies = handler
7754            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
7755            .await
7756            .unwrap();
7757        let queued_by_hello = module_rx.len();
7758
7759        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
7760        let open_handler = handler.clone();
7761        let open = tokio::spawn(async move {
7762            open_handler
7763                .handle_control_frame(
7764                    &client_ctx,
7765                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
7766                )
7767                .await
7768                .unwrap()
7769        });
7770        // Let the route.open run until its route.bind is on the module's queue.
7771        let mut spins = 0;
7772        while module_rx.len() == queued_by_hello {
7773            spins += 1;
7774            assert!(spins < 10_000, "route.open never queued a route.bind");
7775            tokio::task::yield_now().await;
7776        }
7777
7778        // Now the connection loop's half: write the HELLO handler's replies.
7779        for reply in replies {
7780            module_ctx.egress.send(reply).await.unwrap();
7781        }
7782
7783        let first = module_rx.recv().await.unwrap().frame;
7784        assert_eq!(
7785            first.header.ty,
7786            FrameType::HelloAck,
7787            "the first frame a registering module reads must be its HELLO_ACK"
7788        );
7789        assert_eq!(first.header.corr, 7);
7790        let second = module_rx.recv().await.unwrap().frame;
7791        assert_eq!(second.header.ty, FrameType::Request);
7792        assert!(
7793            matches!(
7794                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
7795                ModuleControlRequest::RouteBind { .. }
7796            ),
7797            "the route.bind follows the ack"
7798        );
7799        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
7800
7801        handler
7802            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
7803            .await
7804            .unwrap();
7805        assert!(open.await.unwrap().is_empty());
7806        let _ = client_rx.recv().await.unwrap();
7807    }
7808
7809    #[tokio::test(start_paused = true)]
7810    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
7811        let handler = ControlHandler::with_forwarding(
7812            Arc::new(Registry::default()),
7813            Arc::new(ForwardingTable::default()),
7814        );
7815        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
7816        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
7817        hello_via_sink(
7818            &handler,
7819            &first_ctx,
7820            &mut first_rx,
7821            hello_frame("first", PROTOCOL_VERSION, 1),
7822        )
7823        .await;
7824        hello_via_sink(
7825            &handler,
7826            &second_ctx,
7827            &mut second_rx,
7828            hello_frame("second", PROTOCOL_VERSION, 2),
7829        )
7830        .await;
7831        let root = unique_project_root("second-only");
7832        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
7833        let cloned = handler.clone();
7834        let open = tokio::spawn(async move {
7835            cloned
7836                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
7837                .await
7838                .unwrap()
7839        });
7840        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
7841            .await
7842            .unwrap()
7843            .unwrap();
7844        let first = query_live_roots(&handler, &first_ctx).await;
7845        let second = query_live_roots(&handler, &second_ctx).await;
7846        assert!(
7847            matches!(
7848                first,
7849                ModuleControlResponseToModule::LiveRoots {
7850                    total_bindings: 0,
7851                    ..
7852                }
7853            ),
7854            "cross-module scope must not expose another module's roots"
7855        );
7856        assert!(
7857            matches!(
7858                second,
7859                ModuleControlResponseToModule::LiveRoots {
7860                    total_bindings: 1,
7861                    ..
7862                }
7863            ),
7864            "second module must see its pending route"
7865        );
7866        handler
7867            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
7868            .await
7869            .unwrap();
7870        assert!(open.await.unwrap().is_empty());
7871    }
7872
7873    #[tokio::test(start_paused = true)]
7874    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
7875        let handler = ControlHandler::with_forwarding(
7876            Arc::new(Registry::default()),
7877            Arc::new(ForwardingTable::default()),
7878        );
7879        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
7880        hello_via_sink(
7881            &handler,
7882            &target_ctx,
7883            &mut target_rx,
7884            hello_frame("target", PROTOCOL_VERSION, 1),
7885        )
7886        .await;
7887        let actual = query_live_roots(&handler, &target_ctx).await;
7888        let ModuleControlResponseToModule::LiveRoots {
7889            roots,
7890            unknown_root_bindings,
7891            total_bindings,
7892        } = actual
7893        else {
7894            panic!("expected live roots")
7895        };
7896        assert!(roots.is_empty());
7897        assert_eq!(unknown_root_bindings, 0);
7898        assert_eq!(total_bindings, 0);
7899        assert_eq!(
7900            total_bindings,
7901            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7902        );
7903    }
7904
7905    /// Read the vendored fed corpus rather than hand-building a package.
7906    ///
7907    /// A hand-built object encodes what the test author believed the carrier
7908    /// emits. These vectors are what it actually emits, and one of them exists
7909    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
7910    /// ignores additive unknown fields at the traversal emit terminus."
7911    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
7912        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7913            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
7914        let text = std::fs::read_to_string(&path)
7915            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
7916        let vectors: Vec<(String, Value)> = text
7917            .lines()
7918            .filter(|line| !line.trim().is_empty())
7919            .map(|line| {
7920                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
7921                let id = entry["corpus_id"]
7922                    .as_str()
7923                    .expect("every vector carries a corpus_id")
7924                    .to_string();
7925                (id, entry["package"].clone())
7926            })
7927            .collect();
7928        // Pin the count: a corpus that silently shrinks would take its coverage
7929        // with it, and a suite reading N-1 vectors reports the same clean pass
7930        // as one reading N.
7931        assert_eq!(
7932            vectors.len(),
7933            3,
7934            "vendored fed corpus changed size; re-sync from subc-federation"
7935        );
7936
7937        // Pin what makes the corpus DISCRIMINATING, not just present.
7938        //
7939        // The relay test below takes its expected value from the corpus, so the
7940        // corpus supplies the test's power to detect a lossy relay rather than
7941        // its correctness. A relay that dropped unrecognised fields would still
7942        // be caught -- but only by a package carrying fields it does not know.
7943        // Shrink every package to the handful of keys any implementation would
7944        // recognise and the test keeps passing over an input that can no longer
7945        // fail, which is the same clean green as a corpus that shrank away.
7946        //
7947        // So assert the precondition rather than duplicating the packages here:
7948        // at least one vector must carry a field beyond the small common set.
7949        // That is one claim to maintain instead of nine, and it fails loudly if
7950        // a re-sync ever flattens the corpus.
7951        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
7952        let richest = vectors
7953            .iter()
7954            .filter_map(|(_, package)| package.as_object())
7955            .map(|object| {
7956                object
7957                    .keys()
7958                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
7959                    .count()
7960            })
7961            .max()
7962            .unwrap_or(0);
7963        assert!(
7964            richest >= 2,
7965            "vendored corpus no longer carries a package with unmodelled fields, \
7966             so the relay test can no longer distinguish a verbatim relay from a lossy one"
7967        );
7968
7969        vectors
7970    }
7971
7972    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
7973    ///
7974    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
7975    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
7976    /// three-key object, so a relay that quietly dropped fields it did not
7977    /// recognise would satisfy it. These vectors carry nine keys including ones
7978    /// this crate has no type for, so a typed relay fails here and only here.
7979    #[tokio::test]
7980    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
7981        for (corpus_id, package) in fed_admission_facts_vectors() {
7982            let registry = Arc::new(Registry::default());
7983            let forwarding = Arc::new(ForwardingTable::default());
7984            let supervisor = SupervisorHandle::new();
7985            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7986            let handler =
7987                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
7988                    .with_supervisor(supervisor)
7989                    .with_admission_facts_config(
7990                        Some("fed".to_string()),
7991                        Some(vec!["target".to_string()]),
7992                    );
7993
7994            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
7995            hello_via_sink(
7996                &handler,
7997                &target_ctx,
7998                &mut target_rx,
7999                hello_frame("target", PROTOCOL_VERSION, 1),
8000            )
8001            .await;
8002
8003            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
8004            let route_handler = handler.clone();
8005            let expected = package.clone();
8006            let route_task = tokio::spawn(async move {
8007                route_handler
8008                    .handle_control_frame(
8009                        &client_ctx,
8010                        route_open_frame_with_admission_facts(
8011                            20,
8012                            "target",
8013                            unique_project_root("admission-facts"),
8014                            Some(subc_control::ConsumerIdentity {
8015                                module_id: "fed".to_string(),
8016                                launch_nonce: "fed-nonce".to_string(),
8017                            }),
8018                            Some(package),
8019                        ),
8020                    )
8021                    .await
8022                    .unwrap()
8023            });
8024
8025            let bind_frame = target_rx.recv().await.unwrap();
8026            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8027            let ModuleControlRequest::RouteBind {
8028                admission_facts, ..
8029            } = bind
8030            else {
8031                panic!("{corpus_id}: expected route.bind")
8032            };
8033            assert_eq!(
8034                admission_facts,
8035                Some(expected),
8036                "{corpus_id}: relay must not add, drop or reshape any field"
8037            );
8038
8039            handler
8040                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8041                .await
8042                .unwrap();
8043            route_task.await.unwrap();
8044        }
8045    }
8046
8047    #[tokio::test]
8048    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
8049        let registry = Arc::new(Registry::default());
8050        let forwarding = Arc::new(ForwardingTable::default());
8051        let supervisor = SupervisorHandle::new();
8052        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8053        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
8054        let handler =
8055            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8056                .with_supervisor(supervisor)
8057                .with_admission_facts_config(
8058                    Some("fed".to_string()),
8059                    Some(vec!["target".to_string()]),
8060                );
8061
8062        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
8063        hello_via_sink(
8064            &handler,
8065            &target_ctx,
8066            &mut target_rx,
8067            hello_frame("target", PROTOCOL_VERSION, 1),
8068        )
8069        .await;
8070        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
8071        hello_via_sink(
8072            &handler,
8073            &other_ctx,
8074            &mut other_rx,
8075            hello_frame("other", PROTOCOL_VERSION, 2),
8076        )
8077        .await;
8078
8079        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
8080        let expected_facts = facts.clone();
8081        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
8082        let route_handler = handler.clone();
8083        let route_task = tokio::spawn(async move {
8084            route_handler
8085                .handle_control_frame(
8086                    &client_ctx,
8087                    route_open_frame_with_admission_facts(
8088                        10,
8089                        "target",
8090                        unique_project_root("admission-facts"),
8091                        Some(subc_control::ConsumerIdentity {
8092                            module_id: "fed".to_string(),
8093                            launch_nonce: "fed-nonce".to_string(),
8094                        }),
8095                        Some(facts.clone()),
8096                    ),
8097                )
8098                .await
8099                .unwrap()
8100        });
8101        let bind_frame = target_rx.recv().await.unwrap();
8102        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8103        let ModuleControlRequest::RouteBind {
8104            admission_facts, ..
8105        } = bind
8106        else {
8107            panic!("expected route.bind")
8108        };
8109        assert_eq!(admission_facts, Some(expected_facts));
8110        handler
8111            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8112            .await
8113            .unwrap();
8114        assert!(route_task.await.unwrap().is_empty());
8115        assert!(matches!(
8116            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
8117                .unwrap(),
8118            ClientControlResponse::RouteOpen { .. }
8119        ));
8120
8121        let direct = handler
8122            .handle_control_frame(
8123                &route_ctx(ConnectionId::new(73)).0,
8124                route_open_frame_with_admission_facts(
8125                    11,
8126                    "target",
8127                    unique_project_root("admission-facts"),
8128                    None,
8129                    Some(json!({"x": 1})),
8130                ),
8131            )
8132            .await
8133            .unwrap();
8134        assert_eq!(
8135            parse_error(&direct[0])["code"],
8136            "admission_facts_not_permitted"
8137        );
8138
8139        let different_reserved = handler
8140            .handle_control_frame(
8141                &route_ctx(ConnectionId::new(77)).0,
8142                route_open_frame_with_admission_facts(
8143                    15,
8144                    "target",
8145                    unique_project_root("admission-facts"),
8146                    Some(subc_control::ConsumerIdentity {
8147                        module_id: "other".to_string(),
8148                        launch_nonce: "other-nonce".to_string(),
8149                    }),
8150                    Some(json!({"x": 1})),
8151                ),
8152            )
8153            .await
8154            .unwrap();
8155        assert_eq!(
8156            parse_error(&different_reserved[0])["code"],
8157            "admission_facts_not_permitted"
8158        );
8159
8160        let other_target = handler
8161            .handle_control_frame(
8162                &route_ctx(ConnectionId::new(74)).0,
8163                route_open_frame_with_admission_facts(
8164                    12,
8165                    "other",
8166                    unique_project_root("admission-facts"),
8167                    Some(subc_control::ConsumerIdentity {
8168                        module_id: "fed".to_string(),
8169                        launch_nonce: "fed-nonce".to_string(),
8170                    }),
8171                    Some(json!({"x": 1})),
8172                ),
8173            )
8174            .await
8175            .unwrap();
8176        assert_eq!(
8177            parse_error(&other_target[0])["code"],
8178            "admission_facts_target_not_allowed"
8179        );
8180
8181        let nonexistent = handler
8182            .handle_control_frame(
8183                &route_ctx(ConnectionId::new(75)).0,
8184                route_open_frame_with_admission_facts(
8185                    13,
8186                    "missing",
8187                    unique_project_root("admission-facts"),
8188                    None,
8189                    Some(json!({"x": 1})),
8190                ),
8191            )
8192            .await
8193            .unwrap();
8194        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
8195
8196        let described = handler
8197            .handle_control_frame(
8198                &route_ctx(ConnectionId::new(76)).0,
8199                Frame::build(
8200                    FrameType::Request,
8201                    control_flags(),
8202                    0,
8203                    0,
8204                    14,
8205                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8206                )
8207                .unwrap(),
8208            )
8209            .await
8210            .unwrap();
8211        let ClientControlResponse::ServerDescribe { capabilities, .. } =
8212            serde_json::from_slice(&described[0].body).unwrap()
8213        else {
8214            panic!("expected server.describe response")
8215        };
8216        assert!(capabilities
8217            .iter()
8218            .any(|cap| cap == "admission_facts_relay_v1"));
8219    }
8220
8221    #[tokio::test]
8222    async fn admission_facts_without_configured_carrier_are_rejected() {
8223        let registry = Arc::new(Registry::default());
8224        let forwarding = Arc::new(ForwardingTable::default());
8225        let handler = ControlHandler::with_forwarding(registry, forwarding);
8226        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
8227        hello_via_sink(
8228            &handler,
8229            &target_ctx,
8230            &mut target_rx,
8231            hello_frame("target", PROTOCOL_VERSION, 1),
8232        )
8233        .await;
8234
8235        let responses = handler
8236            .handle_control_frame(
8237                &route_ctx(ConnectionId::new(79)).0,
8238                route_open_frame_with_admission_facts(
8239                    16,
8240                    "target",
8241                    unique_project_root("admission-facts"),
8242                    None,
8243                    Some(json!({"x": 1})),
8244                ),
8245            )
8246            .await
8247            .unwrap();
8248        assert_eq!(
8249            parse_error(&responses[0])["code"],
8250            "admission_facts_not_permitted"
8251        );
8252    }
8253
8254    #[tokio::test]
8255    async fn route_open_relays_consumer_capabilities_verbatim() {
8256        let registry = Arc::new(Registry::default());
8257        let forwarding = Arc::new(ForwardingTable::default());
8258        let handler =
8259            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
8260        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
8261        hello_via_sink(
8262            &handler,
8263            &module_ctx,
8264            &mut module_rx,
8265            hello_frame("aft", PROTOCOL_VERSION, 7),
8266        )
8267        .await;
8268
8269        let expected = vec!["elicitation".to_string(), "roots".to_string()];
8270        let expected_for_request = expected.clone();
8271        let project_root = unique_project_root("consumer-capabilities-present");
8272        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
8273        let route_handler = handler.clone();
8274        let route_task = tokio::spawn(async move {
8275            route_handler
8276                .handle_control_frame(
8277                    &client_ctx,
8278                    route_open_frame_with_consumer_capabilities(
8279                        401,
8280                        "aft",
8281                        project_root,
8282                        Some(expected_for_request),
8283                    ),
8284                )
8285                .await
8286                .unwrap()
8287        });
8288        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8289            .await
8290            .unwrap()
8291            .unwrap();
8292        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8293        let ModuleControlRequest::RouteBind {
8294            consumer_capabilities,
8295            ..
8296        } = bind
8297        else {
8298            panic!("expected route.bind request, got {bind:?}");
8299        };
8300        assert_eq!(consumer_capabilities, Some(expected.clone()));
8301
8302        handler
8303            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8304            .await
8305            .unwrap();
8306        let route_response = route_task.await.unwrap();
8307        assert!(route_response.is_empty());
8308        let published = client_rx.recv().await.unwrap();
8309        assert!(matches!(
8310            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8311            ClientControlResponse::RouteOpen { .. }
8312        ));
8313    }
8314
8315    #[tokio::test]
8316    async fn route_open_without_consumer_capabilities_relays_none() {
8317        let registry = Arc::new(Registry::default());
8318        let forwarding = Arc::new(ForwardingTable::default());
8319        let handler =
8320            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
8321        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
8322        hello_via_sink(
8323            &handler,
8324            &module_ctx,
8325            &mut module_rx,
8326            hello_frame("aft", PROTOCOL_VERSION, 7),
8327        )
8328        .await;
8329
8330        let project_root = unique_project_root("consumer-capabilities-absent");
8331        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
8332        let route_handler = handler.clone();
8333        let route_task = tokio::spawn(async move {
8334            route_handler
8335                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
8336                .await
8337                .unwrap()
8338        });
8339        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8340            .await
8341            .unwrap()
8342            .unwrap();
8343        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8344        let ModuleControlRequest::RouteBind {
8345            consumer_capabilities,
8346            ..
8347        } = bind
8348        else {
8349            panic!("expected route.bind request, got {bind:?}");
8350        };
8351        assert_eq!(consumer_capabilities, None);
8352
8353        handler
8354            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8355            .await
8356            .unwrap();
8357        let route_response = route_task.await.unwrap();
8358        assert!(route_response.is_empty());
8359        let published = client_rx.recv().await.unwrap();
8360        assert!(matches!(
8361            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8362            ClientControlResponse::RouteOpen { .. }
8363        ));
8364    }
8365
8366    #[tokio::test]
8367    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
8368        let registry = Arc::new(Registry::default());
8369        let forwarding = Arc::new(ForwardingTable::default());
8370        let handler =
8371            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8372                .with_health_probe_timeout(Duration::from_secs(5));
8373        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
8374        hello_via_sink(
8375            &handler,
8376            &module_ctx,
8377            &mut module_rx,
8378            non_routable_hello_frame_with_control_ops(
8379                "mcp",
8380                300,
8381                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8382            ),
8383        )
8384        .await;
8385        assert!(registry
8386            .get_module("mcp")
8387            .unwrap()
8388            .unwrap()
8389            .manifest
8390            .provides
8391            .is_empty());
8392
8393        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
8394        let route_response = handler
8395            .handle_control_frame(
8396                &route_client_ctx,
8397                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
8398            )
8399            .await
8400            .unwrap();
8401        assert_eq!(route_response[0].header.ty, FrameType::Error);
8402        assert_eq!(
8403            parse_error(&route_response[0])["code"],
8404            "target_unavailable"
8405        );
8406        assert!(parse_error(&route_response[0])["message"]
8407            .as_str()
8408            .unwrap()
8409            .contains("does not provide the requested target"));
8410        assert!(module_rx.try_recv().is_err());
8411
8412        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
8413        let health_handler = handler.clone();
8414        let health_task = tokio::spawn(async move {
8415            health_handler
8416                .handle_control_frame(
8417                    &health_client_ctx,
8418                    supervisor_health_probe_frame(302, "mcp"),
8419                )
8420                .await
8421                .unwrap()
8422        });
8423        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8424            .await
8425            .unwrap()
8426            .unwrap();
8427        assert_eq!(
8428            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
8429            ModuleControlRequest::HealthCheck {}
8430        );
8431        handler
8432            .handle_control_frame(
8433                &module_ctx,
8434                health_response(health_frame.header.corr, HealthStatus::Ok),
8435            )
8436            .await
8437            .unwrap();
8438        let health_response = health_task.await.unwrap();
8439        assert_eq!(health_response[0].header.ty, FrameType::Response);
8440        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
8441            ClientControlResponse::SupervisorHealthProbe {
8442                module_id, status, ..
8443            } => {
8444                assert_eq!(module_id, "mcp");
8445                assert_eq!(status, HealthStatus::Ok);
8446            }
8447            other => panic!("unexpected health response: {other:?}"),
8448        }
8449
8450        // Exercise the forwarding cleanup path directly while leaving the registry
8451        // advertisement in place. If cleanup leaves a stale control sink behind,
8452        // the next probe will enqueue onto it and wait for the long probe timeout
8453        // instead of returning an immediate no-connection error.
8454        forwarding
8455            .cleanup_connection(module_ctx.connection_id)
8456            .unwrap();
8457        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
8458        let cleanup_response = tokio::time::timeout(
8459            Duration::from_millis(200),
8460            handler.handle_control_frame(
8461                &cleanup_probe_ctx,
8462                supervisor_health_probe_frame(303, "mcp"),
8463            ),
8464        )
8465        .await
8466        .expect("probe should fail immediately when the control lane is gone")
8467        .unwrap();
8468        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
8469        assert_eq!(
8470            parse_error(&cleanup_response[0])["code"],
8471            "target_unavailable"
8472        );
8473        assert!(parse_error(&cleanup_response[0])["message"]
8474            .as_str()
8475            .unwrap()
8476            .contains("no module connection"));
8477
8478        handler
8479            .cleanup_connection(module_ctx.connection_id)
8480            .unwrap();
8481    }
8482
8483    #[tokio::test]
8484    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
8485        let registry = Arc::new(Registry::default());
8486        let supervisor_handle = SupervisorHandle::new();
8487        let supervisor =
8488            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
8489                .with_handle(supervisor_handle.clone())
8490                .with_connection_file_path(
8491                    std::env::temp_dir()
8492                        .join(format!("subc-route-open-warming-{}", std::process::id())),
8493                );
8494        let module = supervisor
8495            .supervise_configured(
8496                ModuleSpec {
8497                    module_id: "warming".to_string(),
8498                    program: fake_aft_stub_path(),
8499                    args: Vec::new(),
8500                    env: Vec::new(),
8501                    reserved: false,
8502                    reserved_prefixes: Vec::new(),
8503                    protocol: ModuleProtocol::Subc,
8504                    overlap: Default::default(),
8505                },
8506                true,
8507            )
8508            .unwrap();
8509        assert_eq!(module.state().unwrap(), ModuleState::Running);
8510
8511        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8512        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
8513        let response = handler
8514            .handle_control_frame(
8515                &ctx,
8516                route_open_frame(304, "warming", unique_project_root("warming")),
8517            )
8518            .await
8519            .unwrap();
8520        module.stop().await.unwrap();
8521
8522        assert_eq!(response[0].header.ty, FrameType::Error);
8523        let error = parse_error(&response[0]);
8524        assert_eq!(error["code"], "module_warming");
8525        assert!(error["message"]
8526            .as_str()
8527            .unwrap()
8528            .contains("state=running, enabled=true, live=false"));
8529    }
8530
8531    /// One wire code has several senders, so the refusal line names the check
8532    /// that refused. This drives the shared refusal path (every non-supervised
8533    /// refusal goes through `route_open_refusal_frame`) with an unregistered
8534    /// target and requires the branch label on the event.
8535    #[tokio::test]
8536    async fn route_open_refusal_names_the_check_that_refused() {
8537        let handler = ControlHandler::new(Arc::new(Registry::default()));
8538        let capture = EventCapture::default();
8539        let _subscriber =
8540            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8541        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
8542        let response = handler
8543            .handle_control_frame(
8544                &ctx,
8545                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
8546            )
8547            .await
8548            .unwrap();
8549
8550        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
8551        let event = capture
8552            .events()
8553            .into_iter()
8554            .find(|event| {
8555                event.target == "control"
8556                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
8557            })
8558            .expect("route.open refusal event");
8559        assert_eq!(
8560            event.fields.get("reason"),
8561            Some(&"\"not_registered\"".to_string())
8562        );
8563    }
8564
8565    #[tokio::test]
8566    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
8567        let registry = Arc::new(Registry::default());
8568        let supervisor_handle = SupervisorHandle::new();
8569        let supervisor =
8570            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
8571                .with_handle(supervisor_handle.clone())
8572                .with_connection_file_path(std::env::temp_dir().join(format!(
8573                    "subc-route-open-refusal-info-{}",
8574                    std::process::id()
8575                )));
8576        let module = supervisor
8577            .supervise_configured(
8578                ModuleSpec {
8579                    module_id: "warming".to_string(),
8580                    program: fake_aft_stub_path(),
8581                    args: Vec::new(),
8582                    env: Vec::new(),
8583                    reserved: false,
8584                    reserved_prefixes: Vec::new(),
8585                    protocol: ModuleProtocol::Subc,
8586                    overlap: Default::default(),
8587                },
8588                true,
8589            )
8590            .unwrap();
8591        assert_eq!(module.state().unwrap(), ModuleState::Running);
8592
8593        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8594        assert!(handler
8595            .counters()
8596            .snapshot()
8597            .get("route_open_refused_by_code")
8598            .is_none());
8599        let capture = EventCapture::default();
8600        let _subscriber =
8601            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8602        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
8603        let response = handler
8604            .handle_control_frame(
8605                &ctx,
8606                route_open_frame(394, "warming", unique_project_root("refusal-info")),
8607            )
8608            .await
8609            .unwrap();
8610        module.stop().await.unwrap();
8611
8612        assert_eq!(parse_error(&response[0])["code"], "module_warming");
8613        let event = capture
8614            .events()
8615            .into_iter()
8616            .find(|event| {
8617                event.target == "control"
8618                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
8619            })
8620            .expect("route.open refusal event");
8621        assert_eq!(
8622            event.fields.get("module_id"),
8623            Some(&"\"warming\"".to_string())
8624        );
8625        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
8626        assert_eq!(
8627            event.fields.get("reason"),
8628            Some(&"\"supervised_not_registered\"".to_string())
8629        );
8630        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
8631        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
8632        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
8633        assert_eq!(
8634            handler.counters().snapshot()["route_open_refused_by_code"],
8635            json!({ "module_warming": 1 })
8636        );
8637    }
8638
8639    #[tokio::test(flavor = "current_thread")]
8640    async fn route_open_unknown_module_escapes_target_module_id() {
8641        let handler = ControlHandler::new(Arc::new(Registry::default()));
8642        let capture = EventCapture::default();
8643        let _subscriber =
8644            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8645        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
8646        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
8647        let response = handler
8648            .handle_control_frame(
8649                &ctx,
8650                route_open_frame(
8651                    395,
8652                    hostile_module_id,
8653                    unique_project_root("hostile-target-module-id"),
8654                ),
8655            )
8656            .await
8657            .unwrap();
8658
8659        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
8660        let event = capture
8661            .events()
8662            .into_iter()
8663            .find(|event| {
8664                event.target == "control"
8665                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
8666            })
8667            .expect("route.open unknown-module refusal event");
8668        let logged = event.fields.get("module_id").expect("module_id field");
8669        assert!(!logged.bytes().any(|byte| byte < 0x20));
8670        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
8671    }
8672
8673    #[tokio::test(flavor = "current_thread")]
8674    async fn route_open_module_rejection_uses_daemon_counter_key() {
8675        let registry = Arc::new(Registry::default());
8676        let forwarding = Arc::new(ForwardingTable::default());
8677        let handler =
8678            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
8679        let module_connection = ConnectionId::new(95);
8680        let (module_ctx, mut module_rx) = route_ctx(module_connection);
8681        hello_via_sink(
8682            &handler,
8683            &module_ctx,
8684            &mut module_rx,
8685            hello_frame("aft", PROTOCOL_VERSION, 395),
8686        )
8687        .await;
8688
8689        let client_connection = ConnectionId::new(96);
8690        let (client_ctx, _client_rx) = route_ctx(client_connection);
8691        let capture = EventCapture::default();
8692        let _subscriber =
8693            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8694        let (route_task, bind) = relay_route_open(
8695            &handler,
8696            client_connection,
8697            &client_ctx.egress,
8698            &mut module_rx,
8699            396,
8700            "aft",
8701            "hostile-module-code",
8702        )
8703        .await;
8704        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
8705        let rejection = Frame::build(
8706            FrameType::Error,
8707            control_flags(),
8708            0,
8709            0,
8710            bind.header.corr,
8711            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
8712        )
8713        .unwrap();
8714        handler
8715            .handle_control_frame(&module_ctx, rejection)
8716            .await
8717            .unwrap();
8718
8719        let response = route_task.await.unwrap();
8720        assert_eq!(parse_error(&response[0])["code"], hostile_code);
8721        let counters = handler.counters().snapshot();
8722        assert_eq!(
8723            counters["route_open_refused_by_code"],
8724            json!({ "module_rejected": 1 })
8725        );
8726        assert!(counters["route_open_refused_by_code"]
8727            .get(hostile_code)
8728            .is_none());
8729
8730        let event = capture
8731            .events()
8732            .into_iter()
8733            .find(|event| {
8734                event.target == "control"
8735                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
8736            })
8737            .expect("route.open module-rejection refusal event");
8738        let logged = event.fields.get("module_code").expect("module_code field");
8739        assert!(!logged.bytes().any(|byte| byte < 0x20));
8740        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
8741    }
8742
8743    #[tokio::test]
8744    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
8745        let registry = Arc::new(Registry::default());
8746        let supervisor_handle = SupervisorHandle::new();
8747        let missing_program = std::env::temp_dir().join(format!(
8748            "subc-route-open-missing-program-{}",
8749            std::process::id()
8750        ));
8751        let supervisor =
8752            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
8753                .with_handle(supervisor_handle.clone());
8754        let module = supervisor
8755            .supervise_configured(
8756                ModuleSpec {
8757                    module_id: "failed".to_string(),
8758                    program: missing_program,
8759                    args: Vec::new(),
8760                    env: Vec::new(),
8761                    reserved: false,
8762                    reserved_prefixes: Vec::new(),
8763                    protocol: ModuleProtocol::Subc,
8764                    overlap: Default::default(),
8765                },
8766                true,
8767            )
8768            .unwrap();
8769        assert_eq!(module.state().unwrap(), ModuleState::Failed);
8770
8771        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8772        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
8773        let response = handler
8774            .handle_control_frame(
8775                &ctx,
8776                route_open_frame(305, "failed", unique_project_root("failed")),
8777            )
8778            .await
8779            .unwrap();
8780
8781        assert_eq!(response[0].header.ty, FrameType::Error);
8782        let error = parse_error(&response[0]);
8783        assert_eq!(error["code"], "target_unavailable");
8784        assert!(error["message"]
8785            .as_str()
8786            .unwrap()
8787            .contains("state=failed, enabled=true, live=false"));
8788    }
8789
8790    #[tokio::test]
8791    async fn route_open_role_mismatch_remains_target_unavailable() {
8792        let registry = Arc::new(Registry::default());
8793        let handler = ControlHandler::new(Arc::clone(&registry));
8794        handler
8795            .handle_control(
8796                ConnectionId::new(41),
8797                non_routable_hello_frame_with_control_ops("health-only", 306, None),
8798            )
8799            .unwrap();
8800
8801        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
8802        let response = handler
8803            .handle_control_frame(
8804                &ctx,
8805                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
8806            )
8807            .await
8808            .unwrap();
8809
8810        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8811        assert!(parse_error(&response[0])["message"]
8812            .as_str()
8813            .unwrap()
8814            .contains("does not provide the requested target"));
8815    }
8816
8817    #[tokio::test]
8818    async fn route_open_inactive_registration_remains_target_unavailable() {
8819        let registry = Arc::new(Registry::default());
8820        let handler = ControlHandler::new(Arc::clone(&registry));
8821        handler
8822            .handle_control(
8823                ConnectionId::new(43),
8824                hello_frame("inactive", PROTOCOL_VERSION, 308),
8825            )
8826            .unwrap();
8827        assert!(registry
8828            .set_module_state_for_test("inactive", ChannelState::Closed)
8829            .unwrap());
8830
8831        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
8832        let response = handler
8833            .handle_control_frame(
8834                &ctx,
8835                route_open_frame(309, "inactive", unique_project_root("inactive")),
8836            )
8837            .await
8838            .unwrap();
8839
8840        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8841        assert!(parse_error(&response[0])["message"]
8842            .as_str()
8843            .unwrap()
8844            .contains("is not active"));
8845    }
8846
8847    #[tokio::test]
8848    async fn late_health_reply_is_recorded_through_the_module_response_path() {
8849        let registry = Arc::new(Registry::default());
8850        let forwarding = Arc::new(ForwardingTable::default());
8851        let supervisor_handle = SupervisorHandle::new();
8852        let supervisor = Supervisor::new(Arc::clone(&registry), crate::RestartPolicy::default())
8853            .with_forwarding(Arc::clone(&forwarding))
8854            .with_handle(supervisor_handle.clone());
8855        let module = supervisor
8856            .supervise_configured(
8857                crate::ModuleSpec {
8858                    module_id: "late-health-response".to_string(),
8859                    program: PathBuf::from("disabled-module"),
8860                    args: Vec::new(),
8861                    env: Vec::new(),
8862                    reserved: false,
8863                    reserved_prefixes: Vec::new(),
8864                    protocol: ModuleProtocol::Subc,
8865                    overlap: Default::default(),
8866                },
8867                false,
8868            )
8869            .unwrap();
8870        let handler =
8871            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8872                .with_supervisor(supervisor_handle);
8873        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
8874        handler
8875            .handle_control_frame(
8876                &module_ctx,
8877                hello_frame_with_control_ops(
8878                    "late-health-response",
8879                    PROTOCOL_VERSION,
8880                    7,
8881                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8882                ),
8883            )
8884            .await
8885            .unwrap();
8886        let probe_started_at = Instant::now() - Duration::from_millis(80);
8887        let pending = forwarding
8888            .begin_health_probe_rpc_for(
8889                "late-health-response",
8890                MODULE_CONTROL_OP_HEALTH_CHECK,
8891                probe_started_at,
8892                Instant::now() - Duration::from_millis(1),
8893            )
8894            .unwrap();
8895        assert!(forwarding
8896            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
8897            .unwrap());
8898
8899        let responses = handler
8900            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
8901            .await
8902            .unwrap();
8903
8904        assert!(responses.is_empty());
8905        let health = module.status().unwrap().health;
8906        assert_eq!(health.late_answer_count, 1);
8907        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
8908    }
8909
8910    #[tokio::test]
8911    async fn health_probe_timeout_and_module_death_are_typed() {
8912        let registry = Arc::new(Registry::default());
8913        let forwarding = Arc::new(ForwardingTable::default());
8914        let handler =
8915            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8916                .with_health_probe_timeout(Duration::from_millis(50));
8917        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
8918        hello_via_sink(
8919            &handler,
8920            &module_ctx,
8921            &mut module_rx,
8922            hello_frame_with_control_ops(
8923                "aft",
8924                PROTOCOL_VERSION,
8925                7,
8926                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8927            ),
8928        )
8929        .await;
8930
8931        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
8932        let responses = handler
8933            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
8934            .await
8935            .unwrap();
8936        assert_eq!(responses[0].header.ty, FrameType::Error);
8937        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
8938        let _ = module_rx.try_recv();
8939
8940        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
8941        let health_handler = handler.clone();
8942        let death_task = tokio::spawn(async move {
8943            health_handler
8944                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
8945                .await
8946                .unwrap()
8947        });
8948        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8949            .await
8950            .unwrap()
8951            .unwrap();
8952        handler
8953            .cleanup_connection(module_ctx.connection_id)
8954            .unwrap();
8955        let responses = death_task.await.unwrap();
8956        assert_eq!(responses[0].header.ty, FrameType::Error);
8957        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
8958    }
8959
8960    #[test]
8961    fn hello_requires_exact_protocol_version() {
8962        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
8963            let registry = Arc::new(Registry::default());
8964            let handler = ControlHandler::new(Arc::clone(&registry));
8965            let responses = handler
8966                .handle_control(
8967                    ConnectionId::new(connection),
8968                    hello_frame("aft", offered, 9),
8969                )
8970                .unwrap();
8971
8972            assert_eq!(responses.len(), 1);
8973            assert_eq!(responses[0].header.ty, FrameType::Error);
8974            let error = parse_error(&responses[0]);
8975            assert_eq!(error["code"], "version_unsupported");
8976            assert!(registry.get_module("aft").unwrap().is_none());
8977            assert_eq!(registry.active_registration_count().unwrap(), 0);
8978        }
8979    }
8980
8981    #[test]
8982    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
8983        let registry = Arc::new(Registry::default());
8984        let forwarding = Arc::new(ForwardingTable::default());
8985        let handler =
8986            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
8987        let module_connection = ConnectionId::new(301);
8988        let registration = registry
8989            .register_with_control_ops(
8990                manifest("aft-push", PROTOCOL_VERSION),
8991                PROTOCOL_VERSION,
8992                module_connection,
8993                module_baseline_control_ops(),
8994            )
8995            .unwrap();
8996        let (module_tx, _module_rx) = mpsc::channel(8);
8997        let endpoint = forwarding
8998            .register_module_connection(
8999                module_connection,
9000                "aft-push".to_string(),
9001                PROTOCOL_VERSION,
9002                manifest_concurrency(&registration.manifest),
9003                FrameSink::new(module_tx),
9004            )
9005            .unwrap();
9006
9007        // A push op this version does not know is ignored (forward-compat), not errored.
9008        let unknown = Frame::build(
9009            FrameType::Push,
9010            control_flags(),
9011            0,
9012            0,
9013            5,
9014            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
9015        )
9016        .unwrap();
9017        let out = handler.handle_status_update(endpoint, unknown).unwrap();
9018        assert!(
9019            out.is_empty(),
9020            "unknown push op must be ignored, got {out:?}"
9021        );
9022
9023        // A malformed body for a KNOWN op is a real error worth surfacing.
9024        let malformed = Frame::build(
9025            FrameType::Push,
9026            control_flags(),
9027            0,
9028            0,
9029            6,
9030            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
9031        )
9032        .unwrap();
9033        let out = handler.handle_status_update(endpoint, malformed).unwrap();
9034        assert_eq!(out.len(), 1);
9035        assert_eq!(out[0].header.ty, FrameType::Error);
9036        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
9037    }
9038
9039    #[test]
9040    fn hello_rejected_when_connection_already_owns_client_routes() {
9041        let registry = Arc::new(Registry::default());
9042        let forwarding = Arc::new(ForwardingTable::default());
9043        let handler =
9044            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9045        // Commits a client route on connection 202 (bound to a module on conn 101).
9046        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
9047        let client_connection = ConnectionId::new(202);
9048
9049        // That same connection now tries to register as a module: rejected, so one
9050        // connection never holds both client-route and module-endpoint state.
9051        let responses = handler
9052            .handle_control(
9053                client_connection,
9054                hello_frame("aft-second", PROTOCOL_VERSION, 9),
9055            )
9056            .unwrap();
9057        assert_eq!(responses[0].header.ty, FrameType::Error);
9058        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
9059        assert!(registry.get_module("aft-second").unwrap().is_none());
9060    }
9061
9062    #[test]
9063    fn reserved_module_hello_requires_matching_launch_nonce() {
9064        let registry = Arc::new(Registry::default());
9065        let supervisor = SupervisorHandle::new();
9066        // The supervisor recorded the nonce it injected when it spawned the reserved
9067        // module; the HELLO verifier checks against the same shared handle.
9068        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9069        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
9070
9071        // A HELLO with NO nonce is rejected.
9072        let no_nonce = handler
9073            .handle_control(
9074                ConnectionId::new(1),
9075                hello_frame("vault", PROTOCOL_VERSION, 1),
9076            )
9077            .unwrap();
9078        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
9079        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
9080        assert!(registry.get_module("vault").unwrap().is_none());
9081
9082        // A HELLO with the WRONG nonce is rejected.
9083        let wrong = handler
9084            .handle_control(
9085                ConnectionId::new(2),
9086                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
9087            )
9088            .unwrap();
9089        assert_eq!(wrong[0].header.ty, FrameType::Error);
9090        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
9091        assert!(registry.get_module("vault").unwrap().is_none());
9092
9093        // A HELLO with the CORRECT nonce registers.
9094        let ok = handler
9095            .handle_control(
9096                ConnectionId::new(3),
9097                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
9098            )
9099            .unwrap();
9100        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
9101        assert!(registry.get_module("vault").unwrap().is_some());
9102    }
9103
9104    #[test]
9105    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
9106        let registry = Arc::new(Registry::default());
9107        let supervisor = SupervisorHandle::new();
9108        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
9109        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
9110        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
9111
9112        let squat = handler
9113            .handle_control(
9114                ConnectionId::new(1),
9115                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
9116            )
9117            .unwrap();
9118        assert_eq!(squat[0].header.ty, FrameType::Error);
9119        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
9120        assert!(parse_error(&squat[0])["message"]
9121            .as_str()
9122            .unwrap()
9123            .contains("fed:"));
9124
9125        let accepted_peer = handler
9126            .handle_control(
9127                ConnectionId::new(2),
9128                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
9129            )
9130            .unwrap();
9131        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
9132
9133        let accepted_short = handler
9134            .handle_control(
9135                ConnectionId::new(3),
9136                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
9137            )
9138            .unwrap();
9139        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
9140
9141        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
9142            let response = handler
9143                .handle_control(
9144                    ConnectionId::new(conn),
9145                    hello_frame(module_id, PROTOCOL_VERSION, conn),
9146                )
9147                .unwrap();
9148            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
9149        }
9150    }
9151
9152    #[test]
9153    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
9154        let registry = Arc::new(Registry::default());
9155        let supervisor = SupervisorHandle::new();
9156        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
9157        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
9158        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
9159        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
9160
9161        let owner_nonce = handler
9162            .handle_control(
9163                ConnectionId::new(1),
9164                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
9165            )
9166            .unwrap();
9167        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
9168        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
9169        assert!(registry.get_module("fed:special").unwrap().is_none());
9170
9171        let exact_nonce = handler
9172            .handle_control(
9173                ConnectionId::new(2),
9174                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
9175            )
9176            .unwrap();
9177        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
9178        assert!(registry.get_module("fed:special").unwrap().is_some());
9179    }
9180
9181    #[test]
9182    fn non_reserved_module_ignores_launch_nonce() {
9183        let registry = Arc::new(Registry::default());
9184        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
9185        // registration succeeds whether a spawned process echoes a nonce or not.
9186        let handler = ControlHandler::new(Arc::clone(&registry));
9187        let no_nonce = handler
9188            .handle_control(
9189                ConnectionId::new(1),
9190                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
9191            )
9192            .unwrap();
9193        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
9194        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
9195
9196        let echoed_nonce = handler
9197            .handle_control(
9198                ConnectionId::new(2),
9199                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
9200            )
9201            .unwrap();
9202        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
9203        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
9204    }
9205
9206    #[test]
9207    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
9208        let handler = ControlHandler::default();
9209        let conn = ConnectionId::new(1);
9210        let malformed = Frame::build(
9211            FrameType::Hello,
9212            control_flags(),
9213            0,
9214            0,
9215            3,
9216            b"{not json".to_vec(),
9217        )
9218        .unwrap();
9219
9220        let error = handler.handle_control(conn, malformed).unwrap();
9221        assert_eq!(error[0].header.ty, FrameType::Error);
9222        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
9223
9224        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
9225        let pong = handler.handle_control(conn, ping).unwrap();
9226        assert_eq!(pong[0].header.ty, FrameType::Pong);
9227        assert_eq!(pong[0].header.corr, 4);
9228    }
9229
9230    #[test]
9231    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
9232        let registry = Arc::new(Registry::default());
9233        let handler = ControlHandler::new(Arc::clone(&registry));
9234
9235        handler
9236            .handle_control(
9237                ConnectionId::new(1),
9238                hello_frame("aft", PROTOCOL_VERSION, 1),
9239            )
9240            .unwrap();
9241        let duplicate = handler
9242            .handle_control(
9243                ConnectionId::new(2),
9244                hello_frame("aft", PROTOCOL_VERSION, 2),
9245            )
9246            .unwrap();
9247
9248        assert_eq!(duplicate[0].header.ty, FrameType::Error);
9249        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
9250        let registration = registry.get_module("aft").unwrap().unwrap();
9251        assert_eq!(registration.connection_id, ConnectionId::new(1));
9252    }
9253
9254    #[test]
9255    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
9256        let registry = Arc::new(Registry::default());
9257        let forwarding = Arc::new(ForwardingTable::default());
9258        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
9259        let handler =
9260            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9261                .with_process_liveness(process_liveness);
9262        let (ctx, route_channel, route_epoch) =
9263            bind_liveness_route(&registry, &forwarding, "aft-dead");
9264        let responses = handler
9265            .handle_route_poll(
9266                &ctx,
9267                route_poll_frame(41, PollKind::Liveness, route_channel),
9268                route_channel,
9269                route_epoch,
9270                PollKind::Liveness,
9271            )
9272            .unwrap();
9273
9274        assert_eq!(responses.len(), 1);
9275        assert_eq!(responses[0].header.ty, FrameType::Response);
9276        assert_route_poll_liveness(&responses[0], false);
9277    }
9278
9279    #[test]
9280    fn liveness_poll_without_process_source_uses_bound_route() {
9281        let registry = Arc::new(Registry::default());
9282        let forwarding = Arc::new(ForwardingTable::default());
9283        let handler =
9284            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9285        let (ctx, route_channel, route_epoch) =
9286            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
9287        let responses = handler
9288            .handle_route_poll(
9289                &ctx,
9290                route_poll_frame(42, PollKind::Liveness, route_channel),
9291                route_channel,
9292                route_epoch,
9293                PollKind::Liveness,
9294            )
9295            .unwrap();
9296
9297        assert_route_poll_liveness(&responses[0], true);
9298    }
9299
9300    #[test]
9301    fn liveness_poll_untracked_process_source_uses_bound_route() {
9302        let registry = Arc::new(Registry::default());
9303        let forwarding = Arc::new(ForwardingTable::default());
9304        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
9305        let handler =
9306            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9307                .with_process_liveness(process_liveness);
9308        let (ctx, route_channel, route_epoch) =
9309            bind_liveness_route(&registry, &forwarding, "aft-untracked");
9310        let responses = handler
9311            .handle_route_poll(
9312                &ctx,
9313                route_poll_frame(43, PollKind::Liveness, route_channel),
9314                route_channel,
9315                route_epoch,
9316                PollKind::Liveness,
9317            )
9318            .unwrap();
9319
9320        assert_route_poll_liveness(&responses[0], true);
9321    }
9322
9323    #[tokio::test]
9324    async fn unknown_op_returns_unknown_control_op() {
9325        let handler = ControlHandler::default();
9326        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
9327        let request = Frame::build(
9328            FrameType::Request,
9329            control_flags(),
9330            0,
9331            0,
9332            55,
9333            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
9334        )
9335        .unwrap();
9336
9337        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9338
9339        assert_eq!(response.len(), 1);
9340        assert_eq!(response[0].header.ty, FrameType::Error);
9341        assert_eq!(response[0].header.corr, 55);
9342        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
9343    }
9344
9345    #[tokio::test]
9346    async fn supervisor_provenance_rejects_unknown_exact_module() {
9347        let handler = ControlHandler::default();
9348        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
9349        let request = Frame::build(
9350            FrameType::Request,
9351            control_flags(),
9352            0,
9353            0,
9354            57,
9355            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
9356        )
9357        .unwrap();
9358
9359        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9360
9361        assert_eq!(response.len(), 1);
9362        assert_eq!(response[0].header.ty, FrameType::Error);
9363        assert_eq!(response[0].header.corr, 57);
9364        let error = parse_error(&response[0]);
9365        assert_eq!(error["code"], "unknown_module");
9366        assert_eq!(error["message"], "module_id 'missing' is not supervised");
9367    }
9368
9369    #[test]
9370    fn provenance_probe_override_keeps_handler_tests_deterministic() {
9371        let expected = subc_control::RunningImageAgreement::Unavailable {
9372            reason: subc_control::RunningImageUnavailableReason::HashFailed,
9373        };
9374        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
9375        assert_eq!(handler.provenance_probe_override, Some(expected));
9376    }
9377
9378    #[test]
9379    fn reload_verdict_detects_configured_program_different_from_spawned_path() {
9380        let verdict = reload_verdict(
9381            std::path::Path::new("/bin/new"),
9382            Some(std::path::Path::new("/bin/old")),
9383            subc_control::RunningImageAgreement::Unavailable {
9384                reason: subc_control::RunningImageUnavailableReason::HashFailed,
9385            },
9386        );
9387        assert!(matches!(
9388            verdict.path,
9389            subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
9390                if configured == std::path::Path::new("/bin/new")
9391                    && spawned_from == std::path::Path::new("/bin/old")
9392        ));
9393    }
9394
9395    #[test]
9396    fn reload_verdict_detects_replaced_image_at_same_path() {
9397        let image = subc_control::RunningImageAgreement::Mismatch {
9398            running: subc_control::RunningImageEvidence::LinuxProcSha256 {
9399                digest: "old".into(),
9400            },
9401            disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
9402                digest: "new".into(),
9403            },
9404        };
9405        let verdict = reload_verdict(
9406            std::path::Path::new("/bin/same"),
9407            Some(std::path::Path::new("/bin/same")),
9408            image.clone(),
9409        );
9410        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
9411        assert_eq!(verdict.image, image);
9412    }
9413
9414    #[test]
9415    fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
9416        let image = subc_control::RunningImageAgreement::Unavailable {
9417            reason: subc_control::RunningImageUnavailableReason::NotRunning,
9418        };
9419        let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
9420        assert_eq!(
9421            verdict.path,
9422            subc_control::ReloadPathAgreement::Unavailable {
9423                reason: subc_control::ReloadPathUnavailableReason::NotRunning,
9424            }
9425        );
9426        assert_eq!(verdict.image, image);
9427
9428        let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
9429            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
9430        };
9431        let verdict = reload_verdict(
9432            std::path::Path::new("/bin/same"),
9433            Some(std::path::Path::new("/bin/same")),
9434            unconfirmed.clone(),
9435        );
9436        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
9437        assert_eq!(verdict.image, unconfirmed);
9438    }
9439
9440    #[test]
9441    fn reload_verdict_preserves_each_image_unavailability_reason() {
9442        use subc_control::RunningImageUnavailableReason as Reason;
9443
9444        for reason in [
9445            Reason::NotRunning,
9446            Reason::UnsupportedPlatform,
9447            Reason::RunningExecutableUnreadable,
9448            Reason::SpawnedPathUnreadable,
9449            Reason::HashFailed,
9450            Reason::ProcessIdentityUnconfirmed,
9451            Reason::Unknown("future_probe_reason".to_string()),
9452        ] {
9453            let image = subc_control::RunningImageAgreement::Unavailable {
9454                reason: reason.clone(),
9455            };
9456            let verdict = reload_verdict(
9457                std::path::Path::new("/bin/same"),
9458                Some(std::path::Path::new("/bin/same")),
9459                image.clone(),
9460            );
9461            assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
9462            assert_eq!(verdict.image, image, "{reason:?}");
9463        }
9464    }
9465
9466    #[tokio::test]
9467    async fn malformed_control_bodies_return_invalid_control_body() {
9468        let handler = ControlHandler::default();
9469        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
9470
9471        for (corr, body) in [
9472            (56, br#"{"route_channel":1}"#.as_slice()),
9473            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
9474            (
9475                58,
9476                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
9477            ),
9478        ] {
9479            let request = Frame::build(
9480                FrameType::Request,
9481                control_flags(),
9482                0,
9483                0,
9484                corr,
9485                body.to_vec(),
9486            )
9487            .unwrap();
9488            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9489
9490            assert_eq!(response.len(), 1);
9491            assert_eq!(response[0].header.ty, FrameType::Error);
9492            assert_eq!(response[0].header.corr, corr);
9493            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
9494        }
9495    }
9496
9497    #[tokio::test]
9498    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
9499        let registry = Arc::new(Registry::default());
9500        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
9501        let router = Router::with_control_handler(Arc::clone(&control));
9502        let connection = router.begin_connection();
9503        let (ctx, mut rx) = route_ctx(connection.id());
9504
9505        router
9506            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
9507            .await
9508            .unwrap();
9509        let response = rx.recv().await.unwrap();
9510        let ack = parse_ack(&response);
9511        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9512        let channel = 1;
9513
9514        let goodbye =
9515            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
9516        router.route_for_connection(&ctx, goodbye).await.unwrap();
9517        assert!(rx.try_recv().is_err());
9518        assert!(registry.get_module("aft").unwrap().is_none());
9519
9520        router
9521            .route_for_connection(&ctx, channel_request(channel, 13))
9522            .await
9523            .unwrap();
9524        let error_frame = rx.recv().await.unwrap();
9525        assert_eq!(error_frame.header.ty, FrameType::Error);
9526        assert_eq!(error_frame.header.channel, channel);
9527    }
9528
9529    #[tokio::test]
9530    async fn dropping_router_connection_releases_registration() {
9531        let registry = Arc::new(Registry::default());
9532        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
9533        let router = Router::with_control_handler(control);
9534        let connection = router.begin_connection();
9535        let (ctx, mut rx) = route_ctx(connection.id());
9536
9537        router
9538            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
9539            .await
9540            .unwrap();
9541        let response = rx.recv().await.unwrap();
9542        let ack = parse_ack(&response);
9543        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9544        assert!(registry.get_module("aft").unwrap().is_some());
9545
9546        drop(connection);
9547
9548        assert!(registry.get_module("aft").unwrap().is_none());
9549        assert_eq!(registry.active_registration_count().unwrap(), 0);
9550    }
9551
9552    fn capability_manifest(
9553        module_id: &str,
9554        provides: &[&str],
9555        must_never_reach: &[&str],
9556    ) -> ModuleManifest {
9557        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
9558        manifest.capabilities = Some(CapabilityDeclarations {
9559            provides: provides
9560                .iter()
9561                .map(|capability| (*capability).to_string())
9562                .collect(),
9563            requires: Vec::new(),
9564            must_never_reach: must_never_reach
9565                .iter()
9566                .map(|capability| (*capability).to_string())
9567                .collect(),
9568        });
9569        manifest
9570    }
9571
9572    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
9573        Frame::build(
9574            FrameType::Hello,
9575            control_flags(),
9576            0,
9577            0,
9578            corr,
9579            serde_json::to_vec(&ModuleHelloBody {
9580                protocol_ver: manifest.protocol_ver,
9581                manifest,
9582                control_ops: None,
9583                launch_nonce: None,
9584            })
9585            .expect("capability test HELLO serializes"),
9586        )
9587        .expect("capability test HELLO frame builds")
9588    }
9589
9590    fn catalog_update_with_capabilities_frame(
9591        corr: u64,
9592        capabilities: CapabilityDeclarations,
9593    ) -> Frame {
9594        Frame::build(
9595            FrameType::Request,
9596            control_flags(),
9597            0,
9598            0,
9599            corr,
9600            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
9601                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
9602                capabilities: Some(capabilities),
9603                ready: None,
9604            })
9605            .expect("capability catalog.update serializes"),
9606        )
9607        .expect("capability catalog.update frame builds")
9608    }
9609
9610    async fn register_capability_manifest(
9611        handler: &ControlHandler,
9612        ctx: &RouteCtx,
9613        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9614        manifest: ModuleManifest,
9615        corr: u64,
9616    ) {
9617        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
9618    }
9619
9620    async fn open_route_for_capability_test(
9621        handler: &ControlHandler,
9622        target_ctx: &RouteCtx,
9623        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9624        client_connection_id: u64,
9625        corr: u64,
9626        target_module_id: &str,
9627        consumer_identity: Option<ConsumerIdentity>,
9628    ) -> (
9629        mpsc::Receiver<crate::router::OutboundFrame>,
9630        ModuleControlRequest,
9631    ) {
9632        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
9633        let route_handler = handler.clone();
9634        let target_module_id = target_module_id.to_string();
9635        let route_task = tokio::spawn(async move {
9636            route_handler
9637                .handle_control_frame(
9638                    &client_ctx,
9639                    route_open_frame_with_admission_facts(
9640                        corr,
9641                        &target_module_id,
9642                        unique_project_root("admission-facts"),
9643                        consumer_identity,
9644                        None,
9645                    ),
9646                )
9647                .await
9648                .expect("capability test route.open succeeds")
9649        });
9650        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
9651            .await
9652            .expect("capability test route.open must reach route.bind")
9653            .expect("target control receiver stays open");
9654        let bind_request: ModuleControlRequest =
9655            serde_json::from_slice(&bind.body).expect("route.bind decodes");
9656        handler
9657            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
9658            .await
9659            .expect("capability test route.bind ACK succeeds");
9660        assert!(route_task.await.expect("route.open task joins").is_empty());
9661        let opened = client_rx
9662            .recv()
9663            .await
9664            .expect("successful route.open publishes a response");
9665        assert!(matches!(
9666            serde_json::from_slice::<ClientControlResponse>(&opened.body),
9667            Ok(ClientControlResponse::RouteOpen { .. })
9668        ));
9669        (client_rx, bind_request)
9670    }
9671
9672    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
9673        assert_eq!(frame.header.ty, FrameType::Push);
9674        assert_eq!(frame.header.channel, 0);
9675        assert_eq!(
9676            serde_json::from_slice::<ClientControlPush>(&frame.body)
9677                .expect("route.closed control push decodes"),
9678            ClientControlPush::RouteClosed {
9679                module_id: target_module_id.to_string(),
9680                reason: RouteCloseReason::CapabilityDenied,
9681                drained: false,
9682                abandoned: 0,
9683                excluded_subscriptions: 0,
9684                terminal: Some(false),
9685            }
9686        );
9687    }
9688
9689    #[tokio::test]
9690    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
9691        let registry = Arc::new(Registry::default());
9692        let forwarding = Arc::new(ForwardingTable::default());
9693        let supervisor = SupervisorHandle::new();
9694        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9695        let handler =
9696            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9697                .with_supervisor(supervisor);
9698        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
9699        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
9700        register_capability_manifest(
9701            &handler,
9702            &target_ctx,
9703            &mut target_rx,
9704            capability_manifest("target", &["credentials-provider/v1"], &[]),
9705            1,
9706        )
9707        .await;
9708        register_capability_manifest(
9709            &handler,
9710            &opener_ctx,
9711            &mut opener_rx,
9712            capability_manifest("opener", &[], &["credentials-provider/v1"]),
9713            2,
9714        )
9715        .await;
9716
9717        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
9718        let replies = handler
9719            .handle_control_frame(
9720                &client_ctx,
9721                route_open_frame_with_admission_facts(
9722                    3,
9723                    "target",
9724                    unique_project_root("admission-facts"),
9725                    Some(ConsumerIdentity {
9726                        module_id: "opener".to_string(),
9727                        launch_nonce: "opener-nonce".to_string(),
9728                    }),
9729                    None,
9730                ),
9731            )
9732            .await
9733            .expect("denied route.open returns a typed frame");
9734        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
9735        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9736        assert!(
9737            target_rx.try_recv().is_err(),
9738            "forbidden route.open must not relay route.bind"
9739        );
9740    }
9741
9742    #[tokio::test]
9743    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
9744        let registry = Arc::new(Registry::default());
9745        let forwarding = Arc::new(ForwardingTable::default());
9746        let supervisor = SupervisorHandle::new();
9747        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9748        let handler =
9749            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9750                .with_supervisor(supervisor);
9751        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
9752        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
9753        register_capability_manifest(
9754            &handler,
9755            &target_ctx,
9756            &mut target_rx,
9757            capability_manifest("target", &["credentials-provider/v1"], &[]),
9758            1,
9759        )
9760        .await;
9761        register_capability_manifest(
9762            &handler,
9763            &old_opener_ctx,
9764            &mut old_opener_rx,
9765            capability_manifest("opener", &[], &[]),
9766            2,
9767        )
9768        .await;
9769        let (mut client_rx, _) = open_route_for_capability_test(
9770            &handler,
9771            &target_ctx,
9772            &mut target_rx,
9773            712,
9774            3,
9775            "target",
9776            Some(ConsumerIdentity {
9777                module_id: "opener".to_string(),
9778                launch_nonce: "opener-nonce".to_string(),
9779            }),
9780        )
9781        .await;
9782        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9783
9784        handler
9785            .cleanup_connection(old_opener_ctx.connection_id)
9786            .expect("old opener registration cleans up");
9787        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
9788        register_capability_manifest(
9789            &handler,
9790            &new_opener_ctx,
9791            &mut new_opener_rx,
9792            capability_manifest("opener", &[], &["credentials-provider/v1"]),
9793            4,
9794        )
9795        .await;
9796
9797        assert_capability_denied_push(
9798            client_rx
9799                .try_recv()
9800                .expect("HELLO deny addition must emit route.closed")
9801                .frame,
9802            "target",
9803        );
9804        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9805        assert!(matches!(
9806            target_rx.try_recv(),
9807            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
9808        ));
9809    }
9810
9811    #[tokio::test]
9812    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
9813        let registry = Arc::new(Registry::default());
9814        let forwarding = Arc::new(ForwardingTable::default());
9815        let supervisor = SupervisorHandle::new();
9816        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9817        let handler =
9818            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9819                .with_supervisor(supervisor);
9820        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
9821        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
9822        register_capability_manifest(
9823            &handler,
9824            &target_ctx,
9825            &mut target_rx,
9826            capability_manifest("target", &[], &[]),
9827            1,
9828        )
9829        .await;
9830        register_capability_manifest(
9831            &handler,
9832            &opener_ctx,
9833            &mut opener_rx,
9834            capability_manifest("opener", &[], &["credentials-provider/v1"]),
9835            2,
9836        )
9837        .await;
9838        let (mut client_rx, _) = open_route_for_capability_test(
9839            &handler,
9840            &target_ctx,
9841            &mut target_rx,
9842            722,
9843            3,
9844            "target",
9845            Some(ConsumerIdentity {
9846                module_id: "opener".to_string(),
9847                launch_nonce: "opener-nonce".to_string(),
9848            }),
9849        )
9850        .await;
9851        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9852
9853        let replies = handler
9854            .handle_control_frame(
9855                &target_ctx,
9856                catalog_update_with_capabilities_frame(
9857                    4,
9858                    CapabilityDeclarations {
9859                        provides: vec!["credentials-provider/v1".to_string()],
9860                        requires: Vec::new(),
9861                        must_never_reach: Vec::new(),
9862                    },
9863                ),
9864            )
9865            .await
9866            .expect("claim catalog.update succeeds");
9867        assert!(matches!(
9868            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
9869            Ok(ModuleControlResponseToModule::CatalogUpdate {})
9870        ));
9871        assert_capability_denied_push(
9872            client_rx
9873                .try_recv()
9874                .expect("claim addition must emit route.closed")
9875                .frame,
9876            "target",
9877        );
9878        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9879        assert!(matches!(
9880            target_rx.try_recv(),
9881            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
9882        ));
9883    }
9884
9885    #[tokio::test]
9886    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
9887        let registry = Arc::new(Registry::default());
9888        let forwarding = Arc::new(ForwardingTable::default());
9889        let supervisor = SupervisorHandle::new();
9890        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9891        let handler =
9892            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9893                .with_supervisor(supervisor);
9894        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
9895        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
9896        register_capability_manifest(
9897            &handler,
9898            &target_ctx,
9899            &mut target_rx,
9900            capability_manifest("target", &["credentials-provider/v1"], &[]),
9901            1,
9902        )
9903        .await;
9904        register_capability_manifest(
9905            &handler,
9906            &opener_ctx,
9907            &mut opener_rx,
9908            capability_manifest("opener", &[], &[]),
9909            2,
9910        )
9911        .await;
9912        let (mut client_rx, _) = open_route_for_capability_test(
9913            &handler,
9914            &target_ctx,
9915            &mut target_rx,
9916            732,
9917            3,
9918            "target",
9919            Some(ConsumerIdentity {
9920                module_id: "opener".to_string(),
9921                launch_nonce: "opener-nonce".to_string(),
9922            }),
9923        )
9924        .await;
9925        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9926
9927        handler
9928            .handle_control_frame(
9929                &target_ctx,
9930                catalog_update_with_capabilities_frame(
9931                    4,
9932                    CapabilityDeclarations {
9933                        provides: Vec::new(),
9934                        requires: Vec::new(),
9935                        must_never_reach: Vec::new(),
9936                    },
9937                ),
9938            )
9939            .await
9940            .expect("claim removal catalog.update succeeds");
9941        assert_eq!(
9942            forwarding.active_binding_count().unwrap(),
9943            1,
9944            "removing an attested target claim must leave the route census unchanged"
9945        );
9946        assert!(
9947            client_rx.try_recv().is_err(),
9948            "claim removal must not emit route.closed capability_denied"
9949        );
9950        assert!(
9951            target_rx.try_recv().is_err(),
9952            "claim removal must not send the target a route GOODBYE"
9953        );
9954    }
9955
9956    /// A direct client may open a route to a denied capability provider; this
9957    /// policy applies only to attested supervised module origins, not to direct clients.
9958    #[tokio::test]
9959    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
9960        let registry = Arc::new(Registry::default());
9961        let forwarding = Arc::new(ForwardingTable::default());
9962        let supervisor = SupervisorHandle::new();
9963        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9964        let handler =
9965            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9966                .with_supervisor(supervisor);
9967        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
9968        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
9969        register_capability_manifest(
9970            &handler,
9971            &target_ctx,
9972            &mut target_rx,
9973            capability_manifest("target", &["credentials-provider/v1"], &[]),
9974            1,
9975        )
9976        .await;
9977        register_capability_manifest(
9978            &handler,
9979            &opener_ctx,
9980            &mut opener_rx,
9981            capability_manifest("opener", &[], &["credentials-provider/v1"]),
9982            2,
9983        )
9984        .await;
9985
9986        let (_client_rx, bind) = open_route_for_capability_test(
9987            &handler,
9988            &target_ctx,
9989            &mut target_rx,
9990            742,
9991            3,
9992            "target",
9993            None,
9994        )
9995        .await;
9996        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
9997            panic!("direct scope-honesty route must bind");
9998        };
9999        assert_eq!(principal, Some(Principal::Direct));
10000        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10001    }
10002
10003    /// A module that denies a capability receives no self-route exemption when it
10004    /// also attestedly provides that capability.
10005    #[tokio::test]
10006    async fn must_never_reach_self_route_is_capability_forbidden() {
10007        let registry = Arc::new(Registry::default());
10008        let forwarding = Arc::new(ForwardingTable::default());
10009        let supervisor = SupervisorHandle::new();
10010        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
10011        let handler =
10012            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10013                .with_supervisor(supervisor);
10014        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
10015        register_capability_manifest(
10016            &handler,
10017            &self_ctx,
10018            &mut self_rx,
10019            capability_manifest(
10020                "self-provider",
10021                &["credentials-provider/v1"],
10022                &["credentials-provider/v1"],
10023            ),
10024            1,
10025        )
10026        .await;
10027
10028        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
10029        let replies = handler
10030            .handle_control_frame(
10031                &client_ctx,
10032                route_open_frame_with_admission_facts(
10033                    2,
10034                    "self-provider",
10035                    unique_project_root("admission-facts"),
10036                    Some(ConsumerIdentity {
10037                        module_id: "self-provider".to_string(),
10038                        launch_nonce: "self-nonce".to_string(),
10039                    }),
10040                    None,
10041                ),
10042            )
10043            .await
10044            .expect("self-route refusal returns a typed frame");
10045        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
10046        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10047        assert!(
10048            self_rx.try_recv().is_err(),
10049            "self denial must not relay route.bind"
10050        );
10051    }
10052
10053    #[test]
10054    fn unsupported_channel_zero_frame_returns_error() {
10055        let handler = ControlHandler::default();
10056        let request = Frame::build(
10057            FrameType::Request,
10058            control_flags(),
10059            0,
10060            0,
10061            21,
10062            b"opaque".to_vec(),
10063        )
10064        .unwrap();
10065
10066        let response = handler
10067            .handle_control(ConnectionId::new(1), request)
10068            .unwrap();
10069
10070        assert_eq!(response[0].header.ty, FrameType::Error);
10071        assert_eq!(
10072            parse_error(&response[0])["code"],
10073            "unsupported_control_frame"
10074        );
10075    }
10076
10077    /// Blue/green swap at the control-plane boundary. The supervisor that opens
10078    /// a swap is not wired yet, so the candidate is registered here directly
10079    /// into the registry and forwarding candidate slots, the way the swap's
10080    /// HELLO admission will.
10081    mod swap {
10082        use super::*;
10083
10084        const INCUMBENT: ConnectionId = ConnectionId::new(30);
10085        const CANDIDATE: ConnectionId = ConnectionId::new(40);
10086
10087        struct Swap {
10088            registry: Arc<Registry>,
10089            forwarding: Arc<ForwardingTable>,
10090            handler: ControlHandler,
10091            incumbent_ctx: RouteCtx,
10092            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
10093            candidate_ctx: RouteCtx,
10094            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
10095        }
10096
10097        async fn swap_with_incumbent() -> Swap {
10098            let registry = Arc::new(Registry::default());
10099            let forwarding = Arc::new(ForwardingTable::default());
10100            let handler =
10101                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10102            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
10103            hello_via_sink(
10104                &handler,
10105                &incumbent_ctx,
10106                &mut incumbent_rx,
10107                hello_frame("aft", PROTOCOL_VERSION, 7),
10108            )
10109            .await;
10110            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
10111            Swap {
10112                registry,
10113                forwarding,
10114                handler,
10115                incumbent_ctx,
10116                incumbent_rx,
10117                candidate_ctx,
10118                candidate_rx,
10119            }
10120        }
10121
10122        fn register_candidate(swap: &Swap, ready: Option<bool>) {
10123            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
10124            candidate_manifest.ready = ready;
10125            let registration = swap
10126                .registry
10127                .register_candidate_with_control_ops(
10128                    candidate_manifest,
10129                    PROTOCOL_VERSION,
10130                    CANDIDATE,
10131                    module_baseline_control_ops(),
10132                )
10133                .unwrap();
10134            swap.forwarding
10135                .register_candidate_module_connection(
10136                    CANDIDATE,
10137                    "aft".to_string(),
10138                    PROTOCOL_VERSION,
10139                    manifest_concurrency(&registration.manifest),
10140                    swap.candidate_ctx.egress.clone(),
10141                )
10142                .unwrap();
10143        }
10144
10145        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
10146            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
10147            swap.registry.promote_candidate("aft").unwrap().unwrap();
10148            cutover.incumbent.unwrap()
10149        }
10150
10151        fn keyed_total(counters: &Value, key: &str) -> u64 {
10152            counters[key]
10153                .as_object()
10154                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
10155                .unwrap_or(0)
10156        }
10157
10158        /// An ack from the incumbent for a bind it was sent before cutover,
10159        /// arriving before the incumbent is drained. The incumbent is the live
10160        /// connection carrying every other client's routes, so the ack must
10161        /// not end it: the waiting client is told to retry, the reservation is
10162        /// given back, and the incumbent is told to drop just that binding.
10163        #[tokio::test]
10164        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
10165            let mut swap = swap_with_incumbent().await;
10166            let handler = swap.handler.clone();
10167
10168            // A co-tenant route, bound on the incumbent before the swap.
10169            let cotenant = ConnectionId::new(31);
10170            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
10171            let (cotenant_task, cotenant_bind) = relay_route_open(
10172                &handler,
10173                cotenant,
10174                &cotenant_ctx.egress,
10175                &mut swap.incumbent_rx,
10176                100,
10177                "aft",
10178                "swap-cotenant",
10179            )
10180            .await;
10181            handler
10182                .handle_control_frame(
10183                    &swap.incumbent_ctx,
10184                    route_bind_ack(cotenant_bind.header.corr),
10185                )
10186                .await
10187                .unwrap();
10188            assert!(cotenant_task.await.unwrap().is_empty());
10189            let (cotenant_channel, cotenant_epoch) =
10190                published_route(&cotenant_rx.recv().await.unwrap());
10191
10192            // A second route.open, relayed to the incumbent and not yet acked.
10193            let caller = ConnectionId::new(32);
10194            let (caller_ctx, mut caller_rx) = route_ctx(caller);
10195            let (caller_task, caller_bind) = relay_route_open(
10196                &handler,
10197                caller,
10198                &caller_ctx.egress,
10199                &mut swap.incumbent_rx,
10200                101,
10201                "aft",
10202                "swap-caller",
10203            )
10204            .await;
10205            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
10206
10207            register_candidate(&swap, None);
10208            cutover(&swap);
10209
10210            // The incumbent acks after promotion and before any drain.
10211            let ack = handler
10212                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
10213                .await;
10214            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
10215            if module_loop_error.is_some() {
10216                // What the connection loop does with an untranslated router
10217                // error: end the connection, releasing every route on it.
10218                handler.cleanup_connection(INCUMBENT).unwrap();
10219            }
10220
10221            // 1. The incumbent's other routes survive.
10222            assert!(
10223                cotenant_rx.try_recv().is_err(),
10224                "the co-tenant route on the incumbent was torn down by one late ack: \
10225                 {module_loop_error:?}"
10226            );
10227            assert!(matches!(
10228                swap.forwarding
10229                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
10230                    .unwrap(),
10231                DataRoute::Client(DataRouteState::Bound(_))
10232            ));
10233            assert_eq!(module_loop_error, None);
10234            assert!(swap
10235                .registry
10236                .get_module_by_connection(INCUMBENT)
10237                .unwrap()
10238                .is_some());
10239
10240            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
10241            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
10242                .await
10243                .expect("the incumbent is told to drop the abandoned binding")
10244                .unwrap()
10245                .frame;
10246            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10247            assert_eq!(goodbye.header.channel, abandoned_channel);
10248            assert_eq!(goodbye.header.epoch, abandoned_epoch);
10249            assert!(swap.incumbent_rx.try_recv().is_err());
10250
10251            // 3. The waiting client gets a retryable refusal and no route.
10252            let response = caller_task.await.unwrap();
10253            assert_eq!(response.len(), 1);
10254            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
10255            assert!(caller_rx.try_recv().is_err());
10256
10257            // 4. The reservation pair is given back, and the pending bind
10258            //    settled exactly once: one accepted open (the co-tenant) and one
10259            //    refused open (the caller), nothing counted twice.
10260            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
10261            let counters = handler.counters().snapshot();
10262            assert_eq!(
10263                keyed_total(&counters, "route_open_accepted_by_principal"),
10264                1
10265            );
10266            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
10267            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
10268        }
10269
10270        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
10271        /// id would resolve to the promoted candidate and every new route.open
10272        /// would be refused as reloading, leaving neither process routable.
10273        #[tokio::test]
10274        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
10275            let mut swap = swap_with_incumbent().await;
10276            register_candidate(&swap, None);
10277            let incumbent = cutover(&swap);
10278            swap.forwarding
10279                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
10280                .unwrap()
10281                .expect("the incumbent is still registered");
10282
10283            let client = ConnectionId::new(33);
10284            let (client_ctx, mut client_rx) = route_ctx(client);
10285            let route_handler = swap.handler.clone();
10286            let open_ctx = RouteCtx {
10287                connection_id: client,
10288                egress: client_ctx.egress.clone(),
10289            };
10290            let mut route_task = tokio::spawn(async move {
10291                route_handler
10292                    .handle_control_frame(
10293                        &open_ctx,
10294                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
10295                    )
10296                    .await
10297                    .unwrap()
10298            });
10299            let bind = tokio::select! {
10300                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
10301                response = &mut route_task => {
10302                    let response = response.unwrap();
10303                    panic!(
10304                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
10305                        parse_error(&response[0])["code"]
10306                    );
10307                }
10308            };
10309            swap.handler
10310                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
10311                .await
10312                .unwrap();
10313            assert!(route_task.await.unwrap().is_empty());
10314            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
10315            match swap
10316                .forwarding
10317                .lookup_data_route(client, channel, epoch)
10318                .unwrap()
10319            {
10320                DataRoute::Client(DataRouteState::Bound(route)) => {
10321                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
10322                }
10323                other => panic!("expected a bound route on the candidate, got {other:?}"),
10324            }
10325            assert!(swap.incumbent_rx.try_recv().is_err());
10326        }
10327
10328        /// A candidate declares itself ready with `catalog.update` on its own
10329        /// connection. If the connection-keyed registry lookups searched only the
10330        /// active slot, this would answer `not_registered` and the candidate
10331        /// would never become ready.
10332        #[tokio::test]
10333        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
10334            let swap = swap_with_incumbent().await;
10335            register_candidate(&swap, Some(false));
10336            let update = Frame::build(
10337                FrameType::Request,
10338                control_flags(),
10339                0,
10340                0,
10341                55,
10342                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
10343                    provides: manifest("aft", PROTOCOL_VERSION).provides,
10344                    capabilities: None,
10345                    ready: Some(true),
10346                })
10347                .unwrap(),
10348            )
10349            .unwrap();
10350
10351            let replies = swap
10352                .handler
10353                .handle_control_frame(&swap.candidate_ctx, update)
10354                .await
10355                .unwrap();
10356
10357            assert_eq!(replies.len(), 1);
10358            assert_eq!(
10359                replies[0].header.ty,
10360                FrameType::Response,
10361                "candidate catalog.update was refused: {:?}",
10362                serde_json::from_slice::<Value>(&replies[0].body).ok()
10363            );
10364            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
10365            assert_eq!(
10366                swap.registry
10367                    .get_module("aft")
10368                    .unwrap()
10369                    .unwrap()
10370                    .connection_id,
10371                INCUMBENT
10372            );
10373        }
10374    }
10375
10376    /// The HELLO gate while the supervisor has a swap open: only the nonce it
10377    /// minted for the candidate admits a second process, into the candidate
10378    /// slot, and that check runs ahead of the reserved-module gate.
10379    mod swap_admission {
10380        use super::*;
10381
10382        const INCUMBENT_NONCE: &str = "incumbent-nonce";
10383        const CANDIDATE_NONCE: &str = "candidate-nonce";
10384
10385        fn handler_with_incumbent(
10386            module_id: &str,
10387            reserved: bool,
10388        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
10389            let registry = Arc::new(Registry::default());
10390            let supervisor = SupervisorHandle::new();
10391            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
10392            if reserved {
10393                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
10394            }
10395            let handler =
10396                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
10397            let incumbent = handler
10398                .handle_control(
10399                    ConnectionId::new(1),
10400                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
10401                )
10402                .unwrap();
10403            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
10404            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
10405            (registry, supervisor, handler)
10406        }
10407
10408        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
10409        /// admits every nonce, so while a swap is open the swap gate is the only
10410        /// thing between a key-holder and the candidate slot. A nonce the
10411        /// supervisor did not mint, or none at all, is refused, and neither the
10412        /// incumbent's registration nor the candidate slot moves.
10413        #[test]
10414        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
10415            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
10416
10417            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
10418                let replies = handler
10419                    .handle_control(
10420                        ConnectionId::new(connection),
10421                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
10422                    )
10423                    .unwrap();
10424                assert_eq!(replies[0].header.ty, FrameType::Error);
10425                assert_eq!(
10426                    parse_error(&replies[0])["code"],
10427                    "swap_token_invalid",
10428                    "nonce {nonce:?}"
10429                );
10430            }
10431            assert!(registry.get_candidate("aft").unwrap().is_none());
10432            assert_eq!(
10433                registry.get_module("aft").unwrap().unwrap().connection_id,
10434                ConnectionId::new(1)
10435            );
10436
10437            // Control: the minted token is admitted, into the candidate slot,
10438            // and only once.
10439            let admitted = handler
10440                .handle_control(
10441                    ConnectionId::new(4),
10442                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
10443                )
10444                .unwrap();
10445            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
10446            assert_eq!(
10447                registry
10448                    .get_candidate("aft")
10449                    .unwrap()
10450                    .unwrap()
10451                    .connection_id,
10452                ConnectionId::new(4)
10453            );
10454            assert_eq!(
10455                registry.get_module("aft").unwrap().unwrap().connection_id,
10456                ConnectionId::new(1),
10457                "the candidate must not take the active slot"
10458            );
10459            let replayed = handler
10460                .handle_control(
10461                    ConnectionId::new(5),
10462                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
10463                )
10464                .unwrap();
10465            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
10466
10467            // The case only this gate covers: the incumbent has died mid-swap,
10468            // so its duplicate refusal is gone too, and without the gate a
10469            // key-holder would take the id's ACTIVE slot.
10470            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
10471            let squatter = handler
10472                .handle_control(
10473                    ConnectionId::new(6),
10474                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
10475                )
10476                .unwrap();
10477            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
10478            assert!(
10479                registry.get_module("aft").unwrap().is_none(),
10480                "a squatter took the active slot of an id being swapped"
10481            );
10482        }
10483
10484        /// Design mutation arm (iii). A reserved module's candidate presents a
10485        /// nonce the reserved gate has never seen (that gate holds the
10486        /// incumbent's), so the swap gate must run first or the candidate is
10487        /// refused `reserved_module` and a reserved module can never be swapped.
10488        #[test]
10489        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
10490            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
10491
10492            let replies = handler
10493                .handle_control(
10494                    ConnectionId::new(2),
10495                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
10496                )
10497                .unwrap();
10498
10499            assert_eq!(
10500                replies[0].header.ty,
10501                FrameType::HelloAck,
10502                "reserved candidate refused: {:?}",
10503                serde_json::from_slice::<Value>(&replies[0].body).ok()
10504            );
10505            assert_eq!(
10506                registry
10507                    .get_candidate("vault")
10508                    .unwrap()
10509                    .unwrap()
10510                    .connection_id,
10511                ConnectionId::new(2)
10512            );
10513        }
10514
10515        /// With no swap open the gate is inert: the incumbent's reserved gate
10516        /// and duplicate refusal behave exactly as before.
10517        #[test]
10518        fn without_an_open_swap_the_ordinary_gates_decide() {
10519            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
10520            supervisor.close_swap("vault");
10521
10522            let candidate = handler
10523                .handle_control(
10524                    ConnectionId::new(2),
10525                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
10526                )
10527                .unwrap();
10528            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
10529            let duplicate = handler
10530                .handle_control(
10531                    ConnectionId::new(3),
10532                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
10533                )
10534                .unwrap();
10535            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10536            assert!(registry.get_candidate("vault").unwrap().is_none());
10537        }
10538    }
10539}
10540
10541#[cfg(test)]
10542mod concurrency_default_exposure_tests {
10543    use super::*;
10544
10545    fn hello_body(role_json: &str) -> Vec<u8> {
10546        format!(
10547            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
10548        )
10549        .into_bytes()
10550    }
10551
10552    fn manifest_from(body: &[u8]) -> ModuleManifest {
10553        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
10554        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
10555            .expect("manifest parses")
10556    }
10557
10558    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
10559
10560    #[test]
10561    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
10562        let body = hello_body(&format!(
10563            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
10564        ));
10565        let manifest = manifest_from(&body);
10566        // Precondition: serde really resolved it to the default, so the typed
10567        // manifest alone cannot answer the question this probe exists for.
10568        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
10569        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
10570    }
10571
10572    #[test]
10573    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
10574        let body = hello_body(&format!(
10575            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
10576        ));
10577        let manifest = manifest_from(&body);
10578        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
10579        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
10580    }
10581
10582    #[test]
10583    fn non_management_roles_are_never_reported() {
10584        let body = hello_body(
10585            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
10586        );
10587        let manifest = manifest_from(&body);
10588        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
10589    }
10590}