Skip to main content

subc_daemon/
control.rs

1use std::{
2    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3    fmt,
4    path::PathBuf,
5    sync::{Arc, Mutex},
6    time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13    ModuleDeclaredProvenance, ModuleProtocol, PollKind, RouteCloseReason, SpawnCursor,
14    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
15    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
16    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
17};
18use subc_protocol::{
19    error_codes,
20    manifest::{
21        validate_hello_capability_grammar, validate_hello_self_signal_declarations,
22        CapabilityDeclarations, Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
23    },
24    session::{
25        HealthReport, ModuleControlPush, ModuleControlRequest, ModuleControlRequestFromModule,
26        ModuleControlResponse, ModuleControlResponseToModule, MODULE_CONTROL_OP_HEALTH_CHECK,
27        MODULE_TO_SUBC_OP_CATALOG_UPDATE,
28    },
29    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
30    Priority, RouteTarget, PROTOCOL_VERSION,
31};
32use tokio::time::{timeout_at, Instant};
33use tracing::{debug, info, warn};
34
35use crate::{
36    capability_requirements::{
37        log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
38        DuplicateClaimSource, RegisteredModule, RequirementStatus, RuntimeModule,
39    },
40    daemon_config::RestartRequiredSection,
41    forwarding::{
42        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
43        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
44        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
45    },
46    observability::ROUTE_OPEN_REFUSED_DECLARED_NOT_READY,
47    provenance::{
48        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
49    },
50    registry::{ChannelState, ConnectionId, Registry, RegistryError},
51    router::{RouteCtx, RouterError},
52    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
53    stderr_tail::{CaptureState, TailEntry},
54    supervise::{
55        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
56        SupervisorHandle, SwapHelloAdmission,
57    },
58    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
59};
60
61/// Lowest envelope version this subc build will negotiate.
62///
63/// Module HELLO negotiation is exact: peers must use the daemon's locked
64/// protocol version. Older and newer peers receive `version_unsupported` and
65/// are not registered.
66pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
67
68const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
69const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
70const CAP_PING_PONG: &str = "ping_pong_v1";
71const CAP_SESSION_ATTACH: &str = "session_attach_v1";
72const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
73
74const SUBC_CONTROL_OPS: &[&str] = &[
75    ops::SERVER_DESCRIBE,
76    ops::CATALOG_LIST,
77    ops::ROUTE_OPEN,
78    ops::ROUTE_POLL,
79    ops::ROUTE_CLOSING,
80    ops::ROUTE_CLOSED,
81    ops::SUPERVISOR_LIST,
82    ops::SUPERVISOR_RESTART,
83    ops::SUPERVISOR_SWAP,
84    ops::SUPERVISOR_RELOAD,
85    ops::SUPERVISOR_RESCAN,
86    ops::SUPERVISOR_RELEASE_RESERVED,
87    ops::SUPERVISOR_SET_ENABLED,
88    ops::SUPERVISOR_HEALTH_PROBE,
89    ops::SUPERVISOR_HEALTH,
90    ops::SUPERVISOR_STDERR_TAIL,
91    ops::SUPERVISOR_TERMINALS,
92    ops::SUPERVISOR_ROUTES,
93    ops::SUPERVISOR_PROVENANCE,
94    ops::SUPERVISOR_SPAWN_SNAPSHOT,
95    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
96];
97
98const MODULE_TO_SUBC_CONTROL_OPS: &[&str] =
99    &[MODULE_TO_SUBC_OP_CATALOG_UPDATE, "supervisor.live_roots"];
100
101const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
102
103/// How long subc waits for a module to ack a relayed route.bind before returning
104/// `module_timeout`. The ack waits on the module's own configure, which for AFT
105/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
106/// and DB-open work — on a cold page cache or a large repo that legitimately
107/// exceeds a couple of seconds. The default is generous because rejecting a VALID
108/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
109/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
110pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
111
112/// How many CONSECUTIVE full-budget relay timeouts against one target module
113/// open that module's bind-relay breaker.
114///
115/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
116/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
117/// so three consecutive full-budget timeouts take ~36s to observe: every client
118/// whose open contributed to opening the breaker had already given up on its
119/// own. That is what makes opening the breaker unable to turn a call that would
120/// have succeeded into a refusal — it can only make an already-failing module
121/// fail faster.
122///
123/// Two would be reachable inside one default deadline. One would convict a
124/// module on a single cold-cache bind, which is exactly the valid-but-slow case
125/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
126pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
127
128/// How long a module's bind-relay breaker stays open before exactly one
129/// `route.open` is let through as a probe.
130///
131/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
132/// re-pays a full-budget stall almost continuously, and the breaker stops being
133/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
134/// deadline: a client that starts retrying after the module recovers has to get
135/// a probe opportunity inside its own deadline, or the breaker converts a
136/// recovered module into a failed call — the failure it exists to prevent,
137/// pointed the other way.
138///
139/// 20s sits between those with room on both sides, and it caps what a wedged
140/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
141/// rather than one per `route.open` per connection. The stall that motivated
142/// this, with its measurements, is written up in
143/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
144/// waited the whole budget out.
145pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
146
147const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
148const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
149
150#[derive(Clone)]
151struct DaemonProvenanceFacts {
152    build: DaemonBuildProvenance,
153    pid: Option<u32>,
154    started_at_ms: Option<u64>,
155    start_clock: Option<crate::clock::StartClock>,
156    executable_path: Option<PathBuf>,
157    executable_identity: Option<SpawnedFileIdentity>,
158    process_start_time: Option<u64>,
159    probe: ExecutableIdentityProbe,
160}
161
162impl Default for DaemonProvenanceFacts {
163    fn default() -> Self {
164        Self {
165            build: DaemonBuildProvenance {
166                build_git_sha: None,
167                build_lock_digest: None,
168            },
169            pid: None,
170            started_at_ms: None,
171            start_clock: None,
172            executable_path: None,
173            executable_identity: None,
174            process_start_time: None,
175            probe: ExecutableIdentityProbe::default(),
176        }
177    }
178}
179
180#[derive(Debug, Clone)]
181struct SupervisorRescanContext {
182    supervisor: Supervisor,
183    config_path: PathBuf,
184    configured_port: Option<u16>,
185    storage_config: Option<crate::daemon_config::StorageConfig>,
186    admission_facts_carrier_module_id: Option<String>,
187    admission_facts_targets: Option<Vec<String>>,
188}
189
190/// Real channel-0 control handler for subc itself.
191#[derive(Clone)]
192pub struct ControlHandler {
193    registry: Arc<Registry>,
194    forwarding: Arc<ForwardingTable>,
195    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
196    supervisor: SupervisorHandle,
197    subc_capabilities: Arc<[String]>,
198    /// Daemon-wide route.bind relay budget. Used as the fallback when the
199    /// target module has no per-module override in
200    /// `route_bind_relay_timeouts`.
201    route_bind_relay_timeout: Duration,
202    /// Per-module route.bind relay budget overrides, keyed by module id. When
203    /// `handle_route_open` resolves the deadline for a target module, a
204    /// per-module entry wins over the daemon-wide value above.
205    route_bind_relay_timeouts: BTreeMap<String, Duration>,
206    /// Per-target-module bind-relay breaker state. Shared with the forwarding
207    /// table, which is where a new module connection resets it.
208    route_bind_breakers: RouteBindBreakers,
209    /// Live relay admissions keyed by target module. Shared through the
210    /// forwarding table so cloned or separately built handlers enforce one cap.
211    route_bind_concurrency: RouteBindConcurrency,
212    /// Consecutive relay timeouts that open a module's breaker.
213    route_bind_breaker_threshold: u32,
214    /// How long a breaker stays open before one probe is admitted.
215    route_bind_breaker_cooldown: Duration,
216    health_probe_timeout: Duration,
217    /// Central storage policy. When set, each registering module receives its
218    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
219    storage_config: Option<crate::daemon_config::StorageConfig>,
220    admission_facts_carrier_module_id: Option<String>,
221    admission_facts_targets: Option<Vec<String>>,
222    rescan: Option<SupervisorRescanContext>,
223    connected_clients: ConnectedClients,
224    counters: DaemonCounters,
225    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
226    daemon_provenance: DaemonProvenanceFacts,
227    #[cfg(test)]
228    control_dispatch_delay: Option<Duration>,
229    #[cfg(test)]
230    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
231}
232
233impl fmt::Debug for ControlHandler {
234    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
235        f.debug_struct("ControlHandler")
236            .field("registry", &self.registry)
237            .field("forwarding", &self.forwarding)
238            .field("process_liveness", &self.process_liveness.is_some())
239            .field("supervisor", &self.supervisor)
240            .field("subc_capabilities", &self.subc_capabilities)
241            .finish()
242    }
243}
244
245struct RouteOpenRequest {
246    target: RouteTarget,
247    identity: BindIdentity,
248    consumer_identity: Option<ConsumerIdentity>,
249    consumer_capabilities: Option<Vec<String>>,
250    admission_facts: Option<serde_json::Value>,
251}
252
253struct RouteBindReservationGuard {
254    forwarding: Arc<ForwardingTable>,
255    endpoint: ModuleEndpointId,
256    relay_corr: u64,
257    armed: bool,
258}
259
260struct ModuleControlRpcGuard {
261    forwarding: Arc<ForwardingTable>,
262    endpoint: ModuleEndpointId,
263    corr: u64,
264    armed: bool,
265}
266
267impl ModuleControlRpcGuard {
268    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
269        Self {
270            forwarding,
271            endpoint,
272            corr,
273            armed: true,
274        }
275    }
276
277    fn disarm(&mut self) {
278        self.armed = false;
279    }
280}
281
282impl Drop for ModuleControlRpcGuard {
283    fn drop(&mut self) {
284        if self.armed {
285            let _ = self
286                .forwarding
287                .cancel_module_control_rpc(self.endpoint, self.corr);
288        }
289    }
290}
291
292impl RouteBindReservationGuard {
293    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
294        Self {
295            forwarding,
296            endpoint,
297            relay_corr,
298            armed: true,
299        }
300    }
301
302    fn release_and_disarm(&mut self) {
303        if !self.armed {
304            return;
305        }
306        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
307            self.endpoint,
308            self.relay_corr,
309            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
310        ) {
311            send_goodbye_target_best_effort(&target, "canceled route.bind");
312        }
313        self.armed = false;
314    }
315
316    fn disarm(&mut self) {
317        self.armed = false;
318    }
319}
320
321impl Drop for RouteBindReservationGuard {
322    fn drop(&mut self) {
323        self.release_and_disarm();
324    }
325}
326
327/// Per-target-module circuit breaker around the `route.bind` relay.
328///
329/// The connection reader is serial per connection, so a module whose `on_bind`
330/// sits on the ack blocks every LATER frame on the connections that call it,
331/// including calls to unrelated modules. This does not make any module's bind
332/// fast; it stops the daemon paying the full budget again and again for a
333/// condition it has already observed.
334///
335/// State is keyed by TARGET MODULE and shared by every connection: a wedged
336/// module wedges everyone, so what one connection learned should protect the
337/// rest.
338///
339/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
340/// relay to that module has actually timed out, and is removed again when a
341/// relay is accepted or the module reconnects, so it cannot grow with traffic
342/// or with modules that behave.
343///
344/// # Why a `std` mutex here is not the head-of-line defect again
345///
346/// Acquisition never awaits. The critical section is a hash lookup plus a few
347/// integer updates, with no I/O and no `.await` inside it, so a reader task
348/// cannot be descheduled behind it the way it can behind
349/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
350/// primitive, held for the same kind of work, as the refusal counter this very
351/// path already increments.
352///
353/// It is also NOT on the data-plane splice path: only `route.open` and module
354/// registration touch it, so bound-route frames gain no state check and no
355/// contention.
356#[derive(Debug, Clone, Default)]
357pub(crate) struct RouteBindBreakers {
358    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
359}
360
361#[derive(Debug, Clone, Default)]
362pub(crate) struct RouteBindConcurrency {
363    modules: Arc<Mutex<HashMap<String, usize>>>,
364}
365
366struct RouteBindConcurrencyGuard {
367    concurrency: RouteBindConcurrency,
368    module_id: String,
369}
370
371impl RouteBindConcurrency {
372    /// Admit without waiting. Waiting here would move the bind stall from the
373    /// module reply to a semaphore and restore reader head-of-line blocking.
374    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
375        let mut modules = self
376            .modules
377            .lock()
378            .expect("route.bind concurrency mutex poisoned");
379        let in_flight = modules.entry(module_id.to_string()).or_default();
380        if *in_flight >= limit {
381            return Err(*in_flight);
382        }
383        *in_flight += 1;
384        Ok(RouteBindConcurrencyGuard {
385            concurrency: self.clone(),
386            module_id: module_id.to_string(),
387        })
388    }
389}
390
391impl Drop for RouteBindConcurrencyGuard {
392    fn drop(&mut self) {
393        let mut modules = self
394            .concurrency
395            .modules
396            .lock()
397            .expect("route.bind concurrency mutex poisoned");
398        let remove = {
399            let in_flight = modules
400                .get_mut(&self.module_id)
401                .expect("admitted route.bind has a concurrency entry");
402            *in_flight -= 1;
403            *in_flight == 0
404        };
405        if remove {
406            modules.remove(&self.module_id);
407        }
408    }
409}
410
411#[derive(Debug, Default)]
412struct ModuleBreakerState {
413    /// Relay timeouts observed with no accepted relay in between.
414    consecutive_timeouts: u32,
415    /// `Some` while the breaker is open: the instant the cooldown expires and
416    /// the next arrival may probe. `None` means closed.
417    cooldown_until: Option<Instant>,
418    /// A half-open probe has been admitted and has not settled yet. This is
419    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
420    /// that read the cooldown, so concurrent opens arriving at the moment the
421    /// cooldown expires cannot all decide that they are the probe.
422    probe_in_flight: bool,
423}
424
425/// What the breaker decided for one `route.open`, before any relay work.
426enum RouteBindAdmission<'a> {
427    Admitted {
428        guard: RouteBindBreakerGuard<'a>,
429        /// This open is the single half-open probe, so the transition is worth
430        /// one log line.
431        probe: bool,
432    },
433    Refused {
434        consecutive_timeouts: u32,
435        /// What is left of the cooldown. Zero when the refusal is because the
436        /// one probe is already in flight rather than because the cooldown has
437        /// not elapsed.
438        retry_in: Duration,
439        probe_in_flight: bool,
440    },
441}
442
443/// An outstanding admission, which must be told how its relay settled.
444///
445/// `Drop` settles it as inconclusive, so an early return between admission and
446/// the relay -- or the whole handler being cancelled when the client
447/// disconnects -- releases a half-open probe slot instead of leaving the
448/// breaker wedged half-open with no further probes.
449struct RouteBindBreakerGuard<'a> {
450    breakers: RouteBindBreakers,
451    module_id: &'a str,
452    settled: bool,
453}
454
455impl RouteBindBreakerGuard<'_> {
456    /// The module answered within the budget and took the bind. THE ONLY
457    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
458    /// breaker, which is a transition worth logging.
459    fn record_accepted(&mut self) -> bool {
460        self.settled = true;
461        self.breakers.record_accepted(self.module_id)
462    }
463
464    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
465    /// COUNTS TOWARD OPENING.
466    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
467        self.settled = true;
468        self.breakers
469            .record_timeout(self.module_id, threshold, cooldown)
470    }
471
472    /// Everything else: the module REJECTED the bind, its connection went away
473    /// mid-relay, or the waiter was cancelled.
474    ///
475    /// None of these is evidence that a module is slow, and each already has
476    /// its own refusal with its own code. A module that rejects a bind in
477    /// microseconds is healthy and must never be convicted for it; a module
478    /// that died has said nothing about the module that replaces it. So these
479    /// neither increment nor reset the count -- they only release a probe slot.
480    fn record_inconclusive(&mut self) {
481        self.settled = true;
482        self.breakers.record_inconclusive(self.module_id);
483    }
484}
485
486impl Drop for RouteBindBreakerGuard<'_> {
487    fn drop(&mut self) {
488        if !self.settled {
489            self.breakers.record_inconclusive(self.module_id);
490        }
491    }
492}
493
494/// The breaker moved to open, reported so the caller can log it outside the
495/// lock. Opening is rare and load-bearing; the refusals that follow are
496/// frequent and are counted rather than logged.
497struct BreakerOpened {
498    consecutive_timeouts: u32,
499    /// True when a failed probe re-opened an already-open breaker, which reads
500    /// very differently in a log from a first opening.
501    reopened_after_probe: bool,
502}
503
504impl RouteBindBreakers {
505    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
506        self.modules
507            .lock()
508            .expect("route.bind breaker mutex poisoned")
509    }
510
511    /// Decide whether this `route.open` may attempt its relay. Takes the map
512    /// lock and nothing else, and never awaits.
513    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
514        let admitted = |probe| RouteBindAdmission::Admitted {
515            guard: RouteBindBreakerGuard {
516                breakers: self.clone(),
517                module_id,
518                settled: false,
519            },
520            probe,
521        };
522
523        let mut modules = self.lock();
524        let Some(state) = modules.get_mut(module_id) else {
525            return admitted(false);
526        };
527        let Some(cooldown_until) = state.cooldown_until else {
528            return admitted(false);
529        };
530        if state.probe_in_flight {
531            return RouteBindAdmission::Refused {
532                consecutive_timeouts: state.consecutive_timeouts,
533                retry_in: Duration::ZERO,
534                probe_in_flight: true,
535            };
536        }
537        let now = Instant::now();
538        if now < cooldown_until {
539            return RouteBindAdmission::Refused {
540                consecutive_timeouts: state.consecutive_timeouts,
541                retry_in: cooldown_until - now,
542                probe_in_flight: false,
543            };
544        }
545        state.probe_in_flight = true;
546        admitted(true)
547    }
548
549    fn record_accepted(&self, module_id: &str) -> bool {
550        self.lock()
551            .remove(module_id)
552            .is_some_and(|state| state.cooldown_until.is_some())
553    }
554
555    fn record_timeout(
556        &self,
557        module_id: &str,
558        threshold: u32,
559        cooldown: Duration,
560    ) -> Option<BreakerOpened> {
561        let mut modules = self.lock();
562        let state = modules.entry(module_id.to_string()).or_default();
563        let was_open = state.cooldown_until.is_some();
564        let was_probe = state.probe_in_flight;
565        state.probe_in_flight = false;
566        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
567        if state.consecutive_timeouts < threshold {
568            return None;
569        }
570        state.cooldown_until = Some(Instant::now() + cooldown);
571        Some(BreakerOpened {
572            consecutive_timeouts: state.consecutive_timeouts,
573            reopened_after_probe: was_open && was_probe,
574        })
575    }
576
577    fn record_inconclusive(&self, module_id: &str) {
578        if let Some(state) = self.lock().get_mut(module_id) {
579            state.probe_in_flight = false;
580        }
581    }
582
583    /// Discard what was learned about a module, because the process it was
584    /// learned about is gone. Returns the discarded count when it was non-zero.
585    ///
586    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
587    /// `module_id` is a configuration identity that outlives any particular
588    /// child; what the breaker observed was the process behind the module
589    /// connection of the moment. When a new connection registers under that id
590    /// the verdict's subject no longer exists, so the verdict is stale by
591    /// construction rather than merely likely to be wrong. Keeping it would
592    /// apply a dead process's record to a live one, which is the same defect
593    /// class this breaker exists to stop the daemon committing.
594    ///
595    /// A half-open probe in flight is discarded with the rest: it was a
596    /// question about the old process.
597    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
598        self.lock()
599            .remove(module_id)
600            .map(|state| state.consecutive_timeouts)
601            .filter(|discarded| *discarded > 0)
602    }
603
604    /// Open breakers, for the `server.describe` counters object. `None` when
605    /// none is open, so the key stays absent rather than present-and-empty.
606    ///
607    /// This is the operator's answer to "is this module refusing instantly or
608    /// is it fine?", which look identical from a client that retries and then
609    /// succeeds.
610    fn open_snapshot(&self) -> Option<serde_json::Value> {
611        let now = Instant::now();
612        let modules = self.lock();
613        let open = modules
614            .iter()
615            .filter_map(|(module_id, state)| {
616                let cooldown_until = state.cooldown_until?;
617                Some((
618                    module_id.clone(),
619                    serde_json::json!({
620                        "consecutive_timeouts": state.consecutive_timeouts,
621                        "cooldown_remaining_ms":
622                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
623                        "probe_in_flight": state.probe_in_flight,
624                    }),
625                ))
626            })
627            .collect::<serde_json::Map<String, serde_json::Value>>();
628        (!open.is_empty()).then_some(serde_json::Value::Object(open))
629    }
630}
631
632impl ControlHandler {
633    pub fn new(registry: Arc<Registry>) -> Self {
634        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
635    }
636
637    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
638        let counters = forwarding.counters();
639        // Taken from the forwarding table rather than created here, so that the
640        // breaker a `route.open` consults is the same one a module's
641        // registration resets, however many handlers are built over one table.
642        let route_bind_breakers = forwarding.route_bind_breakers();
643        let route_bind_concurrency = forwarding.route_bind_concurrency();
644        Self {
645            registry,
646            forwarding,
647            process_liveness: None,
648            supervisor: SupervisorHandle::new(),
649            subc_capabilities: Arc::from([
650                CAP_MANIFEST_REGISTRATION.to_string(),
651                CAP_CHANNEL_LIFECYCLE.to_string(),
652                CAP_PING_PONG.to_string(),
653                CAP_SESSION_ATTACH.to_string(),
654                CAP_ADMISSION_FACTS_RELAY.to_string(),
655            ]),
656            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
657            route_bind_relay_timeouts: BTreeMap::new(),
658            route_bind_breakers,
659            route_bind_concurrency,
660            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
661            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
662            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
663            storage_config: None,
664            admission_facts_carrier_module_id: None,
665            admission_facts_targets: None,
666            rescan: None,
667            connected_clients: ConnectedClients::new(),
668            counters,
669            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
670            daemon_provenance: DaemonProvenanceFacts::default(),
671            #[cfg(test)]
672            control_dispatch_delay: None,
673            #[cfg(test)]
674            provenance_probe_override: None,
675        }
676    }
677
678    /// Set the central storage policy: registering modules then receive their
679    /// resolved storage descriptor in HELLO_ACK.
680    pub fn with_storage_config(
681        mut self,
682        storage_config: Option<crate::daemon_config::StorageConfig>,
683    ) -> Self {
684        self.storage_config = storage_config;
685        self
686    }
687
688    /// Configure the exact reserved module and target ids permitted to relay
689    /// opaque admission facts. Config-file loading validates this authority;
690    /// this builder keeps the same policy available to embedded test daemons.
691    pub fn with_admission_facts_config(
692        mut self,
693        carrier_module_id: Option<String>,
694        targets: Option<Vec<String>>,
695    ) -> Self {
696        self.admission_facts_carrier_module_id = carrier_module_id;
697        self.admission_facts_targets = targets;
698        self
699    }
700
701    /// Override the route.bind relay timeout. Used by tests that assert the
702    /// timeout path so they don't block on the production-safe default.
703    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
704        self.route_bind_relay_timeout = timeout;
705        self
706    }
707
708    /// Install per-module route.bind relay budget overrides. A module id
709    /// listed here wins over the daemon-wide default set via
710    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
711    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
712    /// the same `Duration` the bind path will use.
713    pub fn with_route_bind_relay_timeouts(
714        mut self,
715        timeouts: impl IntoIterator<Item = (String, Duration)>,
716    ) -> Self {
717        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
718        self
719    }
720
721    /// Resolve the route.bind relay budget for a specific target module id.
722    /// Per-module overrides win; the daemon-wide value (set via
723    /// `with_route_bind_relay_timeout` or the built-in default) is the
724    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
725    /// the same resolution `handle_route_open` will use.
726    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
727        self.route_bind_relay_timeouts
728            .get(module_id)
729            .copied()
730            .unwrap_or(self.route_bind_relay_timeout)
731    }
732
733    /// Override the per-module bind-relay breaker policy.
734    ///
735    /// Used by tests, which cannot spend three production budgets opening a
736    /// breaker or twenty seconds waiting for its cooldown. The production
737    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
738    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
739    /// reasoning for the numbers.
740    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
741        self.route_bind_breaker_threshold = threshold.max(1);
742        self.route_bind_breaker_cooldown = cooldown;
743        self
744    }
745
746    #[cfg(test)]
747    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
748        self.health_probe_timeout = timeout;
749        self
750    }
751
752    #[cfg(test)]
753    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
754        self.control_dispatch_delay = Some(delay);
755        self
756    }
757
758    pub fn with_process_liveness(
759        mut self,
760        process_liveness: Arc<dyn ModuleProcessLiveness>,
761    ) -> Self {
762        self.process_liveness = Some(process_liveness);
763        self
764    }
765
766    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
767        self.supervisor = supervisor;
768        self
769    }
770
771    pub fn with_daemon_provenance(
772        mut self,
773        pid: u32,
774        started_at_ms: u64,
775        executable_path: Option<PathBuf>,
776        build_git_sha: Option<String>,
777        build_lock_digest: Option<String>,
778    ) -> Self {
779        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
780        let process_start_time = process_start_time(pid);
781        self.daemon_provenance = DaemonProvenanceFacts {
782            build: DaemonBuildProvenance {
783                build_git_sha,
784                build_lock_digest,
785            },
786            pid: Some(pid),
787            started_at_ms: Some(started_at_ms),
788            start_clock: None,
789            executable_path,
790            executable_identity,
791            process_start_time,
792            probe: ExecutableIdentityProbe::default(),
793        };
794        self
795    }
796
797    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
798        self.daemon_provenance.start_clock = Some(clock);
799        self
800    }
801
802    #[cfg(test)]
803    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
804        self.provenance_probe_override = Some(result);
805        self
806    }
807
808    /// Install the configured module set and its reserved capability bindings.
809    /// Bindings are configuration-scoped and may point at a provider that has not
810    /// been installed yet, so this does not require the bound module to exist.
811    pub fn with_capability_config(
812        self,
813        modules: impl IntoIterator<Item = (String, bool)>,
814        reserved_capabilities: BTreeMap<String, String>,
815    ) -> Self {
816        self.capability_evaluator
817            .configure(modules, reserved_capabilities);
818        self
819    }
820
821    pub fn with_supervisor_rescan(
822        mut self,
823        supervisor: Supervisor,
824        config_path: impl Into<PathBuf>,
825        configured_port: Option<u16>,
826    ) -> Self {
827        self.rescan = Some(SupervisorRescanContext {
828            supervisor,
829            config_path: config_path.into(),
830            configured_port,
831            storage_config: self.storage_config.clone(),
832            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
833            admission_facts_targets: self.admission_facts_targets.clone(),
834        });
835        self
836    }
837
838    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
839        self.connected_clients = connected_clients;
840        self
841    }
842
843    pub fn forwarding(&self) -> Arc<ForwardingTable> {
844        Arc::clone(&self.forwarding)
845    }
846
847    pub(crate) fn counters(&self) -> DaemonCounters {
848        self.counters.clone()
849    }
850
851    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
852    /// requirement event without depending on an operator polling a status command.
853    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
854        tokio::spawn(async move {
855            loop {
856                self.capability_evaluator
857                    .wait_for_change_or_deadline()
858                    .await;
859                self.refresh_capability_requirements();
860            }
861        });
862    }
863
864    fn runtime_capability_snapshot(
865        &self,
866    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
867        let runtime = self
868            .supervisor
869            .list()
870            .into_iter()
871            .map(|module| {
872                let status = module.status().map_err(|err| {
873                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
874                })?;
875                Ok(RuntimeModule {
876                    module_id: status.module_id,
877                    state: status.state,
878                    enabled: status.enabled,
879                })
880            })
881            .collect::<Result<Vec<_>, RouterError>>()?;
882        let (_, registrations) = self.registry.list_modules().map_err(|err| {
883            RouterError::backend(
884                0,
885                0,
886                format!("failed to list capability registrations: {err}"),
887            )
888        })?;
889        let registrations = registrations
890            .into_iter()
891            .map(|registration| RegisteredModule {
892                module_id: registration.manifest.module_id,
893                module_version: registration.manifest.module_version,
894                capabilities: registration.manifest.capabilities,
895            })
896            .collect();
897        Ok((runtime, registrations))
898    }
899
900    /// The capability side effects of a module becoming the active registration
901    /// for its id: cache its manifest (warning if its claims drifted), run the
902    /// deny census when its declarations call for one, and recompute the
903    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
904    /// candidate's does not, and the supervisor does it at promotion instead,
905    /// through [`crate::supervise::SwapPromotionObserver`].
906    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
907        let cached_registration = RegisteredModule {
908            module_id: registration.manifest.module_id.clone(),
909            module_version: registration.manifest.module_version.clone(),
910            capabilities: registration.manifest.capabilities.clone(),
911        };
912        if self.capability_evaluator.record_hello(&cached_registration) {
913            warn!(
914                module_id = %cached_registration.module_id,
915                "capability claims drifted from the cached manifest"
916            );
917        }
918        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
919            self.enforce_capability_denies();
920        }
921        self.refresh_capability_requirements();
922    }
923
924    /// Point the shared supervisor handle at this handler for swap promotions.
925    /// Called wherever a handler is put behind the `Arc` the router serves, so
926    /// it can be held weakly.
927    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
928        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
929            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
930        self.supervisor.set_swap_promotion_observer(observer);
931    }
932
933    pub fn refresh_capability_requirements(&self) {
934        match self.runtime_capability_snapshot() {
935            Ok((runtime, registrations)) => {
936                log_requirement_events(
937                    self.capability_evaluator
938                        .evaluate_now(&runtime, &registrations),
939                );
940            }
941            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
942        }
943    }
944
945    /// Reconcile only live, attested route bindings after a capability deny edge
946    /// or target claim was added. This is deliberately a control-plane census:
947    /// the opaque forwarding hot path must not grow a per-frame capability check.
948    fn enforce_capability_denies(&self) {
949        let (_, registrations) = match self.registry.list_modules() {
950            Ok(snapshot) => snapshot,
951            Err(err) => {
952                warn!(error = %err, "failed to read registrations for capability deny census");
953                return;
954            }
955        };
956        let manifests = registrations
957            .into_iter()
958            .map(|registration| {
959                (
960                    registration.manifest.module_id.clone(),
961                    registration.manifest,
962                )
963            })
964            .collect::<BTreeMap<_, _>>();
965        let census = match self.forwarding.route_census(None) {
966            Ok(census) => census,
967            Err(err) => {
968                warn!(error = %err, "failed to read route census for capability deny enforcement");
969                return;
970            }
971        };
972
973        for (target_module_id, routes) in census {
974            let Some(target_manifest) = manifests.get(&target_module_id) else {
975                continue;
976            };
977            let mut closed_routes = Vec::new();
978            let mut module_goodbyes = Vec::new();
979            for route in routes {
980                let Principal::Reserved {
981                    module_id: opening_module_id,
982                } = &route.principal
983                else {
984                    continue;
985                };
986                let Some(opening_manifest) = manifests.get(opening_module_id) else {
987                    continue;
988                };
989                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
990                    continue;
991                };
992
993                match self.forwarding.release_client_route(
994                    route.goodbye_target.connection_id,
995                    route.goodbye_target.channel,
996                    route.goodbye_target.epoch,
997                ) {
998                    Ok(RouteRelease::Removed(module_goodbye)) => {
999                        warn!(
1000                            opening_module_id,
1001                            target_module_id,
1002                            capability,
1003                            "force-closing route because an attested capability deny edge now matches"
1004                        );
1005                        closed_routes.push(route);
1006                        module_goodbyes.push(module_goodbye);
1007                    }
1008                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1009                    Err(err) => warn!(
1010                        opening_module_id,
1011                        target_module_id,
1012                        capability,
1013                        error = %err,
1014                        "failed to force-close capability-denied route"
1015                    ),
1016                }
1017            }
1018
1019            if closed_routes.is_empty() {
1020                continue;
1021            }
1022            send_route_control_pushes(
1023                &self.forwarding,
1024                closed_routes,
1025                ClientControlPush::RouteClosed {
1026                    module_id: target_module_id,
1027                    reason: RouteCloseReason::CapabilityDenied,
1028                    drained: false,
1029                    abandoned: 0,
1030                    excluded_subscriptions: 0,
1031                    terminal: Some(false),
1032                },
1033            );
1034            self.emit_route_goodbyes(module_goodbyes);
1035        }
1036    }
1037
1038    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1039        self.capability_evaluator
1040            .statuses()
1041            .into_iter()
1042            .map(capability_requirement_status)
1043            .collect()
1044    }
1045
1046    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1047    /// registration-release watch. The signal is what the supervisor waits on
1048    /// before spawning a replacement, so it must only fire once forwarding
1049    /// teardown is also done (see [`Self::cleanup_connection`] /
1050    /// [`Self::handle_goodbye`]). Used directly only where there is no forwarding
1051    /// state to tear down (a HELLO that failed before module registration).
1052    fn deregister_connection(
1053        &self,
1054        connection_id: ConnectionId,
1055    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1056        self.registry.deregister_connection(connection_id)
1057    }
1058
1059    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1060        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1061            return None;
1062        }
1063        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1064            parse_client_control_request(&frame.body)
1065        else {
1066            return None;
1067        };
1068        Some(target_module_id(&target).to_string())
1069    }
1070
1071    pub(crate) fn route_open_capacity_refusal(
1072        &self,
1073        ctx: &RouteCtx,
1074        frame: &Frame,
1075        target_module_id: &str,
1076        limit: usize,
1077    ) -> Result<Frame, RouterError> {
1078        self.route_open_admission_refusal_frame(
1079            ctx,
1080            frame,
1081            target_module_id,
1082            format!(
1083                "connection already has {limit} route.open binds in flight; retry after one settles"
1084            ),
1085        )
1086    }
1087
1088    /// Admission pressure clears as existing binds settle, so its refusal must
1089    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
1090    /// `module_reloading`, `module_warming`, `target_unavailable`, or
1091    /// `module_timeout`. `target_unavailable` is honest for an attempt that
1092    /// cannot currently reach its target; `module_timeout` would falsely claim
1093    /// that a wait expired. A new, cleaner code would be terminal to deployed
1094    /// clients, so it requires a client-tolerance rollout before daemon emission.
1095    fn route_open_admission_refusal_frame(
1096        &self,
1097        ctx: &RouteCtx,
1098        frame: &Frame,
1099        target_module_id: &str,
1100        message: impl Into<String>,
1101    ) -> Result<Frame, RouterError> {
1102        self.route_open_refusal_frame(
1103            ctx,
1104            frame,
1105            target_module_id,
1106            "open_admission_full",
1107            error_codes::TARGET_UNAVAILABLE,
1108            message,
1109        )
1110    }
1111
1112    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
1113    ///
1114    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
1115    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
1116    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
1117    #[cfg(test)]
1118    pub fn handle_control(
1119        &self,
1120        connection_id: ConnectionId,
1121        frame: Frame,
1122    ) -> Result<Vec<Frame>, RouterError> {
1123        match frame.header.ty {
1124            FrameType::Ping => Ok(vec![pong(&frame)?]),
1125            FrameType::Hello => self.handle_hello(connection_id, None, frame),
1126            FrameType::Goodbye => self.handle_goodbye(connection_id),
1127            ty => Ok(vec![control_error_frame(
1128                &frame,
1129                "unsupported_control_frame",
1130                format!("unsupported channel-0 frame {ty:?}"),
1131            )?]),
1132        }
1133    }
1134
1135    pub async fn handle_control_frame(
1136        &self,
1137        ctx: &RouteCtx,
1138        frame: Frame,
1139    ) -> Result<Vec<Frame>, RouterError> {
1140        self.handle_control_frame_timed(ctx, frame, None).await
1141    }
1142
1143    pub(crate) async fn handle_control_frame_timed(
1144        &self,
1145        ctx: &RouteCtx,
1146        frame: Frame,
1147        dispatch_started_at: Option<StdInstant>,
1148    ) -> Result<Vec<Frame>, RouterError> {
1149        match frame.header.ty {
1150            FrameType::Ping => Ok(vec![pong(&frame)?]),
1151            FrameType::Hello => {
1152                self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1153            }
1154            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1155            FrameType::Cancel => {
1156                if self
1157                    .supervisor
1158                    .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1159                {
1160                    Ok(Vec::new())
1161                } else {
1162                    Ok(vec![control_error_frame(
1163                        &frame,
1164                        "unknown_subscription",
1165                        "no supervisor spawn subscription has this correlation id",
1166                    )?])
1167                }
1168            }
1169            FrameType::Request => {
1170                if self
1171                    .forwarding
1172                    .module_endpoint_for_connection(ctx.connection_id)
1173                    .map_err(RouterError::Forwarding)?
1174                    .is_some()
1175                {
1176                    if !is_known_module_request_op(&frame.body) {
1177                        return Ok(vec![control_error_frame(
1178                            &frame,
1179                            "unsupported_control_frame",
1180                            "module-originated channel-0 REQUEST is not supported",
1181                        )?]);
1182                    }
1183                    let request = match parse_module_control_request_from_module(&frame.body) {
1184                        Ok(request) => request,
1185                        Err((err, ControlRequestBodyError::UnknownOp)) => {
1186                            return Ok(vec![control_error_frame(
1187                                &frame,
1188                                "unsupported_control_frame",
1189                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
1190                            )?])
1191                        }
1192                        Err((err, ControlRequestBodyError::InvalidBody)) => {
1193                            return Ok(vec![control_error_frame(
1194                                &frame,
1195                                "invalid_control_body",
1196                                format!("malformed module control body: {err}"),
1197                            )?])
1198                        }
1199                    };
1200                    let op = module_control_request_op(&request);
1201                    let corr = frame.header.corr;
1202                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
1203                    let result =
1204                        self.handle_module_control_request(ctx.connection_id, frame, request);
1205                    log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1206                    return result;
1207                }
1208
1209                if is_known_module_request_op(&frame.body) {
1210                    return Ok(vec![control_error_frame(
1211                        &frame,
1212                        "not_registered",
1213                        "catalog.update requires an active module registration owned by this connection",
1214                    )?]);
1215                }
1216
1217                let request = match parse_client_control_request(&frame.body) {
1218                    Ok(request) => request,
1219                    Err((err, ControlRequestBodyError::UnknownOp)) => {
1220                        return Ok(vec![control_error_frame(
1221                            &frame,
1222                            "unknown_control_op",
1223                            format!("unknown client control op: {err}"),
1224                        )?])
1225                    }
1226                    Err((err, ControlRequestBodyError::InvalidBody)) => {
1227                        return Ok(vec![control_error_frame(
1228                            &frame,
1229                            "invalid_control_body",
1230                            format!("malformed client control body: {err}"),
1231                        )?])
1232                    }
1233                };
1234                let op = client_control_request_op(&request);
1235                let corr = frame.header.corr;
1236                log_control_dispatch_arrival(op, ctx.connection_id, corr);
1237                #[cfg(test)]
1238                if let Some(delay) = self.control_dispatch_delay {
1239                    tokio::time::sleep(delay).await;
1240                }
1241                let result = self
1242                    .handle_client_control_request(ctx, frame, request)
1243                    .await;
1244                log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1245                result
1246            }
1247            FrameType::Push => {
1248                let Some(endpoint) = self
1249                    .forwarding
1250                    .module_endpoint_for_connection(ctx.connection_id)
1251                    .map_err(RouterError::Forwarding)?
1252                else {
1253                    return Ok(vec![control_error_frame(
1254                        &frame,
1255                        "unsupported_control_frame",
1256                        "client-originated channel-0 PUSH is not supported",
1257                    )?]);
1258                };
1259                self.handle_status_update(endpoint, frame)
1260            }
1261            FrameType::Response | FrameType::Error
1262                if self
1263                    .forwarding
1264                    .module_endpoint_for_connection(ctx.connection_id)
1265                    .map_err(RouterError::Forwarding)?
1266                    .is_some() =>
1267            {
1268                self.handle_module_relay_response(ctx.connection_id, frame)
1269            }
1270            ty => Ok(vec![control_error_frame(
1271                &frame,
1272                "unsupported_control_frame",
1273                format!("unsupported channel-0 frame {ty:?}"),
1274            )?]),
1275        }
1276    }
1277
1278    pub fn cleanup_connection(
1279        &self,
1280        connection_id: ConnectionId,
1281    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1282        let crash_closed = self
1283            .registry
1284            .get_module_by_connection(connection_id)?
1285            .and_then(|registration| {
1286                self.forwarding
1287                    .module_endpoint_for_connection(connection_id)
1288                    .ok()
1289                    .flatten()
1290                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1291                    .map(|routes| (registration.manifest.module_id, routes))
1292            });
1293        if let Some((module_id, routes)) = crash_closed {
1294            let terminal = match self.supervisor.get(&module_id) {
1295                None => false,
1296                Some(module) => match module.will_recover_after_connection_loss() {
1297                    Ok(will_recover) => !will_recover,
1298                    Err(err) => {
1299                        warn!(
1300                            %module_id,
1301                            error = %err,
1302                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
1303                        );
1304                        false
1305                    }
1306                },
1307            };
1308            send_route_control_pushes(
1309                &self.forwarding,
1310                routes,
1311                ClientControlPush::RouteClosed {
1312                    module_id,
1313                    reason: RouteCloseReason::Crash,
1314                    drained: false,
1315                    abandoned: 0,
1316                    excluded_subscriptions: 0,
1317                    terminal: Some(terminal),
1318                },
1319            );
1320        }
1321        let registrations = self.deregister_connection(connection_id);
1322        if let Ok(released_routes) = self.forwarding.cleanup_connection(connection_id) {
1323            self.emit_route_goodbyes(released_routes);
1324        }
1325        // Signal the registration-release watch only now that BOTH registry and
1326        // forwarding teardown are done, so a supervisor waiting to spawn a
1327        // replacement never observes release while old routes still exist.
1328        if matches!(&registrations, Ok(r) if !r.is_empty()) {
1329            crate::supervise::notify_registration_release();
1330            self.capability_evaluator.wake_deadline_loop();
1331            self.refresh_capability_requirements();
1332        }
1333        self.supervisor.remove_spawn_subscribers(connection_id);
1334        registrations
1335    }
1336
1337    pub(crate) fn handle_route_goodbye(
1338        &self,
1339        connection_id: ConnectionId,
1340        route_channel: u16,
1341        route_epoch: u32,
1342    ) -> Result<bool, RouterError> {
1343        debug!(
1344            connection_id = connection_id.get(),
1345            route_channel, route_epoch, "handling route GOODBYE"
1346        );
1347        let RouteRelease::Removed(released_route) = self
1348            .forwarding
1349            .release_client_route(connection_id, route_channel, route_epoch)
1350            .map_err(RouterError::Forwarding)?
1351        else {
1352            return Ok(false);
1353        };
1354        self.emit_route_goodbyes(vec![released_route]);
1355        Ok(true)
1356    }
1357
1358    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1359        for released in released_routes {
1360            let frame = match Frame::build_with_version(
1361                released.negotiated_ver,
1362                FrameType::Goodbye,
1363                control_flags(),
1364                released.channel,
1365                released.epoch,
1366                0,
1367                Vec::new(),
1368            ) {
1369                Ok(frame) => frame,
1370                Err(err) => {
1371                    warn!(
1372                        route_channel = released.channel,
1373                        error = %err,
1374                        "failed to build route GOODBYE frame"
1375                    );
1376                    continue;
1377                }
1378            };
1379            if let Err(err) = released.sink.try_send(frame) {
1380                if released.close_on_delivery_failure() {
1381                    warn!(
1382                        target_connection_id = released.connection_id.get(),
1383                        route_channel = released.channel,
1384                        error = %err,
1385                        "route GOODBYE was not delivered to client; closing target connection"
1386                    );
1387                    if self
1388                        .forwarding
1389                        .escalate_client_delivery_failure(
1390                            released.connection_id,
1391                            released.channel,
1392                            released.epoch,
1393                            CloseReason::new(
1394                                "route_goodbye_delivery_failed",
1395                                format!(
1396                                    "failed to enqueue route GOODBYE for channel {}: {err}",
1397                                    released.channel
1398                                ),
1399                            ),
1400                        )
1401                        .unwrap_or(false)
1402                    {
1403                        self.counters.increment_goodbye_relay_client_failed();
1404                    }
1405                } else {
1406                    self.counters
1407                        .increment_goodbye_relay_module_dropped(released.module_id.as_deref());
1408                    warn!(
1409                        target_connection_id = released.connection_id.get(),
1410                        route_channel = released.channel,
1411                        error = %err,
1412                        "route GOODBYE to module dropped under backpressure; not closing shared module connection"
1413                    );
1414                }
1415            }
1416        }
1417    }
1418
1419    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
1420    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
1421    /// own commit failed after the module had already accepted). Without this, a
1422    /// module that accepts late keeps a binding subc has torn down, so a later
1423    /// frame on that module channel could misdeliver if the channel is reused.
1424    ///
1425    /// Never closes the shared module connection on failure: a dropped notification
1426    /// only wastes a bounded amount of warm module-side state, which the module's
1427    /// own idle reaper reclaims. Only call this once the route.bind relay was
1428    /// actually enqueued to the module — if the relay send itself failed, the
1429    /// module never created a binding and there is nothing to tear down.
1430    fn send_abandoned_route_bind_goodbye(
1431        &self,
1432        module_sink: &crate::FrameSink,
1433        negotiated_ver: u8,
1434        module_channel: u16,
1435        module_epoch: u32,
1436    ) {
1437        let frame = match Frame::build_with_version(
1438            negotiated_ver,
1439            FrameType::Goodbye,
1440            control_flags(),
1441            module_channel,
1442            module_epoch,
1443            0,
1444            Vec::new(),
1445        ) {
1446            Ok(frame) => frame,
1447            Err(err) => {
1448                warn!(
1449                    route_channel = module_channel,
1450                    error = %err,
1451                    "failed to build GOODBYE for abandoned route.bind"
1452                );
1453                return;
1454            }
1455        };
1456        if let Err(err) = module_sink.try_send(frame) {
1457            warn!(
1458                route_channel = module_channel,
1459                error = %err,
1460                "GOODBYE for abandoned route.bind dropped; module idle reaper will reclaim the binding"
1461            );
1462        }
1463    }
1464
1465    fn handle_hello(
1466        &self,
1467        connection_id: ConnectionId,
1468        sink: Option<crate::FrameSink>,
1469        frame: Frame,
1470    ) -> Result<Vec<Frame>, RouterError> {
1471        debug!(
1472            connection_id = connection_id.get(),
1473            corr = frame.header.corr,
1474            "handling HELLO"
1475        );
1476        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1477            Ok(value) => value,
1478            Err(err) => {
1479                return Ok(vec![control_error_frame(
1480                    &frame,
1481                    "invalid_hello",
1482                    format!("malformed HELLO body: {err}"),
1483                )?])
1484            }
1485        };
1486        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1487            return Ok(vec![control_error_frame(
1488                &frame,
1489                "invalid_capability_grammar",
1490                err.to_string(),
1491            )?]);
1492        }
1493        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
1494            return Ok(vec![control_error_frame(
1495                &frame,
1496                "invalid_manifest",
1497                err.to_string(),
1498            )?]);
1499        }
1500        if let Some(provenance) = hello_value
1501            .get("manifest")
1502            .and_then(|manifest| manifest.get("provenance"))
1503        {
1504            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
1505                return Ok(vec![control_error_frame(
1506                    &frame,
1507                    "invalid_manifest",
1508                    format!("malformed manifest provenance: {err}"),
1509                )?]);
1510            }
1511        }
1512        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
1513            Ok(hello) => hello,
1514            Err(err) => {
1515                return Ok(vec![control_error_frame(
1516                    &frame,
1517                    "invalid_hello",
1518                    format!("malformed HELLO body: {err}"),
1519                )?])
1520            }
1521        };
1522
1523        if hello.protocol_ver != hello.manifest.protocol_ver {
1524            return Ok(vec![control_error_frame(
1525                &frame,
1526                "invalid_manifest",
1527                format!(
1528                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
1529                    hello.protocol_ver, hello.manifest.protocol_ver
1530                ),
1531            )?]);
1532        }
1533
1534        if hello.manifest.module_id.trim().is_empty() {
1535            return Ok(vec![control_error_frame(
1536                &frame,
1537                "invalid_manifest",
1538                "manifest module_id must not be empty",
1539            )?]);
1540        }
1541
1542        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
1543            Ok(negotiated_ver) => negotiated_ver,
1544            Err(message) => {
1545                return Ok(vec![control_error_frame(
1546                    &frame,
1547                    "version_unsupported",
1548                    message,
1549                )?])
1550            }
1551        };
1552
1553        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
1554        // swap is open for this id, the only HELLO admitted as a second process
1555        // is the one carrying the candidate's launch nonce (the swap token), and
1556        // it registers into the candidate slot rather than being refused as a
1557        // duplicate. Run after the reserved gate, a reserved module's candidate
1558        // would be refused `reserved_module` for presenting a nonce that gate
1559        // does not know. See `SupervisorHandle::swap_hello_admission`.
1560        let swap_admission = self
1561            .supervisor
1562            .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
1563        if swap_admission == SwapHelloAdmission::Refused {
1564            warn!(
1565                module_id = %hello.manifest.module_id,
1566                connection_id = connection_id.get(),
1567                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
1568            );
1569            return Ok(vec![control_error_frame(
1570                &frame,
1571                "swap_token_invalid",
1572                format!(
1573                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
1574                    hello.manifest.module_id
1575                ),
1576            )?]);
1577        }
1578        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
1579
1580        // Reserved-module identity gate: a module_id configured `reserved` may be
1581        // registered ONLY by the process subc spawned for it, proven by echoing the
1582        // one-time launch nonce subc injected. A non-reserved id has no recorded
1583        // nonce and always passes. This blocks a key-holder from impersonating a
1584        // security-boundary module (e.g. the credential vault) while the real one is
1585        // down/restarting and its registration slot is momentarily free. A swap
1586        // candidate has already proven the same thing with its own nonce above.
1587        if let Some(rejection) = (!swap_candidate)
1588            .then(|| {
1589                self.supervisor.reserved_hello_rejection(
1590                    &hello.manifest.module_id,
1591                    hello.launch_nonce.as_deref(),
1592                )
1593            })
1594            .flatten()
1595        {
1596            let message = match rejection {
1597                ReservedHelloRejection::Exact { module_id } => format!(
1598                    "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
1599                ),
1600                ReservedHelloRejection::Prefix {
1601                    prefix,
1602                    owner_module_id,
1603                } => format!(
1604                    "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
1605                    hello.manifest.module_id
1606                ),
1607            };
1608            return Ok(vec![control_error_frame(
1609                &frame,
1610                "reserved_module",
1611                message,
1612            )?]);
1613        }
1614
1615        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
1616            &hello.manifest.module_id,
1617            hello.manifest.capabilities.as_ref(),
1618        );
1619        if let Some(refusal) = reserved_capability_refusals.first() {
1620            let capability = refusal.capability.clone();
1621            let bound_module = refusal.claimants[0].clone();
1622            log_duplicate_claim_events(reserved_capability_refusals);
1623            return Ok(vec![control_error_frame(
1624                &frame,
1625                "reserved_capability",
1626                format!(
1627                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
1628                    capability, bound_module, hello.manifest.module_id
1629                ),
1630            )?]);
1631        }
1632
1633        // A connection that already opened client routes must not also register as
1634        // a module: cleanup would then release only one side and leak the other.
1635        if self
1636            .forwarding
1637            .connection_has_client_routes(connection_id)
1638            .map_err(RouterError::Forwarding)?
1639        {
1640            return Ok(vec![control_error_frame(
1641                &frame,
1642                "invalid_hello",
1643                "connection has open client routes and cannot also register as a module",
1644            )?]);
1645        }
1646
1647        let control_ops = effective_module_control_ops(hello.control_ops);
1648        if swap_candidate {
1649            return self.register_swap_candidate(
1650                connection_id,
1651                sink,
1652                &frame,
1653                hello.manifest,
1654                negotiated_ver,
1655                control_ops,
1656            );
1657        }
1658        let registration = match self.registry.register_with_control_ops(
1659            hello.manifest,
1660            negotiated_ver,
1661            connection_id,
1662            control_ops,
1663        ) {
1664            Ok(registration) => registration,
1665            Err(RegistryError::DuplicateModuleId { module_id }) => {
1666                return Ok(vec![control_error_frame(
1667                    &frame,
1668                    "duplicate_module_id",
1669                    format!(
1670                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
1671                    ),
1672                )?])
1673            }
1674            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
1675                return Ok(vec![control_error_frame(
1676                    &frame,
1677                    "invalid_module_id",
1678                    err.to_string(),
1679                )?])
1680            }
1681            Err(err) => {
1682                return Ok(vec![control_error_frame(
1683                    &frame,
1684                    "registry_error",
1685                    err.to_string(),
1686                )?])
1687            }
1688        };
1689
1690        if let Some(sink) = sink {
1691            // The forwarding table's module store is also the daemon-to-module
1692            // control-RPC lane, so every HELLO gets a live endpoint even when the
1693            // manifest has no routable provider role. Non-routable modules still
1694            // cannot receive route.bind in production: `handle_route_open` checks
1695            // the registry manifest with `target_has_required_role` before the
1696            // only production call to `begin_route_bind_relay_for` below that
1697            // route.open path. The remaining direct relay callers are unit tests
1698            // and benchmark harnesses that construct forwarding state explicitly.
1699            let concurrency = manifest_concurrency(&registration.manifest);
1700            if let Err(err) = self.forwarding.register_module_connection(
1701                connection_id,
1702                registration.manifest.module_id.clone(),
1703                negotiated_ver,
1704                concurrency,
1705                sink,
1706            ) {
1707                // Forwarding registration failed, so there is no forwarding
1708                // state to tear down. Remove the registry entry and signal the
1709                // release watch directly.
1710                if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
1711                    crate::supervise::notify_registration_release();
1712                }
1713                return Ok(vec![control_error_frame(
1714                    &frame,
1715                    forwarding_error_code(&err),
1716                    err.to_string(),
1717                )?]);
1718            }
1719        }
1720
1721        // Exposure over assumption: Concurrency's serde default is pinned to the
1722        // pre-field behavior (ModuleManaged), so a management surface that is
1723        // genuinely Serial and just never declared it inherits concurrent
1724        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
1725        // turns "no module has been bitten yet" into the checkable claim "no
1726        // module is exposed" -- one read of the boot log instead of a fleet
1727        // audit. Detected from the raw HELLO bytes because the serde default
1728        // deliberately erases the absent/declared distinction from the type.
1729        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
1730            info!(
1731                module_id = %registration.manifest.module_id,
1732                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
1733            );
1734        }
1735
1736        self.apply_registration_capabilities(&registration);
1737
1738        info!(
1739            module_id = %registration.manifest.module_id,
1740            module_version = %registration.manifest.module_version,
1741            negotiated_ver,
1742            routable_provider = manifest_provides_routable_role(&registration.manifest),
1743            connection_id = connection_id.get(),
1744            "module registered"
1745        );
1746
1747        self.hello_ack(&frame, negotiated_ver, &registration.manifest.module_id)
1748    }
1749
1750    /// Register a HELLO the swap gate admitted into the candidate slot of the
1751    /// registry and of forwarding, where it is reachable over its own
1752    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
1753    /// nothing routes to it until the supervisor cuts over.
1754    ///
1755    /// Registry first, then forwarding, the same order as an ordinary HELLO;
1756    /// a forwarding failure removes the registry entry again. The capability
1757    /// census is not run: it describes routable modules, and this one is not
1758    /// routable until promotion.
1759    fn register_swap_candidate(
1760        &self,
1761        connection_id: ConnectionId,
1762        sink: Option<crate::FrameSink>,
1763        frame: &Frame,
1764        manifest: ModuleManifest,
1765        negotiated_ver: u8,
1766        control_ops: Vec<String>,
1767    ) -> Result<Vec<Frame>, RouterError> {
1768        let module_id = manifest.module_id.clone();
1769        let registration = match self.registry.register_candidate_with_control_ops(
1770            manifest,
1771            negotiated_ver,
1772            connection_id,
1773            control_ops,
1774        ) {
1775            Ok(registration) => registration,
1776            Err(RegistryError::DuplicateModuleId { module_id }) => {
1777                return Ok(vec![control_error_frame(
1778                    frame,
1779                    "duplicate_module_id",
1780                    format!(
1781                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
1782                    ),
1783                )?])
1784            }
1785            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
1786                return Ok(vec![control_error_frame(
1787                    frame,
1788                    "invalid_module_id",
1789                    err.to_string(),
1790                )?])
1791            }
1792            Err(err) => {
1793                return Ok(vec![control_error_frame(
1794                    frame,
1795                    "registry_error",
1796                    err.to_string(),
1797                )?])
1798            }
1799        };
1800        if let Some(sink) = sink {
1801            let concurrency = manifest_concurrency(&registration.manifest);
1802            if let Err(err) = self.forwarding.register_candidate_module_connection(
1803                connection_id,
1804                module_id.clone(),
1805                negotiated_ver,
1806                concurrency,
1807                sink,
1808            ) {
1809                if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
1810                    crate::supervise::notify_registration_release();
1811                }
1812                return Ok(vec![control_error_frame(
1813                    frame,
1814                    forwarding_error_code(&err),
1815                    err.to_string(),
1816                )?]);
1817            }
1818        }
1819        self.supervisor.mark_swap_candidate_admitted(&module_id);
1820        info!(
1821            module_id = %module_id,
1822            module_version = %registration.manifest.module_version,
1823            negotiated_ver,
1824            ready = registration.ready,
1825            connection_id = connection_id.get(),
1826            "swap candidate registered; not routable until cutover"
1827        );
1828        self.hello_ack(frame, negotiated_ver, &module_id)
1829    }
1830
1831    fn hello_ack(
1832        &self,
1833        frame: &Frame,
1834        negotiated_ver: u8,
1835        module_id: &str,
1836    ) -> Result<Vec<Frame>, RouterError> {
1837        let ack = ModuleHelloAckBody {
1838            negotiated_ver,
1839            subc_ops: module_subc_ops(),
1840            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
1841            storage: self
1842                .storage_config
1843                .as_ref()
1844                .map(|cfg| cfg.descriptor_for(module_id)),
1845        };
1846        let body = serde_json::to_vec(&ack).map_err(|err| {
1847            RouterError::backend(
1848                0,
1849                frame.header.corr,
1850                format!("failed to encode HELLO_ACK: {err}"),
1851            )
1852        })?;
1853
1854        Ok(vec![Frame::build_with_version(
1855            negotiated_ver,
1856            FrameType::HelloAck,
1857            control_flags(),
1858            0,
1859            0,
1860            frame.header.corr,
1861            body,
1862        )
1863        .map_err(RouterError::FrameBuild)?])
1864    }
1865
1866    async fn handle_client_control_request(
1867        &self,
1868        ctx: &RouteCtx,
1869        frame: Frame,
1870        request: ClientControlRequest,
1871    ) -> Result<Vec<Frame>, RouterError> {
1872        match request {
1873            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
1874            ClientControlRequest::CatalogList { module_id } => {
1875                self.handle_catalog_list(frame, module_id)
1876            }
1877            ClientControlRequest::RouteOpen {
1878                target,
1879                identity,
1880                consumer_identity,
1881                consumer_capabilities,
1882                admission_facts,
1883            } => {
1884                self.handle_route_open(
1885                    ctx,
1886                    frame,
1887                    RouteOpenRequest {
1888                        target,
1889                        identity,
1890                        consumer_identity,
1891                        consumer_capabilities,
1892                        admission_facts,
1893                    },
1894                )
1895                .await
1896            }
1897            ClientControlRequest::RoutePoll {
1898                route_channel,
1899                route_epoch,
1900                kind,
1901            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
1902            ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame),
1903            ClientControlRequest::SupervisorSpawnSnapshot {} => {
1904                self.handle_supervisor_spawn_snapshot(frame)
1905            }
1906            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
1907                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
1908            }
1909            ClientControlRequest::SupervisorRestart {
1910                module_id,
1911                drain_timeout_ms,
1912            } => {
1913                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
1914                    .await
1915            }
1916            ClientControlRequest::SupervisorSwap {
1917                module_id,
1918                ready_timeout_ms,
1919            } => {
1920                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
1921                    .await
1922            }
1923            ClientControlRequest::SupervisorReload { module_id } => {
1924                self.handle_supervisor_reload(frame, module_id).await
1925            }
1926            ClientControlRequest::SupervisorRescan { preview } => {
1927                self.handle_supervisor_rescan(frame, preview).await
1928            }
1929            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
1930                self.handle_supervisor_release_reserved(frame, module_id)
1931                    .await
1932            }
1933            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
1934                self.handle_supervisor_set_enabled(frame, module_id, enabled)
1935                    .await
1936            }
1937            ClientControlRequest::SupervisorHealthProbe { module_id } => {
1938                self.handle_supervisor_health_probe(frame, module_id).await
1939            }
1940            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
1941            ClientControlRequest::SupervisorRoutes { module_id } => {
1942                self.handle_supervisor_routes(frame, module_id)
1943            }
1944            ClientControlRequest::SupervisorProvenance { module_id } => {
1945                self.handle_supervisor_provenance(frame, module_id).await
1946            }
1947            ClientControlRequest::SupervisorStderrTail {
1948                module_id,
1949                max_lines,
1950                max_bytes,
1951            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
1952            ClientControlRequest::SupervisorTerminals { module_id } => {
1953                self.handle_supervisor_terminals(frame, module_id)
1954            }
1955        }
1956    }
1957
1958    fn handle_module_control_request(
1959        &self,
1960        connection_id: ConnectionId,
1961        frame: Frame,
1962        request: ModuleControlRequestFromModule,
1963    ) -> Result<Vec<Frame>, RouterError> {
1964        match request {
1965            ModuleControlRequestFromModule::CatalogUpdate {
1966                provides,
1967                capabilities,
1968                ready,
1969            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
1970            ModuleControlRequestFromModule::LiveRoots {} => {
1971                let registered = self
1972                    .registry
1973                    .get_module_by_connection(connection_id)
1974                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
1975                let Some(registration) = registered else {
1976                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
1977                };
1978                let response = self
1979                    .forwarding
1980                    .live_roots(&registration.manifest.module_id)
1981                    .map_err(RouterError::Forwarding)?;
1982                Ok(vec![control_response_body_frame(
1983                    &frame,
1984                    &response,
1985                    "ModuleControlResponseToModule::LiveRoots",
1986                )?])
1987            }
1988        }
1989    }
1990
1991    fn handle_catalog_update(
1992        &self,
1993        connection_id: ConnectionId,
1994        frame: Frame,
1995        provides: Vec<ProviderRole>,
1996        capabilities: Option<CapabilityDeclarations>,
1997        ready: Option<bool>,
1998    ) -> Result<Vec<Frame>, RouterError> {
1999        self.refresh_capability_requirements();
2000        let Some(registration) = self
2001            .registry
2002            .get_module_by_connection(connection_id)
2003            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2004        else {
2005            return Ok(vec![control_error_frame(
2006                &frame,
2007                "not_registered",
2008                "catalog.update requires an active module registration owned by this connection",
2009            )?]);
2010        };
2011
2012        if let Some(message) =
2013            catalog_update_frozen_field_message(&registration.manifest, &provides)
2014        {
2015            return Ok(vec![control_error_frame(
2016                &frame,
2017                "catalog_update_frozen_field",
2018                message,
2019            )?]);
2020        }
2021
2022        let mut candidate = registration.manifest.clone();
2023        candidate.provides = provides.clone();
2024        candidate.capabilities = capabilities
2025            .clone()
2026            .or_else(|| registration.manifest.capabilities.clone());
2027        if let Err(err) = candidate.validate_capability_grammar() {
2028            return Ok(vec![control_error_frame(
2029                &frame,
2030                "invalid_capability_grammar",
2031                err.to_string(),
2032            )?]);
2033        }
2034
2035        let updated = self
2036            .registry
2037            .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
2038            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2039        if updated.is_none() {
2040            return Ok(vec![control_error_frame(
2041                &frame,
2042                "not_registered",
2043                "catalog.update requires an active module registration owned by this connection",
2044            )?]);
2045        }
2046        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
2047            log_duplicate_claim_events(
2048                self.capability_evaluator
2049                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
2050            );
2051        }
2052        if capability_census_trigger(
2053            registration.manifest.capabilities.as_ref(),
2054            updated
2055                .as_ref()
2056                .and_then(|entry| entry.manifest.capabilities.as_ref()),
2057        ) {
2058            self.enforce_capability_denies();
2059        }
2060        self.refresh_capability_requirements();
2061
2062        let response = ModuleControlResponseToModule::CatalogUpdate {};
2063        control_response_body_frame(
2064            &frame,
2065            &response,
2066            "ModuleControlResponseToModule::CatalogUpdate",
2067        )
2068        .map(|frame| vec![frame])
2069    }
2070
2071    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2072        self.refresh_capability_requirements();
2073        // A bare connection count is ambiguous between many clients holding a
2074        // route each and one client accumulating hundreds, so publish the
2075        // concentration alongside it. Route state is best-effort here: a
2076        // diagnostic endpoint must still answer if the forwarding lock is
2077        // contended.
2078        let mut counters = self.counters.snapshot();
2079        if let (Ok((connections_with_routes, max)), Some(obj)) = (
2080            self.forwarding.client_route_concentration(),
2081            counters.as_object_mut(),
2082        ) {
2083            obj.insert(
2084                "client_connections_with_routes".into(),
2085                connections_with_routes.into(),
2086            );
2087            obj.insert("max_routes_on_one_connection".into(), max.into());
2088        }
2089        // A module that is being fast-refused and a module that is fine look
2090        // identical from a client that retries and succeeds, so name the open
2091        // breakers here. This rides the existing free-form counters object
2092        // rather than a new wire field, so no sibling that deserializes
2093        // `ServerDescribe` has to be rebuilt to keep reading it.
2094        if let (Some(open_breakers), Some(obj)) = (
2095            self.route_bind_breakers.open_snapshot(),
2096            counters.as_object_mut(),
2097        ) {
2098            obj.insert("route_bind_breakers_open".into(), open_breakers);
2099        }
2100        let response = ClientControlResponse::ServerDescribe {
2101            protocol_ver: PROTOCOL_VERSION,
2102            subc_ops: subc_ops(),
2103            capabilities: self.subc_capabilities.as_ref().to_vec(),
2104            connected_clients: self.connected_clients.count(),
2105            counters: Some(counters),
2106            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
2107            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
2108            capability_requirements: self.capability_requirement_statuses(),
2109        };
2110        Ok(vec![control_response_body_frame(
2111            &frame,
2112            &response,
2113            "ClientControlResponse::ServerDescribe",
2114        )?])
2115    }
2116
2117    fn handle_catalog_list(
2118        &self,
2119        frame: Frame,
2120        module_id: Option<String>,
2121    ) -> Result<Vec<Frame>, RouterError> {
2122        let (generation, modules) = self.registry.list_modules().map_err(|err| {
2123            RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
2124        })?;
2125        let entries = modules
2126            .into_iter()
2127            .filter(|registration| {
2128                module_id
2129                    .as_deref()
2130                    .map(|wanted| registration.manifest.module_id == wanted)
2131                    .unwrap_or(true)
2132            })
2133            .map(|registration| {
2134                let roles = registration.manifest.provides;
2135                CatalogEntry {
2136                    module_id: registration.manifest.module_id,
2137                    ready: registration.ready,
2138                    module_version: Some(registration.manifest.module_version),
2139                    roles,
2140                    control_ops: registration.control_ops,
2141                    capabilities: registration.manifest.capabilities,
2142                    self_signals: registration.manifest.self_signals,
2143                }
2144            })
2145            .collect();
2146        let response = ClientControlResponse::CatalogList {
2147            generation,
2148            modules: entries,
2149            subc_ops: subc_ops(),
2150        };
2151        Ok(vec![control_response_body_frame(
2152            &frame,
2153            &response,
2154            "ClientControlResponse::CatalogList",
2155        )?])
2156    }
2157
2158    fn route_open_principal(
2159        &self,
2160        frame: &Frame,
2161        consumer_identity: Option<ConsumerIdentity>,
2162    ) -> Result<Result<Principal, Frame>, RouterError> {
2163        let Some(consumer_identity) = consumer_identity else {
2164            return Ok(Ok(Principal::Direct));
2165        };
2166
2167        if self.supervisor.spawned_consumer_authorized(
2168            &consumer_identity.module_id,
2169            &consumer_identity.launch_nonce,
2170        ) {
2171            return Ok(Ok(Principal::Reserved {
2172                module_id: consumer_identity.module_id,
2173            }));
2174        }
2175
2176        Ok(Err(control_error_frame(
2177            frame,
2178            "bad_consumer_identity",
2179            format!(
2180                "consumer_identity for module_id '{}' did not match a supervised launch nonce",
2181                consumer_identity.module_id
2182            ),
2183        )?))
2184    }
2185
2186    /// Every refusal of a `route.open` goes through here so the daemon can
2187    /// attest which code it sent: without the event, a client's "the daemon
2188    /// refused me" and the daemon's own view could only be reconciled by
2189    /// argument. Malformed input (`invalid_project_root`) does not come here;
2190    /// rejecting a request that was never a valid open is not a refusal of one.
2191    fn route_open_refusal_frame(
2192        &self,
2193        ctx: &RouteCtx,
2194        frame: &Frame,
2195        module_id: &str,
2196        reason: &'static str,
2197        code: &'static str,
2198        message: impl Into<String>,
2199    ) -> Result<Frame, RouterError> {
2200        self.observe_route_open_refusal(ctx, module_id, reason, code);
2201        control_error_frame(frame, code, message.into())
2202    }
2203
2204    /// Refuse a `route.open` because the target module's bind-relay breaker is
2205    /// open, without attempting the relay.
2206    ///
2207    /// The wire code is `module_timeout`, which is the truth (the module has
2208    /// not been answering binds) and which both SDKs already classify as
2209    /// retryable with capped backoff. Reusing it is what keeps this change out
2210    /// of both SDKs; the daemon-side distinction lives in the counter key
2211    /// instead.
2212    ///
2213    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
2214    /// While a breaker is open this fires on every open to that module, and the
2215    /// stall written up in `docs/designs/route-open-head-of-line.md` already
2216    /// produced 261 lines about a single module inside 3000 lines of daemon
2217    /// log. The rare transitions are logged at warn/info instead and the volume
2218    /// is carried by the counter, so the evidence survives without the flood.
2219    /// The debug line keeps a per-refusal record reachable for whoever turns
2220    /// the level up.
2221    fn route_open_breaker_refusal_frame(
2222        &self,
2223        ctx: &RouteCtx,
2224        frame: &Frame,
2225        module_id: &str,
2226        consecutive_timeouts: u32,
2227        retry_in: Duration,
2228        probe_in_flight: bool,
2229    ) -> Result<Frame, RouterError> {
2230        self.counters
2231            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
2232        debug!(
2233            target: "control",
2234            code = "module_timeout",
2235            module_id = ?module_id,
2236            connection_id = ctx.connection_id.get(),
2237            consecutive_timeouts,
2238            retry_in_ms = retry_in.as_millis() as u64,
2239            probe_in_flight,
2240            "route.open refused by open bind-relay breaker"
2241        );
2242        let detail = if probe_in_flight {
2243            "one probe bind is already in flight; retry once it settles".to_string()
2244        } else {
2245            format!("not relaying for another {retry_in:?}")
2246        };
2247        control_error_frame(
2248            frame,
2249            "module_timeout",
2250            format!(
2251                "module_id '{module_id}' failed {consecutive_timeouts} consecutive route.bind \
2252                 relays; {detail}"
2253            ),
2254        )
2255    }
2256
2257    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
2258    /// requester's bytes (an unknown target is whatever the client sent) and
2259    /// is Debug-formatted so control characters land in the log escaped
2260    /// rather than as terminal sequences for whoever tails it.
2261    ///
2262    /// `reason` names the check that refused, because one wire code has
2263    /// several senders: after a module registers, `target_unavailable` can
2264    /// come from a missing role, an inactive registration, a supervisor that
2265    /// has not marked the process live, a missing forwarding connection, or a
2266    /// failed relay, and a log that records only the code cannot say which of
2267    /// them fired. It is a static, daemon-chosen label per branch, so it is
2268    /// safe to print plainly and stays a closed set.
2269    fn observe_route_open_refusal(
2270        &self,
2271        ctx: &RouteCtx,
2272        module_id: &str,
2273        reason: &'static str,
2274        code: &'static str,
2275    ) {
2276        self.counters.increment_route_open_refused(code);
2277        info!(
2278            target: "control",
2279            code,
2280            reason,
2281            module_id = ?module_id,
2282            connection_id = ctx.connection_id.get(),
2283            "route.open refused"
2284        );
2285    }
2286
2287    /// Record an ACCEPTED route.open.
2288    ///
2289    /// Refusals have been logged and counted since the attestation work; accepts
2290    /// were invisible, so the daemon knew every principal it stamped and wrote
2291    /// none of them down. The party that attests the identity was the only party
2292    /// not recording it, which left a credential vault unable to name the sender
2293    /// of a call that reached it (claustrum #43) and left the launch-nonce
2294    /// concurrency question unanswerable from the outside.
2295    ///
2296    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
2297    /// `code`/`module_id`/`connection_id` returns both directions of the same
2298    /// decision rather than two shapes a reader has to join by hand.
2299    ///
2300    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
2301    /// and the difference carries information rather than being an
2302    /// inconsistency. This line is only reachable after a successful bind to a
2303    /// REGISTERED module, so the value has already passed HELLO validation
2304    /// including the path-hazard refusal and cannot contain control bytes. A
2305    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
2306    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
2307    ///
2308    /// Bare is also what every other daemon line already emits (`module
2309    /// registered`, `configured module supervised`). Shipping `?module_id` here
2310    /// made this instrument the only one in the file whose ids did not answer
2311    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
2312    /// line whose whole purpose is being grepped beside its sibling.
2313    ///
2314    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
2315    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
2316    /// forwards every field type through it and a bare `&str` and a `?`-escaped
2317    /// one are recorded identically. A test written against that harness passes
2318    /// either way -- I wrote one, measured it, and deleted it rather than ship a
2319    /// green assertion that cannot fail. The same limit applies to the escaping
2320    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
2321    /// it reads as a guard on the Debug escaping and cannot detect its removal.
2322    /// Fencing either needs the real formatter, not the capture layer.
2323    ///
2324    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
2325    /// unix-socket options and subc is loopback TCP, so there is no peer identity
2326    /// to record. The ephemeral port would decay within minutes and answer only a
2327    /// live question. The identity question is instead answered by counting
2328    /// distinct live connections presenting one module's `consumer_identity` --
2329    /// "is anyone else holding this secret" rather than "is this the right
2330    /// process".
2331    fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
2332        self.counters.increment_route_open_accepted(principal);
2333        info!(
2334            target: "control",
2335            principal,
2336            module_id,
2337            connection_id = ctx.connection_id.get(),
2338            "route.open accepted"
2339        );
2340    }
2341
2342    fn supervised_absent_route_open_refusal_frame(
2343        &self,
2344        ctx: &RouteCtx,
2345        frame: &Frame,
2346        module_id: &str,
2347        code: &'static str,
2348        status: &crate::supervise::ModuleStatus,
2349    ) -> Result<Frame, RouterError> {
2350        self.counters.increment_route_open_refused(code);
2351        info!(
2352            target: "control",
2353            code,
2354            reason = "supervised_not_registered",
2355            module_id = ?module_id,
2356            connection_id = ctx.connection_id.get(),
2357            state = %status.state,
2358            enabled = status.enabled,
2359            live = status.live,
2360            "route.open refused"
2361        );
2362        control_error_frame(
2363            frame,
2364            code,
2365            format!(
2366                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
2367                status.state, status.enabled, status.live
2368            ),
2369        )
2370    }
2371
2372    async fn handle_route_open(
2373        &self,
2374        ctx: &RouteCtx,
2375        frame: Frame,
2376        request: RouteOpenRequest,
2377    ) -> Result<Vec<Frame>, RouterError> {
2378        let RouteOpenRequest {
2379            target,
2380            mut identity,
2381            consumer_identity,
2382            consumer_capabilities,
2383            admission_facts,
2384        } = request;
2385        let target_module_id = target_module_id(&target).to_string();
2386        debug!(
2387            connection_id = ctx.connection_id.get(),
2388            corr = frame.header.corr,
2389            module_id = %target_module_id,
2390            "handling route.open"
2391        );
2392
2393        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
2394        // opposite. Below, a caller learns whether a module is unregistered,
2395        // supervised-but-down (with state/enabled/live), or registered without the
2396        // requested role. Elsewhere that is an enumeration leak: a probe learning
2397        // the shape of a fleet it cannot otherwise see.
2398        //
2399        // It is not one here, and the reason is the ACCESS MODEL rather than
2400        // anything about these errors. Reaching route.open requires the
2401        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
2402        // connection file, so any caller who completes it already runs as this
2403        // user -- and can read subc.jsonc for the module list and `ck module
2404        // status` for live state. The reply discloses nothing the caller cannot
2405        // read more easily from disk, while the precision is load-bearing:
2406        // `unknown_module` is retryable and a missing role is not.
2407        //
2408        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
2409        // remote transport, a sandboxed caller, a shared-host mode -- THAT
2410        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
2411        let Some(registration) = self
2412            .registry
2413            .get_module(&target_module_id)
2414            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2415        else {
2416            if let Some((status, warming)) =
2417                self.supervisor_status(&target_module_id, frame.header.corr)?
2418            {
2419                // BEFORE the two availability codes below, because for a module
2420                // that speaks no subc wire both of them are false comfort: they
2421                // say "not right now" and are retried, and this module will
2422                // never register no matter how long the caller waits. The
2423                // absence here is the declaration being honoured, not a module
2424                // that is late.
2425                if status.protocol == ModuleProtocol::None {
2426                    return Ok(vec![self.route_open_refusal_frame(
2427                        ctx,
2428                        &frame,
2429                        &target_module_id,
2430                        "protocol_none",
2431                        error_codes::MODULE_NO_PROTOCOL,
2432                        format!(
2433                            "module_id '{target_module_id}' is declared protocol: none; \
2434                             it speaks no subc wire and serves no routes"
2435                        ),
2436                    )?]);
2437                }
2438                let code = if warming {
2439                    "module_warming"
2440                } else {
2441                    "target_unavailable"
2442                };
2443                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
2444                    ctx,
2445                    &frame,
2446                    &target_module_id,
2447                    code,
2448                    &status,
2449                )?]);
2450            }
2451            if let Some(removed_ago_ms) =
2452                self.supervisor.removal_tombstone_age_ms(&target_module_id)
2453            {
2454                return Ok(vec![self.route_open_refusal_frame(
2455                    ctx,
2456                    &frame,
2457                    &target_module_id,
2458                    "removed",
2459                    error_codes::MODULE_REMOVED,
2460                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
2461                )?]);
2462            }
2463            return Ok(vec![self.route_open_refusal_frame(
2464                ctx,
2465                &frame,
2466                &target_module_id,
2467                "not_registered",
2468                error_codes::UNKNOWN_MODULE,
2469                format!("module_id '{target_module_id}' is not registered"),
2470            )?]);
2471        };
2472
2473        // Best-effort only: registry readiness and forwarding reservation use
2474        // different locks, so a module can flip readiness between this read and
2475        // the relay. Modules must still tolerate an `on_bind` while not ready.
2476        if !registration.ready {
2477            self.counters
2478                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
2479            info!(
2480                target: "control",
2481                code = error_codes::MODULE_WARMING,
2482                module_id = ?target_module_id,
2483                connection_id = ctx.connection_id.get(),
2484                reason = "declared_not_ready",
2485                "route.open refused"
2486            );
2487            return Ok(vec![control_error_body_frame(
2488                &frame,
2489                ErrorBody {
2490                    code: error_codes::MODULE_WARMING.to_string(),
2491                    message: format!(
2492                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
2493                    ),
2494                    detail: Some(serde_json::json!({
2495                        "reason": "declared_not_ready"
2496                    })),
2497                },
2498            )?]);
2499        }
2500
2501        if !target_has_required_role(&target, &registration.manifest.provides) {
2502            return Ok(vec![self.route_open_refusal_frame(
2503                ctx,
2504                &frame,
2505                &target_module_id,
2506                "role_not_provided",
2507                "target_unavailable",
2508                format!("module_id '{target_module_id}' does not provide the requested target"),
2509            )?]);
2510        }
2511
2512        if registration.state != ChannelState::Active {
2513            return Ok(vec![self.route_open_refusal_frame(
2514                ctx,
2515                &frame,
2516                &target_module_id,
2517                "registration_not_active",
2518                "target_unavailable",
2519                format!("module_id '{target_module_id}' is not active"),
2520            )?]);
2521        }
2522
2523        if self
2524            .forwarding
2525            .module_is_draining(&target_module_id)
2526            .map_err(RouterError::Forwarding)?
2527        {
2528            return Ok(vec![self.route_open_refusal_frame(
2529                ctx,
2530                &frame,
2531                &target_module_id,
2532                "reloading",
2533                "module_reloading",
2534                format!("module_id '{target_module_id}' is reloading"),
2535            )?]);
2536        }
2537
2538        if self
2539            .process_liveness
2540            .as_ref()
2541            .and_then(|process_liveness| process_liveness.process_live(&target_module_id))
2542            == Some(false)
2543        {
2544            return Ok(vec![self.route_open_refusal_frame(
2545                ctx,
2546                &frame,
2547                &target_module_id,
2548                "supervisor_not_live",
2549                "target_unavailable",
2550                format!("module_id '{target_module_id}' is not live"),
2551            )?]);
2552        }
2553
2554        if !self
2555            .forwarding
2556            .has_live_module_connection(&target_module_id)
2557            .map_err(RouterError::Forwarding)?
2558        {
2559            return Ok(vec![self.route_open_refusal_frame(
2560                ctx,
2561                &frame,
2562                &target_module_id,
2563                "no_forwarding_connection",
2564                "target_unavailable",
2565                format!("module_id '{target_module_id}' has no live forwarding connection"),
2566            )?]);
2567        }
2568
2569        if let Some(error) =
2570            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
2571        {
2572            self.observe_route_open_refusal(
2573                ctx,
2574                &target_module_id,
2575                "op_not_allowed",
2576                "op_not_allowed",
2577            );
2578            return Ok(vec![error]);
2579        }
2580
2581        let principal = match self.route_open_principal(&frame, consumer_identity)? {
2582            Ok(principal) => principal,
2583            Err(error) => {
2584                self.observe_route_open_refusal(
2585                    ctx,
2586                    &target_module_id,
2587                    "bad_consumer_identity",
2588                    "bad_consumer_identity",
2589                );
2590                return Ok(vec![error]);
2591            }
2592        };
2593
2594        // This is attested, control-plane policy for supervised module origins.
2595        // Keep it before route reservation and out of the opaque forwarding hot
2596        // path: data frames must never acquire a per-frame capability check.
2597        if let Principal::Reserved {
2598            module_id: opening_module_id,
2599        } = &principal
2600        {
2601            if let Some(opening_registration) = self
2602                .registry
2603                .get_module(opening_module_id)
2604                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2605            {
2606                if let Some(capability) =
2607                    denied_capability(&opening_registration.manifest, &registration.manifest)
2608                {
2609                    warn!(
2610                        opening_module_id,
2611                        target_module_id,
2612                        capability,
2613                        "refusing route.open because an attested capability deny edge matches"
2614                    );
2615                    return Ok(vec![self.route_open_refusal_frame(
2616                        ctx,
2617                        &frame,
2618                        &target_module_id,
2619                        "capability_deny_edge",
2620                        "capability_forbidden",
2621                        format!(
2622                            "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
2623                        ),
2624                    )?]);
2625                }
2626            }
2627        }
2628
2629        if admission_facts.is_some() {
2630            let carrier_matches = matches!(
2631                &principal,
2632                Principal::Reserved { module_id }
2633                    if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
2634            );
2635            if !carrier_matches {
2636                return Ok(vec![self.route_open_refusal_frame(
2637                    ctx,
2638                    &frame,
2639                    &target_module_id,
2640                    "admission_facts_carrier_not_permitted",
2641                    "admission_facts_not_permitted",
2642                    "admission facts may only be carried by the configured reserved module",
2643                )?]);
2644            }
2645
2646            let target_allowed = self
2647                .admission_facts_targets
2648                .as_ref()
2649                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
2650            if !target_allowed {
2651                return Ok(vec![self.route_open_refusal_frame(
2652                    ctx,
2653                    &frame,
2654                    &target_module_id,
2655                    "admission_facts_target_not_listed",
2656                    "admission_facts_target_not_allowed",
2657                    format!(
2658                        "admission facts are not permitted for target module_id '{target_module_id}'"
2659                    ),
2660                )?]);
2661            }
2662
2663            // Keep the value opaque to subc. The downstream admission validator owns
2664            // schema and semantic checks; this daemon only enforces carrier authority
2665            // and the configured destination allowlist.
2666        }
2667
2668        // Bind admits a root that no longer exists on disk, because refusing here
2669        // closes the only exit from a paused run: cancel needs a bound route, and a
2670        // renamed or reclaimed directory makes that route unopenable forever. The
2671        // run itself is intact and still addressable by its recorded identity.
2672        //
2673        // This does NOT relax the rule the strict constructor protects. That rule is
2674        // that no root is ever aliased into NEW durable state -- a missing component
2675        // can reappear as a symlink elsewhere, which would move the identity and
2676        // split a session's history across two of them. The engine now refuses the
2677        // two operations that create such state (send and import) at admission,
2678        // which is a narrower way to hold the same invariant: reads and terminations
2679        // are admitted, writes are not. That refusal had to ship before this line
2680        // changed, or there is an interval where a send commits under a provisional
2681        // identity -- the exact failure the original policy existed to prevent.
2682        //
2683        // Resolution follows realpath rather than lexical cleanup: the longest
2684        // existing ancestor is canonicalized and the missing tail re-appended, so a
2685        // live root is unchanged and a vanished leaf keeps the identity it was
2686        // admitted under. Lexical cleanup would mint a DIFFERENT identity for the
2687        // same caller the moment the directory vanished, which strands the run more
2688        // quietly than refusing it.
2689        let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
2690            Ok(project_root) => project_root,
2691            Err(err) => {
2692                return Ok(vec![control_error_frame(
2693                    &frame,
2694                    "invalid_project_root",
2695                    err.to_string(),
2696                )?])
2697            }
2698        };
2699        identity.project_root = project_root.as_path().to_path_buf();
2700
2701        // Last gate before any relay work, and deliberately after the cheap
2702        // registry and availability checks above: those name a more precise
2703        // condition (unknown, removed, reloading) and a caller is better served
2704        // by the precise code than by this one.
2705        //
2706        // Everything below this point costs an egress permit, a reserved handle
2707        // pair and, if the module does not answer, the whole relay budget. The
2708        // reader no longer waits for that budget, so cap each target explicitly;
2709        // serial dispatch used to provide the accidental cap of one relay per
2710        // connection. Admission is a mutex-protected count and never waits.
2711        let _concurrency_guard = match self
2712            .route_bind_concurrency
2713            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
2714        {
2715            Ok(guard) => guard,
2716            Err(in_flight) => {
2717                return Ok(vec![self.route_open_admission_refusal_frame(
2718                    ctx,
2719                    &frame,
2720                    &target_module_id,
2721                    format!(
2722                        "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
2723                    ),
2724                )?]);
2725            }
2726        };
2727
2728        // A module that has already burned the whole budget `threshold` times
2729        // in a row does not get to charge it again until a probe says it recovered.
2730        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
2731            RouteBindAdmission::Admitted { guard, probe } => {
2732                if probe {
2733                    info!(
2734                        module_id = %target_module_id,
2735                        connection_id = ctx.connection_id.get(),
2736                        "route.bind breaker half-open: admitting one probe"
2737                    );
2738                }
2739                guard
2740            }
2741            RouteBindAdmission::Refused {
2742                consecutive_timeouts,
2743                retry_in,
2744                probe_in_flight,
2745            } => {
2746                return Ok(vec![self.route_open_breaker_refusal_frame(
2747                    ctx,
2748                    &frame,
2749                    &target_module_id,
2750                    consecutive_timeouts,
2751                    retry_in,
2752                    probe_in_flight,
2753                )?]);
2754            }
2755        };
2756
2757        // Resolve the per-module budget here so the wait matches the operator's
2758        // intent for this specific target. A per-module override in
2759        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
2760        // daemons) wins over the daemon-wide default.
2761        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
2762        let relay_deadline = Instant::now() + route_bind_relay_timeout;
2763        let pending = match self
2764            .forwarding
2765            .begin_route_bind_relay_for(
2766                ctx.connection_id,
2767                ctx.egress.clone(),
2768                response_version(&frame),
2769                frame.header.corr,
2770                &target_module_id,
2771                principal.clone(),
2772                Some(project_root),
2773                relay_deadline,
2774            )
2775            .await
2776        {
2777            Ok(pending) => pending,
2778            Err(err) => {
2779                return Ok(vec![self.route_open_refusal_frame(
2780                    ctx,
2781                    &frame,
2782                    &target_module_id,
2783                    "relay_reservation_failed",
2784                    forwarding_error_code(&err),
2785                    err.to_string(),
2786                )?])
2787            }
2788        };
2789        let crate::forwarding::PendingRouteBindRelay {
2790            endpoint,
2791            module_sink,
2792            negotiated_ver,
2793            client_channel,
2794            client_epoch,
2795            module_channel,
2796            module_epoch,
2797            corr: relay_corr,
2798            receiver,
2799        } = pending;
2800        let mut reservation =
2801            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
2802
2803        debug!(
2804            connection_id = ctx.connection_id.get(),
2805            client_channel,
2806            client_epoch,
2807            module_channel,
2808            module_epoch,
2809            "reserved route handle pair"
2810        );
2811        // Rendered BEFORE the move into the relay, because the accept arm below
2812        // is where it is logged and the principal is gone by then.
2813        let principal_label = match &principal {
2814            Principal::Reserved { module_id } => format!("reserved:{module_id}"),
2815            Principal::Direct => "direct".to_string(),
2816            other => format!("{other:?}"),
2817        };
2818        let relay = ModuleControlRequest::RouteBind {
2819            route_channel: module_channel,
2820            epoch: module_epoch,
2821            target,
2822            identity,
2823            principal: Some(principal),
2824            consumer_capabilities,
2825            admission_facts,
2826        };
2827        let relay_body = serde_json::to_vec(&relay).map_err(|err| {
2828            RouterError::backend(
2829                0,
2830                frame.header.corr,
2831                format!("failed to encode route.bind request: {err}"),
2832            )
2833        })?;
2834        let relay_frame = Frame::build_with_version(
2835            negotiated_ver,
2836            FrameType::Request,
2837            control_flags(),
2838            0,
2839            0,
2840            relay_corr,
2841            relay_body,
2842        )
2843        .map_err(RouterError::FrameBuild)?;
2844
2845        if let Err(err) = module_sink.send(relay_frame).await {
2846            reservation.release_and_disarm();
2847            return Ok(vec![self.route_open_refusal_frame(
2848                ctx,
2849                &frame,
2850                &target_module_id,
2851                "relay_send_failed",
2852                "target_unavailable",
2853                err.to_string(),
2854            )?]);
2855        }
2856
2857        if !self
2858            .forwarding
2859            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
2860            .map_err(RouterError::Forwarding)?
2861        {
2862            self.send_abandoned_route_bind_goodbye(
2863                &module_sink,
2864                negotiated_ver,
2865                module_channel,
2866                module_epoch,
2867            );
2868        }
2869
2870        match timeout_at(relay_deadline, receiver).await {
2871            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
2872                reservation.disarm();
2873                if breaker.record_accepted() {
2874                    info!(
2875                        module_id = %target_module_id,
2876                        "route.bind breaker closed: the probe was accepted"
2877                    );
2878                }
2879                self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
2880                Ok(Vec::new())
2881            }
2882            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
2883                reservation.release_and_disarm();
2884                // A module that says no in microseconds is healthy. Rejection
2885                // is a different condition with its own refusal and must not
2886                // move the breaker.
2887                breaker.record_inconclusive();
2888                self.counters
2889                    .increment_route_open_refused("module_rejected");
2890                info!(
2891                    target: "control",
2892                    code = "module_rejected",
2893                    module_code = ?body.code,
2894                    module_id = ?target_module_id,
2895                    connection_id = ctx.connection_id.get(),
2896                    "route.open refused"
2897                );
2898                Ok(vec![control_error_body_frame(&frame, body)?])
2899            }
2900            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
2901                reservation.release_and_disarm();
2902                breaker.record_inconclusive();
2903                // Fires when the module's connection closes while a relayed
2904                // bind is pending -- typically a caller racing a module restart
2905                // whose bind was relayed BEFORE the drain mark went up. Logged
2906                // because the caller sees only its own error and the fleet has
2907                // already spent one diagnosis round unable to tell this arm
2908                // from a relay timeout without daemon-side evidence.
2909                tracing::warn!(
2910                    module_id = %target_module_id,
2911                    "route.bind relay abandoned: {message}"
2912                );
2913                Ok(vec![self.route_open_refusal_frame(
2914                    ctx,
2915                    &frame,
2916                    &target_module_id,
2917                    "relay_abandoned",
2918                    "target_unavailable",
2919                    message,
2920                )?])
2921            }
2922            Ok(Err(_)) => {
2923                reservation.release_and_disarm();
2924                breaker.record_inconclusive();
2925                Ok(vec![self.route_open_refusal_frame(
2926                    ctx,
2927                    &frame,
2928                    &target_module_id,
2929                    "relay_waiter_canceled",
2930                    "target_unavailable",
2931                    "route.bind relay waiter was canceled before the module responded",
2932                )?])
2933            }
2934            Err(_) => {
2935                reservation.release_and_disarm();
2936                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
2937                // answer at all is the one condition a fast refusal can
2938                // usefully stand in for; every other arm already answered.
2939                if let Some(opened) = breaker.record_timeout(
2940                    self.route_bind_breaker_threshold,
2941                    self.route_bind_breaker_cooldown,
2942                ) {
2943                    warn!(
2944                        module_id = %target_module_id,
2945                        consecutive_timeouts = opened.consecutive_timeouts,
2946                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
2947                        reopened_after_probe = opened.reopened_after_probe,
2948                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
2949                    );
2950                }
2951                // The generous budget just burned to no answer: the module is
2952                // registered and its connection is up, but its bind handler sat
2953                // on the ack for the full budget (warm-on-bind, cold configure,
2954                // or a wedged handler). Every earlier unavailability shape
2955                // fast-refuses BEFORE the relay, so this arm firing means the
2956                // slowness is module-side -- log it so the per-module timeline
2957                // is reconstructable without client audit rows.
2958                tracing::warn!(
2959                    module_id = %target_module_id,
2960                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
2961                    "route.bind relay timed out: module did not ack within budget"
2962                );
2963                Ok(vec![self.route_open_refusal_frame(
2964                    ctx,
2965                    &frame,
2966                    &target_module_id,
2967                    "relay_timed_out",
2968                    "module_timeout",
2969                    format!(
2970                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
2971                        route_bind_relay_timeout
2972                    ),
2973                )?])
2974            }
2975        }
2976    }
2977
2978    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2979        let response = ClientControlResponse::SupervisorSpawnSnapshot {
2980            snapshot: self.supervisor.spawn_snapshot(),
2981        };
2982        Ok(vec![control_response_body_frame(
2983            &frame,
2984            &response,
2985            "ClientControlResponse::SupervisorSpawnSnapshot",
2986        )?])
2987    }
2988
2989    fn handle_supervisor_spawn_subscribe(
2990        &self,
2991        ctx: &RouteCtx,
2992        frame: Frame,
2993        since: Option<SpawnCursor>,
2994    ) -> Result<Vec<Frame>, RouterError> {
2995        match self.supervisor.subscribe_spawns(
2996            ctx.connection_id,
2997            frame.header.corr,
2998            response_version(&frame),
2999            since,
3000            ctx.egress.clone(),
3001        ) {
3002            Ok(()) => Ok(Vec::new()),
3003            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
3004                Ok(vec![control_error_body_frame(
3005                    &frame,
3006                    ErrorBody {
3007                        code: "spawn_cursor_incarnation_mismatch".to_string(),
3008                        message: "spawn cursor belongs to a different daemon incarnation"
3009                            .to_string(),
3010                        detail: Some(serde_json::json!({
3011                            "current_daemon_incarnation": current
3012                        })),
3013                    },
3014                )?])
3015            }
3016            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
3017                &frame,
3018                ErrorBody {
3019                    code: "spawn_cursor_too_old".to_string(),
3020                    message: "spawn cursor predates the retained event ring".to_string(),
3021                    detail: Some(serde_json::json!({
3022                        "oldest_retained_cursor": oldest
3023                    })),
3024                },
3025            )?]),
3026            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
3027                0,
3028                frame.header.corr,
3029                format!("failed to open supervisor spawn subscription: {error}"),
3030            )),
3031        }
3032    }
3033
3034    fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3035        let generation = self
3036            .registry
3037            .generation()
3038            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3039        let modules = self
3040            .supervisor
3041            .list()
3042            .into_iter()
3043            .map(|module| {
3044                let status = module.status_for_control("list").map_err(|err| {
3045                    RouterError::backend(
3046                        0,
3047                        frame.header.corr,
3048                        format!("failed to read supervisor status: {err}"),
3049                    )
3050                })?;
3051                Ok(SupervisorEntry {
3052                    module_id: status.module_id,
3053                    state: status.state.to_string(),
3054                    enabled: status.enabled,
3055                    live: status.live,
3056                    protocol: status.protocol,
3057                    health: status.health.status,
3058                    last_probe_ms: status.health.last_probe_ms,
3059                    last_exit_code: status.last_exit.as_ref().and_then(|e| e.code),
3060                    last_exit_signal: status.last_exit.as_ref().and_then(|e| e.signal),
3061                    last_exit_ms: status.last_exit.as_ref().map(|e| e.at_ms),
3062                    last_exit_kind: status.last_exit.as_ref().map(|e| e.kind.into()),
3063                    restart_count: Some(status.restart_count),
3064                    max_restarts: Some(status.max_restarts),
3065                    lifetime_restarts: Some(status.lifetime_restarts),
3066                    spawn_generation: Some(status.spawn_generation),
3067                    restart_window_secs: Some(status.restart_window.as_secs()),
3068                    drain_timeout_ms: Some(status.drain_timeout.as_millis() as u64),
3069                    restart_backoff_ms: Some(status.restart_backoff.as_millis() as u64),
3070                    restart_max_backoff_ms: Some(status.restart_max_backoff.as_millis() as u64),
3071                })
3072            })
3073            .collect::<Result<Vec<_>, RouterError>>()?;
3074        let response = ClientControlResponse::SupervisorList {
3075            generation,
3076            modules,
3077        };
3078        Ok(vec![control_response_body_frame(
3079            &frame,
3080            &response,
3081            "ClientControlResponse::SupervisorList",
3082        )?])
3083    }
3084
3085    fn handle_supervisor_stderr_tail(
3086        &self,
3087        frame: Frame,
3088        module_id: String,
3089        max_lines: Option<u32>,
3090        max_bytes: Option<u32>,
3091    ) -> Result<Vec<Frame>, RouterError> {
3092        let Some(module) = self.supervisor.get(&module_id) else {
3093            return Ok(vec![control_error_frame(
3094                &frame,
3095                "unknown_module",
3096                format!("module_id '{module_id}' is not supervised"),
3097            )?]);
3098        };
3099
3100        let snapshot = module.stderr_tail(
3101            max_lines.map(|value| value as usize),
3102            max_bytes.map(|value| value as usize),
3103        );
3104
3105        let response = ClientControlResponse::SupervisorStderrTail {
3106            module_id,
3107            tail: StderrTail {
3108                capture: match snapshot.capture {
3109                    CaptureState::Captured => StderrCaptureState::Captured,
3110                    CaptureState::Incomplete { reason } => {
3111                        StderrCaptureState::Incomplete { reason }
3112                    }
3113                    CaptureState::NotCaptured { reason } => {
3114                        StderrCaptureState::NotCaptured { reason }
3115                    }
3116                },
3117                entries: snapshot
3118                    .entries
3119                    .into_iter()
3120                    .map(|entry| match entry {
3121                        TailEntry::Line { text, truncated } => {
3122                            StderrTailEntry::Line { text, truncated }
3123                        }
3124                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
3125                    })
3126                    .collect(),
3127                dropped_lines: snapshot.dropped_lines,
3128            },
3129        };
3130        Ok(vec![control_response_body_frame(
3131            &frame,
3132            &response,
3133            "ClientControlResponse::SupervisorStderrTail",
3134        )?])
3135    }
3136
3137    fn handle_supervisor_terminals(
3138        &self,
3139        frame: Frame,
3140        module_id: String,
3141    ) -> Result<Vec<Frame>, RouterError> {
3142        let Some(module) = self.supervisor.get(&module_id) else {
3143            return Ok(vec![control_error_frame(
3144                &frame,
3145                "unknown_module",
3146                format!("module_id '{module_id}' is not supervised"),
3147            )?]);
3148        };
3149
3150        let response = ClientControlResponse::SupervisorTerminals {
3151            module_id,
3152            terminals: module.durable_terminal_history(),
3153        };
3154        Ok(vec![control_response_body_frame(
3155            &frame,
3156            &response,
3157            "ClientControlResponse::SupervisorTerminals",
3158        )?])
3159    }
3160
3161    fn handle_supervisor_routes(
3162        &self,
3163        frame: Frame,
3164        module_id: Option<String>,
3165    ) -> Result<Vec<Frame>, RouterError> {
3166        let modules = self
3167            .forwarding
3168            .route_census(module_id.as_deref())
3169            .map_err(RouterError::Forwarding)?
3170            .into_iter()
3171            .map(|(module_id, routes)| SupervisorRouteModule {
3172                module_id,
3173                routes: routes
3174                    .into_iter()
3175                    .map(|route| SupervisorRoute {
3176                        consumer: match route.principal {
3177                            Principal::Reserved { module_id } => {
3178                                SupervisorRouteConsumer::Reserved { module_id }
3179                            }
3180                            Principal::Direct | Principal::Unverified => {
3181                                SupervisorRouteConsumer::Direct {
3182                                    connection_id: route.goodbye_target.connection_id.get(),
3183                                }
3184                            }
3185                        },
3186                        age_ms: Instant::now()
3187                            .saturating_duration_since(route.bound_at)
3188                            .as_millis()
3189                            .try_into()
3190                            .unwrap_or(u64::MAX),
3191                        draining: route.draining,
3192                        drain_reason: route.drain_reason,
3193                    })
3194                    .collect(),
3195            })
3196            .collect();
3197        let response = ClientControlResponse::SupervisorRoutes { modules };
3198        Ok(vec![control_response_body_frame(
3199            &frame,
3200            &response,
3201            "ClientControlResponse::SupervisorRoutes",
3202        )?])
3203    }
3204
3205    async fn handle_supervisor_provenance(
3206        &self,
3207        frame: Frame,
3208        module_id: Option<String>,
3209    ) -> Result<Vec<Frame>, RouterError> {
3210        let mut selected = if let Some(module_id) = module_id {
3211            let Some(module) = self.supervisor.get(&module_id) else {
3212                return Ok(vec![control_error_frame(
3213                    &frame,
3214                    "unknown_module",
3215                    format!("module_id '{module_id}' is not supervised"),
3216                )?]);
3217            };
3218            vec![module]
3219        } else {
3220            self.supervisor.list()
3221        };
3222
3223        let mut modules = Vec::with_capacity(selected.len());
3224        for module in selected.drain(..) {
3225            let status = module.status().map_err(|err| {
3226                RouterError::backend(
3227                    0,
3228                    frame.header.corr,
3229                    format!("failed to read supervisor status: {err}"),
3230                )
3231            })?;
3232            let module_declared = self
3233                .registry
3234                .get_module(&status.module_id)
3235                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3236                .and_then(|registration| registration.manifest.provenance)
3237                .map(|build| ModuleDeclaredProvenance::Reported { build })
3238                .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
3239            #[cfg(test)]
3240            let running_image = match &self.provenance_probe_override {
3241                Some(result) => result.clone(),
3242                None => module.running_image_agreement().await,
3243            };
3244            #[cfg(not(test))]
3245            let running_image = module.running_image_agreement().await;
3246            modules.push(SupervisorModuleProvenance {
3247                module_id: status.module_id,
3248                module_declared,
3249                daemon_observed: SupervisorObservedProcess {
3250                    pid: status.pid,
3251                    spawned_at_ms: status.spawned_at_ms,
3252                    spawned_from: status.spawned_from,
3253                    running_image,
3254                },
3255            });
3256        }
3257        let daemon = SupervisorDaemonProvenance {
3258            daemon_build: self.daemon_provenance.build.clone(),
3259            daemon_observed: DaemonObservedProcess {
3260                pid: self.daemon_provenance.pid,
3261                started_at_ms: self
3262                    .daemon_provenance
3263                    .start_clock
3264                    .map(|clock| clock.started_at_ms())
3265                    .or(self.daemon_provenance.started_at_ms),
3266                running_image: self
3267                    .daemon_provenance
3268                    .probe
3269                    .observe(
3270                        self.daemon_provenance.pid,
3271                        self.daemon_provenance.executable_path.as_deref(),
3272                        self.daemon_provenance.executable_identity,
3273                        self.daemon_provenance.process_start_time,
3274                    )
3275                    .await,
3276            },
3277        };
3278        let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
3279        Ok(vec![control_response_body_frame(
3280            &frame,
3281            &response,
3282            "ClientControlResponse::SupervisorProvenance",
3283        )?])
3284    }
3285
3286    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3287        self.refresh_capability_requirements();
3288        let generation = self
3289            .registry
3290            .generation()
3291            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3292        let modules = self
3293            .supervisor
3294            .list()
3295            .into_iter()
3296            .map(|module| {
3297                let status = module.status_for_control("health").map_err(|err| {
3298                    RouterError::backend(
3299                        0,
3300                        frame.header.corr,
3301                        format!("failed to read supervisor health: {err}"),
3302                    )
3303                })?;
3304                let module_id = status.module_id;
3305                let capability_detail = self
3306                    .capability_evaluator
3307                    .required_problem_detail(&module_id);
3308                Ok(SupervisorHealthEntry {
3309                    module_id,
3310                    status: status.health.status,
3311                    detail: append_capability_problem_detail(
3312                        status.health.detail,
3313                        capability_detail,
3314                    ),
3315                    metrics: status.health.metrics,
3316                    consecutive_failures: status.health.consecutive_failures,
3317                    late_answer_count: status.health.late_answer_count,
3318                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
3319                    last_action: status.health.last_action,
3320                    last_action_ms: status.health.last_action_ms,
3321                    last_probe_ms: status.health.last_probe_ms,
3322                })
3323            })
3324            .collect::<Result<Vec<_>, RouterError>>()?;
3325        let response = ClientControlResponse::SupervisorHealth {
3326            generation,
3327            modules,
3328        };
3329        Ok(vec![control_response_body_frame(
3330            &frame,
3331            &response,
3332            "ClientControlResponse::SupervisorHealth",
3333        )?])
3334    }
3335
3336    async fn handle_supervisor_restart(
3337        &self,
3338        frame: Frame,
3339        module_id: String,
3340        drain_timeout_ms: Option<u64>,
3341    ) -> Result<Vec<Frame>, RouterError> {
3342        let operation_lock = self.supervisor.operation_lock();
3343        let _operation_guard = operation_lock.lock().await;
3344        let Some(module) = self.supervisor.get(&module_id) else {
3345            return Ok(vec![control_error_frame(
3346                &frame,
3347                "unknown_module",
3348                format!("module_id '{module_id}' is not supervised"),
3349            )?]);
3350        };
3351
3352        if let Err(err) = module.restart(drain_timeout_ms).await {
3353            let (code, message) = match err {
3354                crate::supervise::SuperviseError::Disabled { .. } => {
3355                    ("module_disabled", err.to_string())
3356                }
3357                crate::supervise::SuperviseError::SwapInProgress { .. } => {
3358                    ("swap_in_progress", err.to_string())
3359                }
3360                _ => (
3361                    "target_unavailable",
3362                    format!("failed to restart module_id '{module_id}': {err}"),
3363                ),
3364            };
3365            return Ok(vec![control_error_frame(&frame, code, message)?]);
3366        }
3367
3368        let response = ClientControlResponse::SupervisorAck {
3369            module_id,
3370            applied: true,
3371        };
3372        Ok(vec![control_response_body_frame(
3373            &frame,
3374            &response,
3375            "ClientControlResponse::SupervisorAck",
3376        )?])
3377    }
3378
3379    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
3380    /// when the old process has finished draining: a caller whose own lane
3381    /// rides the old process must get its reply before that drain waits on it.
3382    async fn handle_supervisor_swap(
3383        &self,
3384        frame: Frame,
3385        module_id: String,
3386        ready_timeout_ms: Option<u64>,
3387    ) -> Result<Vec<Frame>, RouterError> {
3388        // The daemon-wide operation lock is held only to resolve the handle,
3389        // not across the swap. The swap can take its whole readiness budget,
3390        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
3391        // holding it here would park an operator's stop behind the swap it is
3392        // meant to abort. A rescan or stop that reaches the module during the
3393        // swap is served by the swap itself (see `supervise_swap`).
3394        let module = {
3395            let operation_lock = self.supervisor.operation_lock();
3396            let _operation_guard = operation_lock.lock().await;
3397            self.supervisor.get(&module_id)
3398        };
3399        let Some(module) = module else {
3400            return Ok(vec![control_error_frame(
3401                &frame,
3402                "unknown_module",
3403                format!("module_id '{module_id}' is not supervised"),
3404            )?]);
3405        };
3406
3407        if let Err(err) = module
3408            .swap(ready_timeout_ms.map(Duration::from_millis))
3409            .await
3410        {
3411            use crate::supervise::SuperviseError;
3412            let message = err.to_string();
3413            let error = match err {
3414                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
3415                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
3416                    code: "swap_refused".to_string(),
3417                    message,
3418                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
3419                },
3420                SuperviseError::SwapFailed {
3421                    arm,
3422                    candidate_exit,
3423                    ..
3424                } => ErrorBody {
3425                    code: "swap_failed".to_string(),
3426                    message,
3427                    detail: Some(serde_json::json!({
3428                        "arm": arm.as_str(),
3429                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
3430                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
3431                    })),
3432                },
3433                _ => ErrorBody::new(
3434                    "target_unavailable",
3435                    format!("failed to swap module_id '{module_id}': {message}"),
3436                ),
3437            };
3438            return Ok(vec![control_error_body_frame(&frame, error)?]);
3439        }
3440
3441        let response = ClientControlResponse::SupervisorAck {
3442            module_id,
3443            applied: true,
3444        };
3445        Ok(vec![control_response_body_frame(
3446            &frame,
3447            &response,
3448            "ClientControlResponse::SupervisorAck",
3449        )?])
3450    }
3451
3452    async fn handle_supervisor_reload(
3453        &self,
3454        frame: Frame,
3455        module_id: String,
3456    ) -> Result<Vec<Frame>, RouterError> {
3457        let operation_lock = self.supervisor.operation_lock();
3458        let _operation_guard = operation_lock.lock().await;
3459        let Some(module) = self.supervisor.get(&module_id) else {
3460            return Ok(vec![control_error_frame(
3461                &frame,
3462                "unknown_module",
3463                format!("module_id '{module_id}' is not supervised"),
3464            )?]);
3465        };
3466
3467        if let Err(err) = module.reload().await {
3468            let (code, message) = match err {
3469                crate::supervise::SuperviseError::Disabled { .. } => {
3470                    ("module_disabled", err.to_string())
3471                }
3472                crate::supervise::SuperviseError::SwapInProgress { .. } => {
3473                    ("swap_in_progress", err.to_string())
3474                }
3475                _ => (
3476                    "reload_failed",
3477                    format!("failed to reload module_id '{module_id}': {err}"),
3478                ),
3479            };
3480            return Ok(vec![control_error_frame(&frame, code, message)?]);
3481        }
3482
3483        let response = ClientControlResponse::SupervisorAck {
3484            module_id,
3485            applied: true,
3486        };
3487        Ok(vec![control_response_body_frame(
3488            &frame,
3489            &response,
3490            "ClientControlResponse::SupervisorAck",
3491        )?])
3492    }
3493
3494    async fn handle_supervisor_rescan(
3495        &self,
3496        frame: Frame,
3497        preview: bool,
3498    ) -> Result<Vec<Frame>, RouterError> {
3499        let Some(context) = self.rescan.clone() else {
3500            return Ok(vec![control_error_frame(
3501                &frame,
3502                "rescan_unavailable",
3503                "the daemon was not started with a reloadable config path".to_string(),
3504            )?]);
3505        };
3506
3507        let operation_lock = self.supervisor.operation_lock();
3508        let _operation_guard = operation_lock.lock().await;
3509        let loaded = match crate::daemon_config::load(&context.config_path) {
3510            Ok(config) => config,
3511            Err(err) => {
3512                return Ok(vec![control_error_frame(
3513                    &frame,
3514                    "invalid_daemon_config",
3515                    format!("supervisor rescan rejected daemon config: {err}"),
3516                )?])
3517            }
3518        };
3519        // `load` reports a missing file as Ok(None), which is correct at boot
3520        // (no config, nothing to supervise) and catastrophic here: rescan treats
3521        // "not in the config" as "remove it", so an absent file would read as an
3522        // empty module list and retire the entire running fleet. An editor
3523        // writing via write-new-then-rename, or a half-finished edit, is enough
3524        // to open that window. Refuse instead: a config that cannot be read
3525        // carries no instruction to remove anything.
3526        let Some(config) = loaded else {
3527            return Ok(vec![control_error_frame(
3528                &frame,
3529                "invalid_daemon_config",
3530                format!(
3531                    "daemon config not found at {}; refusing to rescan (an absent config would \
3532                     retire every supervised module)",
3533                    context.config_path.display()
3534                ),
3535            )?]);
3536        };
3537        let (
3538            configured_port,
3539            storage_config,
3540            admission_facts_carrier_module_id,
3541            admission_facts_targets,
3542            modules,
3543            reserved_capabilities,
3544        ) = (
3545            config.port,
3546            config.storage,
3547            config.admission_facts_carrier_module_id,
3548            config.admission_facts_targets,
3549            config.modules,
3550            config.reserved_capabilities,
3551        );
3552
3553        // Collect the sections rescan cannot apply, so the REPLY carries them.
3554        //
3555        // The warning below has always been correct and has always gone only to
3556        // the journal -- addressed to whoever reads logs, while the person who
3557        // just edited the config is looking at the CLI. Naming each section
3558        // individually rather than setting a flag: "something outside modules
3559        // changed" sends the operator back to diffing their own file, which is
3560        // the work this is meant to save.
3561        let mut restart_required = Vec::new();
3562        for section in RestartRequiredSection::ALL {
3563            let changed = match section {
3564                RestartRequiredSection::Port => configured_port != context.configured_port,
3565                RestartRequiredSection::Storage => storage_config != context.storage_config,
3566                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
3567                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
3568                }
3569                RestartRequiredSection::AdmissionFactsTargets => {
3570                    admission_facts_targets != context.admission_facts_targets
3571                }
3572            };
3573            if changed {
3574                restart_required.push(section.label().to_string());
3575            }
3576        }
3577        if !restart_required.is_empty() {
3578            warn!(
3579                config_path = %context.config_path.display(),
3580                sections = %restart_required.join(", "),
3581                "daemon config changed outside the modules section; restart the daemon to apply those changes"
3582            );
3583        }
3584
3585        for configured in &modules {
3586            if let Err(err) = validate_spec(&configured.module_spec()) {
3587                return Ok(vec![control_error_frame(
3588                    &frame,
3589                    "invalid_daemon_config",
3590                    format!("supervisor rescan rejected daemon config: {err}"),
3591                )?]);
3592            }
3593        }
3594
3595        let configured_capabilities = modules
3596            .iter()
3597            .map(|module| (module.module_id.clone(), module.enabled))
3598            .collect::<Vec<_>>();
3599        let preview_capability_warnings = if preview {
3600            let (_, registrations) = self.runtime_capability_snapshot()?;
3601            let current_modules = self
3602                .supervisor
3603                .list()
3604                .into_iter()
3605                .map(|module| module.module_id().to_string())
3606                .collect::<BTreeSet<_>>();
3607            let resulting_modules = configured_capabilities.clone();
3608            let removed = current_modules
3609                .into_iter()
3610                .filter(|module_id| {
3611                    !resulting_modules
3612                        .iter()
3613                        .any(|(configured_id, _)| configured_id == module_id)
3614                })
3615                .collect::<Vec<_>>();
3616            self.capability_evaluator.preview_removal_warnings(
3617                resulting_modules,
3618                &removed,
3619                &registrations,
3620            )
3621        } else {
3622            Vec::new()
3623        };
3624        let result = match self
3625            .reconcile_supervised_modules(&context.supervisor, modules, preview)
3626            .await
3627        {
3628            Ok(result) => result,
3629            Err(message) => {
3630                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
3631            }
3632        };
3633        if !preview {
3634            self.capability_evaluator
3635                .configure(configured_capabilities, reserved_capabilities);
3636            self.capability_evaluator.wake_deadline_loop();
3637            self.refresh_capability_requirements();
3638        }
3639        let mut result = result;
3640        result.restart_required = restart_required;
3641        result.capability_warnings = preview_capability_warnings;
3642        let response = ClientControlResponse::SupervisorRescan { result };
3643        Ok(vec![control_response_body_frame(
3644            &frame,
3645            &response,
3646            "ClientControlResponse::SupervisorRescan",
3647        )?])
3648    }
3649
3650    async fn handle_supervisor_release_reserved(
3651        &self,
3652        frame: Frame,
3653        module_id: String,
3654    ) -> Result<Vec<Frame>, RouterError> {
3655        let Some(context) = self.rescan.clone() else {
3656            return Ok(vec![control_error_frame(
3657                &frame,
3658                "release_unavailable",
3659                "reserved-id release requires a daemon started with a reloadable config path",
3660            )?]);
3661        };
3662        let operation_lock = self.supervisor.operation_lock();
3663        let _operation_guard = operation_lock.lock().await;
3664        let loaded = match crate::daemon_config::load(&context.config_path) {
3665            Ok(Some(config)) => config,
3666            Ok(None) => {
3667                return Ok(vec![control_error_frame(
3668                    &frame,
3669                    "invalid_daemon_config",
3670                    format!(
3671                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
3672                        context.config_path.display()
3673                    ),
3674                )?])
3675            }
3676            Err(err) => {
3677                return Ok(vec![control_error_frame(
3678                    &frame,
3679                    "invalid_daemon_config",
3680                    format!("unable to verify reserved-id release against daemon config: {err}"),
3681                )?])
3682            }
3683        };
3684        if loaded
3685            .modules
3686            .iter()
3687            .any(|configured| configured.module_id == module_id)
3688        {
3689            return Ok(vec![control_error_frame(
3690                &frame,
3691                "reserved_module_configured",
3692                format!(
3693                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
3694                ),
3695            )?]);
3696        }
3697        if !self.supervisor.release_retained_reserved_gate(&module_id) {
3698            return Ok(vec![control_error_frame(
3699                &frame,
3700                "reserved_gate_not_retained",
3701                format!(
3702                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
3703                ),
3704            )?]);
3705        }
3706
3707        let response = ClientControlResponse::SupervisorAck {
3708            module_id,
3709            applied: true,
3710        };
3711        Ok(vec![control_response_body_frame(
3712            &frame,
3713            &response,
3714            "ClientControlResponse::SupervisorAck",
3715        )?])
3716    }
3717
3718    /// Reconcile the running module set against the configured one.
3719    ///
3720    /// With `preview` set, the diff is computed and returned WITHOUT applying any
3721    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
3722    /// deliberately shares this function with the executing path rather than
3723    /// computing the same diff somewhere else -- two implementations of one
3724    /// decision agree until they do not, and the whole value of a preview is that
3725    /// it describes the operation that will actually run.
3726    async fn reconcile_supervised_modules(
3727        &self,
3728        supervisor: &Supervisor,
3729        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
3730        preview: bool,
3731    ) -> Result<SupervisorRescanResult, String> {
3732        let mut current = BTreeMap::new();
3733        for module in self.supervisor.list() {
3734            let (spec, health) = module.configuration().map_err(|err| {
3735                format!(
3736                    "failed to read configuration for module_id '{}': {err}",
3737                    module.module_id()
3738                )
3739            })?;
3740            let enabled = module
3741                .status()
3742                .map_err(|err| {
3743                    format!(
3744                        "failed to read status for module_id '{}': {err}",
3745                        module.module_id()
3746                    )
3747                })?
3748                .enabled;
3749            current.insert(
3750                module.module_id().to_string(),
3751                (module, spec, health, enabled),
3752            );
3753        }
3754        let configured = configured_modules
3755            .into_iter()
3756            .map(|module| (module.module_id.clone(), module))
3757            .collect::<BTreeMap<_, _>>();
3758
3759        let added = configured
3760            .keys()
3761            .filter(|module_id| !current.contains_key(*module_id))
3762            .cloned()
3763            .collect::<Vec<_>>();
3764        let removed = current
3765            .keys()
3766            .filter(|module_id| !configured.contains_key(*module_id))
3767            .cloned()
3768            .collect::<Vec<_>>();
3769        let mut changed_pending_reload = Vec::new();
3770        let mut configuration_changes = BTreeSet::new();
3771        let mut enabled_changes = BTreeSet::new();
3772        let mut unchanged = 0_u32;
3773
3774        for (module_id, configured_module) in &configured {
3775            let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
3776            else {
3777                continue;
3778            };
3779            let configuration_changed = *current_spec != configured_module.module_spec()
3780                || *current_health != configured_module.health;
3781            let enabled_changed = *current_enabled != configured_module.enabled;
3782            if configuration_changed {
3783                configuration_changes.insert(module_id.clone());
3784                changed_pending_reload.push(module_id.clone());
3785            }
3786            if enabled_changed {
3787                enabled_changes.insert(module_id.clone());
3788            }
3789            if !configuration_changed && !enabled_changed {
3790                unchanged = unchanged.saturating_add(1);
3791            }
3792        }
3793
3794        // Everything above this point is pure computation over two snapshots.
3795        // Everything below MUTATES. The preview returns here so the boundary is a
3796        // single early return rather than a condition repeated at each mutation
3797        // site, where one missed guard would apply part of a change the caller was
3798        // told would not happen.
3799        if preview {
3800            return Ok(SupervisorRescanResult {
3801                added,
3802                removed,
3803                changed_pending_reload,
3804                enabled_changes: enabled_changes.iter().cloned().collect(),
3805                unchanged,
3806                preview: true,
3807                // Filled by the caller on both paths, so the preview reports
3808                // restart-required sections identically to an executed rescan --
3809                // the preview is where an operator is most likely to be looking.
3810                restart_required: Vec::new(),
3811                capability_warnings: Vec::new(),
3812            });
3813        }
3814
3815        for module_id in &removed {
3816            let module = &current
3817                .get(module_id)
3818                .expect("removed module came from current supervisor state")
3819                .0;
3820            module.retire().await.map_err(|err| {
3821                format!("failed to retire module_id '{module_id}' during rescan: {err}")
3822            })?;
3823            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
3824            //
3825            // `handle_route_open` resolves an absent module in three steps:
3826            // registry, then supervisor status, then tombstone. Retiring first
3827            // opens a window where ALL THREE ARE ABSENT -- the registry entry
3828            // went with the teardown above, the supervisor entry went with
3829            // `retire`, and the tombstone does not exist yet -- so a route.open
3830            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
3831            // it") for a module that was deliberately removed and whose caller
3832            // should get `module_removed` (TERMINAL, carrying a removal age).
3833            //
3834            // Writing the tombstone first closes it: during the window the
3835            // supervisor entry still answers, so the caller gets
3836            // `target_unavailable` -- retryable, and TRUE, because the module
3837            // is mid-teardown. After both statements it is `module_removed`.
3838            // No instant remains where a removed module reads as one that
3839            // never existed.
3840            //
3841            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
3842            // the absence of a test beside a fix invites deletion: these are
3843            // two sync statements with no await between them, so reaching the
3844            // window needs a second worker thread to land exactly between them
3845            // and there is no hook to force it. MEASURED: the 25 daemon_config
3846            // tests pass identically with the old order and the new one, so
3847            // the existing suite cannot see this and a green run is not
3848            // evidence either way. What the suite does hold is the
3849            // post-condition -- a removed module answers `module_removed` --
3850            // which this preserves.
3851            //
3852            // Found by an Athena panel reading the shipped tree against a
3853            // design note (2026-09-19), as the one concrete instance of that
3854            // note's class that survived contact with source. Direction is
3855            // benign: retryable where terminal was intended, never the reverse.
3856            self.supervisor.record_rescan_removal(module_id);
3857            self.supervisor.retire(module_id);
3858        }
3859
3860        for module_id in configured.keys() {
3861            let Some((module, _, _, _)) = current.get(module_id) else {
3862                continue;
3863            };
3864            let configured_module = configured
3865                .get(module_id)
3866                .expect("configured module id came from configured map");
3867            if configuration_changes.contains(module_id) {
3868                module
3869                    .update_configuration(
3870                        configured_module.module_spec(),
3871                        configured_module.health,
3872                        configured_module.drain_timeout_ms,
3873                    )
3874                    .await
3875                    .map_err(|err| {
3876                        format!(
3877                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
3878                        )
3879                    })?;
3880            }
3881            if enabled_changes.contains(module_id) {
3882                module
3883                    .set_enabled(configured_module.enabled)
3884                    .await
3885                    .map_err(|err| {
3886                        format!(
3887                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
3888                            configured_module.enabled
3889                        )
3890                    })?;
3891            }
3892        }
3893
3894        for module_id in &added {
3895            let configured_module = configured
3896                .get(module_id)
3897                .expect("added module id came from configured map");
3898            supervisor
3899                .supervise_configured_with_health(
3900                    configured_module.module_spec(),
3901                    configured_module.enabled,
3902                    configured_module.health,
3903                    configured_module.drain_timeout_ms,
3904                    configured_module.restart,
3905                )
3906                .map_err(|err| {
3907                    format!("failed to add module_id '{module_id}' during rescan: {err}")
3908                })?;
3909        }
3910
3911        Ok(SupervisorRescanResult {
3912            added,
3913            removed,
3914            changed_pending_reload,
3915            enabled_changes: enabled_changes.iter().cloned().collect(),
3916            unchanged,
3917            preview: false,
3918            // Filled by the caller, which is the only layer that can see the
3919            // previous config to diff against.
3920            restart_required: Vec::new(),
3921            capability_warnings: Vec::new(),
3922        })
3923    }
3924
3925    async fn handle_supervisor_set_enabled(
3926        &self,
3927        frame: Frame,
3928        module_id: String,
3929        enabled: bool,
3930    ) -> Result<Vec<Frame>, RouterError> {
3931        let operation_lock = self.supervisor.operation_lock();
3932        let _operation_guard = operation_lock.lock().await;
3933        let Some(module) = self.supervisor.get(&module_id) else {
3934            return Ok(vec![control_error_frame(
3935                &frame,
3936                "unknown_module",
3937                format!("module_id '{module_id}' is not supervised"),
3938            )?]);
3939        };
3940
3941        let applied = match module.set_enabled(enabled).await {
3942            Ok(applied) => applied,
3943            Err(err) => {
3944                return Ok(vec![control_error_frame(
3945                    &frame,
3946                    "target_unavailable",
3947                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
3948                )?])
3949            }
3950        };
3951
3952        self.capability_evaluator.wake_deadline_loop();
3953        self.refresh_capability_requirements();
3954        let response = ClientControlResponse::SupervisorAck { module_id, applied };
3955        Ok(vec![control_response_body_frame(
3956            &frame,
3957            &response,
3958            "ClientControlResponse::SupervisorAck",
3959        )?])
3960    }
3961
3962    async fn handle_supervisor_health_probe(
3963        &self,
3964        frame: Frame,
3965        module_id: String,
3966    ) -> Result<Vec<Frame>, RouterError> {
3967        self.refresh_capability_requirements();
3968        let Some(registration) = self
3969            .registry
3970            .get_module(&module_id)
3971            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3972        else {
3973            return Ok(vec![control_error_frame(
3974                &frame,
3975                "unknown_module",
3976                format!("module_id '{module_id}' is not registered"),
3977            )?]);
3978        };
3979
3980        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
3981        // named for it. Making `module_registration_grants_op` return false
3982        // unconditionally reddens five tests, and every one is named for something
3983        // else -- capability relay, probe/bind demultiplexing, supervision-only
3984        // probing. They exercise a successful advertisement check on the way to their
3985        // own subject.
3986        //
3987        // Real protection, fragile in a specific way: narrowing any of those tests to
3988        // focus on its stated subject would silently remove coverage nobody knows
3989        // they are carrying. Recorded here rather than as a sixth test, because the
3990        // useful fact is WHICH tests hold the guard up -- a new test would add
3991        // coverage without telling the next person what the existing ones quietly do.
3992        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
3993        {
3994            return Ok(vec![control_error_frame(
3995                &frame,
3996                "health_not_advertised",
3997                format!("module_id '{module_id}' did not advertise health.check"),
3998            )?]);
3999        }
4000
4001        let deadline = Instant::now() + self.health_probe_timeout;
4002        let pending = match self.forwarding.begin_module_control_rpc_for(
4003            &module_id,
4004            MODULE_CONTROL_OP_HEALTH_CHECK,
4005            deadline,
4006        ) {
4007            Ok(pending) => pending,
4008            Err(err) => {
4009                return Ok(vec![control_error_frame(
4010                    &frame,
4011                    forwarding_error_code(&err),
4012                    err.to_string(),
4013                )?])
4014            }
4015        };
4016
4017        let PendingModuleControlRpc {
4018            endpoint,
4019            module_sink,
4020            negotiated_ver,
4021            corr: probe_corr,
4022            receiver,
4023        } = pending;
4024        let mut guard =
4025            ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
4026        let probe_body =
4027            serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
4028                RouterError::backend(
4029                    0,
4030                    frame.header.corr,
4031                    format!("failed to encode health.check request: {err}"),
4032                )
4033            })?;
4034        let probe_frame = Frame::build_with_version(
4035            negotiated_ver,
4036            FrameType::Request,
4037            control_flags(),
4038            0,
4039            0,
4040            probe_corr,
4041            probe_body,
4042        )
4043        .map_err(RouterError::FrameBuild)?;
4044
4045        if let Err(err) = module_sink.send(probe_frame).await {
4046            return Ok(vec![control_error_frame(
4047                &frame,
4048                "target_unavailable",
4049                err.to_string(),
4050            )?]);
4051        }
4052
4053        match timeout_at(deadline, receiver).await {
4054            Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
4055                guard.disarm();
4056                let Some(report) = response.health_report() else {
4057                    return Ok(vec![control_error_frame(
4058                        &frame,
4059                        "invalid_control_body",
4060                        "health.check RPC returned a non-health response",
4061                    )?]);
4062                };
4063                // Metrics go out whole here. The supervisor's cached snapshot
4064                // caps this blob (see truncate_health_metrics), and this path
4065                // exists precisely to answer without that cap -- so applying it
4066                // here would leave no way to see what the cached view drops.
4067                let HealthReport {
4068                    status,
4069                    detail,
4070                    metrics,
4071                } = report;
4072                let capability_detail = self
4073                    .capability_evaluator
4074                    .required_problem_detail(&module_id);
4075                let response = ClientControlResponse::SupervisorHealthProbe {
4076                    module_id,
4077                    status,
4078                    detail: append_capability_problem_detail(detail, capability_detail),
4079                    metrics,
4080                };
4081                Ok(vec![control_response_body_frame(
4082                    &frame,
4083                    &response,
4084                    "ClientControlResponse::SupervisorHealthProbe",
4085                )?])
4086            }
4087            Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
4088                guard.disarm();
4089                Ok(vec![control_error_body_frame(&frame, body)?])
4090            }
4091            Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
4092                guard.disarm();
4093                Ok(vec![control_error_frame(
4094                    &frame,
4095                    "target_unavailable",
4096                    message,
4097                )?])
4098            }
4099            Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
4100                guard.disarm();
4101                Ok(vec![control_error_frame(
4102                    &frame,
4103                    "invalid_control_body",
4104                    message,
4105                )?])
4106            }
4107            Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
4108                guard.disarm();
4109                Ok(vec![control_error_frame(
4110                    &frame,
4111                    "invalid_control_body",
4112                    format!("expected module-control op '{expected}', got '{actual}'"),
4113                )?])
4114            }
4115            Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
4116                guard.disarm();
4117                Ok(vec![control_error_frame(
4118                    &frame,
4119                    "module_timeout",
4120                    format!(
4121                        "module_id '{module_id}' answered health.check after {:?}",
4122                        self.health_probe_timeout
4123                    ),
4124                )?])
4125            }
4126            Ok(Err(_)) => Ok(vec![control_error_frame(
4127                &frame,
4128                "target_unavailable",
4129                "health.check waiter was canceled before the module responded",
4130            )?]),
4131            Err(_) => Ok(vec![control_error_frame(
4132                &frame,
4133                "module_timeout",
4134                format!(
4135                    "module_id '{module_id}' did not answer health.check within {:?}",
4136                    self.health_probe_timeout
4137                ),
4138            )?]),
4139        }
4140    }
4141
4142    fn supervisor_status(
4143        &self,
4144        module_id: &str,
4145        corr: u64,
4146    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
4147        self.supervisor
4148            .get(module_id)
4149            .map(|module| {
4150                let warming = module.is_warming_for_control("status").map_err(|err| {
4151                    RouterError::backend(
4152                        0,
4153                        corr,
4154                        format!(
4155                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
4156                        ),
4157                    )
4158                })?;
4159                module.status_for_control("status").map_err(|err| {
4160                    RouterError::backend(
4161                        0,
4162                        corr,
4163                        format!(
4164                            "failed to read supervisor status for module_id '{module_id}': {err}"
4165                        ),
4166                    )
4167                }).map(|status| (status, warming))
4168            })
4169            .transpose()
4170    }
4171
4172    fn guard_module_control_op(
4173        &self,
4174        frame: &Frame,
4175        module_id: &str,
4176        op: &str,
4177    ) -> Result<Option<Frame>, RouterError> {
4178        if self.module_grants_op(module_id, op, frame.header.corr)? {
4179            return Ok(None);
4180        }
4181
4182        Ok(Some(control_error_frame(
4183            frame,
4184            "op_not_allowed",
4185            format!("module_id '{module_id}' did not grant control op '{op}'"),
4186        )?))
4187    }
4188
4189    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
4190        let Some(registration) = self
4191            .registry
4192            .get_module(module_id)
4193            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
4194        else {
4195            return Ok(false);
4196        };
4197        Ok(module_registration_grants_op(&registration.control_ops, op))
4198    }
4199
4200    fn handle_status_update(
4201        &self,
4202        endpoint: ModuleEndpointId,
4203        frame: Frame,
4204    ) -> Result<Vec<Frame>, RouterError> {
4205        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
4206            Ok(update) => update,
4207            Err(err) => {
4208                // Forward-compat: a newer module may push a channel-0 op this subc
4209                // version doesn't know. The control contract says unknown push ops
4210                // are IGNORED, never answered with an error. Only a malformed body
4211                // for an op we DO know is a real error worth surfacing.
4212                if is_known_module_push_op(&frame.body) {
4213                    return Ok(vec![control_error_frame(
4214                        &frame,
4215                        "invalid_control_body",
4216                        format!("malformed module control push body: {err}"),
4217                    )?]);
4218                }
4219                return Ok(Vec::new());
4220            }
4221        };
4222
4223        match update {
4224            ModuleControlPush::RouteStatus {
4225                route_channel,
4226                route_epoch,
4227                status,
4228            } => {
4229                self.forwarding
4230                    .cache_status(endpoint, route_channel, route_epoch, status)
4231                    .map_err(RouterError::Forwarding)?;
4232            }
4233        }
4234        Ok(Vec::new())
4235    }
4236
4237    fn handle_route_poll(
4238        &self,
4239        ctx: &RouteCtx,
4240        frame: Frame,
4241        route_channel: u16,
4242        route_epoch: u32,
4243        kind: PollKind,
4244    ) -> Result<Vec<Frame>, RouterError> {
4245        let snapshot = self
4246            .forwarding
4247            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
4248            .map_err(RouterError::Forwarding)?;
4249        let response = match (kind, snapshot) {
4250            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
4251                ClientControlResponse::RoutePoll {
4252                    route_channel,
4253                    route_epoch,
4254                    status,
4255                    live: None,
4256                }
4257            }
4258            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
4259                route_channel,
4260                route_epoch,
4261                status: None,
4262                live: None,
4263            },
4264            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
4265                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
4266                // is what makes reporting `true` correct rather than a
4267                // confident guess. `process_live` returns None only when the
4268                // module id has no supervisor snapshot at all -- an
4269                // externally-started module the daemon did not spawn -- and
4270                // for those the supervisor has no opinion to offer, ever. It
4271                // is never None for a supervised module in an unknown state:
4272                // a supervised module always has a snapshot, and the answer
4273                // comes from `state == Running && process_alive`.
4274                //
4275                // The route is Bound, so the module completed a HELLO on a
4276                // live connection; "the process this route points at is
4277                // running" is therefore attested by the binding rather than
4278                // assumed. Reporting `false` for an unsupervised module would
4279                // be the actual lie -- it would tell a client its healthy
4280                // route is dead because the daemon does not manage the
4281                // process.
4282                //
4283                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
4284                // module whose liveness is genuinely unknown, e.g. a snapshot
4285                // that has not been populated yet -- THIS DEFAULT BECOMES
4286                // WRONG and must split: unsupervised stays true, unknown
4287                // becomes null so the client can tell the two apart. The
4288                // response field is already `Option<bool>`, so the wire can
4289                // carry that distinction today.
4290                let live = self
4291                    .process_liveness
4292                    .as_ref()
4293                    .and_then(|source| source.process_live(&module_id))
4294                    .unwrap_or(true);
4295                ClientControlResponse::RoutePoll {
4296                    route_channel,
4297                    route_epoch,
4298                    status: None,
4299                    live: Some(live),
4300                }
4301            }
4302            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
4303                route_channel,
4304                route_epoch,
4305                status: None,
4306                live: Some(false),
4307            },
4308        };
4309
4310        Ok(vec![control_response_body_frame(
4311            &frame,
4312            &response,
4313            "ClientControlResponse::RoutePoll",
4314        )?])
4315    }
4316
4317    pub(crate) fn observe_module_control_completion(
4318        &self,
4319        completion: ModuleControlRpcCompletion,
4320    ) -> bool {
4321        match completion {
4322            ModuleControlRpcCompletion::Unknown => false,
4323            ModuleControlRpcCompletion::Settled => true,
4324            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
4325                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
4326                info!(
4327                    module_id = %module_id,
4328                    latency_ms,
4329                    "late health.check answer proves the module is alive"
4330                );
4331                match self
4332                    .supervisor
4333                    .record_late_health_answer(&module_id, latency_ms)
4334                {
4335                    Ok(true) => {}
4336                    Ok(false) => debug!(
4337                        module_id = %module_id,
4338                        latency_ms,
4339                        "late health.check answer has no active supervisor snapshot"
4340                    ),
4341                    Err(err) => warn!(
4342                        module_id = %module_id,
4343                        latency_ms,
4344                        error = %err,
4345                        "failed to record late health.check answer"
4346                    ),
4347                }
4348                true
4349            }
4350        }
4351    }
4352
4353    /// Decide whether a failure while settling a relayed `route.bind` belongs to
4354    /// the module connection whose frame is being handled, or to the client that
4355    /// relay was opened for.
4356    ///
4357    /// This runs on the MODULE connection's frame handler, where returning `Err`
4358    /// ends that connection -- and a module connection carries every client's
4359    /// routes to that module, so ending it costs the whole fleet its tools.
4360    /// `ConnectionClosing` carries the id of the connection that is closing, and
4361    /// when that id is a CLIENT's, the condition is entirely about that one
4362    /// client's route.open. A client-scoped condition has no authority over a
4363    /// shared module connection, so it is logged and the single relay is dropped:
4364    /// the client is going away, and `complete_pending_relay` already removed the
4365    /// relay before failing, so there is nothing left to settle. Anything that
4366    /// relay still reserved is released by that client's own connection teardown,
4367    /// which is already under way -- that is what "closing" means.
4368    ///
4369    /// Every other failure is a statement about THIS connection and stays fatal:
4370    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
4371    /// id in `ConnectionClosing` all mean this connection cannot keep serving
4372    /// frames correctly.
4373    fn refuse_to_end_module_connection_for_a_client(
4374        &self,
4375        module_connection_id: ConnectionId,
4376        corr: u64,
4377        err: ForwardingError,
4378    ) -> Result<(), RouterError> {
4379        if let ForwardingError::ConnectionClosing { connection_id } = err {
4380            if connection_id != module_connection_id {
4381                warn!(
4382                    module_connection_id = module_connection_id.get(),
4383                    client_connection_id = connection_id.get(),
4384                    corr,
4385                    "dropping a route.bind response for a closing client; the module connection keeps serving"
4386                );
4387                return Ok(());
4388            }
4389        }
4390        Err(RouterError::Forwarding(err))
4391    }
4392
4393    fn handle_module_relay_response(
4394        &self,
4395        connection_id: ConnectionId,
4396        frame: Frame,
4397    ) -> Result<Vec<Frame>, RouterError> {
4398        let mut secondary_error = None;
4399        let outcome = match frame.header.ty {
4400            FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
4401                Ok(probe) if probe.op == "route.bind" => {
4402                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
4403                        Ok(ModuleControlResponse::RouteBindAck {}) => {
4404                            RouteBindRelayOutcome::Accepted
4405                        }
4406                        Ok(other) => {
4407                            let message =
4408                                format!("route.bind response carried unexpected body: {other:?}");
4409                            secondary_error = Some(control_error_frame(
4410                                &frame,
4411                                "invalid_control_body",
4412                                message.clone(),
4413                            )?);
4414                            RouteBindRelayOutcome::ModuleGone(message)
4415                        }
4416                        Err(err) => {
4417                            let message = format!("malformed route.bind response body: {err}");
4418                            secondary_error = Some(control_error_frame(
4419                                &frame,
4420                                "invalid_control_body",
4421                                message.clone(),
4422                            )?);
4423                            RouteBindRelayOutcome::ModuleGone(message)
4424                        }
4425                    }
4426                }
4427                Ok(probe) => {
4428                    let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
4429                    {
4430                        Ok(response) => ModuleControlRpcOutcome::Response(response),
4431                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
4432                            "malformed {} response body: {err}",
4433                            probe.op
4434                        )),
4435                    };
4436                    let completion = self
4437                        .forwarding
4438                        .complete_module_control_rpc(
4439                            connection_id,
4440                            frame.header.corr,
4441                            Some(&probe.op),
4442                            outcome,
4443                        )
4444                        .map_err(RouterError::Forwarding)?;
4445                    if !self.observe_module_control_completion(completion) {
4446                        debug!(
4447                            connection_id = connection_id.get(),
4448                            corr = frame.header.corr,
4449                            op = %probe.op,
4450                            "dropping late or unknown module-control RPC response"
4451                        );
4452                    }
4453                    return Ok(Vec::new());
4454                }
4455                Err(err) => {
4456                    if let Some(expected_op) = self
4457                        .forwarding
4458                        .pending_module_control_op(connection_id, frame.header.corr)
4459                        .map_err(RouterError::Forwarding)?
4460                    {
4461                        let completion = self
4462                            .forwarding
4463                            .complete_module_control_rpc(
4464                                connection_id,
4465                                frame.header.corr,
4466                                None,
4467                                ModuleControlRpcOutcome::MalformedResponse(format!(
4468                                    "malformed {expected_op} response body: {err}"
4469                                )),
4470                            )
4471                            .map_err(RouterError::Forwarding)?;
4472                        if !self.observe_module_control_completion(completion) {
4473                            debug!(
4474                                connection_id = connection_id.get(),
4475                                corr = frame.header.corr,
4476                                "dropping late malformed module-control RPC response"
4477                            );
4478                        }
4479                        return Ok(Vec::new());
4480                    }
4481                    let message = format!("malformed route.bind response body: {err}");
4482                    secondary_error = Some(control_error_frame(
4483                        &frame,
4484                        "invalid_control_body",
4485                        message.clone(),
4486                    )?);
4487                    RouteBindRelayOutcome::ModuleGone(message)
4488                }
4489            },
4490            FrameType::Error => {
4491                if self
4492                    .forwarding
4493                    .pending_module_control_op(connection_id, frame.header.corr)
4494                    .map_err(RouterError::Forwarding)?
4495                    .is_some()
4496                {
4497                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
4498                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
4499                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
4500                            "malformed module-control ERROR body: {err}"
4501                        )),
4502                    };
4503                    let completion = self
4504                        .forwarding
4505                        .complete_module_control_rpc(
4506                            connection_id,
4507                            frame.header.corr,
4508                            None,
4509                            outcome,
4510                        )
4511                        .map_err(RouterError::Forwarding)?;
4512                    if !self.observe_module_control_completion(completion) {
4513                        debug!(
4514                            connection_id = connection_id.get(),
4515                            corr = frame.header.corr,
4516                            "dropping late or unknown module-control RPC error"
4517                        );
4518                    }
4519                    return Ok(Vec::new());
4520                }
4521                match serde_json::from_slice::<ErrorBody>(&frame.body) {
4522                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
4523                    Err(err) => {
4524                        let message = format!("malformed route.bind ERROR body: {err}");
4525                        secondary_error = Some(control_error_frame(
4526                            &frame,
4527                            "invalid_control_body",
4528                            message.clone(),
4529                        )?);
4530                        RouteBindRelayOutcome::ModuleGone(message)
4531                    }
4532                }
4533            }
4534            ty => {
4535                return Ok(vec![control_error_frame(
4536                    &frame,
4537                    "unsupported_control_frame",
4538                    format!("unsupported module channel-0 frame {ty:?}"),
4539                )?])
4540            }
4541        };
4542
4543        let settled =
4544            self.forwarding
4545                .complete_pending_relay(connection_id, frame.header.corr, outcome);
4546        let completion = match settled {
4547            Ok(completion) => completion,
4548            Err(err) => {
4549                self.refuse_to_end_module_connection_for_a_client(
4550                    connection_id,
4551                    frame.header.corr,
4552                    err,
4553                )?;
4554                return Ok(secondary_error.into_iter().collect());
4555            }
4556        };
4557        if let Some(target) = completion.abandoned.as_ref() {
4558            send_goodbye_target_best_effort(target, "late accepted route.bind");
4559        }
4560        if !completion.settled {
4561            debug!(
4562                connection_id = connection_id.get(),
4563                corr = frame.header.corr,
4564                frame_type = ?frame.header.ty,
4565                "dropping late or unknown route.bind relay response"
4566            );
4567        }
4568        Ok(secondary_error.into_iter().collect())
4569    }
4570
4571    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
4572        debug!(connection_id = connection_id.get(), "handling GOODBYE");
4573        let registrations = self
4574            .deregister_connection(connection_id)
4575            .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
4576        let released_routes = self
4577            .forwarding
4578            .cleanup_connection(connection_id)
4579            .map_err(RouterError::Forwarding)?;
4580        self.emit_route_goodbyes(released_routes);
4581        // Notify only after forwarding teardown completes (see cleanup_connection).
4582        if !registrations.is_empty() {
4583            crate::supervise::notify_registration_release();
4584        }
4585        Ok(Vec::new())
4586    }
4587}
4588
4589impl Default for ControlHandler {
4590    fn default() -> Self {
4591        Self::new(Arc::new(Registry::default()))
4592    }
4593}
4594
4595impl crate::supervise::SwapPromotionObserver for ControlHandler {
4596    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
4597        self.apply_registration_capabilities(registration);
4598    }
4599}
4600
4601fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
4602    CapabilityRequirementStatus {
4603        consumer: status.consumer,
4604        capability: status.capability,
4605        need: match status.need {
4606            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
4607            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
4608        },
4609        verdict: status.verdict.as_str().to_string(),
4610        episode_seq: status.episode_seq,
4611        config_satisfiable: status.config_satisfiable,
4612        runtime_available: status.runtime_available,
4613        detail: status.detail,
4614    }
4615}
4616
4617fn append_capability_problem_detail(
4618    detail: Option<String>,
4619    capability_detail: Option<String>,
4620) -> Option<String> {
4621    match (detail, capability_detail) {
4622        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
4623        (Some(detail), None) => Some(detail),
4624        (None, Some(capability_detail)) => Some(capability_detail),
4625        (None, None) => None,
4626    }
4627}
4628
4629fn subc_ops() -> Vec<String> {
4630    SUBC_CONTROL_OPS
4631        .iter()
4632        .map(|op| (*op).to_string())
4633        .collect()
4634}
4635
4636fn module_subc_ops() -> Vec<String> {
4637    SUBC_CONTROL_OPS
4638        .iter()
4639        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
4640        .map(|op| (*op).to_string())
4641        .collect()
4642}
4643
4644#[cfg(test)]
4645fn module_baseline_control_ops() -> Vec<String> {
4646    MODULE_BASELINE_CONTROL_OPS
4647        .iter()
4648        .map(|op| (*op).to_string())
4649        .collect()
4650}
4651
4652fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
4653    let mut seen = HashSet::new();
4654    let mut effective = Vec::new();
4655    for op in MODULE_BASELINE_CONTROL_OPS {
4656        if seen.insert((*op).to_string()) {
4657            effective.push((*op).to_string());
4658        }
4659    }
4660    for op in declared.unwrap_or_default() {
4661        if seen.insert(op.clone()) {
4662            effective.push(op);
4663        }
4664    }
4665    effective
4666}
4667
4668fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
4669    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
4670}
4671
4672fn target_module_id(target: &RouteTarget) -> &str {
4673    match target {
4674        RouteTarget::ToolProvider { module_id }
4675        | RouteTarget::ManagementSurface { module_id }
4676        | RouteTarget::InternalService { module_id, .. } => module_id,
4677    }
4678}
4679
4680fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
4681    roles.iter().any(|role| match (target, role) {
4682        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
4683        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
4684        (
4685            RouteTarget::InternalService { service_id, .. },
4686            ProviderRole::InternalService {
4687                service_id: provided,
4688                ..
4689            },
4690        ) => service_id == provided,
4691        _ => false,
4692    })
4693}
4694
4695fn is_routable_role(role: &ProviderRole) -> bool {
4696    matches!(
4697        role,
4698        ProviderRole::ToolProvider { .. }
4699            | ProviderRole::ManagementSurface { .. }
4700            | ProviderRole::InternalService { .. }
4701    )
4702}
4703
4704#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4705enum ControlRequestBodyError {
4706    UnknownOp,
4707    InvalidBody,
4708}
4709
4710#[derive(Debug, Deserialize)]
4711struct ControlOpProbe {
4712    op: String,
4713}
4714
4715/// Channel-0 push ops this subc version understands. A push whose `op` is not in
4716/// this set is treated as a forward-compat unknown and ignored rather than errored.
4717const MODULE_PUSH_OPS: &[&str] = &["route.status"];
4718
4719fn is_known_module_push_op(body: &[u8]) -> bool {
4720    serde_json::from_slice::<ControlOpProbe>(body)
4721        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
4722        .unwrap_or(false)
4723}
4724
4725fn is_known_module_request_op(body: &[u8]) -> bool {
4726    serde_json::from_slice::<ControlOpProbe>(body)
4727        .map(|probe| MODULE_TO_SUBC_CONTROL_OPS.contains(&probe.op.as_str()))
4728        .unwrap_or(false)
4729}
4730
4731fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
4732    debug!(
4733        op = %op,
4734        connection_id = connection_id.get(),
4735        corr,
4736        "control dispatch"
4737    );
4738}
4739
4740fn log_slow_control_dispatch(
4741    dispatch_started_at: Option<StdInstant>,
4742    op: &'static str,
4743    connection_id: ConnectionId,
4744    corr: u64,
4745) {
4746    let Some(dispatch_started_at) = dispatch_started_at else {
4747        return;
4748    };
4749    let elapsed = dispatch_started_at.elapsed();
4750    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
4751        warn!(
4752            op = %op,
4753            connection_id = connection_id.get(),
4754            corr,
4755            elapsed_ms = elapsed.as_millis() as u64,
4756            "slow control dispatch"
4757        );
4758    }
4759}
4760
4761fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
4762    match request {
4763        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
4764        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
4765        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
4766        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
4767        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
4768        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
4769        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
4770        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
4771        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
4772        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
4773        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
4774        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
4775        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
4776        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
4777        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
4778        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
4779        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
4780        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
4781        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
4782    }
4783}
4784
4785fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
4786    match request {
4787        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
4788        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
4789    }
4790}
4791
4792fn parse_client_control_request(
4793    body: &[u8],
4794) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
4795    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
4796        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
4797            Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
4798                ControlRequestBodyError::InvalidBody
4799            }
4800            Ok(_) => ControlRequestBodyError::UnknownOp,
4801            Err(_) => ControlRequestBodyError::InvalidBody,
4802        };
4803        (err, classification)
4804    })
4805}
4806
4807fn parse_module_control_request_from_module(
4808    body: &[u8],
4809) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
4810    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
4811        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
4812            Ok(probe) if MODULE_TO_SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
4813                ControlRequestBodyError::InvalidBody
4814            }
4815            Ok(_) => ControlRequestBodyError::UnknownOp,
4816            Err(_) => ControlRequestBodyError::InvalidBody,
4817        };
4818        (err, classification)
4819    })
4820}
4821
4822#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
4823enum ProviderRoleKind {
4824    ToolProvider,
4825    PipelineStage,
4826    ManagementSurface,
4827    InternalService,
4828}
4829
4830fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
4831    match role {
4832        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
4833        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
4834        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
4835        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
4836    }
4837}
4838
4839fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
4840    roles.iter().map(provider_role_kind).collect()
4841}
4842
4843/// Return whether a catalog change can create a newly violating live route.
4844/// Removing an attested claim is intentionally excluded: it makes fewer routes
4845/// forbidden and therefore must leave the existing route census untouched.
4846fn capability_census_trigger(
4847    old: Option<&CapabilityDeclarations>,
4848    new: Option<&CapabilityDeclarations>,
4849) -> bool {
4850    let old_provides = old
4851        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
4852        .unwrap_or_default();
4853    let old_denies = old
4854        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
4855        .unwrap_or_default();
4856    let new = new.cloned().unwrap_or(CapabilityDeclarations {
4857        provides: Vec::new(),
4858        requires: Vec::new(),
4859        must_never_reach: Vec::new(),
4860    });
4861
4862    new.provides
4863        .iter()
4864        .any(|capability| !old_provides.contains(capability))
4865        || new
4866            .must_never_reach
4867            .iter()
4868            .any(|capability| !old_denies.contains(capability))
4869}
4870
4871/// Find the first capability an attested opener denies that an attested target
4872/// claims. Both manifests are live registry records, never cached or client data.
4873fn denied_capability<'a>(
4874    opening_manifest: &'a ModuleManifest,
4875    target_manifest: &ModuleManifest,
4876) -> Option<&'a str> {
4877    let opening_capabilities = opening_manifest.capabilities.as_ref()?;
4878    let target_capabilities = target_manifest.capabilities.as_ref()?;
4879    opening_capabilities
4880        .must_never_reach
4881        .iter()
4882        .find(|denied| {
4883            target_capabilities
4884                .provides
4885                .iter()
4886                .any(|provided| provided == *denied)
4887        })
4888        .map(String::as_str)
4889}
4890
4891fn catalog_update_frozen_field_message(
4892    registered: &ModuleManifest,
4893    provides: &[ProviderRole],
4894) -> Option<String> {
4895    let old_has_provides = !registered.provides.is_empty();
4896    let new_has_provides = !provides.is_empty();
4897    if old_has_provides != new_has_provides {
4898        return Some(format!(
4899            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
4900            registered.module_id
4901        ));
4902    }
4903
4904    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
4905        return Some(format!(
4906            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
4907            registered.module_id
4908        ));
4909    }
4910
4911    let registered_concurrency = manifest_concurrency(registered);
4912    let mut candidate = registered.clone();
4913    candidate.provides = provides.to_vec();
4914    let candidate_concurrency = manifest_concurrency(&candidate);
4915    if candidate_concurrency != registered_concurrency {
4916        return Some(format!(
4917            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
4918            registered.module_id, registered_concurrency, candidate_concurrency
4919        ));
4920    }
4921
4922    // control_ops live beside the manifest in the HELLO body, not inside
4923    // ModuleManifest, so a provides-only catalog.update cannot change them.
4924    None
4925}
4926
4927fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
4928    manifest.provides.iter().any(is_routable_role)
4929}
4930
4931/// Returns the routable-provider concurrency subc should enforce for this manifest.
4932///
4933/// ToolProvider and ManagementSurface store their delivery concurrency directly.
4934/// InternalService has no role-specific concurrency field, so it retains the
4935/// existing ModuleManaged default for backward compatibility.
4936fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
4937    manifest
4938        .provides
4939        .iter()
4940        .find_map(|provider| match provider {
4941            ProviderRole::ToolProvider { concurrency, .. }
4942            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
4943            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
4944        })
4945        .unwrap_or(Concurrency::ModuleManaged)
4946}
4947
4948/// True when the manifest carries a ManagementSurface role whose concurrency
4949/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
4950/// bytes because the typed manifest deliberately erases that distinction: the
4951/// default exists for wire compatibility, and this probe exists so the default
4952/// stays observable. Any parse irregularity returns false -- the caller only
4953/// logs, and a malformed body already failed registration upstream.
4954fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
4955    let has_management_surface = manifest
4956        .provides
4957        .iter()
4958        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
4959    if !has_management_surface {
4960        return false;
4961    }
4962    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
4963        return false;
4964    };
4965    let Some(provides) = raw
4966        .get("manifest")
4967        .and_then(|manifest| manifest.get("provides"))
4968        .and_then(serde_json::Value::as_array)
4969    else {
4970        return false;
4971    };
4972    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
4973    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
4974    // against the management_surface_manifest_without_concurrency golden, not
4975    // recalled (the externally-tagged guess was this function's first bug).
4976    provides.iter().any(|role| {
4977        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
4978            && role.get("concurrency").is_none()
4979    })
4980}
4981
4982fn negotiate_version(peer_version: u8) -> Result<u8, String> {
4983    if peer_version != PROTOCOL_VERSION {
4984        return Err(format!(
4985            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
4986        ));
4987    }
4988    Ok(PROTOCOL_VERSION)
4989}
4990
4991fn pong(frame: &Frame) -> Result<Frame, RouterError> {
4992    Frame::build_with_version(
4993        response_version(frame),
4994        FrameType::Pong,
4995        frame.header.flags,
4996        0,
4997        0,
4998        frame.header.corr,
4999        Vec::new(),
5000    )
5001    .map_err(RouterError::FrameBuild)
5002}
5003
5004fn control_error_frame(
5005    frame: &Frame,
5006    code: &'static str,
5007    message: impl Into<String>,
5008) -> Result<Frame, RouterError> {
5009    control_error_body_frame(
5010        frame,
5011        ErrorBody {
5012            code: code.to_string(),
5013            message: message.into(),
5014            detail: None,
5015        },
5016    )
5017}
5018
5019fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
5020    let body = serde_json::to_vec(&error).map_err(|err| {
5021        RouterError::backend(
5022            0,
5023            frame.header.corr,
5024            format!("failed to encode control ERROR: {err}"),
5025        )
5026    })?;
5027
5028    Frame::build_with_version(
5029        response_version(frame),
5030        FrameType::Error,
5031        control_flags(),
5032        0,
5033        0,
5034        frame.header.corr,
5035        body,
5036    )
5037    .map_err(RouterError::FrameBuild)
5038}
5039
5040fn control_response_body_frame<T: Serialize>(
5041    frame: &Frame,
5042    reply: &T,
5043    label: &'static str,
5044) -> Result<Frame, RouterError> {
5045    let body = serde_json::to_vec(reply).map_err(|err| {
5046        RouterError::backend(
5047            0,
5048            frame.header.corr,
5049            format!("failed to encode {label}: {err}"),
5050        )
5051    })?;
5052
5053    Frame::build_with_version(
5054        response_version(frame),
5055        FrameType::Response,
5056        control_flags(),
5057        0,
5058        0,
5059        frame.header.corr,
5060        body,
5061    )
5062    .map_err(RouterError::FrameBuild)
5063}
5064
5065/// Map a forwarding failure to the wire code a client sees.
5066///
5067/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
5068/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
5069/// chosen here decides whether a caller retries or gives up.
5070///
5071/// That makes attribution the load-bearing property, not merely having a code. A
5072/// permanent fault published as a retryable one produces a fleet-wide retry storm
5073/// against something that can never recover; a transient fault published as
5074/// permanent gives up on work that would have succeeded. Both look correct in a
5075/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
5076/// the mapping per variant rather than merely asserting that some code exists.
5077///
5078/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
5079/// two codes on the same side of the boundary passes it. Measured rather than
5080/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
5081/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
5082/// a test named for something else that happens to assert the string.
5083///
5084/// That accidental coverage is deliberately left alone rather than promoted to a
5085/// named test, because it guards a property this function does not promise.
5086/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
5087/// specific code within a class, so identity is free to change and only the
5088/// partition is a contract. Splitting it out would assert a guarantee nothing
5089/// depends on — and a suite that promises more than the code does is the harder
5090/// thing to correct later, because the next reader cannot tell which assertions
5091/// are load-bearing.
5092///
5093/// Pin identity here the moment a consumer branches on a specific code.
5094fn forwarding_error_code(err: &ForwardingError) -> &'static str {
5095    match err {
5096        ForwardingError::NoModuleConnection => "target_unavailable",
5097        ForwardingError::ModuleReloading { .. } => "module_reloading",
5098        ForwardingError::ClientRouteChannelExhausted { .. }
5099        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
5100        ForwardingError::StaleModuleEndpoint
5101        | ForwardingError::UnknownReservation { .. }
5102        | ForwardingError::ConnectionClosing { .. }
5103        | ForwardingError::ClientEgressClosed { .. } => "target_unavailable",
5104        // Only a swap candidate's registration can produce this, and it means
5105        // exactly what a second active HELLO for a live id means.
5106        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
5107        ForwardingError::RelayCorrelationExhausted
5108        | ForwardingError::RouteOpenBuild(_)
5109        | ForwardingError::Poisoned => "forwarding_error",
5110    }
5111}
5112
5113fn response_version(frame: &Frame) -> u8 {
5114    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
5115        frame.header.ver
5116    } else {
5117        PROTOCOL_VERSION
5118    }
5119}
5120
5121fn control_flags() -> Flags {
5122    Flags::new(false, Priority::Passive, false)
5123}
5124
5125fn send_goodbye_target_best_effort(target: &GoodbyeTarget, context: &str) {
5126    let Ok(frame) = Frame::build_with_version(
5127        target.negotiated_ver,
5128        FrameType::Goodbye,
5129        control_flags(),
5130        target.channel,
5131        target.epoch,
5132        0,
5133        Vec::new(),
5134    ) else {
5135        return;
5136    };
5137    if let Err(err) = target.sink.try_send(frame) {
5138        warn!(
5139            route_channel = target.channel,
5140            route_epoch = target.epoch,
5141            error = %err,
5142            %context,
5143            "route GOODBYE dropped under backpressure"
5144        );
5145    }
5146}
5147
5148pub(crate) fn send_route_control_pushes(
5149    forwarding: &ForwardingTable,
5150    routes: Vec<EndpointRoute>,
5151    push: ClientControlPush,
5152) {
5153    let body = match serde_json::to_vec(&push) {
5154        Ok(body) => body,
5155        Err(err) => {
5156            warn!(error = %err, "failed to serialize route lifecycle control PUSH");
5157            return;
5158        }
5159    };
5160    let mut targets = Vec::new();
5161    for route in routes {
5162        let target = route.goodbye_target;
5163        if let Some(existing) = targets
5164            .iter()
5165            .find(|existing: &&GoodbyeTarget| existing.connection_id == target.connection_id)
5166        {
5167            debug_assert_eq!(
5168                existing.negotiated_ver, target.negotiated_ver,
5169                "one connection cannot negotiate multiple frame versions"
5170            );
5171            continue;
5172        }
5173        targets.push(target);
5174    }
5175    for target in targets {
5176        let frame = match Frame::build_with_version(
5177            target.negotiated_ver,
5178            FrameType::Push,
5179            control_flags(),
5180            0,
5181            0,
5182            0,
5183            body.clone(),
5184        ) {
5185            Ok(frame) => frame,
5186            Err(err) => {
5187                warn!(
5188                    route_channel = target.channel,
5189                    error = %err,
5190                    "failed to build route lifecycle control PUSH frame"
5191                );
5192                continue;
5193            }
5194        };
5195        if let Err(err) = target.sink.try_send(frame) {
5196            if target.close_on_delivery_failure() {
5197                warn!(
5198                    target_connection_id = target.connection_id.get(),
5199                    route_channel = target.channel,
5200                    error = %err,
5201                    "route lifecycle control PUSH was not delivered to client; closing target connection"
5202                );
5203                let _ = forwarding.escalate_client_delivery_failure(
5204                    target.connection_id,
5205                    target.channel,
5206                    target.epoch,
5207                    CloseReason::new(
5208                        "route_lifecycle_push_delivery_failed",
5209                        format!(
5210                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
5211                            target.channel
5212                        ),
5213                    ),
5214                );
5215            }
5216        }
5217    }
5218}
5219
5220#[cfg(test)]
5221mod tests {
5222    use std::{
5223        collections::BTreeMap,
5224        fmt,
5225        path::PathBuf,
5226        sync::{Arc, Mutex},
5227        time::Duration,
5228    };
5229
5230    use serde_json::{json, Value};
5231    use subc_protocol::{
5232        manifest::{
5233            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
5234            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
5235        },
5236        session::HealthStatus,
5237        FrameType,
5238    };
5239
5240    use super::*;
5241    use crate::{
5242        forwarding::{DataRoute, DataRouteState},
5243        registry::ChannelState,
5244        router::FrameSink,
5245        stderr_tail::DEFAULT_MAX_LINE_BYTES,
5246        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
5247        test_support::TestTempDir,
5248        RouteCtx, Router,
5249    };
5250    use tokio::{
5251        sync::mpsc,
5252        time::{sleep, Instant},
5253    };
5254    use tracing::{
5255        field::{Field, Visit},
5256        Event, Subscriber,
5257    };
5258    use tracing_subscriber::{layer::Context, prelude::*, Layer};
5259
5260    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
5261    ///
5262    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
5263    /// is only populated for `tests/*.rs` integration test binaries -- this file
5264    /// compiles as part of the library target, which gets neither. This test's
5265    /// own executable path is `<target-dir>/<profile>/deps/subc_core-<hash>`,
5266    /// and the sibling binary lives two directories up at
5267    /// `<target-dir>/<profile>/fake-aft-stub`.
5268    ///
5269    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
5270    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
5271    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
5272    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
5273    /// `NotFound`, which reads as a broken test rather than an unbuilt
5274    /// dependency -- so state the cause and the remedy instead. Deliberately a
5275    /// panic and not a silent skip: a test that quietly passes when it could not
5276    /// run is worse than one that fails, because it reports health it never
5277    /// verified.
5278    fn fake_aft_stub_path() -> PathBuf {
5279        let mut path = std::env::current_exe().expect("current_exe available in tests");
5280        path.pop(); // .../deps/
5281        path.pop(); // .../<profile>/
5282        path.push(if cfg!(windows) {
5283            "fake-aft-stub.exe"
5284        } else {
5285            "fake-aft-stub"
5286        });
5287        assert!(
5288            path.exists(),
5289            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
5290             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
5291            path.display()
5292        );
5293        path
5294    }
5295
5296    /// Whether clients retry `code` in place: the predicate itself, never a copy
5297    /// of its set. A copied list breaks silently when a code is added to or
5298    /// removed from the real one, and a stale copy here would let exactly the
5299    /// failure this test exists to catch pass.
5300    fn client_retries(code: &str) -> bool {
5301        subc_protocol::error_codes::is_retryable_route_open(code)
5302    }
5303
5304    /// A code is not a label — clients branch on it, so publishing the wrong KIND
5305    /// of failure is worse than publishing none. A permanent fault dressed as
5306    /// retryable makes every client in the fleet retry forever against something
5307    /// that cannot recover; a transient fault dressed as permanent abandons work
5308    /// that would have succeeded.
5309    ///
5310    /// Asserting "a code exists" cannot catch either, because the string is free
5311    /// to say anything. This enumerates every variant and pins which side of the
5312    /// retry boundary it lands on, so a new variant must be classified here
5313    /// deliberately rather than inheriting whichever arm it was appended to.
5314    #[test]
5315    fn retryability_of_forwarding_codes_matches_the_failure() {
5316        // Transient by nature: the target is booting, reloading, or its endpoint
5317        // was swapped mid-flight. Retrying is how these resolve.
5318        let transient = [
5319            ForwardingError::NoModuleConnection,
5320            ForwardingError::ModuleReloading {
5321                module_id: "m".into(),
5322            },
5323            ForwardingError::StaleModuleEndpoint,
5324            ForwardingError::UnknownReservation {
5325                client_channel: 1,
5326                module_channel: 1,
5327            },
5328            ForwardingError::ConnectionClosing {
5329                connection_id: ConnectionId::new(1),
5330            },
5331            ForwardingError::ClientEgressClosed {
5332                connection_id: ConnectionId::new(1),
5333            },
5334        ];
5335        for err in transient {
5336            let code = forwarding_error_code(&err);
5337            assert!(
5338                client_retries(code),
5339                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
5340            );
5341        }
5342
5343        // Not fixed by retrying. Channel and correlation exhaustion need the
5344        // caller to close routes, and a poisoned lock is a daemon that cannot
5345        // recover at all — the worst thing to advertise as retryable, since every
5346        // client would storm a daemon that will never answer.
5347        let permanent = [
5348            ForwardingError::ClientRouteChannelExhausted {
5349                connection_id: ConnectionId::new(1),
5350            },
5351            ForwardingError::ModuleRouteChannelExhausted {
5352                endpoint: ModuleEndpointId {
5353                    connection_id: ConnectionId::new(1),
5354                    generation: 1,
5355                },
5356            },
5357            ForwardingError::RelayCorrelationExhausted,
5358            ForwardingError::RouteOpenBuild("x".into()),
5359            ForwardingError::Poisoned,
5360        ];
5361        for err in permanent {
5362            let code = forwarding_error_code(&err);
5363            assert!(
5364                !client_retries(code),
5365                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
5366            );
5367        }
5368    }
5369
5370    /// The principal is the daemon's answer to "who is calling", and modules
5371    /// branch on it: aft gates bash on it, cerebellum gates browser control,
5372    /// plexus gates connector invocation. So a stamp is an authorization input in
5373    /// another process, not a label — and both possible answers SUCCEED, which is
5374    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
5375    /// first-party capability to something that never proved it; a supervised one
5376    /// stamped `Direct` silently strips a module of capability it is entitled to.
5377    ///
5378    /// Neither shows up in a test that only checks the bind succeeded. Before this
5379    /// test the only coverage was accidental —
5380    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
5381    /// stamped principal on its way past, so narrowing that wire-shape test to its
5382    /// stated subject would have deleted the last assertion on this value. It
5383    /// still asserts the stamp, which is now redundancy rather than the only
5384    /// guard: both fail under the same mutation, and this one names the reason.
5385    /// SCOPE: this handler's supervisor has spawned nothing, so
5386    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
5387    /// is unreachable here. Both assertions below are refusals, and a mutant that
5388    /// refuses everything would satisfy them.
5389    ///
5390    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
5391    /// spawns a supervised consumer, reads its live nonce, and asserts the module
5392    /// observed `principal.kind == "reserved"` carrying that module_id — verified
5393    /// at source rather than assumed, since a citation is a claim about another
5394    /// file and ages like one. Recorded because a harness that structurally
5395    /// cannot reach an arm reports "none" for that arm identically to one that
5396    /// covers it and found nothing.
5397    #[tokio::test]
5398    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
5399        let handler = ControlHandler::default();
5400        let frame =
5401            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
5402
5403        // Absent consumer_identity is the ordinary case: a human at a terminal, or
5404        // any process holding the connection file. Nothing was proved, so nothing
5405        // may be granted beyond the unattested floor.
5406        let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
5407        assert_eq!(
5408            stamped,
5409            Principal::Direct,
5410            "a caller that proved nothing must not be stamped as a supervised module"
5411        );
5412
5413        // A claimed module_id with a nonce no supervised child was given is a
5414        // forgery attempt, not a weaker caller: it must be REFUSED rather than
5415        // quietly demoted to Direct, or an impersonation attempt looks identical
5416        // to an ordinary unattested connection.
5417        let forged = handler
5418            .route_open_principal(
5419                &frame,
5420                Some(ConsumerIdentity {
5421                    module_id: "aft".to_string(),
5422                    launch_nonce: "not-a-real-nonce".to_string(),
5423                }),
5424            )
5425            .unwrap();
5426        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
5427        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
5428    }
5429
5430    /// The test above hands `route_open_principal` an identity it built itself,
5431    /// which proves the stamping rule and nothing about where the identity comes
5432    /// from. The real producer is a wire body, and the two are joined by a serde
5433    /// field name that nothing else asserts.
5434    ///
5435    /// That join fails quietly in one specific way: an unrecognised key is simply
5436    /// absent after parsing, so a renamed or misspelled `consumer_identity`
5437    /// yields `None` and every supervised module silently drops to `Direct`.
5438    /// Capability-wise that is the safe direction, but it surfaces far from its
5439    /// cause — as a module mysteriously refused bash — and it would pass every
5440    /// test that builds its own input.
5441    ///
5442    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
5443    /// would break every client the moment the daemon gains a field, trading a
5444    /// quiet demotion for a hard refusal on additive change. Asserting the join
5445    /// instead means a rename breaks a test here rather than the fleet.
5446    #[test]
5447    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
5448        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
5449        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
5450        let ClientControlRequest::RouteOpen {
5451            consumer_identity, ..
5452        } = parsed
5453        else {
5454            panic!("route.open body must parse as RouteOpen");
5455        };
5456        assert_eq!(
5457            consumer_identity,
5458            Some(ConsumerIdentity {
5459                module_id: "aft".to_string(),
5460                launch_nonce: "n".to_string(),
5461            }),
5462            "the wire field name must reach the value route_open_principal reads"
5463        );
5464    }
5465
5466    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
5467        ModuleManifest::builder(module_id, "0.1.0")
5468            .protocol_ver(protocol_ver)
5469            .provides(vec![ProviderRole::ToolProvider {
5470                tools: vec![Tool {
5471                    name: "read".to_string(),
5472                    description: None,
5473                    execution_mode: ExecutionMode::Pure,
5474                    schema: json!({"type": "object"}),
5475                }],
5476                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
5477                concurrency: Concurrency::ModuleManaged,
5478                emits_push: true,
5479                sub_supervises: true,
5480            }])
5481            .build()
5482    }
5483
5484    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
5485        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
5486    }
5487
5488    fn hello_frame_with_control_ops(
5489        module_id: &str,
5490        protocol_ver: u8,
5491        corr: u64,
5492        control_ops: Option<Vec<String>>,
5493    ) -> Frame {
5494        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
5495    }
5496
5497    fn hello_frame_with_nonce(
5498        module_id: &str,
5499        protocol_ver: u8,
5500        corr: u64,
5501        launch_nonce: Option<&str>,
5502    ) -> Frame {
5503        hello_frame_full(
5504            module_id,
5505            protocol_ver,
5506            corr,
5507            None,
5508            launch_nonce.map(ToOwned::to_owned),
5509        )
5510    }
5511
5512    fn hello_frame_full(
5513        module_id: &str,
5514        protocol_ver: u8,
5515        corr: u64,
5516        control_ops: Option<Vec<String>>,
5517        launch_nonce: Option<String>,
5518    ) -> Frame {
5519        let body = serde_json::to_vec(&ModuleHelloBody {
5520            manifest: manifest(module_id, protocol_ver),
5521            protocol_ver,
5522            control_ops,
5523            launch_nonce,
5524        })
5525        .unwrap();
5526        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
5527    }
5528
5529    fn non_routable_hello_frame_with_control_ops(
5530        module_id: &str,
5531        corr: u64,
5532        control_ops: Option<Vec<String>>,
5533    ) -> Frame {
5534        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
5535        manifest.provides.clear();
5536        let body = serde_json::to_vec(&ModuleHelloBody {
5537            manifest,
5538            protocol_ver: PROTOCOL_VERSION,
5539            control_ops,
5540            launch_nonce: None,
5541        })
5542        .unwrap();
5543        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
5544    }
5545
5546    fn capability_grammar_hello_frame(
5547        capabilities: Value,
5548        runtime_computed: Option<Value>,
5549        corr: u64,
5550    ) -> Frame {
5551        let mut body = serde_json::to_value(ModuleHelloBody {
5552            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
5553            protocol_ver: PROTOCOL_VERSION,
5554            control_ops: None,
5555            launch_nonce: None,
5556        })
5557        .expect("HELLO body serializes");
5558        body["manifest"]["capabilities"] = capabilities;
5559        if let Some(runtime_computed) = runtime_computed {
5560            body["runtime_computed"] = runtime_computed;
5561        }
5562        Frame::build(
5563            FrameType::Hello,
5564            control_flags(),
5565            0,
5566            0,
5567            corr,
5568            serde_json::to_vec(&body).expect("HELLO body reserializes"),
5569        )
5570        .expect("HELLO frame builds")
5571    }
5572
5573    fn channel_request(channel: u16, corr: u64) -> Frame {
5574        Frame::build(
5575            FrameType::Request,
5576            Flags::new(true, Priority::Interactive, false),
5577            channel,
5578            0,
5579            corr,
5580            b"opaque".to_vec(),
5581        )
5582        .unwrap()
5583    }
5584
5585    fn route_ctx(
5586        connection_id: ConnectionId,
5587    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
5588        let (tx, rx) = mpsc::channel(8);
5589        (
5590            RouteCtx {
5591                connection_id,
5592                egress: FrameSink::new(tx),
5593            },
5594            rx,
5595        )
5596    }
5597
5598    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
5599        serde_json::from_slice(&frame.body).unwrap()
5600    }
5601
5602    fn parse_error(frame: &Frame) -> Value {
5603        serde_json::from_slice(&frame.body).unwrap()
5604    }
5605
5606    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
5607        serde_json::from_slice(&frame.body).unwrap()
5608    }
5609
5610    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
5611        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
5612            route_channel,
5613            route_epoch: 0,
5614            kind,
5615        })
5616        .unwrap();
5617        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5618    }
5619
5620    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
5621        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
5622            module_id: module_id.to_string(),
5623        })
5624        .unwrap();
5625        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5626    }
5627
5628    fn route_open_frame(corr: u64, module_id: &str, project_root: TestTempDir) -> Frame {
5629        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
5630    }
5631
5632    fn route_open_frame_with_consumer_capabilities(
5633        corr: u64,
5634        module_id: &str,
5635        project_root: TestTempDir,
5636        consumer_capabilities: Option<Vec<String>>,
5637    ) -> Frame {
5638        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
5639            target: RouteTarget::ToolProvider {
5640                module_id: module_id.to_string(),
5641            },
5642            identity: BindIdentity::new(
5643                project_root.path().to_path_buf(),
5644                "unit".to_string(),
5645                "session".to_string(),
5646            ),
5647            consumer_identity: None,
5648            consumer_capabilities,
5649            admission_facts: None,
5650        })
5651        .unwrap();
5652        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5653    }
5654
5655    fn route_open_frame_with_admission_facts(
5656        corr: u64,
5657        module_id: &str,
5658        project_root: TestTempDir,
5659        consumer_identity: Option<subc_control::ConsumerIdentity>,
5660        facts: Option<Value>,
5661    ) -> Frame {
5662        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
5663            target: RouteTarget::ToolProvider {
5664                module_id: module_id.to_string(),
5665            },
5666            identity: BindIdentity::new(
5667                project_root.path().to_path_buf(),
5668                "unit".to_string(),
5669                format!("session-{corr}"),
5670            ),
5671            consumer_identity,
5672            consumer_capabilities: None,
5673            admission_facts: facts,
5674        })
5675        .unwrap();
5676        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5677    }
5678
5679    #[derive(Clone, Default)]
5680    struct EventCapture {
5681        events: Arc<Mutex<Vec<CapturedEvent>>>,
5682    }
5683
5684    #[derive(Clone, Debug)]
5685    struct CapturedEvent {
5686        target: String,
5687        fields: BTreeMap<String, String>,
5688    }
5689
5690    impl EventCapture {
5691        fn events(&self) -> Vec<CapturedEvent> {
5692            self.events.lock().unwrap().clone()
5693        }
5694    }
5695
5696    impl<S> Layer<S> for EventCapture
5697    where
5698        S: Subscriber,
5699    {
5700        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
5701            let mut visitor = EventFieldVisitor::default();
5702            event.record(&mut visitor);
5703            self.events.lock().unwrap().push(CapturedEvent {
5704                target: event.metadata().target().to_string(),
5705                fields: visitor.fields,
5706            });
5707        }
5708    }
5709
5710    #[derive(Default)]
5711    struct EventFieldVisitor {
5712        fields: BTreeMap<String, String>,
5713    }
5714
5715    impl Visit for EventFieldVisitor {
5716        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
5717            self.fields
5718                .insert(field.name().to_string(), format!("{value:?}"));
5719        }
5720    }
5721
5722    fn health_response(corr: u64, status: HealthStatus) -> Frame {
5723        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
5724            status,
5725            detail: Some("warming".to_string()),
5726            metrics: Some(json!({"queue_depth": 3})),
5727        })
5728        .unwrap();
5729        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
5730    }
5731
5732    fn route_bind_ack(corr: u64) -> Frame {
5733        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
5734        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
5735    }
5736
5737    fn unique_project_root(label: &str) -> TestTempDir {
5738        TestTempDir::new(label)
5739    }
5740
5741    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
5742        match parse_route_poll(frame) {
5743            ClientControlResponse::RoutePoll {
5744                status: None,
5745                live: Some(live),
5746                ..
5747            } => assert_eq!(live, expected_live),
5748            other => panic!("unexpected route.poll response: {other:?}"),
5749        }
5750    }
5751
5752    fn bind_liveness_route(
5753        registry: &Registry,
5754        forwarding: &ForwardingTable,
5755        module_id: &str,
5756    ) -> (RouteCtx, u16, u32) {
5757        let module_connection = ConnectionId::new(101);
5758        let client_connection = ConnectionId::new(202);
5759        let registration = registry
5760            .register_with_control_ops(
5761                manifest(module_id, PROTOCOL_VERSION),
5762                PROTOCOL_VERSION,
5763                module_connection,
5764                module_baseline_control_ops(),
5765            )
5766            .unwrap();
5767        let (module_tx, _module_rx) = mpsc::channel(8);
5768        let endpoint = forwarding
5769            .register_module_connection(
5770                module_connection,
5771                module_id.to_string(),
5772                PROTOCOL_VERSION,
5773                manifest_concurrency(&registration.manifest),
5774                FrameSink::new(module_tx),
5775            )
5776            .unwrap();
5777        let (client_ctx, _client_rx) = route_ctx(client_connection);
5778        let pending = forwarding
5779            .begin_route_bind_relay_for_test(
5780                client_connection,
5781                client_ctx.egress.clone(),
5782                1,
5783                module_id,
5784            )
5785            .unwrap();
5786        assert_eq!(pending.endpoint, endpoint);
5787        let route_channel = pending.client_channel;
5788        let route_epoch = pending.client_epoch;
5789        forwarding
5790            .complete_pending_relay(
5791                module_connection,
5792                pending.corr,
5793                RouteBindRelayOutcome::Accepted,
5794            )
5795            .unwrap();
5796        (client_ctx, route_channel, route_epoch)
5797    }
5798
5799    struct FakeProcessLiveness {
5800        live: Option<bool>,
5801    }
5802
5803    impl ModuleProcessLiveness for FakeProcessLiveness {
5804        fn process_live(&self, _module_id: &str) -> Option<bool> {
5805            self.live
5806        }
5807    }
5808
5809    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5810    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
5811    {
5812        let registry = Arc::new(Registry::default());
5813        let supervisor_handle = SupervisorHandle::new();
5814        let supervisor = Supervisor::new(
5815            Arc::clone(&registry),
5816            RestartPolicy::new(1, Duration::from_millis(10)),
5817        )
5818        .with_handle(supervisor_handle.clone());
5819        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
5820        let module = supervisor
5821            .spawn(ModuleSpec {
5822                module_id: "stderr-tail-wire".to_string(),
5823                program: fake_aft_stub_path(),
5824                args: Vec::new(),
5825                env: vec![
5826                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
5827                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
5828                ],
5829                reserved: false,
5830                reserved_prefixes: Vec::new(),
5831                protocol: ModuleProtocol::Subc,
5832                overlap: Default::default(),
5833            })
5834            .unwrap();
5835
5836        let deadline = Instant::now() + Duration::from_secs(5);
5837        loop {
5838            let tail = module.stderr_tail(None, None);
5839            if tail
5840                .entries
5841                .iter()
5842                .any(|entry| matches!(entry, TailEntry::ProcessStart))
5843                && tail.entries.iter().any(|entry| {
5844                    matches!(
5845                        entry,
5846                        TailEntry::Line {
5847                            truncated: true,
5848                            ..
5849                        }
5850                    )
5851                })
5852            {
5853                break;
5854            }
5855            assert!(
5856                Instant::now() < deadline,
5857                "module did not produce a truncated line and restart boundary: {tail:?}"
5858            );
5859            sleep(Duration::from_millis(10)).await;
5860        }
5861
5862        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
5863        let request = ClientControlRequest::SupervisorStderrTail {
5864            module_id: "stderr-tail-wire".to_string(),
5865            max_lines: None,
5866            max_bytes: None,
5867        };
5868        let frame = Frame::build(
5869            FrameType::Request,
5870            control_flags(),
5871            0,
5872            0,
5873            1,
5874            serde_json::to_vec(&request).unwrap(),
5875        )
5876        .unwrap();
5877        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
5878        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
5879        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
5880            serde_json::from_slice(&responses[0].body).unwrap()
5881        else {
5882            panic!("expected supervisor.stderr_tail response");
5883        };
5884
5885        assert!(
5886            tail.entries
5887                .iter()
5888                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
5889            "the control response lost the restart boundary"
5890        );
5891        let Some(StderrTailEntry::Line { text, truncated }) = tail.entries.iter().find(|entry| {
5892            matches!(
5893                entry,
5894                StderrTailEntry::Line {
5895                    truncated: true,
5896                    ..
5897                }
5898            )
5899        }) else {
5900            panic!("the control response lost the truncated line");
5901        };
5902        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
5903        assert!(*truncated);
5904    }
5905
5906    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5907    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
5908        let registry = Arc::new(Registry::default());
5909        let supervisor_handle = SupervisorHandle::new();
5910        let supervisor =
5911            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
5912                .with_handle(supervisor_handle.clone());
5913        let module = supervisor
5914            .spawn(ModuleSpec {
5915                module_id: "terminal-golden".to_string(),
5916                program: fake_aft_stub_path(),
5917                args: Vec::new(),
5918                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
5919                reserved: false,
5920                reserved_prefixes: Vec::new(),
5921                protocol: ModuleProtocol::Subc,
5922                overlap: Default::default(),
5923            })
5924            .unwrap();
5925
5926        let deadline = Instant::now() + Duration::from_secs(5);
5927        while module.terminal_history().entries.len() != 2 {
5928            assert!(
5929                Instant::now() < deadline,
5930                "module did not retain two terminal exits: {:?}",
5931                module.terminal_history()
5932            );
5933            sleep(Duration::from_millis(10)).await;
5934        }
5935
5936        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
5937        let request = ClientControlRequest::SupervisorTerminals {
5938            module_id: "terminal-golden".to_string(),
5939        };
5940        let frame = Frame::build(
5941            FrameType::Request,
5942            control_flags(),
5943            0,
5944            0,
5945            1,
5946            serde_json::to_vec(&request).unwrap(),
5947        )
5948        .unwrap();
5949        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
5950        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
5951        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
5952        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
5953            panic!("expected supervisor.terminals response");
5954        };
5955        assert_eq!(terminals.entries.len(), 2);
5956        assert_eq!(terminals.dropped, 0);
5957
5958        let mut rendered = serde_json::to_value(response).unwrap();
5959        // Wall-clock fields are the observation contract, but not stable fixture
5960        // bytes; normalize only them after the real handler has shaped the response.
5961        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
5962        for (index, entry) in rendered["entries"]
5963            .as_array_mut()
5964            .expect("terminal response entries array")
5965            .iter_mut()
5966            .enumerate()
5967        {
5968            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
5969        }
5970
5971        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
5972            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
5973        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
5974        if std::env::var_os("UPDATE_GOLDEN").is_some() {
5975            std::fs::write(&golden_path, &serialized).unwrap();
5976        }
5977        let expected: Value =
5978            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
5979        assert_eq!(rendered, expected);
5980    }
5981
5982    #[test]
5983    fn hello_registers_manifest_and_returns_ack() {
5984        let registry = Arc::new(Registry::default());
5985        let handler = ControlHandler::new(Arc::clone(&registry));
5986        let conn = ConnectionId::new(1);
5987
5988        let responses = handler
5989            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
5990            .unwrap();
5991
5992        assert_eq!(responses.len(), 1);
5993        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
5994        assert_eq!(responses[0].header.channel, 0);
5995        assert_eq!(responses[0].header.corr, 7);
5996        let ack = parse_ack(&responses[0]);
5997        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
5998        assert!(ack
5999            .subc_capabilities
6000            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
6001        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
6002        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
6003        assert!(ack
6004            .subc_ops
6005            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
6006        assert!(ack
6007            .subc_ops
6008            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
6009
6010        let registration = registry.get_module("aft").unwrap().unwrap();
6011        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
6012        assert_eq!(registration.state, ChannelState::Active);
6013        assert_eq!(registration.connection_id, conn);
6014        assert_eq!(registration.control_ops, module_baseline_control_ops());
6015    }
6016
6017    #[test]
6018    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
6019        let invalid_identifiers = [
6020            ("case_change", "credentials-Provider/v1"),
6021            ("leading_zero", "credentials-provider/v01"),
6022            ("trailing_hyphen", "credentials-provider-/v1"),
6023            ("consecutive_hyphens", "credentials--provider/v1"),
6024            ("uppercase", "Credentials-provider/v1"),
6025            ("missing_v", "credentials-provider/1"),
6026            ("whitespace", "credentials provider/v1"),
6027            ("zero_version", "credentials-provider/v0"),
6028            ("out_of_range_version", "credentials-provider/v4294967296"),
6029            (
6030                "overlength_name",
6031                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
6032            ),
6033        ];
6034        let mut cases = invalid_identifiers
6035            .into_iter()
6036            .map(|(name, identifier)| {
6037                (
6038                    format!("identifier_{name}"),
6039                    "capabilities.provides[0]".to_string(),
6040                    identifier.to_string(),
6041                    json!({ "provides": [identifier] }),
6042                    None,
6043                )
6044            })
6045            .collect::<Vec<_>>();
6046        cases.extend([
6047            (
6048                "unknown_need".to_string(),
6049                "capabilities.requires[0].need".to_string(),
6050                "deferred".to_string(),
6051                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
6052                None,
6053            ),
6054            (
6055                "duplicate_provides".to_string(),
6056                "capabilities.provides[1]".to_string(),
6057                "credentials-provider/v1".to_string(),
6058                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
6059                None,
6060            ),
6061            (
6062                "duplicate_must_never_reach".to_string(),
6063                "capabilities.must_never_reach[1]".to_string(),
6064                "credentials-provider/v1".to_string(),
6065                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
6066                None,
6067            ),
6068            (
6069                "duplicate_requires_same_need".to_string(),
6070                "capabilities.requires[1]".to_string(),
6071                "credentials-provider/v1".to_string(),
6072                json!({ "requires": [
6073                    { "capability": "credentials-provider/v1", "need": "required" },
6074                    { "capability": "credentials-provider/v1", "need": "required" }
6075                ] }),
6076                None,
6077            ),
6078            (
6079                "duplicate_requires_conflicting_need".to_string(),
6080                "capabilities.requires[1]".to_string(),
6081                "credentials-provider/v1".to_string(),
6082                json!({ "requires": [
6083                    { "capability": "credentials-provider/v1", "need": "required" },
6084                    { "capability": "credentials-provider/v1", "need": "optional" }
6085                ] }),
6086                None,
6087            ),
6088            (
6089                "capabilities_root_pointer".to_string(),
6090                "runtime_computed[0]".to_string(),
6091                "/capabilities".to_string(),
6092                json!({}),
6093                Some(json!(["/capabilities"])),
6094            ),
6095            (
6096                "capabilities_descendant_pointer".to_string(),
6097                "runtime_computed[0]".to_string(),
6098                "/capabilities/provides".to_string(),
6099                json!({}),
6100                Some(json!(["/capabilities/provides"])),
6101            ),
6102            (
6103                "malformed_pointer_without_leading_slash".to_string(),
6104                "runtime_computed[0]".to_string(),
6105                "capabilities".to_string(),
6106                json!({}),
6107                Some(json!(["capabilities"])),
6108            ),
6109            (
6110                "malformed_pointer_escape".to_string(),
6111                "runtime_computed[0]".to_string(),
6112                "/roles/~2/tools".to_string(),
6113                json!({}),
6114                Some(json!(["/roles/~2/tools"])),
6115            ),
6116            (
6117                "unknown_capabilities_field".to_string(),
6118                "capabilities.future".to_string(),
6119                "<array>".to_string(),
6120                json!({ "future": [] }),
6121                None,
6122            ),
6123        ]);
6124
6125        for (index, (name, field, value, capabilities, runtime_computed)) in
6126            cases.into_iter().enumerate()
6127        {
6128            let registry = Arc::new(Registry::default());
6129            let handler = ControlHandler::new(Arc::clone(&registry));
6130            let response = handler
6131                .handle_control(
6132                    ConnectionId::new((index + 1) as u64),
6133                    capability_grammar_hello_frame(
6134                        capabilities,
6135                        runtime_computed,
6136                        index as u64 + 1,
6137                    ),
6138                )
6139                .expect("invalid HELLO returns a refusal");
6140
6141            assert_eq!(response.len(), 1, "{name} must emit one refusal");
6142            let error = parse_error(&response[0]);
6143            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
6144            let message = error["message"]
6145                .as_str()
6146                .expect("error message is a string");
6147            assert!(
6148                message.contains(&field),
6149                "{name}: field missing from {message}"
6150            );
6151            assert!(
6152                message.contains(&value),
6153                "{name}: value missing from {message}"
6154            );
6155            assert_eq!(
6156                registry
6157                    .active_registration_count()
6158                    .expect("registry reads"),
6159                0,
6160                "{name}: refused HELLO must not create a catalog entry"
6161            );
6162        }
6163    }
6164
6165    #[test]
6166    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
6167        let registry = Arc::new(Registry::default());
6168        let handler = ControlHandler::new(Arc::clone(&registry));
6169        let capabilities = json!({
6170            "provides": ["credentials-provider/v1"],
6171            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
6172            "must_never_reach": ["federation-transport/v1"]
6173        });
6174        let response = handler
6175            .handle_control(
6176                ConnectionId::new(99),
6177                capability_grammar_hello_frame(
6178                    capabilities.clone(),
6179                    Some(json!(["/roles/0/tools"])),
6180                    99,
6181                ),
6182            )
6183            .expect("valid HELLO registers");
6184        assert_eq!(response[0].header.ty, FrameType::HelloAck);
6185
6186        let request = Frame::build(
6187            FrameType::Request,
6188            control_flags(),
6189            0,
6190            0,
6191            100,
6192            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6193                .expect("catalog request serializes"),
6194        )
6195        .expect("catalog request frame builds");
6196        let response = handler
6197            .handle_catalog_list(request, None)
6198            .expect("catalog list succeeds");
6199        let ClientControlResponse::CatalogList { modules, .. } =
6200            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
6201        else {
6202            panic!("catalog request must return catalog.list");
6203        };
6204        assert_eq!(modules.len(), 1);
6205        assert_eq!(
6206            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
6207            capabilities
6208        );
6209    }
6210
6211    #[test]
6212    fn catalog_list_mirrors_management_operation_description() {
6213        let registry = Arc::new(Registry::default());
6214        let handler = ControlHandler::new(Arc::clone(&registry));
6215        let description = "List managed records and return their identifiers and metadata.";
6216        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
6217        manifest.provides = vec![ProviderRole::ManagementSurface {
6218            operations: vec![ManagementOperation {
6219                name: "records.list".to_string(),
6220                kind: ManagementOperationKind::Query,
6221                description: Some(description.to_string()),
6222            }],
6223            config_schema: json!({"type": "object"}),
6224            observability: vec![ObservabilitySurface {
6225                name: "records.stats".to_string(),
6226                kind: ObservabilityKind::Snapshot,
6227            }],
6228            identity_scope: vec![IdentityScope::Project],
6229            concurrency: Concurrency::ModuleManaged,
6230        }];
6231        registry
6232            .register_with_control_ops(
6233                manifest,
6234                PROTOCOL_VERSION,
6235                ConnectionId::new(99),
6236                Vec::new(),
6237            )
6238            .expect("described management manifest registers");
6239
6240        let request = Frame::build(
6241            FrameType::Request,
6242            control_flags(),
6243            0,
6244            0,
6245            100,
6246            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6247                .expect("catalog request serializes"),
6248        )
6249        .expect("catalog request frame builds");
6250        let response = handler
6251            .handle_catalog_list(request, None)
6252            .expect("catalog list succeeds");
6253        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
6254        assert_eq!(
6255            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
6256            "catalog.list must preserve the declared operation description verbatim"
6257        );
6258    }
6259
6260    #[test]
6261    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
6262        let registry = Arc::new(Registry::default());
6263        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
6264            [("vault".to_string(), true), ("squatter".to_string(), true)],
6265            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
6266        );
6267        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
6268        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
6269            provides: vec!["credentials-provider/v1".to_string()],
6270            requires: Vec::new(),
6271            must_never_reach: Vec::new(),
6272        });
6273        let frame = Frame::build(
6274            FrameType::Hello,
6275            control_flags(),
6276            0,
6277            0,
6278            77,
6279            serde_json::to_vec(&ModuleHelloBody {
6280                manifest: squatter,
6281                protocol_ver: PROTOCOL_VERSION,
6282                control_ops: None,
6283                launch_nonce: None,
6284            })
6285            .expect("HELLO serializes"),
6286        )
6287        .expect("HELLO frame builds");
6288        let response = handler
6289            .handle_control(ConnectionId::new(77), frame)
6290            .expect("reserved claim receives a typed refusal");
6291        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
6292        assert_eq!(
6293            registry
6294                .active_registration_count()
6295                .expect("registry reads"),
6296            0,
6297            "a reserved capability refusal must not leave a catalog entry"
6298        );
6299    }
6300
6301    #[test]
6302    fn server_describe_surfaces_required_capability_verdict_fields() {
6303        let registry = Arc::new(Registry::default());
6304        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
6305            [
6306                ("consumer".to_string(), true),
6307                ("provider".to_string(), false),
6308            ],
6309            BTreeMap::new(),
6310        );
6311        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
6312        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
6313            provides: Vec::new(),
6314            requires: vec![subc_protocol::manifest::CapabilityRequirement {
6315                capability: "credentials-provider/v1".to_string(),
6316                need: subc_protocol::manifest::CapabilityNeed::Required,
6317            }],
6318            must_never_reach: Vec::new(),
6319        });
6320        let hello = Frame::build(
6321            FrameType::Hello,
6322            control_flags(),
6323            0,
6324            0,
6325            78,
6326            serde_json::to_vec(&ModuleHelloBody {
6327                manifest: consumer,
6328                protocol_ver: PROTOCOL_VERSION,
6329                control_ops: None,
6330                launch_nonce: None,
6331            })
6332            .expect("HELLO serializes"),
6333        )
6334        .expect("HELLO frame builds");
6335        handler
6336            .handle_control(ConnectionId::new(78), hello)
6337            .expect("consumer registers");
6338        let describe = Frame::build(
6339            FrameType::Request,
6340            control_flags(),
6341            0,
6342            0,
6343            79,
6344            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
6345                .expect("request serializes"),
6346        )
6347        .expect("describe frame builds");
6348        let response = handler
6349            .handle_server_describe(describe)
6350            .expect("server.describe succeeds");
6351        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
6352        let requirement = &rendered["capability_requirements"][0];
6353        assert_eq!(requirement["consumer"], "consumer");
6354        assert_eq!(requirement["verdict"], "never_provided");
6355        assert_eq!(requirement["episode_seq"], 1);
6356        assert_eq!(requirement["config_satisfiable"], false);
6357        assert_eq!(requirement["runtime_available"], false);
6358        assert!(requirement["detail"]
6359            .as_str()
6360            .expect("detail string")
6361            .contains("credentials-provider/v1"));
6362    }
6363
6364    #[test]
6365    fn catalog_list_omits_capabilities_for_legacy_manifest() {
6366        let registry = Arc::new(Registry::default());
6367        let handler = ControlHandler::new(Arc::clone(&registry));
6368        let hello = handler
6369            .handle_control(
6370                ConnectionId::new(101),
6371                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
6372            )
6373            .expect("legacy HELLO registers");
6374        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
6375
6376        let request = Frame::build(
6377            FrameType::Request,
6378            control_flags(),
6379            0,
6380            0,
6381            102,
6382            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6383                .expect("catalog request serializes"),
6384        )
6385        .expect("catalog request frame builds");
6386        let response = handler
6387            .handle_catalog_list(request, None)
6388            .expect("catalog list succeeds");
6389        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
6390        assert!(
6391            body["modules"][0].get("capabilities").is_none(),
6392            "legacy manifest must retain an absent capabilities field on catalog.list"
6393        );
6394    }
6395
6396    #[test]
6397    fn hello_ack_omits_storage_when_no_storage_config() {
6398        let registry = Arc::new(Registry::default());
6399        let handler = ControlHandler::new(Arc::clone(&registry));
6400        let responses = handler
6401            .handle_control(
6402                ConnectionId::new(1),
6403                hello_frame("aft", PROTOCOL_VERSION, 7),
6404            )
6405            .unwrap();
6406        let ack = parse_ack(&responses[0]);
6407        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
6408    }
6409
6410    #[test]
6411    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
6412        // With a central sqlite storage policy, each registering module gets its
6413        // own resolved descriptor in HELLO_ACK, keyed by its module id.
6414        let registry = Arc::new(Registry::default());
6415        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
6416            crate::daemon_config::StorageConfig::Sqlite {
6417                data_home: std::path::PathBuf::from("/data"),
6418            },
6419        ));
6420
6421        let responses = handler
6422            .handle_control(
6423                ConnectionId::new(1),
6424                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
6425            )
6426            .unwrap();
6427        let ack = parse_ack(&responses[0]);
6428        assert_eq!(
6429            ack.storage,
6430            Some(serde_json::json!({
6431                "module_id": "alfonso-routing",
6432                "storage_namespace": "default",
6433                "isolation": { "kind": "module" },
6434                "backend": {
6435                    "backend": "sqlite",
6436                    "path": "/data/cortexkit/alfonso-routing/store.db"
6437                }
6438            })),
6439            "the delivered descriptor is the module's own sqlite store path"
6440        );
6441    }
6442
6443    #[test]
6444    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
6445        let registry = Arc::new(Registry::default());
6446        let handler = ControlHandler::new(Arc::clone(&registry));
6447        let conn = ConnectionId::new(1);
6448        let responses = handler
6449            .handle_control(
6450                conn,
6451                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
6452            )
6453            .unwrap();
6454        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6455        let registration = registry.get_module("aft").unwrap().unwrap();
6456        assert_eq!(registration.control_ops, module_baseline_control_ops());
6457
6458        let frame =
6459            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
6460        assert!(handler
6461            .guard_module_control_op(&frame, "aft", "route.bind")
6462            .unwrap()
6463            .is_none());
6464        let error = handler
6465            .guard_module_control_op(&frame, "aft", "test.synthetic")
6466            .unwrap()
6467            .expect("synthetic ungranted op should be rejected");
6468        assert_eq!(error.header.ty, FrameType::Error);
6469        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
6470    }
6471
6472    #[test]
6473    fn hello_control_ops_some_adds_optional_grants() {
6474        let registry = Arc::new(Registry::default());
6475        let handler = ControlHandler::new(Arc::clone(&registry));
6476        handler
6477            .handle_control(
6478                ConnectionId::new(1),
6479                hello_frame_with_control_ops(
6480                    "aft",
6481                    PROTOCOL_VERSION,
6482                    7,
6483                    Some(vec![
6484                        "future.synthetic".to_string(),
6485                        "route.bind".to_string(),
6486                    ]),
6487                ),
6488            )
6489            .unwrap();
6490        let registration = registry.get_module("aft").unwrap().unwrap();
6491        assert_eq!(
6492            registration.control_ops,
6493            vec![
6494                "route.bind".to_string(),
6495                "route.status".to_string(),
6496                "future.synthetic".to_string(),
6497            ]
6498        );
6499        let frame =
6500            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
6501        assert!(handler
6502            .guard_module_control_op(&frame, "aft", "future.synthetic")
6503            .unwrap()
6504            .is_none());
6505    }
6506
6507    #[tokio::test]
6508    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
6509        let registry = Arc::new(Registry::default());
6510        let forwarding = Arc::new(ForwardingTable::default());
6511        let handler =
6512            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
6513        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
6514        let responses = handler
6515            .handle_control_frame(
6516                &module_ctx,
6517                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
6518            )
6519            .await
6520            .unwrap();
6521        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6522
6523        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
6524        let responses = handler
6525            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
6526            .await
6527            .unwrap();
6528        assert_eq!(responses.len(), 1);
6529        assert_eq!(responses[0].header.ty, FrameType::Error);
6530        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
6531        assert!(module_rx.try_recv().is_err());
6532    }
6533
6534    #[tokio::test]
6535    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
6536        let registry = Arc::new(Registry::default());
6537        let forwarding = Arc::new(ForwardingTable::default());
6538        let handler =
6539            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
6540        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
6541        handler
6542            .handle_control_frame(
6543                &module_ctx,
6544                hello_frame_with_control_ops(
6545                    "aft",
6546                    PROTOCOL_VERSION,
6547                    7,
6548                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
6549                ),
6550            )
6551            .await
6552            .unwrap();
6553
6554        let project_root = unique_project_root("demux");
6555        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
6556        let route_handler = handler.clone();
6557        let route_task = tokio::spawn(async move {
6558            route_handler
6559                .handle_control_frame(
6560                    &route_client_ctx,
6561                    route_open_frame(100, "aft", project_root),
6562                )
6563                .await
6564                .unwrap()
6565        });
6566        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
6567            .await
6568            .unwrap()
6569            .unwrap();
6570        assert!(matches!(
6571            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
6572            ModuleControlRequest::RouteBind { .. }
6573        ));
6574
6575        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
6576        let health_handler = handler.clone();
6577        let health_task = tokio::spawn(async move {
6578            health_handler
6579                .handle_control_frame(
6580                    &health_client_ctx,
6581                    supervisor_health_probe_frame(101, "aft"),
6582                )
6583                .await
6584                .unwrap()
6585        });
6586        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
6587            .await
6588            .unwrap()
6589            .unwrap();
6590        assert_eq!(
6591            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
6592            ModuleControlRequest::HealthCheck {}
6593        );
6594
6595        handler
6596            .handle_control_frame(
6597                &module_ctx,
6598                health_response(health_frame.header.corr, HealthStatus::Degraded),
6599            )
6600            .await
6601            .unwrap();
6602        let health_response = health_task.await.unwrap();
6603        assert_eq!(health_response.len(), 1);
6604        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
6605            ClientControlResponse::SupervisorHealthProbe {
6606                module_id,
6607                status,
6608                detail,
6609                metrics,
6610            } => {
6611                assert_eq!(module_id, "aft");
6612                assert_eq!(status, HealthStatus::Degraded);
6613                assert_eq!(detail.as_deref(), Some("warming"));
6614                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
6615            }
6616            other => panic!("unexpected health response: {other:?}"),
6617        }
6618
6619        handler
6620            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
6621            .await
6622            .unwrap();
6623        let route_response = route_task.await.unwrap();
6624        assert!(route_response.is_empty());
6625        let published = route_client_rx.recv().await.unwrap();
6626        assert!(matches!(
6627            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
6628            ClientControlResponse::RouteOpen { .. }
6629        ));
6630    }
6631
6632    /// Start one `route.open` on `client_connection` and return its still-running
6633    /// handler task together with the `route.bind` the module received for it.
6634    /// The handler blocks until the module answers, so it has to run as a task
6635    /// while the test drives the module side.
6636    async fn relay_route_open(
6637        handler: &ControlHandler,
6638        client_connection: ConnectionId,
6639        client_egress: &FrameSink,
6640        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
6641        corr: u64,
6642        module_id: &str,
6643        project_root_label: &str,
6644    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
6645        let ctx = RouteCtx {
6646            connection_id: client_connection,
6647            egress: client_egress.clone(),
6648        };
6649        let handler = handler.clone();
6650        let project_root = unique_project_root(project_root_label);
6651        let module_id = module_id.to_string();
6652        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
6653        let task = tokio::spawn(async move {
6654            let _guard = tracing::dispatcher::set_default(&dispatch);
6655            handler
6656                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
6657                .await
6658                .unwrap()
6659        });
6660        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
6661            .await
6662            .expect("module receives the relayed route.bind")
6663            .expect("module egress is open");
6664        (task, bind.frame)
6665    }
6666
6667    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
6668        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
6669            ModuleControlRequest::RouteBind {
6670                route_channel,
6671                epoch,
6672                ..
6673            } => (route_channel, epoch),
6674            other => panic!("expected a route.bind request, got {other:?}"),
6675        }
6676    }
6677
6678    fn published_route(frame: &Frame) -> (u16, u32) {
6679        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
6680            ClientControlResponse::RouteOpen {
6681                route_channel,
6682                route_epoch,
6683            } => (route_channel, route_epoch),
6684            other => panic!("expected a route.open response, got {other:?}"),
6685        }
6686    }
6687
6688    /// Reproduction of a production outage. A client had `route.open`s in
6689    /// flight to a module and was already marked closing -- its egress had refused a
6690    /// module frame, so the daemon asked its connection to end -- while its sink
6691    /// was still open. When the module acked those binds, the daemon refused to
6692    /// commit a route for a closing client, and that refusal was returned from
6693    /// the MODULE connection's frame handler, where a router error that has no
6694    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
6695    /// the supervisor correctly did not respawn a clean exit, and every seat lost
6696    /// its tools for hours -- one client's teardown took down a connection
6697    /// carrying ~170 other routes.
6698    ///
6699    /// The window is opened here by calling the production path that opens it
6700    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
6701    /// state that matters is "in `closing_connections`, sink still open, relay
6702    /// still pending", and it lasts only from the close request until the
6703    /// connection loop reacts to it; a socket-level test can flood a client into
6704    /// that escalation but cannot pin the module's ack inside the window. Closing
6705    /// the socket instead takes the other path entirely -- connection teardown
6706    /// removes the pending relay under the same lock, so the ack finds nothing.
6707    #[tokio::test]
6708    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
6709        let registry = Arc::new(Registry::default());
6710        let forwarding = Arc::new(ForwardingTable::default());
6711        let handler =
6712            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
6713
6714        let module_connection = ConnectionId::new(30);
6715        let (module_ctx, mut module_rx) = route_ctx(module_connection);
6716        handler
6717            .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
6718            .await
6719            .unwrap();
6720
6721        let dying_client = ConnectionId::new(31);
6722        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
6723
6724        // A published route on the dying client. The escalation below only marks
6725        // a connection closing for a route it has already published.
6726        let (first_task, first_bind) = relay_route_open(
6727            &handler,
6728            dying_client,
6729            &dying_ctx.egress,
6730            &mut module_rx,
6731            100,
6732            "aft",
6733            "closing-first",
6734        )
6735        .await;
6736        handler
6737            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
6738            .await
6739            .unwrap();
6740        assert!(first_task.await.unwrap().is_empty());
6741        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
6742
6743        // A second route.open from the same client, relayed and awaiting its ack.
6744        let (second_task, second_bind) = relay_route_open(
6745            &handler,
6746            dying_client,
6747            &dying_ctx.egress,
6748            &mut module_rx,
6749            101,
6750            "aft",
6751            "closing-second",
6752        )
6753        .await;
6754        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
6755
6756        // The window: the client is closing, its sink is still open, and its
6757        // second bind is still pending.
6758        assert!(forwarding
6759            .escalate_client_delivery_failure(
6760                dying_client,
6761                first_channel,
6762                first_epoch,
6763                CloseReason::new(
6764                    "module_to_client_delivery_failed",
6765                    "client egress refused a module frame",
6766                ),
6767            )
6768            .unwrap());
6769        assert!(!dying_ctx.egress.is_closed());
6770
6771        // The frame that used to end the module connection.
6772        let ack = handler
6773            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
6774            .await;
6775        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
6776        if module_loop_error.is_some() {
6777            // What the server's connection loop does with a router error that has
6778            // no ERROR-frame translation: end the connection, which releases the
6779            // module's registration and every route on it.
6780            handler.cleanup_connection(module_connection).unwrap();
6781        }
6782        // Read the module's next frame before opening the co-tenant's route, so
6783        // the GOODBYE assertion below is about THIS ack and not about later
6784        // traffic. `None` means the module was told nothing.
6785        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
6786            .await
6787            .ok()
6788            .flatten();
6789
6790        // 1. The module connection is still registered.
6791        assert!(
6792            registry
6793                .get_module_by_connection(module_connection)
6794                .unwrap()
6795                .is_some(),
6796            "one client's closing connection ended the shared module connection: \
6797             {module_loop_error:?}"
6798        );
6799        // ...and still serving: another client can open and use a route on it.
6800        let cotenant = ConnectionId::new(32);
6801        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
6802        let (cotenant_task, cotenant_bind) = relay_route_open(
6803            &handler,
6804            cotenant,
6805            &cotenant_ctx.egress,
6806            &mut module_rx,
6807            102,
6808            "aft",
6809            "closing-cotenant",
6810        )
6811        .await;
6812        handler
6813            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
6814            .await
6815            .unwrap();
6816        assert!(cotenant_task.await.unwrap().is_empty());
6817        let (cotenant_channel, cotenant_epoch) =
6818            published_route(&cotenant_rx.recv().await.unwrap());
6819        assert!(matches!(
6820            forwarding
6821                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
6822                .unwrap(),
6823            DataRoute::Client(DataRouteState::Bound(_))
6824        ));
6825
6826        // 2. The module was told to drop the binding it created for the route
6827        //    that will never be published.
6828        let goodbye = post_ack_module_frame
6829            .expect("module receives a GOODBYE for the abandoned route channel");
6830        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
6831        assert_eq!(goodbye.header.channel, abandoned_channel);
6832        assert_eq!(goodbye.header.epoch, abandoned_epoch);
6833
6834        // 3. The dying client received nothing: no route was ever published to
6835        //    it. Its route.open is answered as unavailable, which the connection
6836        //    loop would write to a socket that is already going away.
6837        assert!(dying_rx.try_recv().is_err());
6838        let second_response = second_task.await.unwrap();
6839        assert_eq!(second_response.len(), 1);
6840        assert_eq!(
6841            parse_error(&second_response[0])["code"],
6842            "target_unavailable"
6843        );
6844    }
6845
6846    /// The fence at the module-loop boundary, stated as its own contract: which
6847    /// forwarding failures are allowed to end the module connection that is being
6848    /// served. A `ConnectionClosing` naming some client is about that client, and
6849    /// a module connection is shared; the same error naming the module's own
6850    /// connection is about this connection and must stay fatal, as must failures
6851    /// that are about the forwarding table itself.
6852    #[test]
6853    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
6854        let handler = ControlHandler::default();
6855        let module_connection = ConnectionId::new(30);
6856        let client_connection = ConnectionId::new(31);
6857
6858        handler
6859            .refuse_to_end_module_connection_for_a_client(
6860                module_connection,
6861                77,
6862                ForwardingError::ConnectionClosing {
6863                    connection_id: client_connection,
6864                },
6865            )
6866            .expect("a closing client must never end the module connection");
6867
6868        assert!(matches!(
6869            handler.refuse_to_end_module_connection_for_a_client(
6870                module_connection,
6871                78,
6872                ForwardingError::ConnectionClosing {
6873                    connection_id: module_connection,
6874                },
6875            ),
6876            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
6877                connection_id
6878            })) if connection_id == module_connection
6879        ));
6880        assert!(matches!(
6881            handler.refuse_to_end_module_connection_for_a_client(
6882                module_connection,
6883                79,
6884                ForwardingError::Poisoned,
6885            ),
6886            Err(RouterError::Forwarding(ForwardingError::Poisoned))
6887        ));
6888        assert!(matches!(
6889            handler.refuse_to_end_module_connection_for_a_client(
6890                module_connection,
6891                80,
6892                ForwardingError::StaleModuleEndpoint,
6893            ),
6894            Err(RouterError::Forwarding(
6895                ForwardingError::StaleModuleEndpoint
6896            ))
6897        ));
6898    }
6899
6900    /// The spawn-attestation guard is what stops a connected module from claiming
6901    /// another module's identity and being stamped `Reserved` for it. Every other
6902    /// test that supplies a consumer_identity supplies a CORRECT one, because a
6903    /// correct one is what the rest of the flow needs -- so the guard's rejection
6904    /// branch was never the subject of an assertion, only its acceptance branch.
6905    ///
6906    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
6907    /// whole subc-core library suite green; only the forwarding integration tests
6908    /// notice, and they notice for unrelated reasons. This test exists so the
6909    /// refusal itself is asserted where the guard lives: it fails if the identity
6910    /// check stops refusing, which is the direction that matters, since a guard
6911    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
6912    #[tokio::test]
6913    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
6914        let registry = Arc::new(Registry::default());
6915        let forwarding = Arc::new(ForwardingTable::default());
6916        let supervisor = SupervisorHandle::new();
6917        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
6918        let handler =
6919            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
6920                .with_supervisor(supervisor);
6921
6922        let (target_ctx, _target_rx) = route_ctx(ConnectionId::new(90));
6923        handler
6924            .handle_control_frame(&target_ctx, hello_frame("target", PROTOCOL_VERSION, 1))
6925            .await
6926            .unwrap();
6927
6928        // A real supervised module id presenting the wrong nonce. This is the
6929        // impersonation case: the attacker knows a privileged module_id, which is
6930        // public, and guesses at the nonce, which is not.
6931        let wrong_nonce = handler
6932            .handle_control_frame(
6933                &route_ctx(ConnectionId::new(91)).0,
6934                route_open_frame_with_admission_facts(
6935                    20,
6936                    "target",
6937                    unique_project_root("admission-facts"),
6938                    Some(subc_control::ConsumerIdentity {
6939                        module_id: "fed".to_string(),
6940                        launch_nonce: "not-the-real-nonce".to_string(),
6941                    }),
6942                    None,
6943                ),
6944            )
6945            .await
6946            .unwrap();
6947        assert_eq!(
6948            parse_error(&wrong_nonce[0])["code"],
6949            "bad_consumer_identity",
6950            "a mismatched launch nonce must be refused, not stamped Reserved"
6951        );
6952
6953        // A module id the supervisor never spawned at all, so no nonce exists to
6954        // compare against. An implementation that treats "no record" as "nothing
6955        // to check" fails open here while passing the case above.
6956        let never_spawned = handler
6957            .handle_control_frame(
6958                &route_ctx(ConnectionId::new(92)).0,
6959                route_open_frame_with_admission_facts(
6960                    21,
6961                    "target",
6962                    unique_project_root("admission-facts"),
6963                    Some(subc_control::ConsumerIdentity {
6964                        module_id: "never-spawned".to_string(),
6965                        launch_nonce: "any-nonce".to_string(),
6966                    }),
6967                    None,
6968                ),
6969            )
6970            .await
6971            .unwrap();
6972        assert_eq!(
6973            parse_error(&never_spawned[0])["code"],
6974            "bad_consumer_identity",
6975            "an unspawned module_id must be refused rather than accepted for lack of a record"
6976        );
6977    }
6978
6979    /// The refusal test above proves the guard says NO. Nothing proved it can say
6980    /// YES, and the difference is not academic: replacing the whole authorization
6981    /// with `false` -- admitting no consumer identity at all, revoking Reserved
6982    /// standing for every supervised module in the fleet -- leaves 110 of the 111
6983    /// library tests GREEN. The one that notices does so by HANGING, because it
6984    /// waits for a bind that can no longer happen.
6985    ///
6986    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
6987    /// or flaky test, invites a RETRY rather than an investigation, and the retry
6988    /// hangs too and gets blamed on the runner. So a total revocation of the
6989    /// daemon's trust grant would have shipped behind a symptom nobody attributes
6990    /// to code.
6991    ///
6992    /// The bias is structural rather than accidental. A REFUSAL looks like a
6993    /// failure someone writes a test for; a GRANT looks like the happy path. Every
6994    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
6995    /// suite for that reason, and this one is the purest case in the daemon.
6996    ///
6997    /// This test asserts the EFFECT rather than the absence of an error: the module
6998    /// receives a RouteBind and it carries `Reserved` naming the attested module.
6999    /// A guard that admitted nobody would produce no bind at all; one that admitted
7000    /// everybody would stamp the wrong principal, which the refusal test catches.
7001    #[tokio::test]
7002    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
7003        let registry = Arc::new(Registry::default());
7004        let forwarding = Arc::new(ForwardingTable::default());
7005        let supervisor = SupervisorHandle::new();
7006        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7007        let handler =
7008            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
7009                .with_supervisor(supervisor);
7010
7011        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
7012        handler
7013            .handle_control_frame(&target_ctx, hello_frame("target", PROTOCOL_VERSION, 1))
7014            .await
7015            .unwrap();
7016
7017        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
7018        let route_handler = handler.clone();
7019        let route_task = tokio::spawn(async move {
7020            route_handler
7021                .handle_control_frame(
7022                    &client_ctx,
7023                    route_open_frame_with_admission_facts(
7024                        30,
7025                        "target",
7026                        unique_project_root("admission-facts"),
7027                        Some(subc_control::ConsumerIdentity {
7028                            module_id: "fed".to_string(),
7029                            launch_nonce: "fed-nonce".to_string(),
7030                        }),
7031                        None,
7032                    ),
7033                )
7034                .await
7035                .unwrap()
7036        });
7037
7038        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
7039        // the very mutation it exists to catch -- a guard that admits nobody -- no
7040        // bind is ever sent, so it HUNG rather than failing. That reproduces the
7041        // exact defect being fixed: a total revocation detected only as a stalled
7042        // suite, which reads as flakiness and invites a retry. An acceptance test
7043        // that waits for an effect must bound the wait, or a red becomes a hang.
7044        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7045            .await
7046            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
7047            .expect("module control channel closed before route.bind");
7048        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
7049        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
7050            panic!("expected route.bind")
7051        };
7052        assert_eq!(
7053            principal,
7054            Some(Principal::Reserved {
7055                module_id: "fed".to_string()
7056            }),
7057            "a correctly attested consumer must be stamped Reserved for its own id"
7058        );
7059
7060        handler
7061            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
7062            .await
7063            .unwrap();
7064        assert!(route_task.await.unwrap().is_empty());
7065        assert!(
7066            matches!(
7067                serde_json::from_slice::<ClientControlResponse>(
7068                    &client_rx.recv().await.unwrap().body
7069                )
7070                .unwrap(),
7071                ClientControlResponse::RouteOpen { .. }
7072            ),
7073            "the route must actually open, not merely avoid an error"
7074        );
7075    }
7076
7077    #[tokio::test(start_paused = true)]
7078    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
7079        let registry = Arc::new(Registry::default());
7080        let forwarding = Arc::new(ForwardingTable::default());
7081        let supervisor = SupervisorHandle::new();
7082        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7083        let handler =
7084            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
7085                .with_supervisor(supervisor);
7086
7087        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
7088        handler
7089            .handle_control_frame(&target_ctx, hello_frame("target", PROTOCOL_VERSION, 1))
7090            .await
7091            .unwrap();
7092
7093        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
7094        let direct_handler = handler.clone();
7095        let direct_open = tokio::spawn(async move {
7096            direct_handler
7097                .handle_control_frame(
7098                    &direct_ctx,
7099                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
7100                )
7101                .await
7102                .unwrap()
7103        });
7104        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7105            .await
7106            .expect("no direct route.bind within 5s")
7107            .expect("target control channel closed before direct route.bind");
7108        handler
7109            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
7110            .await
7111            .unwrap();
7112        assert!(direct_open.await.unwrap().is_empty());
7113        let _ = direct_rx.recv().await.unwrap();
7114
7115        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
7116        let reserved_handler = handler.clone();
7117        let reserved_open = tokio::spawn(async move {
7118            reserved_handler
7119                .handle_control_frame(
7120                    &reserved_ctx,
7121                    route_open_frame_with_admission_facts(
7122                        3,
7123                        "target",
7124                        unique_project_root("admission-facts"),
7125                        Some(ConsumerIdentity {
7126                            module_id: "fed".to_string(),
7127                            launch_nonce: "fed-nonce".to_string(),
7128                        }),
7129                        None,
7130                    ),
7131                )
7132                .await
7133                .unwrap()
7134        });
7135        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7136            .await
7137            .expect("no reserved route.bind within 5s")
7138            .expect("target control channel closed before reserved route.bind");
7139        handler
7140            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
7141            .await
7142            .unwrap();
7143        assert!(reserved_open.await.unwrap().is_empty());
7144        let _ = reserved_rx.recv().await.unwrap();
7145
7146        forwarding
7147            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
7148            .unwrap();
7149        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
7150        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
7151            module_id: Some("target".to_string()),
7152        })
7153        .unwrap();
7154        let census_frame =
7155            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
7156        let response = handler
7157            .handle_control_frame(&census_ctx, census_frame)
7158            .await
7159            .unwrap()
7160            .pop()
7161            .unwrap();
7162        let actual: Value = serde_json::from_slice(&response.body).unwrap();
7163        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
7164        assert!(matches!(
7165            decoded,
7166            ClientControlResponse::SupervisorRoutes { .. }
7167        ));
7168        let routes = actual["modules"][0]["routes"].as_array().unwrap();
7169        assert_eq!(routes.len(), 2);
7170        assert!(routes.iter().all(|route| route["draining"] == true));
7171        // The census carries WHY: the reason the drain was begun with, in the
7172        // route.closing vocabulary, on every draining route this drain marked.
7173        assert!(
7174            routes.iter().all(|route| route["drain_reason"] == "reload"),
7175            "draining routes must name the drain's reason: {routes:?}"
7176        );
7177        assert!(routes.iter().any(|route| {
7178            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
7179        }));
7180        assert!(routes.iter().any(|route| {
7181            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
7182        }));
7183
7184        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7185            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
7186        if std::env::var_os("UPDATE_GOLDEN").is_some() {
7187            std::fs::write(
7188                &golden_path,
7189                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
7190            )
7191            .unwrap();
7192        }
7193        let expected: Value =
7194            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
7195        assert_eq!(actual, expected);
7196    }
7197
7198    async fn query_live_roots(
7199        handler: &ControlHandler,
7200        module_ctx: &RouteCtx,
7201    ) -> ModuleControlResponseToModule {
7202        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
7203        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
7204        let response = handler
7205            .handle_control_frame(module_ctx, frame)
7206            .await
7207            .unwrap()
7208            .pop()
7209            .unwrap();
7210        serde_json::from_slice(&response.body).unwrap()
7211    }
7212
7213    #[tokio::test(start_paused = true)]
7214    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
7215        let registry = Arc::new(Registry::default());
7216        let forwarding = Arc::new(ForwardingTable::default());
7217        let handler = ControlHandler::with_forwarding(registry, forwarding);
7218        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
7219        handler
7220            .handle_control_frame(&target_ctx, hello_frame("target", PROTOCOL_VERSION, 1))
7221            .await
7222            .unwrap();
7223        let root = unique_project_root("live-roots-known");
7224        let path = ProjectRootId::from_path_allowing_missing(root.path())
7225            .unwrap()
7226            .as_path()
7227            .to_path_buf();
7228        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
7229        let open_handler = handler.clone();
7230        let opened = tokio::spawn(async move {
7231            open_handler
7232                .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
7233                .await
7234                .unwrap()
7235        });
7236        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7237            .await
7238            .unwrap()
7239            .unwrap();
7240        handler
7241            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
7242            .await
7243            .unwrap();
7244        assert!(opened.await.unwrap().is_empty());
7245        let _ = client_rx.recv().await.unwrap();
7246
7247        let root = unique_project_root("live-roots-pending");
7248        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
7249            .unwrap()
7250            .as_path()
7251            .to_path_buf();
7252        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
7253        let open_handler = handler.clone();
7254        let pending = tokio::spawn(async move {
7255            open_handler
7256                .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
7257                .await
7258                .unwrap()
7259        });
7260        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7261            .await
7262            .unwrap()
7263            .unwrap();
7264        let actual = query_live_roots(&handler, &target_ctx).await;
7265        let ModuleControlResponseToModule::LiveRoots {
7266            roots,
7267            unknown_root_bindings,
7268            total_bindings,
7269        } = actual
7270        else {
7271            panic!("expected live roots")
7272        };
7273        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
7274        assert_eq!(unknown_root_bindings, 0);
7275        assert_eq!(
7276            roots.len(),
7277            2,
7278            "root-known arm must retain each canonical root"
7279        );
7280        assert_eq!(
7281            total_bindings,
7282            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7283        );
7284        let counts = roots
7285            .iter()
7286            .map(|root| (root.project_root.clone(), root.bound, root.pending))
7287            .collect::<Vec<_>>();
7288        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
7289        expected.sort_by(|a, b| a.0.cmp(&b.0));
7290        assert_eq!(
7291            counts, expected,
7292            "roots must sort by path and count pending separately"
7293        );
7294        handler
7295            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
7296            .await
7297            .unwrap();
7298        assert!(pending.await.unwrap().is_empty());
7299    }
7300
7301    #[tokio::test(start_paused = true)]
7302    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
7303        let forwarding = Arc::new(ForwardingTable::default());
7304        let handler =
7305            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
7306        let (target_ctx, _target_rx) = route_ctx(ConnectionId::new(311));
7307        handler
7308            .handle_control_frame(&target_ctx, hello_frame("target", PROTOCOL_VERSION, 1))
7309            .await
7310            .unwrap();
7311        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
7312        let pending = forwarding
7313            .begin_route_bind_relay_for_test(
7314                client_ctx.connection_id,
7315                client_ctx.egress.clone(),
7316                2,
7317                "target",
7318            )
7319            .unwrap();
7320        forwarding
7321            .complete_pending_relay(
7322                target_ctx.connection_id,
7323                pending.corr,
7324                RouteBindRelayOutcome::Accepted,
7325            )
7326            .unwrap();
7327        let actual = query_live_roots(&handler, &target_ctx).await;
7328        let ModuleControlResponseToModule::LiveRoots {
7329            roots,
7330            unknown_root_bindings,
7331            total_bindings,
7332        } = actual
7333        else {
7334            panic!("expected live roots")
7335        };
7336        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
7337        assert_eq!(
7338            unknown_root_bindings, 1,
7339            "unknown-root arm must not read as no bindings"
7340        );
7341        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
7342        assert_eq!(
7343            total_bindings,
7344            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7345        );
7346    }
7347
7348    #[tokio::test(start_paused = true)]
7349    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
7350        let handler = ControlHandler::with_forwarding(
7351            Arc::new(Registry::default()),
7352            Arc::new(ForwardingTable::default()),
7353        );
7354        let (first_ctx, _first_rx) = route_ctx(ConnectionId::new(315));
7355        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
7356        handler
7357            .handle_control_frame(&first_ctx, hello_frame("first", PROTOCOL_VERSION, 1))
7358            .await
7359            .unwrap();
7360        handler
7361            .handle_control_frame(&second_ctx, hello_frame("second", PROTOCOL_VERSION, 2))
7362            .await
7363            .unwrap();
7364        let root = unique_project_root("second-only");
7365        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
7366        let cloned = handler.clone();
7367        let open = tokio::spawn(async move {
7368            cloned
7369                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
7370                .await
7371                .unwrap()
7372        });
7373        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
7374            .await
7375            .unwrap()
7376            .unwrap();
7377        let first = query_live_roots(&handler, &first_ctx).await;
7378        let second = query_live_roots(&handler, &second_ctx).await;
7379        assert!(
7380            matches!(
7381                first,
7382                ModuleControlResponseToModule::LiveRoots {
7383                    total_bindings: 0,
7384                    ..
7385                }
7386            ),
7387            "cross-module scope must not expose another module's roots"
7388        );
7389        assert!(
7390            matches!(
7391                second,
7392                ModuleControlResponseToModule::LiveRoots {
7393                    total_bindings: 1,
7394                    ..
7395                }
7396            ),
7397            "second module must see its pending route"
7398        );
7399        handler
7400            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
7401            .await
7402            .unwrap();
7403        assert!(open.await.unwrap().is_empty());
7404    }
7405
7406    #[tokio::test(start_paused = true)]
7407    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
7408        let handler = ControlHandler::with_forwarding(
7409            Arc::new(Registry::default()),
7410            Arc::new(ForwardingTable::default()),
7411        );
7412        let (target_ctx, _target_rx) = route_ctx(ConnectionId::new(321));
7413        handler
7414            .handle_control_frame(&target_ctx, hello_frame("target", PROTOCOL_VERSION, 1))
7415            .await
7416            .unwrap();
7417        let actual = query_live_roots(&handler, &target_ctx).await;
7418        let ModuleControlResponseToModule::LiveRoots {
7419            roots,
7420            unknown_root_bindings,
7421            total_bindings,
7422        } = actual
7423        else {
7424            panic!("expected live roots")
7425        };
7426        assert!(roots.is_empty());
7427        assert_eq!(unknown_root_bindings, 0);
7428        assert_eq!(total_bindings, 0);
7429        assert_eq!(
7430            total_bindings,
7431            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7432        );
7433    }
7434
7435    /// Read the vendored fed corpus rather than hand-building a package.
7436    ///
7437    /// A hand-built object encodes what the test author believed the carrier
7438    /// emits. These vectors are what it actually emits, and one of them exists
7439    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
7440    /// ignores additive unknown fields at the traversal emit terminus."
7441    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
7442        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7443            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
7444        let text = std::fs::read_to_string(&path)
7445            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
7446        let vectors: Vec<(String, Value)> = text
7447            .lines()
7448            .filter(|line| !line.trim().is_empty())
7449            .map(|line| {
7450                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
7451                let id = entry["corpus_id"]
7452                    .as_str()
7453                    .expect("every vector carries a corpus_id")
7454                    .to_string();
7455                (id, entry["package"].clone())
7456            })
7457            .collect();
7458        // Pin the count: a corpus that silently shrinks would take its coverage
7459        // with it, and a suite reading N-1 vectors reports the same clean pass
7460        // as one reading N.
7461        assert_eq!(
7462            vectors.len(),
7463            3,
7464            "vendored fed corpus changed size; re-sync from subc-federation"
7465        );
7466
7467        // Pin what makes the corpus DISCRIMINATING, not just present.
7468        //
7469        // The relay test below takes its expected value from the corpus, so the
7470        // corpus supplies the test's power to detect a lossy relay rather than
7471        // its correctness. A relay that dropped unrecognised fields would still
7472        // be caught -- but only by a package carrying fields it does not know.
7473        // Shrink every package to the handful of keys any implementation would
7474        // recognise and the test keeps passing over an input that can no longer
7475        // fail, which is the same clean green as a corpus that shrank away.
7476        //
7477        // So assert the precondition rather than duplicating the packages here:
7478        // at least one vector must carry a field beyond the small common set.
7479        // That is one claim to maintain instead of nine, and it fails loudly if
7480        // a re-sync ever flattens the corpus.
7481        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
7482        let richest = vectors
7483            .iter()
7484            .filter_map(|(_, package)| package.as_object())
7485            .map(|object| {
7486                object
7487                    .keys()
7488                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
7489                    .count()
7490            })
7491            .max()
7492            .unwrap_or(0);
7493        assert!(
7494            richest >= 2,
7495            "vendored corpus no longer carries a package with unmodelled fields, \
7496             so the relay test can no longer distinguish a verbatim relay from a lossy one"
7497        );
7498
7499        vectors
7500    }
7501
7502    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
7503    ///
7504    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
7505    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
7506    /// three-key object, so a relay that quietly dropped fields it did not
7507    /// recognise would satisfy it. These vectors carry nine keys including ones
7508    /// this crate has no type for, so a typed relay fails here and only here.
7509    #[tokio::test]
7510    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
7511        for (corpus_id, package) in fed_admission_facts_vectors() {
7512            let registry = Arc::new(Registry::default());
7513            let forwarding = Arc::new(ForwardingTable::default());
7514            let supervisor = SupervisorHandle::new();
7515            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7516            let handler =
7517                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
7518                    .with_supervisor(supervisor)
7519                    .with_admission_facts_config(
7520                        Some("fed".to_string()),
7521                        Some(vec!["target".to_string()]),
7522                    );
7523
7524            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
7525            handler
7526                .handle_control_frame(&target_ctx, hello_frame("target", PROTOCOL_VERSION, 1))
7527                .await
7528                .unwrap();
7529
7530            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
7531            let route_handler = handler.clone();
7532            let expected = package.clone();
7533            let route_task = tokio::spawn(async move {
7534                route_handler
7535                    .handle_control_frame(
7536                        &client_ctx,
7537                        route_open_frame_with_admission_facts(
7538                            20,
7539                            "target",
7540                            unique_project_root("admission-facts"),
7541                            Some(subc_control::ConsumerIdentity {
7542                                module_id: "fed".to_string(),
7543                                launch_nonce: "fed-nonce".to_string(),
7544                            }),
7545                            Some(package),
7546                        ),
7547                    )
7548                    .await
7549                    .unwrap()
7550            });
7551
7552            let bind_frame = target_rx.recv().await.unwrap();
7553            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
7554            let ModuleControlRequest::RouteBind {
7555                admission_facts, ..
7556            } = bind
7557            else {
7558                panic!("{corpus_id}: expected route.bind")
7559            };
7560            assert_eq!(
7561                admission_facts,
7562                Some(expected),
7563                "{corpus_id}: relay must not add, drop or reshape any field"
7564            );
7565
7566            handler
7567                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
7568                .await
7569                .unwrap();
7570            route_task.await.unwrap();
7571        }
7572    }
7573
7574    #[tokio::test]
7575    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
7576        let registry = Arc::new(Registry::default());
7577        let forwarding = Arc::new(ForwardingTable::default());
7578        let supervisor = SupervisorHandle::new();
7579        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7580        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
7581        let handler =
7582            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
7583                .with_supervisor(supervisor)
7584                .with_admission_facts_config(
7585                    Some("fed".to_string()),
7586                    Some(vec!["target".to_string()]),
7587                );
7588
7589        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
7590        handler
7591            .handle_control_frame(&target_ctx, hello_frame("target", PROTOCOL_VERSION, 1))
7592            .await
7593            .unwrap();
7594        let (other_ctx, _other_rx) = route_ctx(ConnectionId::new(71));
7595        handler
7596            .handle_control_frame(&other_ctx, hello_frame("other", PROTOCOL_VERSION, 2))
7597            .await
7598            .unwrap();
7599
7600        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
7601        let expected_facts = facts.clone();
7602        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
7603        let route_handler = handler.clone();
7604        let route_task = tokio::spawn(async move {
7605            route_handler
7606                .handle_control_frame(
7607                    &client_ctx,
7608                    route_open_frame_with_admission_facts(
7609                        10,
7610                        "target",
7611                        unique_project_root("admission-facts"),
7612                        Some(subc_control::ConsumerIdentity {
7613                            module_id: "fed".to_string(),
7614                            launch_nonce: "fed-nonce".to_string(),
7615                        }),
7616                        Some(facts.clone()),
7617                    ),
7618                )
7619                .await
7620                .unwrap()
7621        });
7622        let bind_frame = target_rx.recv().await.unwrap();
7623        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
7624        let ModuleControlRequest::RouteBind {
7625            admission_facts, ..
7626        } = bind
7627        else {
7628            panic!("expected route.bind")
7629        };
7630        assert_eq!(admission_facts, Some(expected_facts));
7631        handler
7632            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
7633            .await
7634            .unwrap();
7635        assert!(route_task.await.unwrap().is_empty());
7636        assert!(matches!(
7637            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
7638                .unwrap(),
7639            ClientControlResponse::RouteOpen { .. }
7640        ));
7641
7642        let direct = handler
7643            .handle_control_frame(
7644                &route_ctx(ConnectionId::new(73)).0,
7645                route_open_frame_with_admission_facts(
7646                    11,
7647                    "target",
7648                    unique_project_root("admission-facts"),
7649                    None,
7650                    Some(json!({"x": 1})),
7651                ),
7652            )
7653            .await
7654            .unwrap();
7655        assert_eq!(
7656            parse_error(&direct[0])["code"],
7657            "admission_facts_not_permitted"
7658        );
7659
7660        let different_reserved = handler
7661            .handle_control_frame(
7662                &route_ctx(ConnectionId::new(77)).0,
7663                route_open_frame_with_admission_facts(
7664                    15,
7665                    "target",
7666                    unique_project_root("admission-facts"),
7667                    Some(subc_control::ConsumerIdentity {
7668                        module_id: "other".to_string(),
7669                        launch_nonce: "other-nonce".to_string(),
7670                    }),
7671                    Some(json!({"x": 1})),
7672                ),
7673            )
7674            .await
7675            .unwrap();
7676        assert_eq!(
7677            parse_error(&different_reserved[0])["code"],
7678            "admission_facts_not_permitted"
7679        );
7680
7681        let other_target = handler
7682            .handle_control_frame(
7683                &route_ctx(ConnectionId::new(74)).0,
7684                route_open_frame_with_admission_facts(
7685                    12,
7686                    "other",
7687                    unique_project_root("admission-facts"),
7688                    Some(subc_control::ConsumerIdentity {
7689                        module_id: "fed".to_string(),
7690                        launch_nonce: "fed-nonce".to_string(),
7691                    }),
7692                    Some(json!({"x": 1})),
7693                ),
7694            )
7695            .await
7696            .unwrap();
7697        assert_eq!(
7698            parse_error(&other_target[0])["code"],
7699            "admission_facts_target_not_allowed"
7700        );
7701
7702        let nonexistent = handler
7703            .handle_control_frame(
7704                &route_ctx(ConnectionId::new(75)).0,
7705                route_open_frame_with_admission_facts(
7706                    13,
7707                    "missing",
7708                    unique_project_root("admission-facts"),
7709                    None,
7710                    Some(json!({"x": 1})),
7711                ),
7712            )
7713            .await
7714            .unwrap();
7715        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
7716
7717        let described = handler
7718            .handle_control_frame(
7719                &route_ctx(ConnectionId::new(76)).0,
7720                Frame::build(
7721                    FrameType::Request,
7722                    control_flags(),
7723                    0,
7724                    0,
7725                    14,
7726                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
7727                )
7728                .unwrap(),
7729            )
7730            .await
7731            .unwrap();
7732        let ClientControlResponse::ServerDescribe { capabilities, .. } =
7733            serde_json::from_slice(&described[0].body).unwrap()
7734        else {
7735            panic!("expected server.describe response")
7736        };
7737        assert!(capabilities
7738            .iter()
7739            .any(|cap| cap == "admission_facts_relay_v1"));
7740    }
7741
7742    #[tokio::test]
7743    async fn admission_facts_without_configured_carrier_are_rejected() {
7744        let registry = Arc::new(Registry::default());
7745        let forwarding = Arc::new(ForwardingTable::default());
7746        let handler = ControlHandler::with_forwarding(registry, forwarding);
7747        let (target_ctx, _) = route_ctx(ConnectionId::new(78));
7748        handler
7749            .handle_control_frame(&target_ctx, hello_frame("target", PROTOCOL_VERSION, 1))
7750            .await
7751            .unwrap();
7752
7753        let responses = handler
7754            .handle_control_frame(
7755                &route_ctx(ConnectionId::new(79)).0,
7756                route_open_frame_with_admission_facts(
7757                    16,
7758                    "target",
7759                    unique_project_root("admission-facts"),
7760                    None,
7761                    Some(json!({"x": 1})),
7762                ),
7763            )
7764            .await
7765            .unwrap();
7766        assert_eq!(
7767            parse_error(&responses[0])["code"],
7768            "admission_facts_not_permitted"
7769        );
7770    }
7771
7772    #[tokio::test]
7773    async fn route_open_relays_consumer_capabilities_verbatim() {
7774        let registry = Arc::new(Registry::default());
7775        let forwarding = Arc::new(ForwardingTable::default());
7776        let handler =
7777            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
7778        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
7779        handler
7780            .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
7781            .await
7782            .unwrap();
7783
7784        let expected = vec!["elicitation".to_string(), "roots".to_string()];
7785        let expected_for_request = expected.clone();
7786        let project_root = unique_project_root("consumer-capabilities-present");
7787        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
7788        let route_handler = handler.clone();
7789        let route_task = tokio::spawn(async move {
7790            route_handler
7791                .handle_control_frame(
7792                    &client_ctx,
7793                    route_open_frame_with_consumer_capabilities(
7794                        401,
7795                        "aft",
7796                        project_root,
7797                        Some(expected_for_request),
7798                    ),
7799                )
7800                .await
7801                .unwrap()
7802        });
7803        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7804            .await
7805            .unwrap()
7806            .unwrap();
7807        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
7808        let ModuleControlRequest::RouteBind {
7809            consumer_capabilities,
7810            ..
7811        } = bind
7812        else {
7813            panic!("expected route.bind request, got {bind:?}");
7814        };
7815        assert_eq!(consumer_capabilities, Some(expected.clone()));
7816
7817        handler
7818            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
7819            .await
7820            .unwrap();
7821        let route_response = route_task.await.unwrap();
7822        assert!(route_response.is_empty());
7823        let published = client_rx.recv().await.unwrap();
7824        assert!(matches!(
7825            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
7826            ClientControlResponse::RouteOpen { .. }
7827        ));
7828    }
7829
7830    #[tokio::test]
7831    async fn route_open_without_consumer_capabilities_relays_none() {
7832        let registry = Arc::new(Registry::default());
7833        let forwarding = Arc::new(ForwardingTable::default());
7834        let handler =
7835            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
7836        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
7837        handler
7838            .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
7839            .await
7840            .unwrap();
7841
7842        let project_root = unique_project_root("consumer-capabilities-absent");
7843        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
7844        let route_handler = handler.clone();
7845        let route_task = tokio::spawn(async move {
7846            route_handler
7847                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
7848                .await
7849                .unwrap()
7850        });
7851        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7852            .await
7853            .unwrap()
7854            .unwrap();
7855        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
7856        let ModuleControlRequest::RouteBind {
7857            consumer_capabilities,
7858            ..
7859        } = bind
7860        else {
7861            panic!("expected route.bind request, got {bind:?}");
7862        };
7863        assert_eq!(consumer_capabilities, None);
7864
7865        handler
7866            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
7867            .await
7868            .unwrap();
7869        let route_response = route_task.await.unwrap();
7870        assert!(route_response.is_empty());
7871        let published = client_rx.recv().await.unwrap();
7872        assert!(matches!(
7873            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
7874            ClientControlResponse::RouteOpen { .. }
7875        ));
7876    }
7877
7878    #[tokio::test]
7879    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
7880        let registry = Arc::new(Registry::default());
7881        let forwarding = Arc::new(ForwardingTable::default());
7882        let handler =
7883            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
7884                .with_health_probe_timeout(Duration::from_secs(5));
7885        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
7886        let responses = handler
7887            .handle_control_frame(
7888                &module_ctx,
7889                non_routable_hello_frame_with_control_ops(
7890                    "mcp",
7891                    300,
7892                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
7893                ),
7894            )
7895            .await
7896            .unwrap();
7897        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
7898        assert!(registry
7899            .get_module("mcp")
7900            .unwrap()
7901            .unwrap()
7902            .manifest
7903            .provides
7904            .is_empty());
7905
7906        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
7907        let route_response = handler
7908            .handle_control_frame(
7909                &route_client_ctx,
7910                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
7911            )
7912            .await
7913            .unwrap();
7914        assert_eq!(route_response[0].header.ty, FrameType::Error);
7915        assert_eq!(
7916            parse_error(&route_response[0])["code"],
7917            "target_unavailable"
7918        );
7919        assert!(parse_error(&route_response[0])["message"]
7920            .as_str()
7921            .unwrap()
7922            .contains("does not provide the requested target"));
7923        assert!(module_rx.try_recv().is_err());
7924
7925        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
7926        let health_handler = handler.clone();
7927        let health_task = tokio::spawn(async move {
7928            health_handler
7929                .handle_control_frame(
7930                    &health_client_ctx,
7931                    supervisor_health_probe_frame(302, "mcp"),
7932                )
7933                .await
7934                .unwrap()
7935        });
7936        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7937            .await
7938            .unwrap()
7939            .unwrap();
7940        assert_eq!(
7941            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
7942            ModuleControlRequest::HealthCheck {}
7943        );
7944        handler
7945            .handle_control_frame(
7946                &module_ctx,
7947                health_response(health_frame.header.corr, HealthStatus::Ok),
7948            )
7949            .await
7950            .unwrap();
7951        let health_response = health_task.await.unwrap();
7952        assert_eq!(health_response[0].header.ty, FrameType::Response);
7953        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
7954            ClientControlResponse::SupervisorHealthProbe {
7955                module_id, status, ..
7956            } => {
7957                assert_eq!(module_id, "mcp");
7958                assert_eq!(status, HealthStatus::Ok);
7959            }
7960            other => panic!("unexpected health response: {other:?}"),
7961        }
7962
7963        // Exercise the forwarding cleanup path directly while leaving the registry
7964        // advertisement in place. If cleanup leaves a stale control sink behind,
7965        // the next probe will enqueue onto it and wait for the long probe timeout
7966        // instead of returning an immediate no-connection error.
7967        forwarding
7968            .cleanup_connection(module_ctx.connection_id)
7969            .unwrap();
7970        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
7971        let cleanup_response = tokio::time::timeout(
7972            Duration::from_millis(200),
7973            handler.handle_control_frame(
7974                &cleanup_probe_ctx,
7975                supervisor_health_probe_frame(303, "mcp"),
7976            ),
7977        )
7978        .await
7979        .expect("probe should fail immediately when the control lane is gone")
7980        .unwrap();
7981        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
7982        assert_eq!(
7983            parse_error(&cleanup_response[0])["code"],
7984            "target_unavailable"
7985        );
7986        assert!(parse_error(&cleanup_response[0])["message"]
7987            .as_str()
7988            .unwrap()
7989            .contains("no module connection"));
7990
7991        handler
7992            .cleanup_connection(module_ctx.connection_id)
7993            .unwrap();
7994    }
7995
7996    #[tokio::test]
7997    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
7998        let registry = Arc::new(Registry::default());
7999        let supervisor_handle = SupervisorHandle::new();
8000        let supervisor =
8001            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
8002                .with_handle(supervisor_handle.clone())
8003                .with_connection_file_path(
8004                    std::env::temp_dir()
8005                        .join(format!("subc-route-open-warming-{}", std::process::id())),
8006                );
8007        let module = supervisor
8008            .supervise_configured(
8009                ModuleSpec {
8010                    module_id: "warming".to_string(),
8011                    program: fake_aft_stub_path(),
8012                    args: Vec::new(),
8013                    env: Vec::new(),
8014                    reserved: false,
8015                    reserved_prefixes: Vec::new(),
8016                    protocol: ModuleProtocol::Subc,
8017                    overlap: Default::default(),
8018                },
8019                true,
8020            )
8021            .unwrap();
8022        assert_eq!(module.state().unwrap(), ModuleState::Running);
8023
8024        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8025        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
8026        let response = handler
8027            .handle_control_frame(
8028                &ctx,
8029                route_open_frame(304, "warming", unique_project_root("warming")),
8030            )
8031            .await
8032            .unwrap();
8033        module.stop().await.unwrap();
8034
8035        assert_eq!(response[0].header.ty, FrameType::Error);
8036        let error = parse_error(&response[0]);
8037        assert_eq!(error["code"], "module_warming");
8038        assert!(error["message"]
8039            .as_str()
8040            .unwrap()
8041            .contains("state=running, enabled=true, live=false"));
8042    }
8043
8044    /// One wire code has several senders, so the refusal line names the check
8045    /// that refused. This drives the shared refusal path (every non-supervised
8046    /// refusal goes through `route_open_refusal_frame`) with an unregistered
8047    /// target and requires the branch label on the event.
8048    #[tokio::test]
8049    async fn route_open_refusal_names_the_check_that_refused() {
8050        let handler = ControlHandler::new(Arc::new(Registry::default()));
8051        let capture = EventCapture::default();
8052        let _subscriber =
8053            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8054        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
8055        let response = handler
8056            .handle_control_frame(
8057                &ctx,
8058                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
8059            )
8060            .await
8061            .unwrap();
8062
8063        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
8064        let event = capture
8065            .events()
8066            .into_iter()
8067            .find(|event| {
8068                event.target == "control"
8069                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
8070            })
8071            .expect("route.open refusal event");
8072        assert_eq!(
8073            event.fields.get("reason"),
8074            Some(&"\"not_registered\"".to_string())
8075        );
8076    }
8077
8078    #[tokio::test]
8079    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
8080        let registry = Arc::new(Registry::default());
8081        let supervisor_handle = SupervisorHandle::new();
8082        let supervisor =
8083            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
8084                .with_handle(supervisor_handle.clone())
8085                .with_connection_file_path(std::env::temp_dir().join(format!(
8086                    "subc-route-open-refusal-info-{}",
8087                    std::process::id()
8088                )));
8089        let module = supervisor
8090            .supervise_configured(
8091                ModuleSpec {
8092                    module_id: "warming".to_string(),
8093                    program: fake_aft_stub_path(),
8094                    args: Vec::new(),
8095                    env: Vec::new(),
8096                    reserved: false,
8097                    reserved_prefixes: Vec::new(),
8098                    protocol: ModuleProtocol::Subc,
8099                    overlap: Default::default(),
8100                },
8101                true,
8102            )
8103            .unwrap();
8104        assert_eq!(module.state().unwrap(), ModuleState::Running);
8105
8106        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8107        assert!(handler
8108            .counters()
8109            .snapshot()
8110            .get("route_open_refused_by_code")
8111            .is_none());
8112        let capture = EventCapture::default();
8113        let _subscriber =
8114            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8115        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
8116        let response = handler
8117            .handle_control_frame(
8118                &ctx,
8119                route_open_frame(394, "warming", unique_project_root("refusal-info")),
8120            )
8121            .await
8122            .unwrap();
8123        module.stop().await.unwrap();
8124
8125        assert_eq!(parse_error(&response[0])["code"], "module_warming");
8126        let event = capture
8127            .events()
8128            .into_iter()
8129            .find(|event| {
8130                event.target == "control"
8131                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
8132            })
8133            .expect("route.open refusal event");
8134        assert_eq!(
8135            event.fields.get("module_id"),
8136            Some(&"\"warming\"".to_string())
8137        );
8138        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
8139        assert_eq!(
8140            event.fields.get("reason"),
8141            Some(&"\"supervised_not_registered\"".to_string())
8142        );
8143        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
8144        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
8145        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
8146        assert_eq!(
8147            handler.counters().snapshot()["route_open_refused_by_code"],
8148            json!({ "module_warming": 1 })
8149        );
8150    }
8151
8152    #[tokio::test(flavor = "current_thread")]
8153    async fn route_open_unknown_module_escapes_target_module_id() {
8154        let handler = ControlHandler::new(Arc::new(Registry::default()));
8155        let capture = EventCapture::default();
8156        let _subscriber =
8157            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8158        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
8159        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
8160        let response = handler
8161            .handle_control_frame(
8162                &ctx,
8163                route_open_frame(
8164                    395,
8165                    hostile_module_id,
8166                    unique_project_root("hostile-target-module-id"),
8167                ),
8168            )
8169            .await
8170            .unwrap();
8171
8172        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
8173        let event = capture
8174            .events()
8175            .into_iter()
8176            .find(|event| {
8177                event.target == "control"
8178                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
8179            })
8180            .expect("route.open unknown-module refusal event");
8181        let logged = event.fields.get("module_id").expect("module_id field");
8182        assert!(!logged.bytes().any(|byte| byte < 0x20));
8183        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
8184    }
8185
8186    #[tokio::test(flavor = "current_thread")]
8187    async fn route_open_module_rejection_uses_daemon_counter_key() {
8188        let registry = Arc::new(Registry::default());
8189        let forwarding = Arc::new(ForwardingTable::default());
8190        let handler =
8191            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
8192        let module_connection = ConnectionId::new(95);
8193        let (module_ctx, mut module_rx) = route_ctx(module_connection);
8194        handler
8195            .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 395))
8196            .await
8197            .unwrap();
8198
8199        let client_connection = ConnectionId::new(96);
8200        let (client_ctx, _client_rx) = route_ctx(client_connection);
8201        let capture = EventCapture::default();
8202        let _subscriber =
8203            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8204        let (route_task, bind) = relay_route_open(
8205            &handler,
8206            client_connection,
8207            &client_ctx.egress,
8208            &mut module_rx,
8209            396,
8210            "aft",
8211            "hostile-module-code",
8212        )
8213        .await;
8214        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
8215        let rejection = Frame::build(
8216            FrameType::Error,
8217            control_flags(),
8218            0,
8219            0,
8220            bind.header.corr,
8221            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
8222        )
8223        .unwrap();
8224        handler
8225            .handle_control_frame(&module_ctx, rejection)
8226            .await
8227            .unwrap();
8228
8229        let response = route_task.await.unwrap();
8230        assert_eq!(parse_error(&response[0])["code"], hostile_code);
8231        let counters = handler.counters().snapshot();
8232        assert_eq!(
8233            counters["route_open_refused_by_code"],
8234            json!({ "module_rejected": 1 })
8235        );
8236        assert!(counters["route_open_refused_by_code"]
8237            .get(hostile_code)
8238            .is_none());
8239
8240        let event = capture
8241            .events()
8242            .into_iter()
8243            .find(|event| {
8244                event.target == "control"
8245                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
8246            })
8247            .expect("route.open module-rejection refusal event");
8248        let logged = event.fields.get("module_code").expect("module_code field");
8249        assert!(!logged.bytes().any(|byte| byte < 0x20));
8250        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
8251    }
8252
8253    #[tokio::test]
8254    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
8255        let registry = Arc::new(Registry::default());
8256        let supervisor_handle = SupervisorHandle::new();
8257        let missing_program = std::env::temp_dir().join(format!(
8258            "subc-route-open-missing-program-{}",
8259            std::process::id()
8260        ));
8261        let supervisor =
8262            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
8263                .with_handle(supervisor_handle.clone());
8264        let module = supervisor
8265            .supervise_configured(
8266                ModuleSpec {
8267                    module_id: "failed".to_string(),
8268                    program: missing_program,
8269                    args: Vec::new(),
8270                    env: Vec::new(),
8271                    reserved: false,
8272                    reserved_prefixes: Vec::new(),
8273                    protocol: ModuleProtocol::Subc,
8274                    overlap: Default::default(),
8275                },
8276                true,
8277            )
8278            .unwrap();
8279        assert_eq!(module.state().unwrap(), ModuleState::Failed);
8280
8281        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8282        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
8283        let response = handler
8284            .handle_control_frame(
8285                &ctx,
8286                route_open_frame(305, "failed", unique_project_root("failed")),
8287            )
8288            .await
8289            .unwrap();
8290
8291        assert_eq!(response[0].header.ty, FrameType::Error);
8292        let error = parse_error(&response[0]);
8293        assert_eq!(error["code"], "target_unavailable");
8294        assert!(error["message"]
8295            .as_str()
8296            .unwrap()
8297            .contains("state=failed, enabled=true, live=false"));
8298    }
8299
8300    #[tokio::test]
8301    async fn route_open_role_mismatch_remains_target_unavailable() {
8302        let registry = Arc::new(Registry::default());
8303        let handler = ControlHandler::new(Arc::clone(&registry));
8304        handler
8305            .handle_control(
8306                ConnectionId::new(41),
8307                non_routable_hello_frame_with_control_ops("health-only", 306, None),
8308            )
8309            .unwrap();
8310
8311        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
8312        let response = handler
8313            .handle_control_frame(
8314                &ctx,
8315                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
8316            )
8317            .await
8318            .unwrap();
8319
8320        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8321        assert!(parse_error(&response[0])["message"]
8322            .as_str()
8323            .unwrap()
8324            .contains("does not provide the requested target"));
8325    }
8326
8327    #[tokio::test]
8328    async fn route_open_inactive_registration_remains_target_unavailable() {
8329        let registry = Arc::new(Registry::default());
8330        let handler = ControlHandler::new(Arc::clone(&registry));
8331        handler
8332            .handle_control(
8333                ConnectionId::new(43),
8334                hello_frame("inactive", PROTOCOL_VERSION, 308),
8335            )
8336            .unwrap();
8337        assert!(registry
8338            .set_module_state_for_test("inactive", ChannelState::Closed)
8339            .unwrap());
8340
8341        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
8342        let response = handler
8343            .handle_control_frame(
8344                &ctx,
8345                route_open_frame(309, "inactive", unique_project_root("inactive")),
8346            )
8347            .await
8348            .unwrap();
8349
8350        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8351        assert!(parse_error(&response[0])["message"]
8352            .as_str()
8353            .unwrap()
8354            .contains("is not active"));
8355    }
8356
8357    #[tokio::test]
8358    async fn late_health_reply_is_recorded_through_the_module_response_path() {
8359        let registry = Arc::new(Registry::default());
8360        let forwarding = Arc::new(ForwardingTable::default());
8361        let supervisor_handle = SupervisorHandle::new();
8362        let supervisor = Supervisor::new(Arc::clone(&registry), crate::RestartPolicy::default())
8363            .with_forwarding(Arc::clone(&forwarding))
8364            .with_handle(supervisor_handle.clone());
8365        let module = supervisor
8366            .supervise_configured(
8367                crate::ModuleSpec {
8368                    module_id: "late-health-response".to_string(),
8369                    program: PathBuf::from("disabled-module"),
8370                    args: Vec::new(),
8371                    env: Vec::new(),
8372                    reserved: false,
8373                    reserved_prefixes: Vec::new(),
8374                    protocol: ModuleProtocol::Subc,
8375                    overlap: Default::default(),
8376                },
8377                false,
8378            )
8379            .unwrap();
8380        let handler =
8381            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8382                .with_supervisor(supervisor_handle);
8383        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
8384        handler
8385            .handle_control_frame(
8386                &module_ctx,
8387                hello_frame_with_control_ops(
8388                    "late-health-response",
8389                    PROTOCOL_VERSION,
8390                    7,
8391                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8392                ),
8393            )
8394            .await
8395            .unwrap();
8396        let probe_started_at = Instant::now() - Duration::from_millis(80);
8397        let pending = forwarding
8398            .begin_health_probe_rpc_for(
8399                "late-health-response",
8400                MODULE_CONTROL_OP_HEALTH_CHECK,
8401                probe_started_at,
8402                Instant::now() - Duration::from_millis(1),
8403            )
8404            .unwrap();
8405        assert!(forwarding
8406            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
8407            .unwrap());
8408
8409        let responses = handler
8410            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
8411            .await
8412            .unwrap();
8413
8414        assert!(responses.is_empty());
8415        let health = module.status().unwrap().health;
8416        assert_eq!(health.late_answer_count, 1);
8417        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
8418    }
8419
8420    #[tokio::test]
8421    async fn health_probe_timeout_and_module_death_are_typed() {
8422        let registry = Arc::new(Registry::default());
8423        let forwarding = Arc::new(ForwardingTable::default());
8424        let handler =
8425            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8426                .with_health_probe_timeout(Duration::from_millis(50));
8427        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
8428        handler
8429            .handle_control_frame(
8430                &module_ctx,
8431                hello_frame_with_control_ops(
8432                    "aft",
8433                    PROTOCOL_VERSION,
8434                    7,
8435                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8436                ),
8437            )
8438            .await
8439            .unwrap();
8440
8441        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
8442        let responses = handler
8443            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
8444            .await
8445            .unwrap();
8446        assert_eq!(responses[0].header.ty, FrameType::Error);
8447        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
8448        let _ = module_rx.try_recv();
8449
8450        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
8451        let health_handler = handler.clone();
8452        let death_task = tokio::spawn(async move {
8453            health_handler
8454                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
8455                .await
8456                .unwrap()
8457        });
8458        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8459            .await
8460            .unwrap()
8461            .unwrap();
8462        handler
8463            .cleanup_connection(module_ctx.connection_id)
8464            .unwrap();
8465        let responses = death_task.await.unwrap();
8466        assert_eq!(responses[0].header.ty, FrameType::Error);
8467        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
8468    }
8469
8470    #[test]
8471    fn hello_requires_exact_protocol_version() {
8472        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
8473            let registry = Arc::new(Registry::default());
8474            let handler = ControlHandler::new(Arc::clone(&registry));
8475            let responses = handler
8476                .handle_control(
8477                    ConnectionId::new(connection),
8478                    hello_frame("aft", offered, 9),
8479                )
8480                .unwrap();
8481
8482            assert_eq!(responses.len(), 1);
8483            assert_eq!(responses[0].header.ty, FrameType::Error);
8484            let error = parse_error(&responses[0]);
8485            assert_eq!(error["code"], "version_unsupported");
8486            assert!(registry.get_module("aft").unwrap().is_none());
8487            assert_eq!(registry.active_registration_count().unwrap(), 0);
8488        }
8489    }
8490
8491    #[test]
8492    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
8493        let registry = Arc::new(Registry::default());
8494        let forwarding = Arc::new(ForwardingTable::default());
8495        let handler =
8496            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
8497        let module_connection = ConnectionId::new(301);
8498        let registration = registry
8499            .register_with_control_ops(
8500                manifest("aft-push", PROTOCOL_VERSION),
8501                PROTOCOL_VERSION,
8502                module_connection,
8503                module_baseline_control_ops(),
8504            )
8505            .unwrap();
8506        let (module_tx, _module_rx) = mpsc::channel(8);
8507        let endpoint = forwarding
8508            .register_module_connection(
8509                module_connection,
8510                "aft-push".to_string(),
8511                PROTOCOL_VERSION,
8512                manifest_concurrency(&registration.manifest),
8513                FrameSink::new(module_tx),
8514            )
8515            .unwrap();
8516
8517        // A push op this version does not know is ignored (forward-compat), not errored.
8518        let unknown = Frame::build(
8519            FrameType::Push,
8520            control_flags(),
8521            0,
8522            0,
8523            5,
8524            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
8525        )
8526        .unwrap();
8527        let out = handler.handle_status_update(endpoint, unknown).unwrap();
8528        assert!(
8529            out.is_empty(),
8530            "unknown push op must be ignored, got {out:?}"
8531        );
8532
8533        // A malformed body for a KNOWN op is a real error worth surfacing.
8534        let malformed = Frame::build(
8535            FrameType::Push,
8536            control_flags(),
8537            0,
8538            0,
8539            6,
8540            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
8541        )
8542        .unwrap();
8543        let out = handler.handle_status_update(endpoint, malformed).unwrap();
8544        assert_eq!(out.len(), 1);
8545        assert_eq!(out[0].header.ty, FrameType::Error);
8546        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
8547    }
8548
8549    #[test]
8550    fn hello_rejected_when_connection_already_owns_client_routes() {
8551        let registry = Arc::new(Registry::default());
8552        let forwarding = Arc::new(ForwardingTable::default());
8553        let handler =
8554            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
8555        // Commits a client route on connection 202 (bound to a module on conn 101).
8556        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
8557        let client_connection = ConnectionId::new(202);
8558
8559        // That same connection now tries to register as a module: rejected, so one
8560        // connection never holds both client-route and module-endpoint state.
8561        let responses = handler
8562            .handle_control(
8563                client_connection,
8564                hello_frame("aft-second", PROTOCOL_VERSION, 9),
8565            )
8566            .unwrap();
8567        assert_eq!(responses[0].header.ty, FrameType::Error);
8568        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
8569        assert!(registry.get_module("aft-second").unwrap().is_none());
8570    }
8571
8572    #[test]
8573    fn reserved_module_hello_requires_matching_launch_nonce() {
8574        let registry = Arc::new(Registry::default());
8575        let supervisor = SupervisorHandle::new();
8576        // The supervisor recorded the nonce it injected when it spawned the reserved
8577        // module; the HELLO verifier checks against the same shared handle.
8578        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
8579        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
8580
8581        // A HELLO with NO nonce is rejected.
8582        let no_nonce = handler
8583            .handle_control(
8584                ConnectionId::new(1),
8585                hello_frame("vault", PROTOCOL_VERSION, 1),
8586            )
8587            .unwrap();
8588        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
8589        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
8590        assert!(registry.get_module("vault").unwrap().is_none());
8591
8592        // A HELLO with the WRONG nonce is rejected.
8593        let wrong = handler
8594            .handle_control(
8595                ConnectionId::new(2),
8596                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
8597            )
8598            .unwrap();
8599        assert_eq!(wrong[0].header.ty, FrameType::Error);
8600        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
8601        assert!(registry.get_module("vault").unwrap().is_none());
8602
8603        // A HELLO with the CORRECT nonce registers.
8604        let ok = handler
8605            .handle_control(
8606                ConnectionId::new(3),
8607                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
8608            )
8609            .unwrap();
8610        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
8611        assert!(registry.get_module("vault").unwrap().is_some());
8612    }
8613
8614    #[test]
8615    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
8616        let registry = Arc::new(Registry::default());
8617        let supervisor = SupervisorHandle::new();
8618        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
8619        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
8620        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
8621
8622        let squat = handler
8623            .handle_control(
8624                ConnectionId::new(1),
8625                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
8626            )
8627            .unwrap();
8628        assert_eq!(squat[0].header.ty, FrameType::Error);
8629        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
8630        assert!(parse_error(&squat[0])["message"]
8631            .as_str()
8632            .unwrap()
8633            .contains("fed:"));
8634
8635        let accepted_peer = handler
8636            .handle_control(
8637                ConnectionId::new(2),
8638                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
8639            )
8640            .unwrap();
8641        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
8642
8643        let accepted_short = handler
8644            .handle_control(
8645                ConnectionId::new(3),
8646                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
8647            )
8648            .unwrap();
8649        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
8650
8651        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
8652            let response = handler
8653                .handle_control(
8654                    ConnectionId::new(conn),
8655                    hello_frame(module_id, PROTOCOL_VERSION, conn),
8656                )
8657                .unwrap();
8658            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
8659        }
8660    }
8661
8662    #[test]
8663    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
8664        let registry = Arc::new(Registry::default());
8665        let supervisor = SupervisorHandle::new();
8666        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
8667        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
8668        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
8669        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
8670
8671        let owner_nonce = handler
8672            .handle_control(
8673                ConnectionId::new(1),
8674                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
8675            )
8676            .unwrap();
8677        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
8678        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
8679        assert!(registry.get_module("fed:special").unwrap().is_none());
8680
8681        let exact_nonce = handler
8682            .handle_control(
8683                ConnectionId::new(2),
8684                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
8685            )
8686            .unwrap();
8687        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
8688        assert!(registry.get_module("fed:special").unwrap().is_some());
8689    }
8690
8691    #[test]
8692    fn non_reserved_module_ignores_launch_nonce() {
8693        let registry = Arc::new(Registry::default());
8694        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
8695        // registration succeeds whether a spawned process echoes a nonce or not.
8696        let handler = ControlHandler::new(Arc::clone(&registry));
8697        let no_nonce = handler
8698            .handle_control(
8699                ConnectionId::new(1),
8700                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
8701            )
8702            .unwrap();
8703        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
8704        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
8705
8706        let echoed_nonce = handler
8707            .handle_control(
8708                ConnectionId::new(2),
8709                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
8710            )
8711            .unwrap();
8712        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
8713        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
8714    }
8715
8716    #[test]
8717    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
8718        let handler = ControlHandler::default();
8719        let conn = ConnectionId::new(1);
8720        let malformed = Frame::build(
8721            FrameType::Hello,
8722            control_flags(),
8723            0,
8724            0,
8725            3,
8726            b"{not json".to_vec(),
8727        )
8728        .unwrap();
8729
8730        let error = handler.handle_control(conn, malformed).unwrap();
8731        assert_eq!(error[0].header.ty, FrameType::Error);
8732        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
8733
8734        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
8735        let pong = handler.handle_control(conn, ping).unwrap();
8736        assert_eq!(pong[0].header.ty, FrameType::Pong);
8737        assert_eq!(pong[0].header.corr, 4);
8738    }
8739
8740    #[test]
8741    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
8742        let registry = Arc::new(Registry::default());
8743        let handler = ControlHandler::new(Arc::clone(&registry));
8744
8745        handler
8746            .handle_control(
8747                ConnectionId::new(1),
8748                hello_frame("aft", PROTOCOL_VERSION, 1),
8749            )
8750            .unwrap();
8751        let duplicate = handler
8752            .handle_control(
8753                ConnectionId::new(2),
8754                hello_frame("aft", PROTOCOL_VERSION, 2),
8755            )
8756            .unwrap();
8757
8758        assert_eq!(duplicate[0].header.ty, FrameType::Error);
8759        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
8760        let registration = registry.get_module("aft").unwrap().unwrap();
8761        assert_eq!(registration.connection_id, ConnectionId::new(1));
8762    }
8763
8764    #[test]
8765    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
8766        let registry = Arc::new(Registry::default());
8767        let forwarding = Arc::new(ForwardingTable::default());
8768        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
8769        let handler =
8770            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8771                .with_process_liveness(process_liveness);
8772        let (ctx, route_channel, route_epoch) =
8773            bind_liveness_route(&registry, &forwarding, "aft-dead");
8774        let responses = handler
8775            .handle_route_poll(
8776                &ctx,
8777                route_poll_frame(41, PollKind::Liveness, route_channel),
8778                route_channel,
8779                route_epoch,
8780                PollKind::Liveness,
8781            )
8782            .unwrap();
8783
8784        assert_eq!(responses.len(), 1);
8785        assert_eq!(responses[0].header.ty, FrameType::Response);
8786        assert_route_poll_liveness(&responses[0], false);
8787    }
8788
8789    #[test]
8790    fn liveness_poll_without_process_source_uses_bound_route() {
8791        let registry = Arc::new(Registry::default());
8792        let forwarding = Arc::new(ForwardingTable::default());
8793        let handler =
8794            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
8795        let (ctx, route_channel, route_epoch) =
8796            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
8797        let responses = handler
8798            .handle_route_poll(
8799                &ctx,
8800                route_poll_frame(42, PollKind::Liveness, route_channel),
8801                route_channel,
8802                route_epoch,
8803                PollKind::Liveness,
8804            )
8805            .unwrap();
8806
8807        assert_route_poll_liveness(&responses[0], true);
8808    }
8809
8810    #[test]
8811    fn liveness_poll_untracked_process_source_uses_bound_route() {
8812        let registry = Arc::new(Registry::default());
8813        let forwarding = Arc::new(ForwardingTable::default());
8814        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
8815        let handler =
8816            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8817                .with_process_liveness(process_liveness);
8818        let (ctx, route_channel, route_epoch) =
8819            bind_liveness_route(&registry, &forwarding, "aft-untracked");
8820        let responses = handler
8821            .handle_route_poll(
8822                &ctx,
8823                route_poll_frame(43, PollKind::Liveness, route_channel),
8824                route_channel,
8825                route_epoch,
8826                PollKind::Liveness,
8827            )
8828            .unwrap();
8829
8830        assert_route_poll_liveness(&responses[0], true);
8831    }
8832
8833    #[tokio::test]
8834    async fn unknown_op_returns_unknown_control_op() {
8835        let handler = ControlHandler::default();
8836        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
8837        let request = Frame::build(
8838            FrameType::Request,
8839            control_flags(),
8840            0,
8841            0,
8842            55,
8843            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
8844        )
8845        .unwrap();
8846
8847        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
8848
8849        assert_eq!(response.len(), 1);
8850        assert_eq!(response[0].header.ty, FrameType::Error);
8851        assert_eq!(response[0].header.corr, 55);
8852        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
8853    }
8854
8855    #[tokio::test]
8856    async fn supervisor_provenance_rejects_unknown_exact_module() {
8857        let handler = ControlHandler::default();
8858        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
8859        let request = Frame::build(
8860            FrameType::Request,
8861            control_flags(),
8862            0,
8863            0,
8864            57,
8865            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
8866        )
8867        .unwrap();
8868
8869        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
8870
8871        assert_eq!(response.len(), 1);
8872        assert_eq!(response[0].header.ty, FrameType::Error);
8873        assert_eq!(response[0].header.corr, 57);
8874        let error = parse_error(&response[0]);
8875        assert_eq!(error["code"], "unknown_module");
8876        assert_eq!(error["message"], "module_id 'missing' is not supervised");
8877    }
8878
8879    #[test]
8880    fn provenance_probe_override_keeps_handler_tests_deterministic() {
8881        let expected = subc_control::RunningImageAgreement::Unavailable {
8882            reason: subc_control::RunningImageUnavailableReason::HashFailed,
8883        };
8884        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
8885        assert_eq!(handler.provenance_probe_override, Some(expected));
8886    }
8887
8888    #[tokio::test]
8889    async fn malformed_control_bodies_return_invalid_control_body() {
8890        let handler = ControlHandler::default();
8891        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
8892
8893        for (corr, body) in [
8894            (56, br#"{"route_channel":1}"#.as_slice()),
8895            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
8896            (
8897                58,
8898                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
8899            ),
8900        ] {
8901            let request = Frame::build(
8902                FrameType::Request,
8903                control_flags(),
8904                0,
8905                0,
8906                corr,
8907                body.to_vec(),
8908            )
8909            .unwrap();
8910            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
8911
8912            assert_eq!(response.len(), 1);
8913            assert_eq!(response[0].header.ty, FrameType::Error);
8914            assert_eq!(response[0].header.corr, corr);
8915            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
8916        }
8917    }
8918
8919    #[tokio::test]
8920    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
8921        let registry = Arc::new(Registry::default());
8922        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
8923        let router = Router::with_control_handler(Arc::clone(&control));
8924        let connection = router.begin_connection();
8925        let (ctx, mut rx) = route_ctx(connection.id());
8926
8927        router
8928            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
8929            .await
8930            .unwrap();
8931        let response = rx.recv().await.unwrap();
8932        let ack = parse_ack(&response);
8933        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
8934        let channel = 1;
8935
8936        let goodbye =
8937            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
8938        router.route_for_connection(&ctx, goodbye).await.unwrap();
8939        assert!(rx.try_recv().is_err());
8940        assert!(registry.get_module("aft").unwrap().is_none());
8941
8942        router
8943            .route_for_connection(&ctx, channel_request(channel, 13))
8944            .await
8945            .unwrap();
8946        let error_frame = rx.recv().await.unwrap();
8947        assert_eq!(error_frame.header.ty, FrameType::Error);
8948        assert_eq!(error_frame.header.channel, channel);
8949    }
8950
8951    #[tokio::test]
8952    async fn dropping_router_connection_releases_registration() {
8953        let registry = Arc::new(Registry::default());
8954        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
8955        let router = Router::with_control_handler(control);
8956        let connection = router.begin_connection();
8957        let (ctx, mut rx) = route_ctx(connection.id());
8958
8959        router
8960            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
8961            .await
8962            .unwrap();
8963        let response = rx.recv().await.unwrap();
8964        let ack = parse_ack(&response);
8965        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
8966        assert!(registry.get_module("aft").unwrap().is_some());
8967
8968        drop(connection);
8969
8970        assert!(registry.get_module("aft").unwrap().is_none());
8971        assert_eq!(registry.active_registration_count().unwrap(), 0);
8972    }
8973
8974    fn capability_manifest(
8975        module_id: &str,
8976        provides: &[&str],
8977        must_never_reach: &[&str],
8978    ) -> ModuleManifest {
8979        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
8980        manifest.capabilities = Some(CapabilityDeclarations {
8981            provides: provides
8982                .iter()
8983                .map(|capability| (*capability).to_string())
8984                .collect(),
8985            requires: Vec::new(),
8986            must_never_reach: must_never_reach
8987                .iter()
8988                .map(|capability| (*capability).to_string())
8989                .collect(),
8990        });
8991        manifest
8992    }
8993
8994    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
8995        Frame::build(
8996            FrameType::Hello,
8997            control_flags(),
8998            0,
8999            0,
9000            corr,
9001            serde_json::to_vec(&ModuleHelloBody {
9002                protocol_ver: manifest.protocol_ver,
9003                manifest,
9004                control_ops: None,
9005                launch_nonce: None,
9006            })
9007            .expect("capability test HELLO serializes"),
9008        )
9009        .expect("capability test HELLO frame builds")
9010    }
9011
9012    fn catalog_update_with_capabilities_frame(
9013        corr: u64,
9014        capabilities: CapabilityDeclarations,
9015    ) -> Frame {
9016        Frame::build(
9017            FrameType::Request,
9018            control_flags(),
9019            0,
9020            0,
9021            corr,
9022            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
9023                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
9024                capabilities: Some(capabilities),
9025                ready: None,
9026            })
9027            .expect("capability catalog.update serializes"),
9028        )
9029        .expect("capability catalog.update frame builds")
9030    }
9031
9032    async fn register_capability_manifest(
9033        handler: &ControlHandler,
9034        ctx: &RouteCtx,
9035        manifest: ModuleManifest,
9036        corr: u64,
9037    ) {
9038        let replies = handler
9039            .handle_control_frame(ctx, hello_frame_with_manifest(manifest, corr))
9040            .await
9041            .expect("capability test HELLO succeeds");
9042        assert!(
9043            matches!(replies.as_slice(), [Frame { header, .. }] if header.ty == FrameType::HelloAck),
9044            "capability test HELLO must register"
9045        );
9046    }
9047
9048    async fn open_route_for_capability_test(
9049        handler: &ControlHandler,
9050        target_ctx: &RouteCtx,
9051        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9052        client_connection_id: u64,
9053        corr: u64,
9054        target_module_id: &str,
9055        consumer_identity: Option<ConsumerIdentity>,
9056    ) -> (
9057        mpsc::Receiver<crate::router::OutboundFrame>,
9058        ModuleControlRequest,
9059    ) {
9060        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
9061        let route_handler = handler.clone();
9062        let target_module_id = target_module_id.to_string();
9063        let route_task = tokio::spawn(async move {
9064            route_handler
9065                .handle_control_frame(
9066                    &client_ctx,
9067                    route_open_frame_with_admission_facts(
9068                        corr,
9069                        &target_module_id,
9070                        unique_project_root("admission-facts"),
9071                        consumer_identity,
9072                        None,
9073                    ),
9074                )
9075                .await
9076                .expect("capability test route.open succeeds")
9077        });
9078        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
9079            .await
9080            .expect("capability test route.open must reach route.bind")
9081            .expect("target control receiver stays open");
9082        let bind_request: ModuleControlRequest =
9083            serde_json::from_slice(&bind.body).expect("route.bind decodes");
9084        handler
9085            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
9086            .await
9087            .expect("capability test route.bind ACK succeeds");
9088        assert!(route_task.await.expect("route.open task joins").is_empty());
9089        let opened = client_rx
9090            .recv()
9091            .await
9092            .expect("successful route.open publishes a response");
9093        assert!(matches!(
9094            serde_json::from_slice::<ClientControlResponse>(&opened.body),
9095            Ok(ClientControlResponse::RouteOpen { .. })
9096        ));
9097        (client_rx, bind_request)
9098    }
9099
9100    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
9101        assert_eq!(frame.header.ty, FrameType::Push);
9102        assert_eq!(frame.header.channel, 0);
9103        assert_eq!(
9104            serde_json::from_slice::<ClientControlPush>(&frame.body)
9105                .expect("route.closed control push decodes"),
9106            ClientControlPush::RouteClosed {
9107                module_id: target_module_id.to_string(),
9108                reason: RouteCloseReason::CapabilityDenied,
9109                drained: false,
9110                abandoned: 0,
9111                excluded_subscriptions: 0,
9112                terminal: Some(false),
9113            }
9114        );
9115    }
9116
9117    #[tokio::test]
9118    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
9119        let registry = Arc::new(Registry::default());
9120        let forwarding = Arc::new(ForwardingTable::default());
9121        let supervisor = SupervisorHandle::new();
9122        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9123        let handler =
9124            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9125                .with_supervisor(supervisor);
9126        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
9127        let (opener_ctx, _opener_rx) = route_ctx(ConnectionId::new(701));
9128        register_capability_manifest(
9129            &handler,
9130            &target_ctx,
9131            capability_manifest("target", &["credentials-provider/v1"], &[]),
9132            1,
9133        )
9134        .await;
9135        register_capability_manifest(
9136            &handler,
9137            &opener_ctx,
9138            capability_manifest("opener", &[], &["credentials-provider/v1"]),
9139            2,
9140        )
9141        .await;
9142
9143        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
9144        let replies = handler
9145            .handle_control_frame(
9146                &client_ctx,
9147                route_open_frame_with_admission_facts(
9148                    3,
9149                    "target",
9150                    unique_project_root("admission-facts"),
9151                    Some(ConsumerIdentity {
9152                        module_id: "opener".to_string(),
9153                        launch_nonce: "opener-nonce".to_string(),
9154                    }),
9155                    None,
9156                ),
9157            )
9158            .await
9159            .expect("denied route.open returns a typed frame");
9160        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
9161        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9162        assert!(
9163            target_rx.try_recv().is_err(),
9164            "forbidden route.open must not relay route.bind"
9165        );
9166    }
9167
9168    #[tokio::test]
9169    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
9170        let registry = Arc::new(Registry::default());
9171        let forwarding = Arc::new(ForwardingTable::default());
9172        let supervisor = SupervisorHandle::new();
9173        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9174        let handler =
9175            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9176                .with_supervisor(supervisor);
9177        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
9178        let (old_opener_ctx, _old_opener_rx) = route_ctx(ConnectionId::new(711));
9179        register_capability_manifest(
9180            &handler,
9181            &target_ctx,
9182            capability_manifest("target", &["credentials-provider/v1"], &[]),
9183            1,
9184        )
9185        .await;
9186        register_capability_manifest(
9187            &handler,
9188            &old_opener_ctx,
9189            capability_manifest("opener", &[], &[]),
9190            2,
9191        )
9192        .await;
9193        let (mut client_rx, _) = open_route_for_capability_test(
9194            &handler,
9195            &target_ctx,
9196            &mut target_rx,
9197            712,
9198            3,
9199            "target",
9200            Some(ConsumerIdentity {
9201                module_id: "opener".to_string(),
9202                launch_nonce: "opener-nonce".to_string(),
9203            }),
9204        )
9205        .await;
9206        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9207
9208        handler
9209            .cleanup_connection(old_opener_ctx.connection_id)
9210            .expect("old opener registration cleans up");
9211        let (new_opener_ctx, _new_opener_rx) = route_ctx(ConnectionId::new(713));
9212        register_capability_manifest(
9213            &handler,
9214            &new_opener_ctx,
9215            capability_manifest("opener", &[], &["credentials-provider/v1"]),
9216            4,
9217        )
9218        .await;
9219
9220        assert_capability_denied_push(
9221            client_rx
9222                .try_recv()
9223                .expect("HELLO deny addition must emit route.closed")
9224                .frame,
9225            "target",
9226        );
9227        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9228        assert!(matches!(
9229            target_rx.try_recv(),
9230            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
9231        ));
9232    }
9233
9234    #[tokio::test]
9235    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
9236        let registry = Arc::new(Registry::default());
9237        let forwarding = Arc::new(ForwardingTable::default());
9238        let supervisor = SupervisorHandle::new();
9239        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9240        let handler =
9241            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9242                .with_supervisor(supervisor);
9243        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
9244        let (opener_ctx, _opener_rx) = route_ctx(ConnectionId::new(721));
9245        register_capability_manifest(
9246            &handler,
9247            &target_ctx,
9248            capability_manifest("target", &[], &[]),
9249            1,
9250        )
9251        .await;
9252        register_capability_manifest(
9253            &handler,
9254            &opener_ctx,
9255            capability_manifest("opener", &[], &["credentials-provider/v1"]),
9256            2,
9257        )
9258        .await;
9259        let (mut client_rx, _) = open_route_for_capability_test(
9260            &handler,
9261            &target_ctx,
9262            &mut target_rx,
9263            722,
9264            3,
9265            "target",
9266            Some(ConsumerIdentity {
9267                module_id: "opener".to_string(),
9268                launch_nonce: "opener-nonce".to_string(),
9269            }),
9270        )
9271        .await;
9272        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9273
9274        let replies = handler
9275            .handle_control_frame(
9276                &target_ctx,
9277                catalog_update_with_capabilities_frame(
9278                    4,
9279                    CapabilityDeclarations {
9280                        provides: vec!["credentials-provider/v1".to_string()],
9281                        requires: Vec::new(),
9282                        must_never_reach: Vec::new(),
9283                    },
9284                ),
9285            )
9286            .await
9287            .expect("claim catalog.update succeeds");
9288        assert!(matches!(
9289            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
9290            Ok(ModuleControlResponseToModule::CatalogUpdate {})
9291        ));
9292        assert_capability_denied_push(
9293            client_rx
9294                .try_recv()
9295                .expect("claim addition must emit route.closed")
9296                .frame,
9297            "target",
9298        );
9299        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9300        assert!(matches!(
9301            target_rx.try_recv(),
9302            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
9303        ));
9304    }
9305
9306    #[tokio::test]
9307    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
9308        let registry = Arc::new(Registry::default());
9309        let forwarding = Arc::new(ForwardingTable::default());
9310        let supervisor = SupervisorHandle::new();
9311        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9312        let handler =
9313            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9314                .with_supervisor(supervisor);
9315        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
9316        let (opener_ctx, _opener_rx) = route_ctx(ConnectionId::new(731));
9317        register_capability_manifest(
9318            &handler,
9319            &target_ctx,
9320            capability_manifest("target", &["credentials-provider/v1"], &[]),
9321            1,
9322        )
9323        .await;
9324        register_capability_manifest(
9325            &handler,
9326            &opener_ctx,
9327            capability_manifest("opener", &[], &[]),
9328            2,
9329        )
9330        .await;
9331        let (mut client_rx, _) = open_route_for_capability_test(
9332            &handler,
9333            &target_ctx,
9334            &mut target_rx,
9335            732,
9336            3,
9337            "target",
9338            Some(ConsumerIdentity {
9339                module_id: "opener".to_string(),
9340                launch_nonce: "opener-nonce".to_string(),
9341            }),
9342        )
9343        .await;
9344        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9345
9346        handler
9347            .handle_control_frame(
9348                &target_ctx,
9349                catalog_update_with_capabilities_frame(
9350                    4,
9351                    CapabilityDeclarations {
9352                        provides: Vec::new(),
9353                        requires: Vec::new(),
9354                        must_never_reach: Vec::new(),
9355                    },
9356                ),
9357            )
9358            .await
9359            .expect("claim removal catalog.update succeeds");
9360        assert_eq!(
9361            forwarding.active_binding_count().unwrap(),
9362            1,
9363            "removing an attested target claim must leave the route census unchanged"
9364        );
9365        assert!(
9366            client_rx.try_recv().is_err(),
9367            "claim removal must not emit route.closed capability_denied"
9368        );
9369        assert!(
9370            target_rx.try_recv().is_err(),
9371            "claim removal must not send the target a route GOODBYE"
9372        );
9373    }
9374
9375    /// A direct client may open a route to a denied capability provider; this
9376    /// policy applies only to attested supervised module origins, not to direct clients.
9377    #[tokio::test]
9378    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
9379        let registry = Arc::new(Registry::default());
9380        let forwarding = Arc::new(ForwardingTable::default());
9381        let supervisor = SupervisorHandle::new();
9382        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9383        let handler =
9384            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9385                .with_supervisor(supervisor);
9386        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
9387        let (opener_ctx, _opener_rx) = route_ctx(ConnectionId::new(741));
9388        register_capability_manifest(
9389            &handler,
9390            &target_ctx,
9391            capability_manifest("target", &["credentials-provider/v1"], &[]),
9392            1,
9393        )
9394        .await;
9395        register_capability_manifest(
9396            &handler,
9397            &opener_ctx,
9398            capability_manifest("opener", &[], &["credentials-provider/v1"]),
9399            2,
9400        )
9401        .await;
9402
9403        let (_client_rx, bind) = open_route_for_capability_test(
9404            &handler,
9405            &target_ctx,
9406            &mut target_rx,
9407            742,
9408            3,
9409            "target",
9410            None,
9411        )
9412        .await;
9413        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
9414            panic!("direct scope-honesty route must bind");
9415        };
9416        assert_eq!(principal, Some(Principal::Direct));
9417        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9418    }
9419
9420    /// A module that denies a capability receives no self-route exemption when it
9421    /// also attestedly provides that capability.
9422    #[tokio::test]
9423    async fn must_never_reach_self_route_is_capability_forbidden() {
9424        let registry = Arc::new(Registry::default());
9425        let forwarding = Arc::new(ForwardingTable::default());
9426        let supervisor = SupervisorHandle::new();
9427        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
9428        let handler =
9429            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9430                .with_supervisor(supervisor);
9431        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
9432        register_capability_manifest(
9433            &handler,
9434            &self_ctx,
9435            capability_manifest(
9436                "self-provider",
9437                &["credentials-provider/v1"],
9438                &["credentials-provider/v1"],
9439            ),
9440            1,
9441        )
9442        .await;
9443
9444        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
9445        let replies = handler
9446            .handle_control_frame(
9447                &client_ctx,
9448                route_open_frame_with_admission_facts(
9449                    2,
9450                    "self-provider",
9451                    unique_project_root("admission-facts"),
9452                    Some(ConsumerIdentity {
9453                        module_id: "self-provider".to_string(),
9454                        launch_nonce: "self-nonce".to_string(),
9455                    }),
9456                    None,
9457                ),
9458            )
9459            .await
9460            .expect("self-route refusal returns a typed frame");
9461        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
9462        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9463        assert!(
9464            self_rx.try_recv().is_err(),
9465            "self denial must not relay route.bind"
9466        );
9467    }
9468
9469    #[test]
9470    fn unsupported_channel_zero_frame_returns_error() {
9471        let handler = ControlHandler::default();
9472        let request = Frame::build(
9473            FrameType::Request,
9474            control_flags(),
9475            0,
9476            0,
9477            21,
9478            b"opaque".to_vec(),
9479        )
9480        .unwrap();
9481
9482        let response = handler
9483            .handle_control(ConnectionId::new(1), request)
9484            .unwrap();
9485
9486        assert_eq!(response[0].header.ty, FrameType::Error);
9487        assert_eq!(
9488            parse_error(&response[0])["code"],
9489            "unsupported_control_frame"
9490        );
9491    }
9492
9493    /// Blue/green swap at the control-plane boundary. The supervisor that opens
9494    /// a swap is not wired yet, so the candidate is registered here directly
9495    /// into the registry and forwarding candidate slots, the way the swap's
9496    /// HELLO admission will.
9497    mod swap {
9498        use super::*;
9499
9500        const INCUMBENT: ConnectionId = ConnectionId::new(30);
9501        const CANDIDATE: ConnectionId = ConnectionId::new(40);
9502
9503        struct Swap {
9504            registry: Arc<Registry>,
9505            forwarding: Arc<ForwardingTable>,
9506            handler: ControlHandler,
9507            incumbent_ctx: RouteCtx,
9508            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
9509            candidate_ctx: RouteCtx,
9510            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
9511        }
9512
9513        async fn swap_with_incumbent() -> Swap {
9514            let registry = Arc::new(Registry::default());
9515            let forwarding = Arc::new(ForwardingTable::default());
9516            let handler =
9517                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9518            let (incumbent_ctx, incumbent_rx) = route_ctx(INCUMBENT);
9519            handler
9520                .handle_control_frame(&incumbent_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
9521                .await
9522                .unwrap();
9523            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
9524            Swap {
9525                registry,
9526                forwarding,
9527                handler,
9528                incumbent_ctx,
9529                incumbent_rx,
9530                candidate_ctx,
9531                candidate_rx,
9532            }
9533        }
9534
9535        fn register_candidate(swap: &Swap, ready: Option<bool>) {
9536            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
9537            candidate_manifest.ready = ready;
9538            let registration = swap
9539                .registry
9540                .register_candidate_with_control_ops(
9541                    candidate_manifest,
9542                    PROTOCOL_VERSION,
9543                    CANDIDATE,
9544                    module_baseline_control_ops(),
9545                )
9546                .unwrap();
9547            swap.forwarding
9548                .register_candidate_module_connection(
9549                    CANDIDATE,
9550                    "aft".to_string(),
9551                    PROTOCOL_VERSION,
9552                    manifest_concurrency(&registration.manifest),
9553                    swap.candidate_ctx.egress.clone(),
9554                )
9555                .unwrap();
9556        }
9557
9558        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
9559            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
9560            swap.registry.promote_candidate("aft").unwrap().unwrap();
9561            cutover.incumbent.unwrap()
9562        }
9563
9564        fn keyed_total(counters: &Value, key: &str) -> u64 {
9565            counters[key]
9566                .as_object()
9567                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
9568                .unwrap_or(0)
9569        }
9570
9571        /// An ack from the incumbent for a bind it was sent before cutover,
9572        /// arriving before the incumbent is drained. The incumbent is the live
9573        /// connection carrying every other client's routes, so the ack must
9574        /// not end it: the waiting client is told to retry, the reservation is
9575        /// given back, and the incumbent is told to drop just that binding.
9576        #[tokio::test]
9577        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
9578            let mut swap = swap_with_incumbent().await;
9579            let handler = swap.handler.clone();
9580
9581            // A co-tenant route, bound on the incumbent before the swap.
9582            let cotenant = ConnectionId::new(31);
9583            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
9584            let (cotenant_task, cotenant_bind) = relay_route_open(
9585                &handler,
9586                cotenant,
9587                &cotenant_ctx.egress,
9588                &mut swap.incumbent_rx,
9589                100,
9590                "aft",
9591                "swap-cotenant",
9592            )
9593            .await;
9594            handler
9595                .handle_control_frame(
9596                    &swap.incumbent_ctx,
9597                    route_bind_ack(cotenant_bind.header.corr),
9598                )
9599                .await
9600                .unwrap();
9601            assert!(cotenant_task.await.unwrap().is_empty());
9602            let (cotenant_channel, cotenant_epoch) =
9603                published_route(&cotenant_rx.recv().await.unwrap());
9604
9605            // A second route.open, relayed to the incumbent and not yet acked.
9606            let caller = ConnectionId::new(32);
9607            let (caller_ctx, mut caller_rx) = route_ctx(caller);
9608            let (caller_task, caller_bind) = relay_route_open(
9609                &handler,
9610                caller,
9611                &caller_ctx.egress,
9612                &mut swap.incumbent_rx,
9613                101,
9614                "aft",
9615                "swap-caller",
9616            )
9617            .await;
9618            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
9619
9620            register_candidate(&swap, None);
9621            cutover(&swap);
9622
9623            // The incumbent acks after promotion and before any drain.
9624            let ack = handler
9625                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
9626                .await;
9627            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
9628            if module_loop_error.is_some() {
9629                // What the connection loop does with an untranslated router
9630                // error: end the connection, releasing every route on it.
9631                handler.cleanup_connection(INCUMBENT).unwrap();
9632            }
9633
9634            // 1. The incumbent's other routes survive.
9635            assert!(
9636                cotenant_rx.try_recv().is_err(),
9637                "the co-tenant route on the incumbent was torn down by one late ack: \
9638                 {module_loop_error:?}"
9639            );
9640            assert!(matches!(
9641                swap.forwarding
9642                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
9643                    .unwrap(),
9644                DataRoute::Client(DataRouteState::Bound(_))
9645            ));
9646            assert_eq!(module_loop_error, None);
9647            assert!(swap
9648                .registry
9649                .get_module_by_connection(INCUMBENT)
9650                .unwrap()
9651                .is_some());
9652
9653            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
9654            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
9655                .await
9656                .expect("the incumbent is told to drop the abandoned binding")
9657                .unwrap()
9658                .frame;
9659            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9660            assert_eq!(goodbye.header.channel, abandoned_channel);
9661            assert_eq!(goodbye.header.epoch, abandoned_epoch);
9662            assert!(swap.incumbent_rx.try_recv().is_err());
9663
9664            // 3. The waiting client gets a retryable refusal and no route.
9665            let response = caller_task.await.unwrap();
9666            assert_eq!(response.len(), 1);
9667            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
9668            assert!(caller_rx.try_recv().is_err());
9669
9670            // 4. The reservation pair is given back, and the pending bind
9671            //    settled exactly once: one accepted open (the co-tenant) and one
9672            //    refused open (the caller), nothing counted twice.
9673            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
9674            let counters = handler.counters().snapshot();
9675            assert_eq!(
9676                keyed_total(&counters, "route_open_accepted_by_principal"),
9677                1
9678            );
9679            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
9680            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
9681        }
9682
9683        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
9684        /// id would resolve to the promoted candidate and every new route.open
9685        /// would be refused as reloading, leaving neither process routable.
9686        #[tokio::test]
9687        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
9688            let mut swap = swap_with_incumbent().await;
9689            register_candidate(&swap, None);
9690            let incumbent = cutover(&swap);
9691            swap.forwarding
9692                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
9693                .unwrap()
9694                .expect("the incumbent is still registered");
9695
9696            let client = ConnectionId::new(33);
9697            let (client_ctx, mut client_rx) = route_ctx(client);
9698            let route_handler = swap.handler.clone();
9699            let open_ctx = RouteCtx {
9700                connection_id: client,
9701                egress: client_ctx.egress.clone(),
9702            };
9703            let mut route_task = tokio::spawn(async move {
9704                route_handler
9705                    .handle_control_frame(
9706                        &open_ctx,
9707                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
9708                    )
9709                    .await
9710                    .unwrap()
9711            });
9712            let bind = tokio::select! {
9713                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
9714                response = &mut route_task => {
9715                    let response = response.unwrap();
9716                    panic!(
9717                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
9718                        parse_error(&response[0])["code"]
9719                    );
9720                }
9721            };
9722            swap.handler
9723                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
9724                .await
9725                .unwrap();
9726            assert!(route_task.await.unwrap().is_empty());
9727            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
9728            match swap
9729                .forwarding
9730                .lookup_data_route(client, channel, epoch)
9731                .unwrap()
9732            {
9733                DataRoute::Client(DataRouteState::Bound(route)) => {
9734                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
9735                }
9736                other => panic!("expected a bound route on the candidate, got {other:?}"),
9737            }
9738            assert!(swap.incumbent_rx.try_recv().is_err());
9739        }
9740
9741        /// A candidate declares itself ready with `catalog.update` on its own
9742        /// connection. If the connection-keyed registry lookups searched only the
9743        /// active slot, this would answer `not_registered` and the candidate
9744        /// would never become ready.
9745        #[tokio::test]
9746        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
9747            let swap = swap_with_incumbent().await;
9748            register_candidate(&swap, Some(false));
9749            let update = Frame::build(
9750                FrameType::Request,
9751                control_flags(),
9752                0,
9753                0,
9754                55,
9755                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
9756                    provides: manifest("aft", PROTOCOL_VERSION).provides,
9757                    capabilities: None,
9758                    ready: Some(true),
9759                })
9760                .unwrap(),
9761            )
9762            .unwrap();
9763
9764            let replies = swap
9765                .handler
9766                .handle_control_frame(&swap.candidate_ctx, update)
9767                .await
9768                .unwrap();
9769
9770            assert_eq!(replies.len(), 1);
9771            assert_eq!(
9772                replies[0].header.ty,
9773                FrameType::Response,
9774                "candidate catalog.update was refused: {:?}",
9775                serde_json::from_slice::<Value>(&replies[0].body).ok()
9776            );
9777            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
9778            assert_eq!(
9779                swap.registry
9780                    .get_module("aft")
9781                    .unwrap()
9782                    .unwrap()
9783                    .connection_id,
9784                INCUMBENT
9785            );
9786        }
9787    }
9788
9789    /// The HELLO gate while the supervisor has a swap open: only the nonce it
9790    /// minted for the candidate admits a second process, into the candidate
9791    /// slot, and that check runs ahead of the reserved-module gate.
9792    mod swap_admission {
9793        use super::*;
9794
9795        const INCUMBENT_NONCE: &str = "incumbent-nonce";
9796        const CANDIDATE_NONCE: &str = "candidate-nonce";
9797
9798        fn handler_with_incumbent(
9799            module_id: &str,
9800            reserved: bool,
9801        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
9802            let registry = Arc::new(Registry::default());
9803            let supervisor = SupervisorHandle::new();
9804            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
9805            if reserved {
9806                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
9807            }
9808            let handler =
9809                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
9810            let incumbent = handler
9811                .handle_control(
9812                    ConnectionId::new(1),
9813                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
9814                )
9815                .unwrap();
9816            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
9817            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
9818            (registry, supervisor, handler)
9819        }
9820
9821        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
9822        /// admits every nonce, so while a swap is open the swap gate is the only
9823        /// thing between a key-holder and the candidate slot. A nonce the
9824        /// supervisor did not mint, or none at all, is refused, and neither the
9825        /// incumbent's registration nor the candidate slot moves.
9826        #[test]
9827        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
9828            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
9829
9830            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
9831                let replies = handler
9832                    .handle_control(
9833                        ConnectionId::new(connection),
9834                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
9835                    )
9836                    .unwrap();
9837                assert_eq!(replies[0].header.ty, FrameType::Error);
9838                assert_eq!(
9839                    parse_error(&replies[0])["code"],
9840                    "swap_token_invalid",
9841                    "nonce {nonce:?}"
9842                );
9843            }
9844            assert!(registry.get_candidate("aft").unwrap().is_none());
9845            assert_eq!(
9846                registry.get_module("aft").unwrap().unwrap().connection_id,
9847                ConnectionId::new(1)
9848            );
9849
9850            // Control: the minted token is admitted, into the candidate slot,
9851            // and only once.
9852            let admitted = handler
9853                .handle_control(
9854                    ConnectionId::new(4),
9855                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
9856                )
9857                .unwrap();
9858            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
9859            assert_eq!(
9860                registry
9861                    .get_candidate("aft")
9862                    .unwrap()
9863                    .unwrap()
9864                    .connection_id,
9865                ConnectionId::new(4)
9866            );
9867            assert_eq!(
9868                registry.get_module("aft").unwrap().unwrap().connection_id,
9869                ConnectionId::new(1),
9870                "the candidate must not take the active slot"
9871            );
9872            let replayed = handler
9873                .handle_control(
9874                    ConnectionId::new(5),
9875                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
9876                )
9877                .unwrap();
9878            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
9879
9880            // The case only this gate covers: the incumbent has died mid-swap,
9881            // so its duplicate refusal is gone too, and without the gate a
9882            // key-holder would take the id's ACTIVE slot.
9883            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
9884            let squatter = handler
9885                .handle_control(
9886                    ConnectionId::new(6),
9887                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
9888                )
9889                .unwrap();
9890            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
9891            assert!(
9892                registry.get_module("aft").unwrap().is_none(),
9893                "a squatter took the active slot of an id being swapped"
9894            );
9895        }
9896
9897        /// Design mutation arm (iii). A reserved module's candidate presents a
9898        /// nonce the reserved gate has never seen (that gate holds the
9899        /// incumbent's), so the swap gate must run first or the candidate is
9900        /// refused `reserved_module` and a reserved module can never be swapped.
9901        #[test]
9902        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
9903            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
9904
9905            let replies = handler
9906                .handle_control(
9907                    ConnectionId::new(2),
9908                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
9909                )
9910                .unwrap();
9911
9912            assert_eq!(
9913                replies[0].header.ty,
9914                FrameType::HelloAck,
9915                "reserved candidate refused: {:?}",
9916                serde_json::from_slice::<Value>(&replies[0].body).ok()
9917            );
9918            assert_eq!(
9919                registry
9920                    .get_candidate("vault")
9921                    .unwrap()
9922                    .unwrap()
9923                    .connection_id,
9924                ConnectionId::new(2)
9925            );
9926        }
9927
9928        /// With no swap open the gate is inert: the incumbent's reserved gate
9929        /// and duplicate refusal behave exactly as before.
9930        #[test]
9931        fn without_an_open_swap_the_ordinary_gates_decide() {
9932            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
9933            supervisor.close_swap("vault");
9934
9935            let candidate = handler
9936                .handle_control(
9937                    ConnectionId::new(2),
9938                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
9939                )
9940                .unwrap();
9941            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
9942            let duplicate = handler
9943                .handle_control(
9944                    ConnectionId::new(3),
9945                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
9946                )
9947                .unwrap();
9948            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
9949            assert!(registry.get_candidate("vault").unwrap().is_none());
9950        }
9951    }
9952}
9953
9954#[cfg(test)]
9955mod concurrency_default_exposure_tests {
9956    use super::*;
9957
9958    fn hello_body(role_json: &str) -> Vec<u8> {
9959        format!(
9960            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
9961        )
9962        .into_bytes()
9963    }
9964
9965    fn manifest_from(body: &[u8]) -> ModuleManifest {
9966        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
9967        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
9968            .expect("manifest parses")
9969    }
9970
9971    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
9972
9973    #[test]
9974    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
9975        let body = hello_body(&format!(
9976            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
9977        ));
9978        let manifest = manifest_from(&body);
9979        // Precondition: serde really resolved it to the default, so the typed
9980        // manifest alone cannot answer the question this probe exists for.
9981        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
9982        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
9983    }
9984
9985    #[test]
9986    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
9987        let body = hello_body(&format!(
9988            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
9989        ));
9990        let manifest = manifest_from(&body);
9991        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
9992        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
9993    }
9994
9995    #[test]
9996    fn non_management_roles_are_never_reported() {
9997        let body = hello_body(
9998            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
9999        );
10000        let manifest = manifest_from(&body);
10001        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
10002    }
10003}