Skip to main content

subc_daemon/
control.rs

1use std::{
2    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3    fmt,
4    path::PathBuf,
5    sync::{Arc, Mutex},
6    time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PollKind, RouteCloseReason,
14    SpawnCursor, StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance,
15    SupervisorEntry, SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
16    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
17};
18use subc_protocol::{
19    error_codes,
20    manifest::{
21        validate_hello_capability_grammar, validate_hello_self_signal_declarations,
22        CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
23        ProviderRole,
24    },
25    session::{
26        HealthReport, ModuleControlPush, ModuleControlRequest, ModuleControlRequestFromModule,
27        ModuleControlResponse, ModuleControlResponseToModule, MODULE_CONTROL_OP_HEALTH_CHECK,
28        MODULE_TO_SUBC_OP_CATALOG_UPDATE,
29    },
30    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
31    Priority, RouteTarget, PROTOCOL_VERSION,
32};
33use tokio::time::{timeout_at, Instant};
34use tracing::{debug, info, warn};
35
36use crate::{
37    capability_requirements::{
38        log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
39        CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
40        RuntimeModule,
41    },
42    daemon_config::RestartRequiredSection,
43    forwarding::{
44        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
45        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
46        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
47    },
48    observability::{
49        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
50    },
51    provenance::{
52        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
53    },
54    registry::{ChannelState, ConnectionId, Registry, RegistryError},
55    router::{RouteCtx, RouterError},
56    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
57    stderr_tail::{CaptureState, TailEntry},
58    supervise::{
59        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
60        SupervisorHandle, SwapHelloAdmission,
61    },
62    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
63};
64
65/// Lowest envelope version this subc build will negotiate.
66///
67/// Module HELLO negotiation is exact: peers must use the daemon's locked
68/// protocol version. Older and newer peers receive `version_unsupported` and
69/// are not registered.
70pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
71
72const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
73const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
74const CAP_PING_PONG: &str = "ping_pong_v1";
75const CAP_SESSION_ATTACH: &str = "session_attach_v1";
76const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
77
78const SUBC_CONTROL_OPS: &[&str] = &[
79    ops::SERVER_DESCRIBE,
80    ops::CATALOG_LIST,
81    ops::ROUTE_OPEN,
82    ops::ROUTE_POLL,
83    ops::ROUTE_CLOSING,
84    ops::ROUTE_CLOSED,
85    ops::SUPERVISOR_LIST,
86    ops::SUPERVISOR_RESTART,
87    ops::SUPERVISOR_SWAP,
88    ops::SUPERVISOR_RELOAD,
89    ops::SUPERVISOR_RESCAN,
90    ops::SUPERVISOR_RELEASE_RESERVED,
91    ops::SUPERVISOR_SET_ENABLED,
92    ops::SUPERVISOR_HEALTH_PROBE,
93    ops::SUPERVISOR_HEALTH,
94    ops::SUPERVISOR_STDERR_TAIL,
95    ops::SUPERVISOR_TERMINALS,
96    ops::SUPERVISOR_ROUTES,
97    ops::SUPERVISOR_PROVENANCE,
98    ops::SUPERVISOR_SPAWN_SNAPSHOT,
99    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
100];
101
102const MODULE_TO_SUBC_CONTROL_OPS: &[&str] =
103    &[MODULE_TO_SUBC_OP_CATALOG_UPDATE, "supervisor.live_roots"];
104
105const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
106
107/// How long subc waits for a module to ack a relayed route.bind before returning
108/// `module_timeout`. The ack waits on the module's own configure, which for AFT
109/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
110/// and DB-open work — on a cold page cache or a large repo that legitimately
111/// exceeds a couple of seconds. The default is generous because rejecting a VALID
112/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
113/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
114pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
115
116/// How many CONSECUTIVE full-budget relay timeouts against one target module
117/// open that module's bind-relay breaker.
118///
119/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
120/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
121/// so three consecutive full-budget timeouts take ~36s to observe: every client
122/// whose open contributed to opening the breaker had already given up on its
123/// own. That is what makes opening the breaker unable to turn a call that would
124/// have succeeded into a refusal — it can only make an already-failing module
125/// fail faster.
126///
127/// Two would be reachable inside one default deadline. One would convict a
128/// module on a single cold-cache bind, which is exactly the valid-but-slow case
129/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
130pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
131
132/// How long a module's bind-relay breaker stays open before exactly one
133/// `route.open` is let through as a probe.
134///
135/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
136/// re-pays a full-budget stall almost continuously, and the breaker stops being
137/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
138/// deadline: a client that starts retrying after the module recovers has to get
139/// a probe opportunity inside its own deadline, or the breaker converts a
140/// recovered module into a failed call — the failure it exists to prevent,
141/// pointed the other way.
142///
143/// 20s sits between those with room on both sides, and it caps what a wedged
144/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
145/// rather than one per `route.open` per connection. The stall that motivated
146/// this, with its measurements, is written up in
147/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
148/// waited the whole budget out.
149pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
150
151const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
152const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
153
154#[derive(Clone)]
155struct DaemonProvenanceFacts {
156    build: DaemonBuildProvenance,
157    pid: Option<u32>,
158    started_at_ms: Option<u64>,
159    start_clock: Option<crate::clock::StartClock>,
160    executable_path: Option<PathBuf>,
161    executable_identity: Option<SpawnedFileIdentity>,
162    process_start_time: Option<u64>,
163    probe: ExecutableIdentityProbe,
164}
165
166impl Default for DaemonProvenanceFacts {
167    fn default() -> Self {
168        Self {
169            build: DaemonBuildProvenance {
170                build_git_sha: None,
171                build_lock_digest: None,
172            },
173            pid: None,
174            started_at_ms: None,
175            start_clock: None,
176            executable_path: None,
177            executable_identity: None,
178            process_start_time: None,
179            probe: ExecutableIdentityProbe::default(),
180        }
181    }
182}
183
184#[derive(Debug, Clone)]
185struct SupervisorRescanContext {
186    supervisor: Supervisor,
187    config_path: PathBuf,
188    configured_port: Option<u16>,
189    storage_config: Option<crate::daemon_config::StorageConfig>,
190    admission_facts_carrier_module_id: Option<String>,
191    admission_facts_targets: Option<Vec<String>>,
192}
193
194/// Real channel-0 control handler for subc itself.
195#[derive(Clone)]
196pub struct ControlHandler {
197    registry: Arc<Registry>,
198    forwarding: Arc<ForwardingTable>,
199    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
200    supervisor: SupervisorHandle,
201    subc_capabilities: Arc<[String]>,
202    /// Daemon-wide route.bind relay budget. Used as the fallback when the
203    /// target module has no per-module override in
204    /// `route_bind_relay_timeouts`.
205    route_bind_relay_timeout: Duration,
206    /// Per-module route.bind relay budget overrides, keyed by module id. When
207    /// `handle_route_open` resolves the deadline for a target module, a
208    /// per-module entry wins over the daemon-wide value above.
209    route_bind_relay_timeouts: BTreeMap<String, Duration>,
210    /// Per-target-module bind-relay breaker state. Shared with the forwarding
211    /// table, which is where a new module connection resets it.
212    route_bind_breakers: RouteBindBreakers,
213    /// Live relay admissions keyed by target module. Shared through the
214    /// forwarding table so cloned or separately built handlers enforce one cap.
215    route_bind_concurrency: RouteBindConcurrency,
216    /// Consecutive relay timeouts that open a module's breaker.
217    route_bind_breaker_threshold: u32,
218    /// How long a breaker stays open before one probe is admitted.
219    route_bind_breaker_cooldown: Duration,
220    health_probe_timeout: Duration,
221    /// Central storage policy. When set, each registering module receives its
222    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
223    storage_config: Option<crate::daemon_config::StorageConfig>,
224    /// The machine id established at boot, served on every HELLO_ACK and on
225    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
226    /// changes the file, never this value. `None` serves no id.
227    machine_id: Option<crate::machine_id::MachineId>,
228    admission_facts_carrier_module_id: Option<String>,
229    admission_facts_targets: Option<Vec<String>>,
230    rescan: Option<SupervisorRescanContext>,
231    connected_clients: ConnectedClients,
232    counters: DaemonCounters,
233    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
234    daemon_provenance: DaemonProvenanceFacts,
235    #[cfg(test)]
236    control_dispatch_delay: Option<Duration>,
237    #[cfg(test)]
238    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
239}
240
241impl fmt::Debug for ControlHandler {
242    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
243        f.debug_struct("ControlHandler")
244            .field("registry", &self.registry)
245            .field("forwarding", &self.forwarding)
246            .field("process_liveness", &self.process_liveness.is_some())
247            .field("supervisor", &self.supervisor)
248            .field("subc_capabilities", &self.subc_capabilities)
249            .finish()
250    }
251}
252
253struct RouteOpenRequest {
254    target: RouteTarget,
255    identity: BindIdentity,
256    consumer_identity: Option<ConsumerIdentity>,
257    consumer_capabilities: Option<Vec<String>>,
258    admission_facts: Option<serde_json::Value>,
259}
260
261struct RouteBindReservationGuard {
262    forwarding: Arc<ForwardingTable>,
263    endpoint: ModuleEndpointId,
264    relay_corr: u64,
265    armed: bool,
266}
267
268struct ModuleControlRpcGuard {
269    forwarding: Arc<ForwardingTable>,
270    endpoint: ModuleEndpointId,
271    corr: u64,
272    armed: bool,
273}
274
275impl ModuleControlRpcGuard {
276    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
277        Self {
278            forwarding,
279            endpoint,
280            corr,
281            armed: true,
282        }
283    }
284
285    fn disarm(&mut self) {
286        self.armed = false;
287    }
288}
289
290impl Drop for ModuleControlRpcGuard {
291    fn drop(&mut self) {
292        if self.armed {
293            let _ = self
294                .forwarding
295                .cancel_module_control_rpc(self.endpoint, self.corr);
296        }
297    }
298}
299
300impl RouteBindReservationGuard {
301    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
302        Self {
303            forwarding,
304            endpoint,
305            relay_corr,
306            armed: true,
307        }
308    }
309
310    fn release_and_disarm(&mut self) {
311        if !self.armed {
312            return;
313        }
314        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
315            self.endpoint,
316            self.relay_corr,
317            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
318        ) {
319            send_goodbye_target_best_effort(
320                &self.forwarding.counters(),
321                &target,
322                "canceled route.bind",
323            );
324        }
325        self.armed = false;
326    }
327
328    fn disarm(&mut self) {
329        self.armed = false;
330    }
331}
332
333impl Drop for RouteBindReservationGuard {
334    fn drop(&mut self) {
335        self.release_and_disarm();
336    }
337}
338
339/// Per-target-module circuit breaker around the `route.bind` relay.
340///
341/// The connection reader is serial per connection, so a module whose `on_bind`
342/// sits on the ack blocks every LATER frame on the connections that call it,
343/// including calls to unrelated modules. This does not make any module's bind
344/// fast; it stops the daemon paying the full budget again and again for a
345/// condition it has already observed.
346///
347/// State is keyed by TARGET MODULE and shared by every connection: a wedged
348/// module wedges everyone, so what one connection learned should protect the
349/// rest.
350///
351/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
352/// relay to that module has actually timed out, and is removed again when a
353/// relay is accepted or the module reconnects, so it cannot grow with traffic
354/// or with modules that behave.
355///
356/// # Why a `std` mutex here is not the head-of-line defect again
357///
358/// Acquisition never awaits. The critical section is a hash lookup plus a few
359/// integer updates, with no I/O and no `.await` inside it, so a reader task
360/// cannot be descheduled behind it the way it can behind
361/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
362/// primitive, held for the same kind of work, as the refusal counter this very
363/// path already increments.
364///
365/// It is also NOT on the data-plane splice path: only `route.open` and module
366/// registration touch it, so bound-route frames gain no state check and no
367/// contention.
368#[derive(Debug, Clone, Default)]
369pub(crate) struct RouteBindBreakers {
370    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
371}
372
373#[derive(Debug, Clone, Default)]
374pub(crate) struct RouteBindConcurrency {
375    modules: Arc<Mutex<HashMap<String, usize>>>,
376}
377
378struct RouteBindConcurrencyGuard {
379    concurrency: RouteBindConcurrency,
380    module_id: String,
381}
382
383impl RouteBindConcurrency {
384    /// Admit without waiting. Waiting here would move the bind stall from the
385    /// module reply to a semaphore and restore reader head-of-line blocking.
386    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
387        let mut modules = self
388            .modules
389            .lock()
390            .expect("route.bind concurrency mutex poisoned");
391        let in_flight = modules.entry(module_id.to_string()).or_default();
392        if *in_flight >= limit {
393            return Err(*in_flight);
394        }
395        *in_flight += 1;
396        Ok(RouteBindConcurrencyGuard {
397            concurrency: self.clone(),
398            module_id: module_id.to_string(),
399        })
400    }
401}
402
403impl Drop for RouteBindConcurrencyGuard {
404    fn drop(&mut self) {
405        let mut modules = self
406            .concurrency
407            .modules
408            .lock()
409            .expect("route.bind concurrency mutex poisoned");
410        let remove = {
411            let in_flight = modules
412                .get_mut(&self.module_id)
413                .expect("admitted route.bind has a concurrency entry");
414            *in_flight -= 1;
415            *in_flight == 0
416        };
417        if remove {
418            modules.remove(&self.module_id);
419        }
420    }
421}
422
423#[derive(Debug, Default)]
424struct ModuleBreakerState {
425    /// Relay timeouts observed with no accepted relay in between.
426    consecutive_timeouts: u32,
427    /// `Some` while the breaker is open: the instant the cooldown expires and
428    /// the next arrival may probe. `None` means closed.
429    cooldown_until: Option<Instant>,
430    /// A half-open probe has been admitted and has not settled yet. This is
431    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
432    /// that read the cooldown, so concurrent opens arriving at the moment the
433    /// cooldown expires cannot all decide that they are the probe.
434    probe_in_flight: bool,
435}
436
437/// What the breaker decided for one `route.open`, before any relay work.
438enum RouteBindAdmission<'a> {
439    Admitted {
440        guard: RouteBindBreakerGuard<'a>,
441        /// This open is the single half-open probe, so the transition is worth
442        /// one log line.
443        probe: bool,
444    },
445    Refused {
446        consecutive_timeouts: u32,
447        /// What is left of the cooldown. Zero when the refusal is because the
448        /// one probe is already in flight rather than because the cooldown has
449        /// not elapsed.
450        retry_in: Duration,
451        probe_in_flight: bool,
452    },
453}
454
455/// An outstanding admission, which must be told how its relay settled.
456///
457/// `Drop` settles it as inconclusive, so an early return between admission and
458/// the relay -- or the whole handler being cancelled when the client
459/// disconnects -- releases a half-open probe slot instead of leaving the
460/// breaker wedged half-open with no further probes.
461struct RouteBindBreakerGuard<'a> {
462    breakers: RouteBindBreakers,
463    module_id: &'a str,
464    settled: bool,
465}
466
467impl RouteBindBreakerGuard<'_> {
468    /// The module answered within the budget and took the bind. THE ONLY
469    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
470    /// breaker, which is a transition worth logging.
471    fn record_accepted(&mut self) -> bool {
472        self.settled = true;
473        self.breakers.record_accepted(self.module_id)
474    }
475
476    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
477    /// COUNTS TOWARD OPENING.
478    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
479        self.settled = true;
480        self.breakers
481            .record_timeout(self.module_id, threshold, cooldown)
482    }
483
484    /// Everything else: the module REJECTED the bind, its connection went away
485    /// mid-relay, or the waiter was cancelled.
486    ///
487    /// None of these is evidence that a module is slow, and each already has
488    /// its own refusal with its own code. A module that rejects a bind in
489    /// microseconds is healthy and must never be convicted for it; a module
490    /// that died has said nothing about the module that replaces it. So these
491    /// neither increment nor reset the count -- they only release a probe slot.
492    fn record_inconclusive(&mut self) {
493        self.settled = true;
494        self.breakers.record_inconclusive(self.module_id);
495    }
496}
497
498impl Drop for RouteBindBreakerGuard<'_> {
499    fn drop(&mut self) {
500        if !self.settled {
501            self.breakers.record_inconclusive(self.module_id);
502        }
503    }
504}
505
506/// The breaker moved to open, reported so the caller can log it outside the
507/// lock. Opening is rare and load-bearing; the refusals that follow are
508/// frequent and are counted rather than logged.
509struct BreakerOpened {
510    consecutive_timeouts: u32,
511    /// True when a failed probe re-opened an already-open breaker, which reads
512    /// very differently in a log from a first opening.
513    reopened_after_probe: bool,
514}
515
516impl RouteBindBreakers {
517    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
518        self.modules
519            .lock()
520            .expect("route.bind breaker mutex poisoned")
521    }
522
523    /// Decide whether this `route.open` may attempt its relay. Takes the map
524    /// lock and nothing else, and never awaits.
525    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
526        let admitted = |probe| RouteBindAdmission::Admitted {
527            guard: RouteBindBreakerGuard {
528                breakers: self.clone(),
529                module_id,
530                settled: false,
531            },
532            probe,
533        };
534
535        let mut modules = self.lock();
536        let Some(state) = modules.get_mut(module_id) else {
537            return admitted(false);
538        };
539        let Some(cooldown_until) = state.cooldown_until else {
540            return admitted(false);
541        };
542        if state.probe_in_flight {
543            return RouteBindAdmission::Refused {
544                consecutive_timeouts: state.consecutive_timeouts,
545                retry_in: Duration::ZERO,
546                probe_in_flight: true,
547            };
548        }
549        let now = Instant::now();
550        if now < cooldown_until {
551            return RouteBindAdmission::Refused {
552                consecutive_timeouts: state.consecutive_timeouts,
553                retry_in: cooldown_until - now,
554                probe_in_flight: false,
555            };
556        }
557        state.probe_in_flight = true;
558        admitted(true)
559    }
560
561    fn record_accepted(&self, module_id: &str) -> bool {
562        self.lock()
563            .remove(module_id)
564            .is_some_and(|state| state.cooldown_until.is_some())
565    }
566
567    fn record_timeout(
568        &self,
569        module_id: &str,
570        threshold: u32,
571        cooldown: Duration,
572    ) -> Option<BreakerOpened> {
573        let mut modules = self.lock();
574        let state = modules.entry(module_id.to_string()).or_default();
575        let was_open = state.cooldown_until.is_some();
576        let was_probe = state.probe_in_flight;
577        state.probe_in_flight = false;
578        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
579        if state.consecutive_timeouts < threshold {
580            return None;
581        }
582        state.cooldown_until = Some(Instant::now() + cooldown);
583        Some(BreakerOpened {
584            consecutive_timeouts: state.consecutive_timeouts,
585            reopened_after_probe: was_open && was_probe,
586        })
587    }
588
589    fn record_inconclusive(&self, module_id: &str) {
590        if let Some(state) = self.lock().get_mut(module_id) {
591            state.probe_in_flight = false;
592        }
593    }
594
595    /// Discard what was learned about a module, because the process it was
596    /// learned about is gone. Returns the discarded count when it was non-zero.
597    ///
598    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
599    /// `module_id` is a configuration identity that outlives any particular
600    /// child; what the breaker observed was the process behind the module
601    /// connection of the moment. When a new connection registers under that id
602    /// the verdict's subject no longer exists, so the verdict is stale by
603    /// construction rather than merely likely to be wrong. Keeping it would
604    /// apply a dead process's record to a live one, which is the same defect
605    /// class this breaker exists to stop the daemon committing.
606    ///
607    /// A half-open probe in flight is discarded with the rest: it was a
608    /// question about the old process.
609    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
610        self.lock()
611            .remove(module_id)
612            .map(|state| state.consecutive_timeouts)
613            .filter(|discarded| *discarded > 0)
614    }
615
616    /// Open breakers, for the `server.describe` counters object. `None` when
617    /// none is open, so the key stays absent rather than present-and-empty.
618    ///
619    /// This is the operator's answer to "is this module refusing instantly or
620    /// is it fine?", which look identical from a client that retries and then
621    /// succeeds.
622    fn open_snapshot(&self) -> Option<serde_json::Value> {
623        let now = Instant::now();
624        let modules = self.lock();
625        let open = modules
626            .iter()
627            .filter_map(|(module_id, state)| {
628                let cooldown_until = state.cooldown_until?;
629                Some((
630                    module_id.clone(),
631                    serde_json::json!({
632                        "consecutive_timeouts": state.consecutive_timeouts,
633                        "cooldown_remaining_ms":
634                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
635                        "probe_in_flight": state.probe_in_flight,
636                    }),
637                ))
638            })
639            .collect::<serde_json::Map<String, serde_json::Value>>();
640        (!open.is_empty()).then_some(serde_json::Value::Object(open))
641    }
642}
643
644impl ControlHandler {
645    pub fn new(registry: Arc<Registry>) -> Self {
646        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
647    }
648
649    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
650        let counters = forwarding.counters();
651        // Taken from the forwarding table rather than created here, so that the
652        // breaker a `route.open` consults is the same one a module's
653        // registration resets, however many handlers are built over one table.
654        let route_bind_breakers = forwarding.route_bind_breakers();
655        let route_bind_concurrency = forwarding.route_bind_concurrency();
656        Self {
657            registry,
658            forwarding,
659            process_liveness: None,
660            supervisor: SupervisorHandle::new(),
661            subc_capabilities: Arc::from([
662                CAP_MANIFEST_REGISTRATION.to_string(),
663                CAP_CHANNEL_LIFECYCLE.to_string(),
664                CAP_PING_PONG.to_string(),
665                CAP_SESSION_ATTACH.to_string(),
666                CAP_ADMISSION_FACTS_RELAY.to_string(),
667            ]),
668            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
669            route_bind_relay_timeouts: BTreeMap::new(),
670            route_bind_breakers,
671            route_bind_concurrency,
672            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
673            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
674            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
675            storage_config: None,
676            machine_id: None,
677            admission_facts_carrier_module_id: None,
678            admission_facts_targets: None,
679            rescan: None,
680            connected_clients: ConnectedClients::new(),
681            counters,
682            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
683            daemon_provenance: DaemonProvenanceFacts::default(),
684            #[cfg(test)]
685            control_dispatch_delay: None,
686            #[cfg(test)]
687            provenance_probe_override: None,
688        }
689    }
690
691    /// Set the central storage policy: registering modules then receive their
692    /// resolved storage descriptor in HELLO_ACK.
693    pub fn with_storage_config(
694        mut self,
695        storage_config: Option<crate::daemon_config::StorageConfig>,
696    ) -> Self {
697        self.storage_config = storage_config;
698        self
699    }
700
701    /// Set the machine id served to every registering module (HELLO_ACK) and on
702    /// `server.describe`.
703    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
704        self.machine_id = machine_id;
705        self
706    }
707
708    /// Configure the exact reserved module and target ids permitted to relay
709    /// opaque admission facts. Config-file loading validates this authority;
710    /// this builder keeps the same policy available to embedded test daemons.
711    pub fn with_admission_facts_config(
712        mut self,
713        carrier_module_id: Option<String>,
714        targets: Option<Vec<String>>,
715    ) -> Self {
716        self.admission_facts_carrier_module_id = carrier_module_id;
717        self.admission_facts_targets = targets;
718        self
719    }
720
721    /// Override the route.bind relay timeout. Used by tests that assert the
722    /// timeout path so they don't block on the production-safe default.
723    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
724        self.route_bind_relay_timeout = timeout;
725        self
726    }
727
728    /// Install per-module route.bind relay budget overrides. A module id
729    /// listed here wins over the daemon-wide default set via
730    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
731    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
732    /// the same `Duration` the bind path will use.
733    pub fn with_route_bind_relay_timeouts(
734        mut self,
735        timeouts: impl IntoIterator<Item = (String, Duration)>,
736    ) -> Self {
737        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
738        self
739    }
740
741    /// Resolve the route.bind relay budget for a specific target module id.
742    /// Per-module overrides win; the daemon-wide value (set via
743    /// `with_route_bind_relay_timeout` or the built-in default) is the
744    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
745    /// the same resolution `handle_route_open` will use.
746    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
747        self.route_bind_relay_timeouts
748            .get(module_id)
749            .copied()
750            .unwrap_or(self.route_bind_relay_timeout)
751    }
752
753    /// Override the per-module bind-relay breaker policy.
754    ///
755    /// Used by tests, which cannot spend three production budgets opening a
756    /// breaker or twenty seconds waiting for its cooldown. The production
757    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
758    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
759    /// reasoning for the numbers.
760    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
761        self.route_bind_breaker_threshold = threshold.max(1);
762        self.route_bind_breaker_cooldown = cooldown;
763        self
764    }
765
766    #[cfg(test)]
767    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
768        self.health_probe_timeout = timeout;
769        self
770    }
771
772    #[cfg(test)]
773    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
774        self.control_dispatch_delay = Some(delay);
775        self
776    }
777
778    pub fn with_process_liveness(
779        mut self,
780        process_liveness: Arc<dyn ModuleProcessLiveness>,
781    ) -> Self {
782        self.process_liveness = Some(process_liveness);
783        self
784    }
785
786    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
787        self.supervisor = supervisor;
788        self
789    }
790
791    pub fn with_daemon_provenance(
792        mut self,
793        pid: u32,
794        started_at_ms: u64,
795        executable_path: Option<PathBuf>,
796        build_git_sha: Option<String>,
797        build_lock_digest: Option<String>,
798    ) -> Self {
799        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
800        let process_start_time = process_start_time(pid);
801        self.daemon_provenance = DaemonProvenanceFacts {
802            build: DaemonBuildProvenance {
803                build_git_sha,
804                build_lock_digest,
805            },
806            pid: Some(pid),
807            started_at_ms: Some(started_at_ms),
808            start_clock: None,
809            executable_path,
810            executable_identity,
811            process_start_time,
812            probe: ExecutableIdentityProbe::default(),
813        };
814        self
815    }
816
817    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
818        self.daemon_provenance.start_clock = Some(clock);
819        self
820    }
821
822    #[cfg(test)]
823    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
824        self.provenance_probe_override = Some(result);
825        self
826    }
827
828    /// Install the configured module set and its reserved capability bindings.
829    /// Bindings are configuration-scoped and may point at a provider that has not
830    /// been installed yet, so this does not require the bound module to exist.
831    pub fn with_capability_config(
832        self,
833        modules: impl IntoIterator<Item = (String, bool)>,
834        reserved_capabilities: BTreeMap<String, String>,
835    ) -> Self {
836        self.capability_evaluator
837            .configure(modules, reserved_capabilities);
838        self
839    }
840
841    pub fn with_supervisor_rescan(
842        mut self,
843        supervisor: Supervisor,
844        config_path: impl Into<PathBuf>,
845        configured_port: Option<u16>,
846    ) -> Self {
847        self.rescan = Some(SupervisorRescanContext {
848            supervisor,
849            config_path: config_path.into(),
850            configured_port,
851            storage_config: self.storage_config.clone(),
852            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
853            admission_facts_targets: self.admission_facts_targets.clone(),
854        });
855        self
856    }
857
858    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
859        self.connected_clients = connected_clients;
860        self
861    }
862
863    pub fn forwarding(&self) -> Arc<ForwardingTable> {
864        Arc::clone(&self.forwarding)
865    }
866
867    pub(crate) fn counters(&self) -> DaemonCounters {
868        self.counters.clone()
869    }
870
871    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
872    /// requirement event without depending on an operator polling a status command.
873    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
874        tokio::spawn(async move {
875            loop {
876                self.capability_evaluator
877                    .wait_for_change_or_deadline()
878                    .await;
879                self.refresh_capability_requirements();
880            }
881        });
882    }
883
884    fn runtime_capability_snapshot(
885        &self,
886    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
887        let runtime = self
888            .supervisor
889            .list()
890            .into_iter()
891            .map(|module| {
892                let status = module.status().map_err(|err| {
893                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
894                })?;
895                Ok(RuntimeModule {
896                    module_id: status.module_id,
897                    state: status.state,
898                    enabled: status.enabled,
899                })
900            })
901            .collect::<Result<Vec<_>, RouterError>>()?;
902        let (_, registrations) = self.registry.list_modules().map_err(|err| {
903            RouterError::backend(
904                0,
905                0,
906                format!("failed to list capability registrations: {err}"),
907            )
908        })?;
909        let registrations = registrations
910            .into_iter()
911            .map(|registration| RegisteredModule {
912                module_id: registration.manifest.module_id,
913                module_version: registration.manifest.module_version,
914                capabilities: registration.manifest.capabilities,
915            })
916            .collect();
917        Ok((runtime, registrations))
918    }
919
920    /// The capability side effects of a module becoming the active registration
921    /// for its id: cache its manifest (warning if its claims drifted), run the
922    /// deny census when its declarations call for one, and recompute the
923    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
924    /// candidate's does not, and the supervisor does it at promotion instead,
925    /// through [`crate::supervise::SwapPromotionObserver`].
926    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
927        let cached_registration = RegisteredModule {
928            module_id: registration.manifest.module_id.clone(),
929            module_version: registration.manifest.module_version.clone(),
930            capabilities: registration.manifest.capabilities.clone(),
931        };
932        if self.capability_evaluator.record_hello(&cached_registration) {
933            warn!(
934                module_id = %cached_registration.module_id,
935                "capability claims drifted from the cached manifest"
936            );
937        }
938        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
939            self.enforce_capability_denies();
940        }
941        self.refresh_capability_requirements();
942    }
943
944    /// Point the shared supervisor handle at this handler for swap promotions.
945    /// Called wherever a handler is put behind the `Arc` the router serves, so
946    /// it can be held weakly.
947    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
948        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
949            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
950        self.supervisor.set_swap_promotion_observer(observer);
951    }
952
953    pub fn refresh_capability_requirements(&self) {
954        match self.runtime_capability_snapshot() {
955            Ok((runtime, registrations)) => {
956                log_requirement_events(
957                    self.capability_evaluator
958                        .evaluate_now(&runtime, &registrations),
959                );
960            }
961            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
962        }
963    }
964
965    /// Reconcile only live, attested route bindings after a capability deny edge
966    /// or target claim was added. This is deliberately a control-plane census:
967    /// the opaque forwarding hot path must not grow a per-frame capability check.
968    fn enforce_capability_denies(&self) {
969        let (_, registrations) = match self.registry.list_modules() {
970            Ok(snapshot) => snapshot,
971            Err(err) => {
972                warn!(error = %err, "failed to read registrations for capability deny census");
973                return;
974            }
975        };
976        let manifests = registrations
977            .into_iter()
978            .map(|registration| {
979                (
980                    registration.manifest.module_id.clone(),
981                    registration.manifest,
982                )
983            })
984            .collect::<BTreeMap<_, _>>();
985        let census = match self.forwarding.route_census(None) {
986            Ok(census) => census,
987            Err(err) => {
988                warn!(error = %err, "failed to read route census for capability deny enforcement");
989                return;
990            }
991        };
992
993        for (target_module_id, routes) in census {
994            let Some(target_manifest) = manifests.get(&target_module_id) else {
995                continue;
996            };
997            let mut closed_routes = Vec::new();
998            let mut module_goodbyes = Vec::new();
999            for route in routes {
1000                let Principal::Reserved {
1001                    module_id: opening_module_id,
1002                } = &route.principal
1003                else {
1004                    continue;
1005                };
1006                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1007                    continue;
1008                };
1009                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1010                    continue;
1011                };
1012
1013                match self.forwarding.release_client_route(
1014                    route.goodbye_target.connection_id,
1015                    route.goodbye_target.channel,
1016                    route.goodbye_target.epoch,
1017                ) {
1018                    Ok(RouteRelease::Removed(module_goodbye)) => {
1019                        warn!(
1020                            opening_module_id,
1021                            target_module_id,
1022                            capability,
1023                            "force-closing route because an attested capability deny edge now matches"
1024                        );
1025                        closed_routes.push(route);
1026                        module_goodbyes.push(module_goodbye);
1027                    }
1028                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1029                    Err(err) => warn!(
1030                        opening_module_id,
1031                        target_module_id,
1032                        capability,
1033                        error = %err,
1034                        "failed to force-close capability-denied route"
1035                    ),
1036                }
1037            }
1038
1039            if closed_routes.is_empty() {
1040                continue;
1041            }
1042            send_route_control_pushes(
1043                &self.forwarding,
1044                closed_routes,
1045                ClientControlPush::RouteClosed {
1046                    module_id: target_module_id,
1047                    reason: RouteCloseReason::CapabilityDenied,
1048                    drained: false,
1049                    abandoned: 0,
1050                    excluded_subscriptions: 0,
1051                    terminal: Some(false),
1052                },
1053            );
1054            self.emit_route_goodbyes(module_goodbyes);
1055        }
1056    }
1057
1058    /// Why a registered module is not accepting new route binds, or `None` when
1059    /// it is. This is the module's effective readiness: its declared readiness
1060    /// first, then every `need: required` capability it declares evaluating to
1061    /// `provided`. `route.open` and `catalog.list` both read it here so the
1062    /// catalog never reports a module routable that `route.open` would refuse.
1063    fn not_ready_reason(
1064        &self,
1065        registration: &crate::registry::ModuleRegistration,
1066    ) -> Option<NotReadyReason> {
1067        if !registration.ready {
1068            return Some(NotReadyReason {
1069                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1070                capability: None,
1071            });
1072        }
1073        self.first_unprovided_required_capability(registration)
1074            .map(|capability| NotReadyReason {
1075                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1076                capability: Some(capability),
1077            })
1078    }
1079
1080    /// The lexicographically first capability this registration declares
1081    /// `need: required` whose evaluator verdict is not `provided`.
1082    ///
1083    /// The verdicts are the capability evaluator's own; nothing here decides
1084    /// what "provided" means. The evaluator counts a capability provided as
1085    /// soon as a module claiming it has REGISTERED, not once that module is
1086    /// ready. That distinction is what keeps two modules that require each
1087    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1088    /// is ready", each would wait for the other to become ready first and
1089    /// neither ever would. Do not tighten it to readiness.
1090    ///
1091    /// A required capability with no verdict at all means this registration's
1092    /// HELLO or catalog.update landed after the last recompute; recompute once
1093    /// rather than let a missing verdict read as either answer. If it is still
1094    /// missing (the recompute itself failed) the capability counts as
1095    /// unprovided: the refusal is retryable, and routing a module whose
1096    /// required provider is unknown is the outcome this check exists to stop.
1097    fn first_unprovided_required_capability(
1098        &self,
1099        registration: &crate::registry::ModuleRegistration,
1100    ) -> Option<String> {
1101        let required = registration
1102            .manifest
1103            .capabilities
1104            .iter()
1105            .flat_map(|declarations| declarations.requires.iter())
1106            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1107            .map(|requirement| requirement.capability.as_str())
1108            .collect::<BTreeSet<_>>();
1109        if required.is_empty() {
1110            return None;
1111        }
1112        let module_id = registration.manifest.module_id.as_str();
1113        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1114        if required
1115            .iter()
1116            .any(|capability| verdict(capability).is_none())
1117        {
1118            self.refresh_capability_requirements();
1119        }
1120        required
1121            .into_iter()
1122            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1123            .map(str::to_string)
1124    }
1125
1126    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1127        self.capability_evaluator
1128            .statuses()
1129            .into_iter()
1130            .map(capability_requirement_status)
1131            .collect()
1132    }
1133
1134    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1135    /// registration-release watch. The signal is what the supervisor waits on
1136    /// before spawning a replacement, so it must only fire once forwarding
1137    /// teardown is also done (see [`Self::cleanup_connection`] /
1138    /// [`Self::handle_goodbye`]). Used directly only where there is no forwarding
1139    /// state to tear down (a HELLO that failed before module registration).
1140    fn deregister_connection(
1141        &self,
1142        connection_id: ConnectionId,
1143    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1144        self.registry.deregister_connection(connection_id)
1145    }
1146
1147    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1148        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1149            return None;
1150        }
1151        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1152            parse_client_control_request(&frame.body)
1153        else {
1154            return None;
1155        };
1156        Some(target_module_id(&target).to_string())
1157    }
1158
1159    pub(crate) fn route_open_capacity_refusal(
1160        &self,
1161        ctx: &RouteCtx,
1162        frame: &Frame,
1163        target_module_id: &str,
1164        limit: usize,
1165    ) -> Result<Frame, RouterError> {
1166        self.route_open_admission_refusal_frame(
1167            ctx,
1168            frame,
1169            target_module_id,
1170            format!(
1171                "connection already has {limit} route.open binds in flight; retry after one settles"
1172            ),
1173        )
1174    }
1175
1176    /// Admission pressure clears as existing binds settle, so its refusal must
1177    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
1178    /// `module_reloading`, `module_warming`, `target_unavailable`, or
1179    /// `module_timeout`. `target_unavailable` is honest for an attempt that
1180    /// cannot currently reach its target; `module_timeout` would falsely claim
1181    /// that a wait expired. A new, cleaner code would be terminal to deployed
1182    /// clients, so it requires a client-tolerance rollout before daemon emission.
1183    fn route_open_admission_refusal_frame(
1184        &self,
1185        ctx: &RouteCtx,
1186        frame: &Frame,
1187        target_module_id: &str,
1188        message: impl Into<String>,
1189    ) -> Result<Frame, RouterError> {
1190        self.route_open_refusal_frame(
1191            ctx,
1192            frame,
1193            target_module_id,
1194            "open_admission_full",
1195            error_codes::TARGET_UNAVAILABLE,
1196            message,
1197        )
1198    }
1199
1200    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
1201    ///
1202    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
1203    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
1204    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
1205    #[cfg(test)]
1206    pub fn handle_control(
1207        &self,
1208        connection_id: ConnectionId,
1209        frame: Frame,
1210    ) -> Result<Vec<Frame>, RouterError> {
1211        match frame.header.ty {
1212            FrameType::Ping => Ok(vec![pong(&frame)?]),
1213            FrameType::Hello => self.handle_hello(connection_id, None, frame),
1214            FrameType::Goodbye => self.handle_goodbye(connection_id),
1215            ty => Ok(vec![control_error_frame(
1216                &frame,
1217                "unsupported_control_frame",
1218                format!("unsupported channel-0 frame {ty:?}"),
1219            )?]),
1220        }
1221    }
1222
1223    pub async fn handle_control_frame(
1224        &self,
1225        ctx: &RouteCtx,
1226        frame: Frame,
1227    ) -> Result<Vec<Frame>, RouterError> {
1228        self.handle_control_frame_timed(ctx, frame, None).await
1229    }
1230
1231    pub(crate) async fn handle_control_frame_timed(
1232        &self,
1233        ctx: &RouteCtx,
1234        frame: Frame,
1235        dispatch_started_at: Option<StdInstant>,
1236    ) -> Result<Vec<Frame>, RouterError> {
1237        match frame.header.ty {
1238            FrameType::Ping => Ok(vec![pong(&frame)?]),
1239            FrameType::Hello => {
1240                self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1241            }
1242            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1243            FrameType::Cancel => {
1244                if self
1245                    .supervisor
1246                    .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1247                {
1248                    Ok(Vec::new())
1249                } else {
1250                    Ok(vec![control_error_frame(
1251                        &frame,
1252                        "unknown_subscription",
1253                        "no supervisor spawn subscription has this correlation id",
1254                    )?])
1255                }
1256            }
1257            FrameType::Request => {
1258                if self
1259                    .forwarding
1260                    .module_endpoint_for_connection(ctx.connection_id)
1261                    .map_err(RouterError::Forwarding)?
1262                    .is_some()
1263                {
1264                    if !is_known_module_request_op(&frame.body) {
1265                        return Ok(vec![control_error_frame(
1266                            &frame,
1267                            "unsupported_control_frame",
1268                            "module-originated channel-0 REQUEST is not supported",
1269                        )?]);
1270                    }
1271                    let request = match parse_module_control_request_from_module(&frame.body) {
1272                        Ok(request) => request,
1273                        Err((err, ControlRequestBodyError::UnknownOp)) => {
1274                            return Ok(vec![control_error_frame(
1275                                &frame,
1276                                "unsupported_control_frame",
1277                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
1278                            )?])
1279                        }
1280                        Err((err, ControlRequestBodyError::InvalidBody)) => {
1281                            return Ok(vec![control_error_frame(
1282                                &frame,
1283                                "invalid_control_body",
1284                                format!("malformed module control body: {err}"),
1285                            )?])
1286                        }
1287                    };
1288                    let op = module_control_request_op(&request);
1289                    let corr = frame.header.corr;
1290                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
1291                    let result =
1292                        self.handle_module_control_request(ctx.connection_id, frame, request);
1293                    log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1294                    return result;
1295                }
1296
1297                if is_known_module_request_op(&frame.body) {
1298                    return Ok(vec![control_error_frame(
1299                        &frame,
1300                        "not_registered",
1301                        "catalog.update requires an active module registration owned by this connection",
1302                    )?]);
1303                }
1304
1305                let request = match parse_client_control_request(&frame.body) {
1306                    Ok(request) => request,
1307                    Err((err, ControlRequestBodyError::UnknownOp)) => {
1308                        return Ok(vec![control_error_frame(
1309                            &frame,
1310                            "unknown_control_op",
1311                            format!("unknown client control op: {err}"),
1312                        )?])
1313                    }
1314                    Err((err, ControlRequestBodyError::InvalidBody)) => {
1315                        return Ok(vec![control_error_frame(
1316                            &frame,
1317                            "invalid_control_body",
1318                            format!("malformed client control body: {err}"),
1319                        )?])
1320                    }
1321                };
1322                let op = client_control_request_op(&request);
1323                let corr = frame.header.corr;
1324                log_control_dispatch_arrival(op, ctx.connection_id, corr);
1325                #[cfg(test)]
1326                if let Some(delay) = self.control_dispatch_delay {
1327                    tokio::time::sleep(delay).await;
1328                }
1329                let result = self
1330                    .handle_client_control_request(ctx, frame, request)
1331                    .await;
1332                log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1333                result
1334            }
1335            FrameType::Push => {
1336                let Some(endpoint) = self
1337                    .forwarding
1338                    .module_endpoint_for_connection(ctx.connection_id)
1339                    .map_err(RouterError::Forwarding)?
1340                else {
1341                    return Ok(vec![control_error_frame(
1342                        &frame,
1343                        "unsupported_control_frame",
1344                        "client-originated channel-0 PUSH is not supported",
1345                    )?]);
1346                };
1347                self.handle_status_update(endpoint, frame)
1348            }
1349            FrameType::Response | FrameType::Error
1350                if self
1351                    .forwarding
1352                    .module_endpoint_for_connection(ctx.connection_id)
1353                    .map_err(RouterError::Forwarding)?
1354                    .is_some() =>
1355            {
1356                self.handle_module_relay_response(ctx.connection_id, frame)
1357            }
1358            ty => Ok(vec![control_error_frame(
1359                &frame,
1360                "unsupported_control_frame",
1361                format!("unsupported channel-0 frame {ty:?}"),
1362            )?]),
1363        }
1364    }
1365
1366    pub fn cleanup_connection(
1367        &self,
1368        connection_id: ConnectionId,
1369    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1370        let crash_closed = self
1371            .registry
1372            .get_module_by_connection(connection_id)?
1373            .and_then(|registration| {
1374                self.forwarding
1375                    .module_endpoint_for_connection(connection_id)
1376                    .ok()
1377                    .flatten()
1378                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1379                    .map(|routes| (registration.manifest.module_id, routes))
1380            });
1381        if let Some((module_id, routes)) = crash_closed {
1382            let terminal = match self.supervisor.get(&module_id) {
1383                None => false,
1384                Some(module) => match module.will_recover_after_connection_loss() {
1385                    Ok(will_recover) => !will_recover,
1386                    Err(err) => {
1387                        warn!(
1388                            %module_id,
1389                            error = %err,
1390                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
1391                        );
1392                        false
1393                    }
1394                },
1395            };
1396            send_route_control_pushes(
1397                &self.forwarding,
1398                routes,
1399                ClientControlPush::RouteClosed {
1400                    module_id,
1401                    reason: RouteCloseReason::Crash,
1402                    drained: false,
1403                    abandoned: 0,
1404                    excluded_subscriptions: 0,
1405                    terminal: Some(terminal),
1406                },
1407            );
1408        }
1409        let registrations = self.deregister_connection(connection_id);
1410        if let Ok(released_routes) = self.forwarding.cleanup_connection(connection_id) {
1411            self.emit_route_goodbyes(released_routes);
1412        }
1413        // Signal the registration-release watch only now that BOTH registry and
1414        // forwarding teardown are done, so a supervisor waiting to spawn a
1415        // replacement never observes release while old routes still exist.
1416        if matches!(&registrations, Ok(r) if !r.is_empty()) {
1417            crate::supervise::notify_registration_release();
1418            self.capability_evaluator.wake_deadline_loop();
1419            self.refresh_capability_requirements();
1420        }
1421        self.supervisor.remove_spawn_subscribers(connection_id);
1422        registrations
1423    }
1424
1425    pub(crate) fn handle_route_goodbye(
1426        &self,
1427        connection_id: ConnectionId,
1428        route_channel: u16,
1429        route_epoch: u32,
1430    ) -> Result<bool, RouterError> {
1431        debug!(
1432            connection_id = connection_id.get(),
1433            route_channel, route_epoch, "handling route GOODBYE"
1434        );
1435        let RouteRelease::Removed(released_route) = self
1436            .forwarding
1437            .release_client_route(connection_id, route_channel, route_epoch)
1438            .map_err(RouterError::Forwarding)?
1439        else {
1440            return Ok(false);
1441        };
1442        self.emit_route_goodbyes(vec![released_route]);
1443        Ok(true)
1444    }
1445
1446    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1447        for released in released_routes {
1448            let frame = match Frame::build_with_version(
1449                released.negotiated_ver,
1450                FrameType::Goodbye,
1451                control_flags(),
1452                released.channel,
1453                released.epoch,
1454                0,
1455                Vec::new(),
1456            ) {
1457                Ok(frame) => frame,
1458                Err(err) => {
1459                    warn!(
1460                        route_channel = released.channel,
1461                        error = %err,
1462                        "failed to build route GOODBYE frame"
1463                    );
1464                    continue;
1465                }
1466            };
1467            if !released.close_on_delivery_failure() {
1468                crate::forwarding::send_module_route_goodbye(
1469                    &self.counters,
1470                    &released.sink,
1471                    frame,
1472                    released.module_id.as_deref(),
1473                    "client route released",
1474                );
1475                continue;
1476            }
1477            if let Err(err) = released.sink.try_send(frame) {
1478                warn!(
1479                    target_connection_id = released.connection_id.get(),
1480                    route_channel = released.channel,
1481                    error = %err,
1482                    "route GOODBYE was not delivered to client; closing target connection"
1483                );
1484                if self
1485                    .forwarding
1486                    .escalate_client_delivery_failure(
1487                        released.connection_id,
1488                        released.channel,
1489                        released.epoch,
1490                        CloseReason::new(
1491                            "route_goodbye_delivery_failed",
1492                            format!(
1493                                "failed to enqueue route GOODBYE for channel {}: {err}",
1494                                released.channel
1495                            ),
1496                        ),
1497                        crate::forwarding::UndeliveredFrame {
1498                            module_id: released.module_id.as_deref(),
1499                            sink: &released.sink,
1500                        },
1501                    )
1502                    .unwrap_or(false)
1503                {
1504                    self.counters.increment_goodbye_relay_client_failed();
1505                }
1506            }
1507        }
1508    }
1509
1510    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
1511    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
1512    /// own commit failed after the module had already accepted). Without this, a
1513    /// module that accepts late keeps a binding subc has torn down, so a later
1514    /// frame on that module channel could misdeliver if the channel is reused.
1515    ///
1516    /// Never closes the shared module connection on failure: a dropped notification
1517    /// only wastes a bounded amount of warm module-side state, which the module's
1518    /// own idle reaper reclaims. Only call this once the route.bind relay was
1519    /// actually enqueued to the module — if the relay send itself failed, the
1520    /// module never created a binding and there is nothing to tear down.
1521    fn send_abandoned_route_bind_goodbye(
1522        &self,
1523        module_sink: &crate::FrameSink,
1524        negotiated_ver: u8,
1525        module_channel: u16,
1526        module_epoch: u32,
1527    ) {
1528        let frame = match Frame::build_with_version(
1529            negotiated_ver,
1530            FrameType::Goodbye,
1531            control_flags(),
1532            module_channel,
1533            module_epoch,
1534            0,
1535            Vec::new(),
1536        ) {
1537            Ok(frame) => frame,
1538            Err(err) => {
1539                warn!(
1540                    route_channel = module_channel,
1541                    error = %err,
1542                    "failed to build GOODBYE for abandoned route.bind"
1543                );
1544                return;
1545            }
1546        };
1547        crate::forwarding::send_module_route_goodbye(
1548            &self.counters,
1549            module_sink,
1550            frame,
1551            None,
1552            "abandoned route.bind",
1553        );
1554    }
1555
1556    fn handle_hello(
1557        &self,
1558        connection_id: ConnectionId,
1559        sink: Option<crate::FrameSink>,
1560        frame: Frame,
1561    ) -> Result<Vec<Frame>, RouterError> {
1562        debug!(
1563            connection_id = connection_id.get(),
1564            corr = frame.header.corr,
1565            "handling HELLO"
1566        );
1567        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1568            Ok(value) => value,
1569            Err(err) => {
1570                return Ok(vec![control_error_frame(
1571                    &frame,
1572                    "invalid_hello",
1573                    format!("malformed HELLO body: {err}"),
1574                )?])
1575            }
1576        };
1577        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1578            return Ok(vec![control_error_frame(
1579                &frame,
1580                "invalid_capability_grammar",
1581                err.to_string(),
1582            )?]);
1583        }
1584        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
1585            return Ok(vec![control_error_frame(
1586                &frame,
1587                "invalid_manifest",
1588                err.to_string(),
1589            )?]);
1590        }
1591        if let Some(provenance) = hello_value
1592            .get("manifest")
1593            .and_then(|manifest| manifest.get("provenance"))
1594        {
1595            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
1596                return Ok(vec![control_error_frame(
1597                    &frame,
1598                    "invalid_manifest",
1599                    format!("malformed manifest provenance: {err}"),
1600                )?]);
1601            }
1602        }
1603        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
1604            Ok(hello) => hello,
1605            Err(err) => {
1606                return Ok(vec![control_error_frame(
1607                    &frame,
1608                    "invalid_hello",
1609                    format!("malformed HELLO body: {err}"),
1610                )?])
1611            }
1612        };
1613
1614        if hello.protocol_ver != hello.manifest.protocol_ver {
1615            return Ok(vec![control_error_frame(
1616                &frame,
1617                "invalid_manifest",
1618                format!(
1619                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
1620                    hello.protocol_ver, hello.manifest.protocol_ver
1621                ),
1622            )?]);
1623        }
1624
1625        if hello.manifest.module_id.trim().is_empty() {
1626            return Ok(vec![control_error_frame(
1627                &frame,
1628                "invalid_manifest",
1629                "manifest module_id must not be empty",
1630            )?]);
1631        }
1632
1633        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
1634            Ok(negotiated_ver) => negotiated_ver,
1635            Err(message) => {
1636                return Ok(vec![control_error_frame(
1637                    &frame,
1638                    "version_unsupported",
1639                    message,
1640                )?])
1641            }
1642        };
1643
1644        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
1645        // swap is open for this id, the only HELLO admitted as a second process
1646        // is the one carrying the candidate's launch nonce (the swap token), and
1647        // it registers into the candidate slot rather than being refused as a
1648        // duplicate. Run after the reserved gate, a reserved module's candidate
1649        // would be refused `reserved_module` for presenting a nonce that gate
1650        // does not know. See `SupervisorHandle::swap_hello_admission`.
1651        let swap_admission = self
1652            .supervisor
1653            .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
1654        if swap_admission == SwapHelloAdmission::Refused {
1655            warn!(
1656                module_id = %hello.manifest.module_id,
1657                connection_id = connection_id.get(),
1658                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
1659            );
1660            return Ok(vec![control_error_frame(
1661                &frame,
1662                "swap_token_invalid",
1663                format!(
1664                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
1665                    hello.manifest.module_id
1666                ),
1667            )?]);
1668        }
1669        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
1670
1671        // Reserved-module identity gate: a module_id configured `reserved` may be
1672        // registered ONLY by the process subc spawned for it, proven by echoing the
1673        // one-time launch nonce subc injected. A non-reserved id has no recorded
1674        // nonce and always passes. This blocks a key-holder from impersonating a
1675        // security-boundary module (e.g. the credential vault) while the real one is
1676        // down/restarting and its registration slot is momentarily free. A swap
1677        // candidate has already proven the same thing with its own nonce above.
1678        if let Some(rejection) = (!swap_candidate)
1679            .then(|| {
1680                self.supervisor.reserved_hello_rejection(
1681                    &hello.manifest.module_id,
1682                    hello.launch_nonce.as_deref(),
1683                )
1684            })
1685            .flatten()
1686        {
1687            let message = match rejection {
1688                ReservedHelloRejection::Exact { module_id } => format!(
1689                    "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
1690                ),
1691                ReservedHelloRejection::Prefix {
1692                    prefix,
1693                    owner_module_id,
1694                } => format!(
1695                    "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
1696                    hello.manifest.module_id
1697                ),
1698            };
1699            return Ok(vec![control_error_frame(
1700                &frame,
1701                "reserved_module",
1702                message,
1703            )?]);
1704        }
1705
1706        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
1707            &hello.manifest.module_id,
1708            hello.manifest.capabilities.as_ref(),
1709        );
1710        if let Some(refusal) = reserved_capability_refusals.first() {
1711            let capability = refusal.capability.clone();
1712            let bound_module = refusal.claimants[0].clone();
1713            log_duplicate_claim_events(reserved_capability_refusals);
1714            return Ok(vec![control_error_frame(
1715                &frame,
1716                "reserved_capability",
1717                format!(
1718                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
1719                    capability, bound_module, hello.manifest.module_id
1720                ),
1721            )?]);
1722        }
1723
1724        // A connection that already opened client routes must not also register as
1725        // a module: cleanup would then release only one side and leak the other.
1726        if self
1727            .forwarding
1728            .connection_has_client_routes(connection_id)
1729            .map_err(RouterError::Forwarding)?
1730        {
1731            return Ok(vec![control_error_frame(
1732                &frame,
1733                "invalid_hello",
1734                "connection has open client routes and cannot also register as a module",
1735            )?]);
1736        }
1737
1738        let control_ops = effective_module_control_ops(hello.control_ops);
1739        // Built before anything is registered so an encoding failure leaves no
1740        // registry or forwarding state behind.
1741        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
1742        if swap_candidate {
1743            return self.register_swap_candidate(
1744                connection_id,
1745                sink,
1746                &frame,
1747                hello.manifest,
1748                negotiated_ver,
1749                control_ops,
1750                hello_ack,
1751            );
1752        }
1753        let registration = match self.registry.register_with_control_ops(
1754            hello.manifest,
1755            negotiated_ver,
1756            connection_id,
1757            control_ops,
1758        ) {
1759            Ok(registration) => registration,
1760            Err(RegistryError::DuplicateModuleId { module_id }) => {
1761                return Ok(vec![control_error_frame(
1762                    &frame,
1763                    "duplicate_module_id",
1764                    format!(
1765                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
1766                    ),
1767                )?])
1768            }
1769            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
1770                return Ok(vec![control_error_frame(
1771                    &frame,
1772                    "invalid_module_id",
1773                    err.to_string(),
1774                )?])
1775            }
1776            Err(err) => {
1777                return Ok(vec![control_error_frame(
1778                    &frame,
1779                    "registry_error",
1780                    err.to_string(),
1781                )?])
1782            }
1783        };
1784
1785        let reply = if let Some(sink) = sink {
1786            // The forwarding table's module store is also the daemon-to-module
1787            // control-RPC lane, so every HELLO gets a live endpoint even when the
1788            // manifest has no routable provider role. Non-routable modules still
1789            // cannot receive route.bind in production: `handle_route_open` checks
1790            // the registry manifest with `target_has_required_role` before the
1791            // only production call to `begin_route_bind_relay_for` below that
1792            // route.open path. The remaining direct relay callers are unit tests
1793            // and benchmark harnesses that construct forwarding state explicitly.
1794            //
1795            // The HELLO_ACK is queued by the forwarding table itself, before the
1796            // endpoint becomes visible, and is NOT returned as a reply. A module
1797            // reads HELLO_ACK first and exits on anything else; a reply is only
1798            // written after this handler returns, by which time a route.open on
1799            // another connection could already have queued a route.bind request
1800            // for this module ahead of it.
1801            let concurrency = manifest_concurrency(&registration.manifest);
1802            if let Err(err) = self.forwarding.register_module_connection_acked(
1803                connection_id,
1804                registration.manifest.module_id.clone(),
1805                negotiated_ver,
1806                concurrency,
1807                sink,
1808                hello_ack,
1809            ) {
1810                // Forwarding registration failed, so there is no forwarding
1811                // state to tear down. Remove the registry entry and signal the
1812                // release watch directly.
1813                if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
1814                    crate::supervise::notify_registration_release();
1815                }
1816                return Ok(vec![control_error_frame(
1817                    &frame,
1818                    forwarding_error_code(&err),
1819                    err.to_string(),
1820                )?]);
1821            }
1822            Vec::new()
1823        } else {
1824            // No sink means no forwarding endpoint, so nothing can be routed
1825            // ahead of the ack; it goes out as the reply.
1826            vec![hello_ack]
1827        };
1828
1829        // Exposure over assumption: Concurrency's serde default is pinned to the
1830        // pre-field behavior (ModuleManaged), so a management surface that is
1831        // genuinely Serial and just never declared it inherits concurrent
1832        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
1833        // turns "no module has been bitten yet" into the checkable claim "no
1834        // module is exposed" -- one read of the boot log instead of a fleet
1835        // audit. Detected from the raw HELLO bytes because the serde default
1836        // deliberately erases the absent/declared distinction from the type.
1837        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
1838            info!(
1839                module_id = %registration.manifest.module_id,
1840                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
1841            );
1842        }
1843
1844        self.apply_registration_capabilities(&registration);
1845
1846        info!(
1847            module_id = %registration.manifest.module_id,
1848            module_version = %registration.manifest.module_version,
1849            negotiated_ver,
1850            routable_provider = manifest_provides_routable_role(&registration.manifest),
1851            connection_id = connection_id.get(),
1852            "module registered"
1853        );
1854
1855        Ok(reply)
1856    }
1857
1858    /// Register a HELLO the swap gate admitted into the candidate slot of the
1859    /// registry and of forwarding, where it is reachable over its own
1860    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
1861    /// nothing routes to it until the supervisor cuts over.
1862    ///
1863    /// Registry first, then forwarding, the same order as an ordinary HELLO;
1864    /// a forwarding failure removes the registry entry again. The capability
1865    /// census is not run: it describes routable modules, and this one is not
1866    /// routable until promotion.
1867    #[allow(clippy::too_many_arguments)]
1868    fn register_swap_candidate(
1869        &self,
1870        connection_id: ConnectionId,
1871        sink: Option<crate::FrameSink>,
1872        frame: &Frame,
1873        manifest: ModuleManifest,
1874        negotiated_ver: u8,
1875        control_ops: Vec<String>,
1876        hello_ack: Frame,
1877    ) -> Result<Vec<Frame>, RouterError> {
1878        let module_id = manifest.module_id.clone();
1879        let registration = match self.registry.register_candidate_with_control_ops(
1880            manifest,
1881            negotiated_ver,
1882            connection_id,
1883            control_ops,
1884        ) {
1885            Ok(registration) => registration,
1886            Err(RegistryError::DuplicateModuleId { module_id }) => {
1887                return Ok(vec![control_error_frame(
1888                    frame,
1889                    "duplicate_module_id",
1890                    format!(
1891                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
1892                    ),
1893                )?])
1894            }
1895            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
1896                return Ok(vec![control_error_frame(
1897                    frame,
1898                    "invalid_module_id",
1899                    err.to_string(),
1900                )?])
1901            }
1902            Err(err) => {
1903                return Ok(vec![control_error_frame(
1904                    frame,
1905                    "registry_error",
1906                    err.to_string(),
1907                )?])
1908            }
1909        };
1910        let reply = if let Some(sink) = sink {
1911            // Same ordering as an ordinary HELLO: the forwarding table queues
1912            // the HELLO_ACK before the candidate endpoint is inserted, because
1913            // a module exits if its first frame after HELLO is anything else.
1914            let concurrency = manifest_concurrency(&registration.manifest);
1915            if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
1916                connection_id,
1917                module_id.clone(),
1918                negotiated_ver,
1919                concurrency,
1920                sink,
1921                hello_ack,
1922            ) {
1923                if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
1924                    crate::supervise::notify_registration_release();
1925                }
1926                return Ok(vec![control_error_frame(
1927                    frame,
1928                    forwarding_error_code(&err),
1929                    err.to_string(),
1930                )?]);
1931            }
1932            Vec::new()
1933        } else {
1934            vec![hello_ack]
1935        };
1936        self.supervisor.mark_swap_candidate_admitted(&module_id);
1937        info!(
1938            module_id = %module_id,
1939            module_version = %registration.manifest.module_version,
1940            negotiated_ver,
1941            ready = registration.ready,
1942            connection_id = connection_id.get(),
1943            "swap candidate registered; not routable until cutover"
1944        );
1945        Ok(reply)
1946    }
1947
1948    fn build_hello_ack(
1949        &self,
1950        frame: &Frame,
1951        negotiated_ver: u8,
1952        module_id: &str,
1953    ) -> Result<Frame, RouterError> {
1954        let ack = ModuleHelloAckBody {
1955            negotiated_ver,
1956            subc_ops: module_subc_ops(),
1957            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
1958            storage: self
1959                .storage_config
1960                .as_ref()
1961                .map(|cfg| cfg.descriptor_for(module_id)),
1962            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
1963        };
1964        let body = serde_json::to_vec(&ack).map_err(|err| {
1965            RouterError::backend(
1966                0,
1967                frame.header.corr,
1968                format!("failed to encode HELLO_ACK: {err}"),
1969            )
1970        })?;
1971
1972        Frame::build_with_version(
1973            negotiated_ver,
1974            FrameType::HelloAck,
1975            control_flags(),
1976            0,
1977            0,
1978            frame.header.corr,
1979            body,
1980        )
1981        .map_err(RouterError::FrameBuild)
1982    }
1983
1984    async fn handle_client_control_request(
1985        &self,
1986        ctx: &RouteCtx,
1987        frame: Frame,
1988        request: ClientControlRequest,
1989    ) -> Result<Vec<Frame>, RouterError> {
1990        match request {
1991            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
1992            ClientControlRequest::CatalogList { module_id } => {
1993                self.handle_catalog_list(frame, module_id)
1994            }
1995            ClientControlRequest::RouteOpen {
1996                target,
1997                identity,
1998                consumer_identity,
1999                consumer_capabilities,
2000                admission_facts,
2001            } => {
2002                self.handle_route_open(
2003                    ctx,
2004                    frame,
2005                    RouteOpenRequest {
2006                        target,
2007                        identity,
2008                        consumer_identity,
2009                        consumer_capabilities,
2010                        admission_facts,
2011                    },
2012                )
2013                .await
2014            }
2015            ClientControlRequest::RoutePoll {
2016                route_channel,
2017                route_epoch,
2018                kind,
2019            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2020            ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame),
2021            ClientControlRequest::SupervisorSpawnSnapshot {} => {
2022                self.handle_supervisor_spawn_snapshot(frame)
2023            }
2024            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2025                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2026            }
2027            ClientControlRequest::SupervisorRestart {
2028                module_id,
2029                drain_timeout_ms,
2030            } => {
2031                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2032                    .await
2033            }
2034            ClientControlRequest::SupervisorSwap {
2035                module_id,
2036                ready_timeout_ms,
2037            } => {
2038                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2039                    .await
2040            }
2041            ClientControlRequest::SupervisorReload { module_id } => {
2042                self.handle_supervisor_reload(frame, module_id).await
2043            }
2044            ClientControlRequest::SupervisorRescan { preview } => {
2045                self.handle_supervisor_rescan(frame, preview).await
2046            }
2047            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2048                self.handle_supervisor_release_reserved(frame, module_id)
2049                    .await
2050            }
2051            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2052                self.handle_supervisor_set_enabled(frame, module_id, enabled)
2053                    .await
2054            }
2055            ClientControlRequest::SupervisorHealthProbe { module_id } => {
2056                self.handle_supervisor_health_probe(frame, module_id).await
2057            }
2058            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2059            ClientControlRequest::SupervisorRoutes { module_id } => {
2060                self.handle_supervisor_routes(frame, module_id)
2061            }
2062            ClientControlRequest::SupervisorProvenance { module_id } => {
2063                self.handle_supervisor_provenance(frame, module_id).await
2064            }
2065            ClientControlRequest::SupervisorStderrTail {
2066                module_id,
2067                max_lines,
2068                max_bytes,
2069            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2070            ClientControlRequest::SupervisorTerminals { module_id } => {
2071                self.handle_supervisor_terminals(frame, module_id).await
2072            }
2073        }
2074    }
2075
2076    fn handle_module_control_request(
2077        &self,
2078        connection_id: ConnectionId,
2079        frame: Frame,
2080        request: ModuleControlRequestFromModule,
2081    ) -> Result<Vec<Frame>, RouterError> {
2082        match request {
2083            ModuleControlRequestFromModule::CatalogUpdate {
2084                provides,
2085                capabilities,
2086                ready,
2087            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2088            ModuleControlRequestFromModule::LiveRoots {} => {
2089                let registered = self
2090                    .registry
2091                    .get_module_by_connection(connection_id)
2092                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2093                let Some(registration) = registered else {
2094                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2095                };
2096                let response = self
2097                    .forwarding
2098                    .live_roots(&registration.manifest.module_id)
2099                    .map_err(RouterError::Forwarding)?;
2100                Ok(vec![control_response_body_frame(
2101                    &frame,
2102                    &response,
2103                    "ModuleControlResponseToModule::LiveRoots",
2104                )?])
2105            }
2106        }
2107    }
2108
2109    fn handle_catalog_update(
2110        &self,
2111        connection_id: ConnectionId,
2112        frame: Frame,
2113        provides: Vec<ProviderRole>,
2114        capabilities: Option<CapabilityDeclarations>,
2115        ready: Option<bool>,
2116    ) -> Result<Vec<Frame>, RouterError> {
2117        self.refresh_capability_requirements();
2118        let Some(registration) = self
2119            .registry
2120            .get_module_by_connection(connection_id)
2121            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2122        else {
2123            return Ok(vec![control_error_frame(
2124                &frame,
2125                "not_registered",
2126                "catalog.update requires an active module registration owned by this connection",
2127            )?]);
2128        };
2129
2130        if let Some(message) =
2131            catalog_update_frozen_field_message(&registration.manifest, &provides)
2132        {
2133            return Ok(vec![control_error_frame(
2134                &frame,
2135                "catalog_update_frozen_field",
2136                message,
2137            )?]);
2138        }
2139
2140        let mut candidate = registration.manifest.clone();
2141        candidate.provides = provides.clone();
2142        candidate.capabilities = capabilities
2143            .clone()
2144            .or_else(|| registration.manifest.capabilities.clone());
2145        if let Err(err) = candidate.validate_capability_grammar() {
2146            return Ok(vec![control_error_frame(
2147                &frame,
2148                "invalid_capability_grammar",
2149                err.to_string(),
2150            )?]);
2151        }
2152
2153        let updated = self
2154            .registry
2155            .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
2156            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2157        if updated.is_none() {
2158            return Ok(vec![control_error_frame(
2159                &frame,
2160                "not_registered",
2161                "catalog.update requires an active module registration owned by this connection",
2162            )?]);
2163        }
2164        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
2165            log_duplicate_claim_events(
2166                self.capability_evaluator
2167                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
2168            );
2169        }
2170        if capability_census_trigger(
2171            registration.manifest.capabilities.as_ref(),
2172            updated
2173                .as_ref()
2174                .and_then(|entry| entry.manifest.capabilities.as_ref()),
2175        ) {
2176            self.enforce_capability_denies();
2177        }
2178        self.refresh_capability_requirements();
2179
2180        let response = ModuleControlResponseToModule::CatalogUpdate {};
2181        control_response_body_frame(
2182            &frame,
2183            &response,
2184            "ModuleControlResponseToModule::CatalogUpdate",
2185        )
2186        .map(|frame| vec![frame])
2187    }
2188
2189    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2190        self.refresh_capability_requirements();
2191        // A bare connection count is ambiguous between many clients holding a
2192        // route each and one client accumulating hundreds, so publish the
2193        // concentration alongside it. Route state is best-effort here: a
2194        // diagnostic endpoint must still answer if the forwarding lock is
2195        // contended.
2196        let mut counters = self.counters.snapshot();
2197        if let (Ok((connections_with_routes, max)), Some(obj)) = (
2198            self.forwarding.client_route_concentration(),
2199            counters.as_object_mut(),
2200        ) {
2201            obj.insert(
2202                "client_connections_with_routes".into(),
2203                connections_with_routes.into(),
2204            );
2205            obj.insert("max_routes_on_one_connection".into(), max.into());
2206        }
2207        // A module that is being fast-refused and a module that is fine look
2208        // identical from a client that retries and succeeds, so name the open
2209        // breakers here. This rides the existing free-form counters object
2210        // rather than a new wire field, so no sibling that deserializes
2211        // `ServerDescribe` has to be rebuilt to keep reading it.
2212        if let (Some(open_breakers), Some(obj)) = (
2213            self.route_bind_breakers.open_snapshot(),
2214            counters.as_object_mut(),
2215        ) {
2216            obj.insert("route_bind_breakers_open".into(), open_breakers);
2217        }
2218        let response = ClientControlResponse::ServerDescribe {
2219            protocol_ver: PROTOCOL_VERSION,
2220            subc_ops: subc_ops(),
2221            capabilities: self.subc_capabilities.as_ref().to_vec(),
2222            connected_clients: self.connected_clients.count(),
2223            counters: Some(counters),
2224            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
2225            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
2226            capability_requirements: self.capability_requirement_statuses(),
2227            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2228        };
2229        Ok(vec![control_response_body_frame(
2230            &frame,
2231            &response,
2232            "ClientControlResponse::ServerDescribe",
2233        )?])
2234    }
2235
2236    fn handle_catalog_list(
2237        &self,
2238        frame: Frame,
2239        module_id: Option<String>,
2240    ) -> Result<Vec<Frame>, RouterError> {
2241        let (generation, modules) = self.registry.list_modules().map_err(|err| {
2242            RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
2243        })?;
2244        let entries = modules
2245            .into_iter()
2246            .filter(|registration| {
2247                module_id
2248                    .as_deref()
2249                    .map(|wanted| registration.manifest.module_id == wanted)
2250                    .unwrap_or(true)
2251            })
2252            .map(|registration| {
2253                let not_ready = self.not_ready_reason(&registration);
2254                let roles = registration.manifest.provides;
2255                CatalogEntry {
2256                    module_id: registration.manifest.module_id,
2257                    ready: not_ready.is_none(),
2258                    not_ready,
2259                    module_version: Some(registration.manifest.module_version),
2260                    roles,
2261                    control_ops: registration.control_ops,
2262                    capabilities: registration.manifest.capabilities,
2263                    self_signals: registration.manifest.self_signals,
2264                }
2265            })
2266            .collect();
2267        let response = ClientControlResponse::CatalogList {
2268            generation,
2269            modules: entries,
2270            subc_ops: subc_ops(),
2271        };
2272        Ok(vec![control_response_body_frame(
2273            &frame,
2274            &response,
2275            "ClientControlResponse::CatalogList",
2276        )?])
2277    }
2278
2279    fn route_open_principal(
2280        &self,
2281        frame: &Frame,
2282        consumer_identity: Option<ConsumerIdentity>,
2283    ) -> Result<Result<Principal, Frame>, RouterError> {
2284        let Some(consumer_identity) = consumer_identity else {
2285            return Ok(Ok(Principal::Direct));
2286        };
2287
2288        if self.supervisor.spawned_consumer_authorized(
2289            &consumer_identity.module_id,
2290            &consumer_identity.launch_nonce,
2291        ) {
2292            return Ok(Ok(Principal::Reserved {
2293                module_id: consumer_identity.module_id,
2294            }));
2295        }
2296
2297        Ok(Err(control_error_frame(
2298            frame,
2299            "bad_consumer_identity",
2300            format!(
2301                "consumer_identity for module_id '{}' did not match a supervised launch nonce",
2302                consumer_identity.module_id
2303            ),
2304        )?))
2305    }
2306
2307    /// Every refusal of a `route.open` goes through here so the daemon can
2308    /// attest which code it sent: without the event, a client's "the daemon
2309    /// refused me" and the daemon's own view could only be reconciled by
2310    /// argument. Malformed input (`invalid_project_root`) does not come here;
2311    /// rejecting a request that was never a valid open is not a refusal of one.
2312    fn route_open_refusal_frame(
2313        &self,
2314        ctx: &RouteCtx,
2315        frame: &Frame,
2316        module_id: &str,
2317        reason: &'static str,
2318        code: &'static str,
2319        message: impl Into<String>,
2320    ) -> Result<Frame, RouterError> {
2321        self.observe_route_open_refusal(ctx, module_id, reason, code);
2322        control_error_frame(frame, code, message.into())
2323    }
2324
2325    /// Refuse a `route.open` because the target module's bind-relay breaker is
2326    /// open, without attempting the relay.
2327    ///
2328    /// The wire code is `module_timeout`, which is the truth (the module has
2329    /// not been answering binds) and which both SDKs already classify as
2330    /// retryable with capped backoff. Reusing it is what keeps this change out
2331    /// of both SDKs; the daemon-side distinction lives in the counter key
2332    /// instead.
2333    ///
2334    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
2335    /// While a breaker is open this fires on every open to that module, and the
2336    /// stall written up in `docs/designs/route-open-head-of-line.md` already
2337    /// produced 261 lines about a single module inside 3000 lines of daemon
2338    /// log. The rare transitions are logged at warn/info instead and the volume
2339    /// is carried by the counter, so the evidence survives without the flood.
2340    /// The debug line keeps a per-refusal record reachable for whoever turns
2341    /// the level up.
2342    fn route_open_breaker_refusal_frame(
2343        &self,
2344        ctx: &RouteCtx,
2345        frame: &Frame,
2346        module_id: &str,
2347        consecutive_timeouts: u32,
2348        retry_in: Duration,
2349        probe_in_flight: bool,
2350    ) -> Result<Frame, RouterError> {
2351        self.counters
2352            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
2353        debug!(
2354            target: "control",
2355            code = "module_timeout",
2356            module_id = ?module_id,
2357            connection_id = ctx.connection_id.get(),
2358            consecutive_timeouts,
2359            retry_in_ms = retry_in.as_millis() as u64,
2360            probe_in_flight,
2361            "route.open refused by open bind-relay breaker"
2362        );
2363        let detail = if probe_in_flight {
2364            "one probe bind is already in flight; retry once it settles".to_string()
2365        } else {
2366            format!("not relaying for another {retry_in:?}")
2367        };
2368        control_error_frame(
2369            frame,
2370            "module_timeout",
2371            format!(
2372                "module_id '{module_id}' failed {consecutive_timeouts} consecutive route.bind \
2373                 relays; {detail}"
2374            ),
2375        )
2376    }
2377
2378    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
2379    /// requester's bytes (an unknown target is whatever the client sent) and
2380    /// is Debug-formatted so control characters land in the log escaped
2381    /// rather than as terminal sequences for whoever tails it.
2382    ///
2383    /// `reason` names the check that refused, because one wire code has
2384    /// several senders: after a module registers, `target_unavailable` can
2385    /// come from a missing role, an inactive registration, a supervisor that
2386    /// has not marked the process live, a missing forwarding connection, or a
2387    /// failed relay, and a log that records only the code cannot say which of
2388    /// them fired. It is a static, daemon-chosen label per branch, so it is
2389    /// safe to print plainly and stays a closed set.
2390    fn observe_route_open_refusal(
2391        &self,
2392        ctx: &RouteCtx,
2393        module_id: &str,
2394        reason: &'static str,
2395        code: &'static str,
2396    ) {
2397        self.counters.increment_route_open_refused(code);
2398        info!(
2399            target: "control",
2400            code,
2401            reason,
2402            module_id = ?module_id,
2403            connection_id = ctx.connection_id.get(),
2404            "route.open refused"
2405        );
2406    }
2407
2408    /// Record an ACCEPTED route.open.
2409    ///
2410    /// Refusals have been logged and counted since the attestation work; accepts
2411    /// were invisible, so the daemon knew every principal it stamped and wrote
2412    /// none of them down. The party that attests the identity was the only party
2413    /// not recording it, which left a credential vault unable to name the sender
2414    /// of a call that reached it (claustrum #43) and left the launch-nonce
2415    /// concurrency question unanswerable from the outside.
2416    ///
2417    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
2418    /// `code`/`module_id`/`connection_id` returns both directions of the same
2419    /// decision rather than two shapes a reader has to join by hand.
2420    ///
2421    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
2422    /// and the difference carries information rather than being an
2423    /// inconsistency. This line is only reachable after a successful bind to a
2424    /// REGISTERED module, so the value has already passed HELLO validation
2425    /// including the path-hazard refusal and cannot contain control bytes. A
2426    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
2427    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
2428    ///
2429    /// Bare is also what every other daemon line already emits (`module
2430    /// registered`, `configured module supervised`). Shipping `?module_id` here
2431    /// made this instrument the only one in the file whose ids did not answer
2432    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
2433    /// line whose whole purpose is being grepped beside its sibling.
2434    ///
2435    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
2436    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
2437    /// forwards every field type through it and a bare `&str` and a `?`-escaped
2438    /// one are recorded identically. A test written against that harness passes
2439    /// either way -- I wrote one, measured it, and deleted it rather than ship a
2440    /// green assertion that cannot fail. The same limit applies to the escaping
2441    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
2442    /// it reads as a guard on the Debug escaping and cannot detect its removal.
2443    /// Fencing either needs the real formatter, not the capture layer.
2444    ///
2445    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
2446    /// unix-socket options and subc is loopback TCP, so there is no peer identity
2447    /// to record. The ephemeral port would decay within minutes and answer only a
2448    /// live question. The identity question is instead answered by counting
2449    /// distinct live connections presenting one module's `consumer_identity` --
2450    /// "is anyone else holding this secret" rather than "is this the right
2451    /// process".
2452    fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
2453        self.counters.increment_route_open_accepted(principal);
2454        info!(
2455            target: "control",
2456            principal,
2457            module_id,
2458            connection_id = ctx.connection_id.get(),
2459            "route.open accepted"
2460        );
2461    }
2462
2463    fn supervised_absent_route_open_refusal_frame(
2464        &self,
2465        ctx: &RouteCtx,
2466        frame: &Frame,
2467        module_id: &str,
2468        code: &'static str,
2469        status: &crate::supervise::ModuleStatus,
2470    ) -> Result<Frame, RouterError> {
2471        self.counters.increment_route_open_refused(code);
2472        info!(
2473            target: "control",
2474            code,
2475            reason = "supervised_not_registered",
2476            module_id = ?module_id,
2477            connection_id = ctx.connection_id.get(),
2478            state = %status.state,
2479            enabled = status.enabled,
2480            live = status.live,
2481            "route.open refused"
2482        );
2483        control_error_frame(
2484            frame,
2485            code,
2486            format!(
2487                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
2488                status.state, status.enabled, status.live
2489            ),
2490        )
2491    }
2492
2493    async fn handle_route_open(
2494        &self,
2495        ctx: &RouteCtx,
2496        frame: Frame,
2497        request: RouteOpenRequest,
2498    ) -> Result<Vec<Frame>, RouterError> {
2499        let RouteOpenRequest {
2500            target,
2501            mut identity,
2502            consumer_identity,
2503            consumer_capabilities,
2504            admission_facts,
2505        } = request;
2506        let target_module_id = target_module_id(&target).to_string();
2507        debug!(
2508            connection_id = ctx.connection_id.get(),
2509            corr = frame.header.corr,
2510            module_id = %target_module_id,
2511            "handling route.open"
2512        );
2513
2514        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
2515        // opposite. Below, a caller learns whether a module is unregistered,
2516        // supervised-but-down (with state/enabled/live), or registered without the
2517        // requested role. Elsewhere that is an enumeration leak: a probe learning
2518        // the shape of a fleet it cannot otherwise see.
2519        //
2520        // It is not one here, and the reason is the ACCESS MODEL rather than
2521        // anything about these errors. Reaching route.open requires the
2522        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
2523        // connection file, so any caller who completes it already runs as this
2524        // user -- and can read subc.jsonc for the module list and `ck module
2525        // status` for live state. The reply discloses nothing the caller cannot
2526        // read more easily from disk, while the precision is load-bearing:
2527        // `unknown_module` is retryable and a missing role is not.
2528        //
2529        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
2530        // remote transport, a sandboxed caller, a shared-host mode -- THAT
2531        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
2532        let Some(registration) = self
2533            .registry
2534            .get_module(&target_module_id)
2535            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2536        else {
2537            if let Some((status, warming)) =
2538                self.supervisor_status(&target_module_id, frame.header.corr)?
2539            {
2540                // BEFORE the two availability codes below, because for a module
2541                // that speaks no subc wire both of them are false comfort: they
2542                // say "not right now" and are retried, and this module will
2543                // never register no matter how long the caller waits. The
2544                // absence here is the declaration being honoured, not a module
2545                // that is late.
2546                if status.protocol == ModuleProtocol::None {
2547                    return Ok(vec![self.route_open_refusal_frame(
2548                        ctx,
2549                        &frame,
2550                        &target_module_id,
2551                        "protocol_none",
2552                        error_codes::MODULE_NO_PROTOCOL,
2553                        format!(
2554                            "module_id '{target_module_id}' is declared protocol: none; \
2555                             it speaks no subc wire and serves no routes"
2556                        ),
2557                    )?]);
2558                }
2559                let code = if warming {
2560                    "module_warming"
2561                } else {
2562                    "target_unavailable"
2563                };
2564                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
2565                    ctx,
2566                    &frame,
2567                    &target_module_id,
2568                    code,
2569                    &status,
2570                )?]);
2571            }
2572            if let Some(removed_ago_ms) =
2573                self.supervisor.removal_tombstone_age_ms(&target_module_id)
2574            {
2575                return Ok(vec![self.route_open_refusal_frame(
2576                    ctx,
2577                    &frame,
2578                    &target_module_id,
2579                    "removed",
2580                    error_codes::MODULE_REMOVED,
2581                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
2582                )?]);
2583            }
2584            return Ok(vec![self.route_open_refusal_frame(
2585                ctx,
2586                &frame,
2587                &target_module_id,
2588                "not_registered",
2589                error_codes::UNKNOWN_MODULE,
2590                format!("module_id '{target_module_id}' is not registered"),
2591            )?]);
2592        };
2593
2594        // Best-effort only: registry readiness and forwarding reservation use
2595        // different locks, so a module can flip readiness between this read and
2596        // the relay. Modules must still tolerate an `on_bind` while not ready.
2597        if !registration.ready {
2598            self.counters
2599                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
2600            info!(
2601                target: "control",
2602                code = error_codes::MODULE_WARMING,
2603                module_id = ?target_module_id,
2604                connection_id = ctx.connection_id.get(),
2605                reason = "declared_not_ready",
2606                "route.open refused"
2607            );
2608            return Ok(vec![control_error_body_frame(
2609                &frame,
2610                ErrorBody {
2611                    code: error_codes::MODULE_WARMING.to_string(),
2612                    message: format!(
2613                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
2614                    ),
2615                    detail: Some(serde_json::json!({
2616                        "reason": "declared_not_ready"
2617                    })),
2618                },
2619            )?]);
2620        }
2621
2622        // Effective readiness, second half: a module that declares a capability
2623        // `need: required` is not routable while that capability has no
2624        // registered provider. It is enforced HERE, as a retryable routing
2625        // refusal, and deliberately not as spawn ordering or a boot block. The
2626        // module is still started and registered and can make its own calls;
2627        // spawn ordering is a promise that cannot be kept once a provider
2628        // crashes at runtime, and refusing to boot would stop the whole
2629        // machine, including the tools needed to fix its configuration.
2630        //
2631        // "Provided" is the evaluator's verdict, which counts a provider as
2632        // soon as it has REGISTERED, not once it is ready. Two modules that
2633        // require each other's capabilities are therefore both routable once
2634        // both register; counting readiness instead would deadlock them.
2635        //
2636        // Only new opens are refused. Routes already bound when a provider
2637        // goes away stay bound: nothing here tears them down, and the module
2638        // answers them as it can. Like the readiness read above this is
2639        // best-effort against a provider registering or leaving concurrently.
2640        if let Some(capability) = self.first_unprovided_required_capability(&registration) {
2641            self.counters
2642                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
2643            info!(
2644                target: "control",
2645                code = error_codes::MODULE_WARMING,
2646                module_id = ?target_module_id,
2647                connection_id = ctx.connection_id.get(),
2648                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
2649                capability = %capability,
2650                "route.open refused"
2651            );
2652            return Ok(vec![control_error_body_frame(
2653                &frame,
2654                ErrorBody {
2655                    code: error_codes::MODULE_WARMING.to_string(),
2656                    message: format!(
2657                        "module_id '{target_module_id}' requires capability '{capability}', \
2658                         which no registered module provides; retry"
2659                    ),
2660                    detail: Some(serde_json::json!({
2661                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
2662                        "capability": capability,
2663                    })),
2664                },
2665            )?]);
2666        }
2667
2668        if !target_has_required_role(&target, &registration.manifest.provides) {
2669            return Ok(vec![self.route_open_refusal_frame(
2670                ctx,
2671                &frame,
2672                &target_module_id,
2673                "role_not_provided",
2674                "target_unavailable",
2675                format!("module_id '{target_module_id}' does not provide the requested target"),
2676            )?]);
2677        }
2678
2679        if registration.state != ChannelState::Active {
2680            return Ok(vec![self.route_open_refusal_frame(
2681                ctx,
2682                &frame,
2683                &target_module_id,
2684                "registration_not_active",
2685                "target_unavailable",
2686                format!("module_id '{target_module_id}' is not active"),
2687            )?]);
2688        }
2689
2690        if self
2691            .forwarding
2692            .module_is_draining(&target_module_id)
2693            .map_err(RouterError::Forwarding)?
2694        {
2695            return Ok(vec![self.route_open_refusal_frame(
2696                ctx,
2697                &frame,
2698                &target_module_id,
2699                "reloading",
2700                "module_reloading",
2701                format!("module_id '{target_module_id}' is reloading"),
2702            )?]);
2703        }
2704
2705        if self
2706            .process_liveness
2707            .as_ref()
2708            .and_then(|process_liveness| process_liveness.process_live(&target_module_id))
2709            == Some(false)
2710        {
2711            return Ok(vec![self.route_open_refusal_frame(
2712                ctx,
2713                &frame,
2714                &target_module_id,
2715                "supervisor_not_live",
2716                "target_unavailable",
2717                format!("module_id '{target_module_id}' is not live"),
2718            )?]);
2719        }
2720
2721        if !self
2722            .forwarding
2723            .has_live_module_connection(&target_module_id)
2724            .map_err(RouterError::Forwarding)?
2725        {
2726            return Ok(vec![self.route_open_refusal_frame(
2727                ctx,
2728                &frame,
2729                &target_module_id,
2730                "no_forwarding_connection",
2731                "target_unavailable",
2732                format!("module_id '{target_module_id}' has no live forwarding connection"),
2733            )?]);
2734        }
2735
2736        if let Some(error) =
2737            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
2738        {
2739            self.observe_route_open_refusal(
2740                ctx,
2741                &target_module_id,
2742                "op_not_allowed",
2743                "op_not_allowed",
2744            );
2745            return Ok(vec![error]);
2746        }
2747
2748        let principal = match self.route_open_principal(&frame, consumer_identity)? {
2749            Ok(principal) => principal,
2750            Err(error) => {
2751                self.observe_route_open_refusal(
2752                    ctx,
2753                    &target_module_id,
2754                    "bad_consumer_identity",
2755                    "bad_consumer_identity",
2756                );
2757                return Ok(vec![error]);
2758            }
2759        };
2760
2761        // This is attested, control-plane policy for supervised module origins.
2762        // Keep it before route reservation and out of the opaque forwarding hot
2763        // path: data frames must never acquire a per-frame capability check.
2764        if let Principal::Reserved {
2765            module_id: opening_module_id,
2766        } = &principal
2767        {
2768            if let Some(opening_registration) = self
2769                .registry
2770                .get_module(opening_module_id)
2771                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2772            {
2773                if let Some(capability) =
2774                    denied_capability(&opening_registration.manifest, &registration.manifest)
2775                {
2776                    warn!(
2777                        opening_module_id,
2778                        target_module_id,
2779                        capability,
2780                        "refusing route.open because an attested capability deny edge matches"
2781                    );
2782                    return Ok(vec![self.route_open_refusal_frame(
2783                        ctx,
2784                        &frame,
2785                        &target_module_id,
2786                        "capability_deny_edge",
2787                        "capability_forbidden",
2788                        format!(
2789                            "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
2790                        ),
2791                    )?]);
2792                }
2793            }
2794        }
2795
2796        if admission_facts.is_some() {
2797            let carrier_matches = matches!(
2798                &principal,
2799                Principal::Reserved { module_id }
2800                    if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
2801            );
2802            if !carrier_matches {
2803                return Ok(vec![self.route_open_refusal_frame(
2804                    ctx,
2805                    &frame,
2806                    &target_module_id,
2807                    "admission_facts_carrier_not_permitted",
2808                    "admission_facts_not_permitted",
2809                    "admission facts may only be carried by the configured reserved module",
2810                )?]);
2811            }
2812
2813            let target_allowed = self
2814                .admission_facts_targets
2815                .as_ref()
2816                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
2817            if !target_allowed {
2818                return Ok(vec![self.route_open_refusal_frame(
2819                    ctx,
2820                    &frame,
2821                    &target_module_id,
2822                    "admission_facts_target_not_listed",
2823                    "admission_facts_target_not_allowed",
2824                    format!(
2825                        "admission facts are not permitted for target module_id '{target_module_id}'"
2826                    ),
2827                )?]);
2828            }
2829
2830            // Keep the value opaque to subc. The downstream admission validator owns
2831            // schema and semantic checks; this daemon only enforces carrier authority
2832            // and the configured destination allowlist.
2833        }
2834
2835        // Bind admits a root that no longer exists on disk, because refusing here
2836        // closes the only exit from a paused run: cancel needs a bound route, and a
2837        // renamed or reclaimed directory makes that route unopenable forever. The
2838        // run itself is intact and still addressable by its recorded identity.
2839        //
2840        // This does NOT relax the rule the strict constructor protects. That rule is
2841        // that no root is ever aliased into NEW durable state -- a missing component
2842        // can reappear as a symlink elsewhere, which would move the identity and
2843        // split a session's history across two of them. The engine now refuses the
2844        // two operations that create such state (send and import) at admission,
2845        // which is a narrower way to hold the same invariant: reads and terminations
2846        // are admitted, writes are not. That refusal had to ship before this line
2847        // changed, or there is an interval where a send commits under a provisional
2848        // identity -- the exact failure the original policy existed to prevent.
2849        //
2850        // Resolution follows realpath rather than lexical cleanup: the longest
2851        // existing ancestor is canonicalized and the missing tail re-appended, so a
2852        // live root is unchanged and a vanished leaf keeps the identity it was
2853        // admitted under. Lexical cleanup would mint a DIFFERENT identity for the
2854        // same caller the moment the directory vanished, which strands the run more
2855        // quietly than refusing it.
2856        let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
2857            Ok(project_root) => project_root,
2858            Err(err) => {
2859                return Ok(vec![control_error_frame(
2860                    &frame,
2861                    "invalid_project_root",
2862                    err.to_string(),
2863                )?])
2864            }
2865        };
2866        identity.project_root = project_root.as_path().to_path_buf();
2867
2868        // Last gate before any relay work, and deliberately after the cheap
2869        // registry and availability checks above: those name a more precise
2870        // condition (unknown, removed, reloading) and a caller is better served
2871        // by the precise code than by this one.
2872        //
2873        // Everything below this point costs an egress permit, a reserved handle
2874        // pair and, if the module does not answer, the whole relay budget. The
2875        // reader no longer waits for that budget, so cap each target explicitly;
2876        // serial dispatch used to provide the accidental cap of one relay per
2877        // connection. Admission is a mutex-protected count and never waits.
2878        let _concurrency_guard = match self
2879            .route_bind_concurrency
2880            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
2881        {
2882            Ok(guard) => guard,
2883            Err(in_flight) => {
2884                return Ok(vec![self.route_open_admission_refusal_frame(
2885                    ctx,
2886                    &frame,
2887                    &target_module_id,
2888                    format!(
2889                        "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
2890                    ),
2891                )?]);
2892            }
2893        };
2894
2895        // A module that has already burned the whole budget `threshold` times
2896        // in a row does not get to charge it again until a probe says it recovered.
2897        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
2898            RouteBindAdmission::Admitted { guard, probe } => {
2899                if probe {
2900                    info!(
2901                        module_id = %target_module_id,
2902                        connection_id = ctx.connection_id.get(),
2903                        "route.bind breaker half-open: admitting one probe"
2904                    );
2905                }
2906                guard
2907            }
2908            RouteBindAdmission::Refused {
2909                consecutive_timeouts,
2910                retry_in,
2911                probe_in_flight,
2912            } => {
2913                return Ok(vec![self.route_open_breaker_refusal_frame(
2914                    ctx,
2915                    &frame,
2916                    &target_module_id,
2917                    consecutive_timeouts,
2918                    retry_in,
2919                    probe_in_flight,
2920                )?]);
2921            }
2922        };
2923
2924        // Resolve the per-module budget here so the wait matches the operator's
2925        // intent for this specific target. A per-module override in
2926        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
2927        // daemons) wins over the daemon-wide default.
2928        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
2929        let relay_deadline = Instant::now() + route_bind_relay_timeout;
2930        let pending = match self
2931            .forwarding
2932            .begin_route_bind_relay_for(
2933                ctx.connection_id,
2934                ctx.egress.clone(),
2935                response_version(&frame),
2936                frame.header.corr,
2937                &target_module_id,
2938                principal.clone(),
2939                Some(project_root),
2940                relay_deadline,
2941            )
2942            .await
2943        {
2944            Ok(pending) => pending,
2945            Err(err) => {
2946                return Ok(vec![self.route_open_refusal_frame(
2947                    ctx,
2948                    &frame,
2949                    &target_module_id,
2950                    "relay_reservation_failed",
2951                    forwarding_error_code(&err),
2952                    err.to_string(),
2953                )?])
2954            }
2955        };
2956        let crate::forwarding::PendingRouteBindRelay {
2957            endpoint,
2958            module_sink,
2959            negotiated_ver,
2960            client_channel,
2961            client_epoch,
2962            module_channel,
2963            module_epoch,
2964            corr: relay_corr,
2965            receiver,
2966        } = pending;
2967        let mut reservation =
2968            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
2969
2970        debug!(
2971            connection_id = ctx.connection_id.get(),
2972            client_channel,
2973            client_epoch,
2974            module_channel,
2975            module_epoch,
2976            "reserved route handle pair"
2977        );
2978        // Rendered BEFORE the move into the relay, because the accept arm below
2979        // is where it is logged and the principal is gone by then.
2980        let principal_label = match &principal {
2981            Principal::Reserved { module_id } => format!("reserved:{module_id}"),
2982            Principal::Direct => "direct".to_string(),
2983            other => format!("{other:?}"),
2984        };
2985        let relay = ModuleControlRequest::RouteBind {
2986            route_channel: module_channel,
2987            epoch: module_epoch,
2988            target,
2989            identity,
2990            principal: Some(principal),
2991            consumer_capabilities,
2992            admission_facts,
2993        };
2994        let relay_body = serde_json::to_vec(&relay).map_err(|err| {
2995            RouterError::backend(
2996                0,
2997                frame.header.corr,
2998                format!("failed to encode route.bind request: {err}"),
2999            )
3000        })?;
3001        let relay_frame = Frame::build_with_version(
3002            negotiated_ver,
3003            FrameType::Request,
3004            control_flags(),
3005            0,
3006            0,
3007            relay_corr,
3008            relay_body,
3009        )
3010        .map_err(RouterError::FrameBuild)?;
3011
3012        if let Err(err) = module_sink.send(relay_frame).await {
3013            reservation.release_and_disarm();
3014            return Ok(vec![self.route_open_refusal_frame(
3015                ctx,
3016                &frame,
3017                &target_module_id,
3018                "relay_send_failed",
3019                "target_unavailable",
3020                err.to_string(),
3021            )?]);
3022        }
3023
3024        if !self
3025            .forwarding
3026            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
3027            .map_err(RouterError::Forwarding)?
3028        {
3029            self.send_abandoned_route_bind_goodbye(
3030                &module_sink,
3031                negotiated_ver,
3032                module_channel,
3033                module_epoch,
3034            );
3035        }
3036
3037        match timeout_at(relay_deadline, receiver).await {
3038            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
3039                reservation.disarm();
3040                if breaker.record_accepted() {
3041                    info!(
3042                        module_id = %target_module_id,
3043                        "route.bind breaker closed: the probe was accepted"
3044                    );
3045                }
3046                self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
3047                Ok(Vec::new())
3048            }
3049            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
3050                reservation.release_and_disarm();
3051                // A module that says no in microseconds is healthy. Rejection
3052                // is a different condition with its own refusal and must not
3053                // move the breaker.
3054                breaker.record_inconclusive();
3055                self.counters
3056                    .increment_route_open_refused("module_rejected");
3057                info!(
3058                    target: "control",
3059                    code = "module_rejected",
3060                    module_code = ?body.code,
3061                    module_id = ?target_module_id,
3062                    connection_id = ctx.connection_id.get(),
3063                    "route.open refused"
3064                );
3065                Ok(vec![control_error_body_frame(&frame, body)?])
3066            }
3067            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
3068                reservation.release_and_disarm();
3069                breaker.record_inconclusive();
3070                // Fires when the module's connection closes while a relayed
3071                // bind is pending -- typically a caller racing a module restart
3072                // whose bind was relayed BEFORE the drain mark went up. Logged
3073                // because the caller sees only its own error and the fleet has
3074                // already spent one diagnosis round unable to tell this arm
3075                // from a relay timeout without daemon-side evidence.
3076                tracing::warn!(
3077                    module_id = %target_module_id,
3078                    "route.bind relay abandoned: {message}"
3079                );
3080                Ok(vec![self.route_open_refusal_frame(
3081                    ctx,
3082                    &frame,
3083                    &target_module_id,
3084                    "relay_abandoned",
3085                    "target_unavailable",
3086                    message,
3087                )?])
3088            }
3089            Ok(Err(_)) => {
3090                reservation.release_and_disarm();
3091                breaker.record_inconclusive();
3092                Ok(vec![self.route_open_refusal_frame(
3093                    ctx,
3094                    &frame,
3095                    &target_module_id,
3096                    "relay_waiter_canceled",
3097                    "target_unavailable",
3098                    "route.bind relay waiter was canceled before the module responded",
3099                )?])
3100            }
3101            Err(_) => {
3102                reservation.release_and_disarm();
3103                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
3104                // answer at all is the one condition a fast refusal can
3105                // usefully stand in for; every other arm already answered.
3106                if let Some(opened) = breaker.record_timeout(
3107                    self.route_bind_breaker_threshold,
3108                    self.route_bind_breaker_cooldown,
3109                ) {
3110                    warn!(
3111                        module_id = %target_module_id,
3112                        consecutive_timeouts = opened.consecutive_timeouts,
3113                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
3114                        reopened_after_probe = opened.reopened_after_probe,
3115                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
3116                    );
3117                }
3118                // The generous budget just burned to no answer: the module is
3119                // registered and its connection is up, but its bind handler sat
3120                // on the ack for the full budget (warm-on-bind, cold configure,
3121                // or a wedged handler). Every earlier unavailability shape
3122                // fast-refuses BEFORE the relay, so this arm firing means the
3123                // slowness is module-side -- log it so the per-module timeline
3124                // is reconstructable without client audit rows.
3125                tracing::warn!(
3126                    module_id = %target_module_id,
3127                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
3128                    "route.bind relay timed out: module did not ack within budget"
3129                );
3130                Ok(vec![self.route_open_refusal_frame(
3131                    ctx,
3132                    &frame,
3133                    &target_module_id,
3134                    "relay_timed_out",
3135                    "module_timeout",
3136                    format!(
3137                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
3138                        route_bind_relay_timeout
3139                    ),
3140                )?])
3141            }
3142        }
3143    }
3144
3145    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3146        let response = ClientControlResponse::SupervisorSpawnSnapshot {
3147            snapshot: self.supervisor.spawn_snapshot(),
3148        };
3149        Ok(vec![control_response_body_frame(
3150            &frame,
3151            &response,
3152            "ClientControlResponse::SupervisorSpawnSnapshot",
3153        )?])
3154    }
3155
3156    fn handle_supervisor_spawn_subscribe(
3157        &self,
3158        ctx: &RouteCtx,
3159        frame: Frame,
3160        since: Option<SpawnCursor>,
3161    ) -> Result<Vec<Frame>, RouterError> {
3162        match self.supervisor.subscribe_spawns(
3163            ctx.connection_id,
3164            frame.header.corr,
3165            response_version(&frame),
3166            since,
3167            ctx.egress.clone(),
3168        ) {
3169            Ok(()) => Ok(Vec::new()),
3170            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
3171                Ok(vec![control_error_body_frame(
3172                    &frame,
3173                    ErrorBody {
3174                        code: "spawn_cursor_incarnation_mismatch".to_string(),
3175                        message: "spawn cursor belongs to a different daemon incarnation"
3176                            .to_string(),
3177                        detail: Some(serde_json::json!({
3178                            "current_daemon_incarnation": current
3179                        })),
3180                    },
3181                )?])
3182            }
3183            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
3184                &frame,
3185                ErrorBody {
3186                    code: "spawn_cursor_too_old".to_string(),
3187                    message: "spawn cursor predates the retained event ring".to_string(),
3188                    detail: Some(serde_json::json!({
3189                        "oldest_retained_cursor": oldest
3190                    })),
3191                },
3192            )?]),
3193            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
3194                0,
3195                frame.header.corr,
3196                format!("failed to open supervisor spawn subscription: {error}"),
3197            )),
3198        }
3199    }
3200
3201    fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3202        let generation = self
3203            .registry
3204            .generation()
3205            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3206        let modules = self
3207            .supervisor
3208            .list()
3209            .into_iter()
3210            .map(|module| {
3211                let status = module.status_for_control("list").map_err(|err| {
3212                    RouterError::backend(
3213                        0,
3214                        frame.header.corr,
3215                        format!("failed to read supervisor status: {err}"),
3216                    )
3217                })?;
3218                Ok(SupervisorEntry {
3219                    module_id: status.module_id,
3220                    state: status.state.to_string(),
3221                    enabled: status.enabled,
3222                    live: status.live,
3223                    protocol: status.protocol,
3224                    health: status.health.status,
3225                    last_probe_ms: status.health.last_probe_ms,
3226                    last_exit_code: status.last_exit.as_ref().and_then(|e| e.code),
3227                    last_exit_signal: status.last_exit.as_ref().and_then(|e| e.signal),
3228                    last_exit_ms: status.last_exit.as_ref().map(|e| e.at_ms),
3229                    last_exit_kind: status.last_exit.as_ref().map(|e| e.kind.into()),
3230                    restart_count: Some(status.restart_count),
3231                    max_restarts: Some(status.max_restarts),
3232                    lifetime_restarts: Some(status.lifetime_restarts),
3233                    spawn_generation: Some(status.spawn_generation),
3234                    restart_window_secs: Some(status.restart_window.as_secs()),
3235                    drain_timeout_ms: Some(status.drain_timeout.as_millis() as u64),
3236                    restart_backoff_ms: Some(status.restart_backoff.as_millis() as u64),
3237                    restart_max_backoff_ms: Some(status.restart_max_backoff.as_millis() as u64),
3238                })
3239            })
3240            .collect::<Result<Vec<_>, RouterError>>()?;
3241        let response = ClientControlResponse::SupervisorList {
3242            generation,
3243            modules,
3244        };
3245        Ok(vec![control_response_body_frame(
3246            &frame,
3247            &response,
3248            "ClientControlResponse::SupervisorList",
3249        )?])
3250    }
3251
3252    fn handle_supervisor_stderr_tail(
3253        &self,
3254        frame: Frame,
3255        module_id: String,
3256        max_lines: Option<u32>,
3257        max_bytes: Option<u32>,
3258    ) -> Result<Vec<Frame>, RouterError> {
3259        let Some(module) = self.supervisor.get(&module_id) else {
3260            return Ok(vec![control_error_frame(
3261                &frame,
3262                "unknown_module",
3263                format!("module_id '{module_id}' is not supervised"),
3264            )?]);
3265        };
3266
3267        let snapshot = module.stderr_tail(
3268            max_lines.map(|value| value as usize),
3269            max_bytes.map(|value| value as usize),
3270        );
3271
3272        let response = ClientControlResponse::SupervisorStderrTail {
3273            module_id,
3274            tail: StderrTail {
3275                capture: match snapshot.capture {
3276                    CaptureState::Captured => StderrCaptureState::Captured,
3277                    CaptureState::Incomplete { reason } => {
3278                        StderrCaptureState::Incomplete { reason }
3279                    }
3280                    CaptureState::NotCaptured { reason } => {
3281                        StderrCaptureState::NotCaptured { reason }
3282                    }
3283                },
3284                entries: snapshot
3285                    .entries
3286                    .into_iter()
3287                    .map(|entry| match entry {
3288                        TailEntry::Line { text, truncated } => {
3289                            StderrTailEntry::Line { text, truncated }
3290                        }
3291                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
3292                    })
3293                    .collect(),
3294                dropped_lines: snapshot.dropped_lines,
3295            },
3296        };
3297        Ok(vec![control_response_body_frame(
3298            &frame,
3299            &response,
3300            "ClientControlResponse::SupervisorStderrTail",
3301        )?])
3302    }
3303
3304    async fn handle_supervisor_terminals(
3305        &self,
3306        frame: Frame,
3307        module_id: String,
3308    ) -> Result<Vec<Frame>, RouterError> {
3309        let Some(module) = self.supervisor.get(&module_id) else {
3310            return Ok(vec![control_error_frame(
3311                &frame,
3312                "unknown_module",
3313                format!("module_id '{module_id}' is not supervised"),
3314            )?]);
3315        };
3316
3317        // The journal read runs on a blocking thread: it can be megabytes of
3318        // file I/O and must not occupy a runtime worker.
3319        let terminals = module
3320            .read_durable_terminal_history()
3321            .await
3322            .map_err(|error| {
3323                RouterError::backend(
3324                    0,
3325                    frame.header.corr,
3326                    format!("failed to read terminal history: {error}"),
3327                )
3328            })?;
3329        let response = ClientControlResponse::SupervisorTerminals {
3330            module_id,
3331            terminals,
3332        };
3333        Ok(vec![control_response_body_frame(
3334            &frame,
3335            &response,
3336            "ClientControlResponse::SupervisorTerminals",
3337        )?])
3338    }
3339
3340    fn handle_supervisor_routes(
3341        &self,
3342        frame: Frame,
3343        module_id: Option<String>,
3344    ) -> Result<Vec<Frame>, RouterError> {
3345        let modules = self
3346            .forwarding
3347            .route_census(module_id.as_deref())
3348            .map_err(RouterError::Forwarding)?
3349            .into_iter()
3350            .map(|(module_id, routes)| SupervisorRouteModule {
3351                module_id,
3352                routes: routes
3353                    .into_iter()
3354                    .map(|route| SupervisorRoute {
3355                        consumer: match route.principal {
3356                            Principal::Reserved { module_id } => {
3357                                SupervisorRouteConsumer::Reserved { module_id }
3358                            }
3359                            Principal::Direct | Principal::Unverified => {
3360                                SupervisorRouteConsumer::Direct {
3361                                    connection_id: route.goodbye_target.connection_id.get(),
3362                                }
3363                            }
3364                        },
3365                        age_ms: Instant::now()
3366                            .saturating_duration_since(route.bound_at)
3367                            .as_millis()
3368                            .try_into()
3369                            .unwrap_or(u64::MAX),
3370                        draining: route.draining,
3371                        drain_reason: route.drain_reason,
3372                    })
3373                    .collect(),
3374            })
3375            .collect();
3376        let response = ClientControlResponse::SupervisorRoutes { modules };
3377        Ok(vec![control_response_body_frame(
3378            &frame,
3379            &response,
3380            "ClientControlResponse::SupervisorRoutes",
3381        )?])
3382    }
3383
3384    async fn handle_supervisor_provenance(
3385        &self,
3386        frame: Frame,
3387        module_id: Option<String>,
3388    ) -> Result<Vec<Frame>, RouterError> {
3389        let mut selected = if let Some(module_id) = module_id {
3390            let Some(module) = self.supervisor.get(&module_id) else {
3391                return Ok(vec![control_error_frame(
3392                    &frame,
3393                    "unknown_module",
3394                    format!("module_id '{module_id}' is not supervised"),
3395                )?]);
3396            };
3397            vec![module]
3398        } else {
3399            self.supervisor.list()
3400        };
3401
3402        let mut modules = Vec::with_capacity(selected.len());
3403        for module in selected.drain(..) {
3404            let status = module.status().map_err(|err| {
3405                RouterError::backend(
3406                    0,
3407                    frame.header.corr,
3408                    format!("failed to read supervisor status: {err}"),
3409                )
3410            })?;
3411            let module_declared = self
3412                .registry
3413                .get_module(&status.module_id)
3414                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3415                .and_then(|registration| registration.manifest.provenance)
3416                .map(|build| ModuleDeclaredProvenance::Reported { build })
3417                .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
3418            #[cfg(test)]
3419            let running_image = match &self.provenance_probe_override {
3420                Some(result) => result.clone(),
3421                None => module.running_image_agreement().await,
3422            };
3423            #[cfg(not(test))]
3424            let running_image = module.running_image_agreement().await;
3425            modules.push(SupervisorModuleProvenance {
3426                module_id: status.module_id,
3427                module_declared,
3428                daemon_observed: SupervisorObservedProcess {
3429                    pid: status.pid,
3430                    spawned_at_ms: status.spawned_at_ms,
3431                    spawned_from: status.spawned_from,
3432                    running_image,
3433                },
3434            });
3435        }
3436        let daemon = SupervisorDaemonProvenance {
3437            daemon_build: self.daemon_provenance.build.clone(),
3438            daemon_observed: DaemonObservedProcess {
3439                pid: self.daemon_provenance.pid,
3440                started_at_ms: self
3441                    .daemon_provenance
3442                    .start_clock
3443                    .map(|clock| clock.started_at_ms())
3444                    .or(self.daemon_provenance.started_at_ms),
3445                running_image: self
3446                    .daemon_provenance
3447                    .probe
3448                    .observe(
3449                        self.daemon_provenance.pid,
3450                        self.daemon_provenance.executable_path.as_deref(),
3451                        self.daemon_provenance.executable_identity,
3452                        self.daemon_provenance.process_start_time,
3453                    )
3454                    .await,
3455            },
3456        };
3457        let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
3458        Ok(vec![control_response_body_frame(
3459            &frame,
3460            &response,
3461            "ClientControlResponse::SupervisorProvenance",
3462        )?])
3463    }
3464
3465    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3466        self.refresh_capability_requirements();
3467        let generation = self
3468            .registry
3469            .generation()
3470            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3471        let modules = self
3472            .supervisor
3473            .list()
3474            .into_iter()
3475            .map(|module| {
3476                let status = module.status_for_control("health").map_err(|err| {
3477                    RouterError::backend(
3478                        0,
3479                        frame.header.corr,
3480                        format!("failed to read supervisor health: {err}"),
3481                    )
3482                })?;
3483                let module_id = status.module_id;
3484                let capability_detail = self
3485                    .capability_evaluator
3486                    .required_problem_detail(&module_id);
3487                Ok(SupervisorHealthEntry {
3488                    module_id,
3489                    status: status.health.status,
3490                    detail: append_capability_problem_detail(
3491                        status.health.detail,
3492                        capability_detail,
3493                    ),
3494                    metrics: status.health.metrics,
3495                    consecutive_failures: status.health.consecutive_failures,
3496                    late_answer_count: status.health.late_answer_count,
3497                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
3498                    last_action: status.health.last_action,
3499                    last_action_ms: status.health.last_action_ms,
3500                    last_probe_ms: status.health.last_probe_ms,
3501                })
3502            })
3503            .collect::<Result<Vec<_>, RouterError>>()?;
3504        let response = ClientControlResponse::SupervisorHealth {
3505            generation,
3506            modules,
3507        };
3508        Ok(vec![control_response_body_frame(
3509            &frame,
3510            &response,
3511            "ClientControlResponse::SupervisorHealth",
3512        )?])
3513    }
3514
3515    async fn handle_supervisor_restart(
3516        &self,
3517        frame: Frame,
3518        module_id: String,
3519        drain_timeout_ms: Option<u64>,
3520    ) -> Result<Vec<Frame>, RouterError> {
3521        let operation_lock = self.supervisor.operation_lock();
3522        let _operation_guard = operation_lock.lock().await;
3523        let Some(module) = self.supervisor.get(&module_id) else {
3524            return Ok(vec![control_error_frame(
3525                &frame,
3526                "unknown_module",
3527                format!("module_id '{module_id}' is not supervised"),
3528            )?]);
3529        };
3530
3531        if let Err(err) = module.restart(drain_timeout_ms).await {
3532            let (code, message) = match err {
3533                crate::supervise::SuperviseError::Disabled { .. } => {
3534                    ("module_disabled", err.to_string())
3535                }
3536                crate::supervise::SuperviseError::SwapInProgress { .. } => {
3537                    ("swap_in_progress", err.to_string())
3538                }
3539                _ => (
3540                    "target_unavailable",
3541                    format!("failed to restart module_id '{module_id}': {err}"),
3542                ),
3543            };
3544            return Ok(vec![control_error_frame(&frame, code, message)?]);
3545        }
3546
3547        let response = ClientControlResponse::SupervisorAck {
3548            module_id,
3549            applied: true,
3550        };
3551        Ok(vec![control_response_body_frame(
3552            &frame,
3553            &response,
3554            "ClientControlResponse::SupervisorAck",
3555        )?])
3556    }
3557
3558    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
3559    /// when the old process has finished draining: a caller whose own lane
3560    /// rides the old process must get its reply before that drain waits on it.
3561    async fn handle_supervisor_swap(
3562        &self,
3563        frame: Frame,
3564        module_id: String,
3565        ready_timeout_ms: Option<u64>,
3566    ) -> Result<Vec<Frame>, RouterError> {
3567        // The daemon-wide operation lock is held only to resolve the handle,
3568        // not across the swap. The swap can take its whole readiness budget,
3569        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
3570        // holding it here would park an operator's stop behind the swap it is
3571        // meant to abort. A rescan or stop that reaches the module during the
3572        // swap is served by the swap itself (see `supervise_swap`).
3573        let module = {
3574            let operation_lock = self.supervisor.operation_lock();
3575            let _operation_guard = operation_lock.lock().await;
3576            self.supervisor.get(&module_id)
3577        };
3578        let Some(module) = module else {
3579            return Ok(vec![control_error_frame(
3580                &frame,
3581                "unknown_module",
3582                format!("module_id '{module_id}' is not supervised"),
3583            )?]);
3584        };
3585
3586        if let Err(err) = module
3587            .swap(ready_timeout_ms.map(Duration::from_millis))
3588            .await
3589        {
3590            use crate::supervise::SuperviseError;
3591            let message = err.to_string();
3592            let error = match err {
3593                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
3594                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
3595                    code: "swap_refused".to_string(),
3596                    message,
3597                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
3598                },
3599                SuperviseError::SwapFailed {
3600                    arm,
3601                    candidate_exit,
3602                    ..
3603                } => ErrorBody {
3604                    code: "swap_failed".to_string(),
3605                    message,
3606                    detail: Some(serde_json::json!({
3607                        "arm": arm.as_str(),
3608                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
3609                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
3610                    })),
3611                },
3612                _ => ErrorBody::new(
3613                    "target_unavailable",
3614                    format!("failed to swap module_id '{module_id}': {message}"),
3615                ),
3616            };
3617            return Ok(vec![control_error_body_frame(&frame, error)?]);
3618        }
3619
3620        let response = ClientControlResponse::SupervisorAck {
3621            module_id,
3622            applied: true,
3623        };
3624        Ok(vec![control_response_body_frame(
3625            &frame,
3626            &response,
3627            "ClientControlResponse::SupervisorAck",
3628        )?])
3629    }
3630
3631    async fn handle_supervisor_reload(
3632        &self,
3633        frame: Frame,
3634        module_id: String,
3635    ) -> Result<Vec<Frame>, RouterError> {
3636        let operation_lock = self.supervisor.operation_lock();
3637        let _operation_guard = operation_lock.lock().await;
3638        let Some(module) = self.supervisor.get(&module_id) else {
3639            return Ok(vec![control_error_frame(
3640                &frame,
3641                "unknown_module",
3642                format!("module_id '{module_id}' is not supervised"),
3643            )?]);
3644        };
3645
3646        if let Err(err) = module.reload().await {
3647            let (code, message) = match err {
3648                crate::supervise::SuperviseError::Disabled { .. } => {
3649                    ("module_disabled", err.to_string())
3650                }
3651                crate::supervise::SuperviseError::SwapInProgress { .. } => {
3652                    ("swap_in_progress", err.to_string())
3653                }
3654                _ => (
3655                    "reload_failed",
3656                    format!("failed to reload module_id '{module_id}': {err}"),
3657                ),
3658            };
3659            return Ok(vec![control_error_frame(&frame, code, message)?]);
3660        }
3661
3662        let response = ClientControlResponse::SupervisorAck {
3663            module_id,
3664            applied: true,
3665        };
3666        Ok(vec![control_response_body_frame(
3667            &frame,
3668            &response,
3669            "ClientControlResponse::SupervisorAck",
3670        )?])
3671    }
3672
3673    async fn handle_supervisor_rescan(
3674        &self,
3675        frame: Frame,
3676        preview: bool,
3677    ) -> Result<Vec<Frame>, RouterError> {
3678        let Some(context) = self.rescan.clone() else {
3679            return Ok(vec![control_error_frame(
3680                &frame,
3681                "rescan_unavailable",
3682                "the daemon was not started with a reloadable config path".to_string(),
3683            )?]);
3684        };
3685
3686        let operation_lock = self.supervisor.operation_lock();
3687        let _operation_guard = operation_lock.lock().await;
3688        let loaded = match crate::daemon_config::load(&context.config_path) {
3689            Ok(config) => config,
3690            Err(err) => {
3691                return Ok(vec![control_error_frame(
3692                    &frame,
3693                    "invalid_daemon_config",
3694                    format!("supervisor rescan rejected daemon config: {err}"),
3695                )?])
3696            }
3697        };
3698        // `load` reports a missing file as Ok(None), which is correct at boot
3699        // (no config, nothing to supervise) and catastrophic here: rescan treats
3700        // "not in the config" as "remove it", so an absent file would read as an
3701        // empty module list and retire the entire running fleet. An editor
3702        // writing via write-new-then-rename, or a half-finished edit, is enough
3703        // to open that window. Refuse instead: a config that cannot be read
3704        // carries no instruction to remove anything.
3705        let Some(config) = loaded else {
3706            return Ok(vec![control_error_frame(
3707                &frame,
3708                "invalid_daemon_config",
3709                format!(
3710                    "daemon config not found at {}; refusing to rescan (an absent config would \
3711                     retire every supervised module)",
3712                    context.config_path.display()
3713                ),
3714            )?]);
3715        };
3716        let (
3717            configured_port,
3718            storage_config,
3719            admission_facts_carrier_module_id,
3720            admission_facts_targets,
3721            modules,
3722            reserved_capabilities,
3723        ) = (
3724            config.port,
3725            config.storage,
3726            config.admission_facts_carrier_module_id,
3727            config.admission_facts_targets,
3728            config.modules,
3729            config.reserved_capabilities,
3730        );
3731
3732        // Collect the sections rescan cannot apply, so the REPLY carries them.
3733        //
3734        // The warning below has always been correct and has always gone only to
3735        // the journal -- addressed to whoever reads logs, while the person who
3736        // just edited the config is looking at the CLI. Naming each section
3737        // individually rather than setting a flag: "something outside modules
3738        // changed" sends the operator back to diffing their own file, which is
3739        // the work this is meant to save.
3740        let mut restart_required = Vec::new();
3741        for section in RestartRequiredSection::ALL {
3742            let changed = match section {
3743                RestartRequiredSection::Port => configured_port != context.configured_port,
3744                RestartRequiredSection::Storage => storage_config != context.storage_config,
3745                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
3746                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
3747                }
3748                RestartRequiredSection::AdmissionFactsTargets => {
3749                    admission_facts_targets != context.admission_facts_targets
3750                }
3751            };
3752            if changed {
3753                restart_required.push(section.label().to_string());
3754            }
3755        }
3756        if !restart_required.is_empty() {
3757            warn!(
3758                config_path = %context.config_path.display(),
3759                sections = %restart_required.join(", "),
3760                "daemon config changed outside the modules section; restart the daemon to apply those changes"
3761            );
3762        }
3763
3764        for configured in &modules {
3765            if let Err(err) = validate_spec(&configured.module_spec()) {
3766                return Ok(vec![control_error_frame(
3767                    &frame,
3768                    "invalid_daemon_config",
3769                    format!("supervisor rescan rejected daemon config: {err}"),
3770                )?]);
3771            }
3772        }
3773
3774        let configured_capabilities = modules
3775            .iter()
3776            .map(|module| (module.module_id.clone(), module.enabled))
3777            .collect::<Vec<_>>();
3778        let preview_capability_warnings = if preview {
3779            let (_, registrations) = self.runtime_capability_snapshot()?;
3780            let current_modules = self
3781                .supervisor
3782                .list()
3783                .into_iter()
3784                .map(|module| module.module_id().to_string())
3785                .collect::<BTreeSet<_>>();
3786            let resulting_modules = configured_capabilities.clone();
3787            let removed = current_modules
3788                .into_iter()
3789                .filter(|module_id| {
3790                    !resulting_modules
3791                        .iter()
3792                        .any(|(configured_id, _)| configured_id == module_id)
3793                })
3794                .collect::<Vec<_>>();
3795            self.capability_evaluator.preview_removal_warnings(
3796                resulting_modules,
3797                &removed,
3798                &registrations,
3799            )
3800        } else {
3801            Vec::new()
3802        };
3803        let result = match self
3804            .reconcile_supervised_modules(&context.supervisor, modules, preview)
3805            .await
3806        {
3807            Ok(result) => result,
3808            Err(message) => {
3809                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
3810            }
3811        };
3812        if !preview {
3813            self.capability_evaluator
3814                .configure(configured_capabilities, reserved_capabilities);
3815            self.capability_evaluator.wake_deadline_loop();
3816            self.refresh_capability_requirements();
3817        }
3818        let mut result = result;
3819        result.restart_required = restart_required;
3820        result.capability_warnings = preview_capability_warnings;
3821        let response = ClientControlResponse::SupervisorRescan { result };
3822        Ok(vec![control_response_body_frame(
3823            &frame,
3824            &response,
3825            "ClientControlResponse::SupervisorRescan",
3826        )?])
3827    }
3828
3829    async fn handle_supervisor_release_reserved(
3830        &self,
3831        frame: Frame,
3832        module_id: String,
3833    ) -> Result<Vec<Frame>, RouterError> {
3834        let Some(context) = self.rescan.clone() else {
3835            return Ok(vec![control_error_frame(
3836                &frame,
3837                "release_unavailable",
3838                "reserved-id release requires a daemon started with a reloadable config path",
3839            )?]);
3840        };
3841        let operation_lock = self.supervisor.operation_lock();
3842        let _operation_guard = operation_lock.lock().await;
3843        let loaded = match crate::daemon_config::load(&context.config_path) {
3844            Ok(Some(config)) => config,
3845            Ok(None) => {
3846                return Ok(vec![control_error_frame(
3847                    &frame,
3848                    "invalid_daemon_config",
3849                    format!(
3850                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
3851                        context.config_path.display()
3852                    ),
3853                )?])
3854            }
3855            Err(err) => {
3856                return Ok(vec![control_error_frame(
3857                    &frame,
3858                    "invalid_daemon_config",
3859                    format!("unable to verify reserved-id release against daemon config: {err}"),
3860                )?])
3861            }
3862        };
3863        if loaded
3864            .modules
3865            .iter()
3866            .any(|configured| configured.module_id == module_id)
3867        {
3868            return Ok(vec![control_error_frame(
3869                &frame,
3870                "reserved_module_configured",
3871                format!(
3872                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
3873                ),
3874            )?]);
3875        }
3876        if !self.supervisor.release_retained_reserved_gate(&module_id) {
3877            return Ok(vec![control_error_frame(
3878                &frame,
3879                "reserved_gate_not_retained",
3880                format!(
3881                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
3882                ),
3883            )?]);
3884        }
3885
3886        let response = ClientControlResponse::SupervisorAck {
3887            module_id,
3888            applied: true,
3889        };
3890        Ok(vec![control_response_body_frame(
3891            &frame,
3892            &response,
3893            "ClientControlResponse::SupervisorAck",
3894        )?])
3895    }
3896
3897    /// Reconcile the running module set against the configured one.
3898    ///
3899    /// With `preview` set, the diff is computed and returned WITHOUT applying any
3900    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
3901    /// deliberately shares this function with the executing path rather than
3902    /// computing the same diff somewhere else -- two implementations of one
3903    /// decision agree until they do not, and the whole value of a preview is that
3904    /// it describes the operation that will actually run.
3905    async fn reconcile_supervised_modules(
3906        &self,
3907        supervisor: &Supervisor,
3908        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
3909        preview: bool,
3910    ) -> Result<SupervisorRescanResult, String> {
3911        let mut current = BTreeMap::new();
3912        for module in self.supervisor.list() {
3913            let (spec, health) = module.configuration().map_err(|err| {
3914                format!(
3915                    "failed to read configuration for module_id '{}': {err}",
3916                    module.module_id()
3917                )
3918            })?;
3919            let enabled = module
3920                .status()
3921                .map_err(|err| {
3922                    format!(
3923                        "failed to read status for module_id '{}': {err}",
3924                        module.module_id()
3925                    )
3926                })?
3927                .enabled;
3928            current.insert(
3929                module.module_id().to_string(),
3930                (module, spec, health, enabled),
3931            );
3932        }
3933        let configured = configured_modules
3934            .into_iter()
3935            .map(|module| (module.module_id.clone(), module))
3936            .collect::<BTreeMap<_, _>>();
3937
3938        let added = configured
3939            .keys()
3940            .filter(|module_id| !current.contains_key(*module_id))
3941            .cloned()
3942            .collect::<Vec<_>>();
3943        let removed = current
3944            .keys()
3945            .filter(|module_id| !configured.contains_key(*module_id))
3946            .cloned()
3947            .collect::<Vec<_>>();
3948        let mut changed_pending_reload = Vec::new();
3949        let mut configuration_changes = BTreeSet::new();
3950        let mut enabled_changes = BTreeSet::new();
3951        let mut unchanged = 0_u32;
3952
3953        for (module_id, configured_module) in &configured {
3954            let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
3955            else {
3956                continue;
3957            };
3958            let configuration_changed = *current_spec != configured_module.module_spec()
3959                || *current_health != configured_module.health;
3960            let enabled_changed = *current_enabled != configured_module.enabled;
3961            if configuration_changed {
3962                configuration_changes.insert(module_id.clone());
3963                changed_pending_reload.push(module_id.clone());
3964            }
3965            if enabled_changed {
3966                enabled_changes.insert(module_id.clone());
3967            }
3968            if !configuration_changed && !enabled_changed {
3969                unchanged = unchanged.saturating_add(1);
3970            }
3971        }
3972
3973        // Everything above this point is pure computation over two snapshots.
3974        // Everything below MUTATES. The preview returns here so the boundary is a
3975        // single early return rather than a condition repeated at each mutation
3976        // site, where one missed guard would apply part of a change the caller was
3977        // told would not happen.
3978        if preview {
3979            return Ok(SupervisorRescanResult {
3980                added,
3981                removed,
3982                changed_pending_reload,
3983                enabled_changes: enabled_changes.iter().cloned().collect(),
3984                unchanged,
3985                preview: true,
3986                // Filled by the caller on both paths, so the preview reports
3987                // restart-required sections identically to an executed rescan --
3988                // the preview is where an operator is most likely to be looking.
3989                restart_required: Vec::new(),
3990                capability_warnings: Vec::new(),
3991            });
3992        }
3993
3994        for module_id in &removed {
3995            let module = &current
3996                .get(module_id)
3997                .expect("removed module came from current supervisor state")
3998                .0;
3999            module.retire().await.map_err(|err| {
4000                format!("failed to retire module_id '{module_id}' during rescan: {err}")
4001            })?;
4002            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
4003            //
4004            // `handle_route_open` resolves an absent module in three steps:
4005            // registry, then supervisor status, then tombstone. Retiring first
4006            // opens a window where ALL THREE ARE ABSENT -- the registry entry
4007            // went with the teardown above, the supervisor entry went with
4008            // `retire`, and the tombstone does not exist yet -- so a route.open
4009            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
4010            // it") for a module that was deliberately removed and whose caller
4011            // should get `module_removed` (TERMINAL, carrying a removal age).
4012            //
4013            // Writing the tombstone first closes it: during the window the
4014            // supervisor entry still answers, so the caller gets
4015            // `target_unavailable` -- retryable, and TRUE, because the module
4016            // is mid-teardown. After both statements it is `module_removed`.
4017            // No instant remains where a removed module reads as one that
4018            // never existed.
4019            //
4020            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
4021            // the absence of a test beside a fix invites deletion: these are
4022            // two sync statements with no await between them, so reaching the
4023            // window needs a second worker thread to land exactly between them
4024            // and there is no hook to force it. MEASURED: the 25 daemon_config
4025            // tests pass identically with the old order and the new one, so
4026            // the existing suite cannot see this and a green run is not
4027            // evidence either way. What the suite does hold is the
4028            // post-condition -- a removed module answers `module_removed` --
4029            // which this preserves.
4030            //
4031            // Found by an Athena panel reading the shipped tree against a
4032            // design note (2026-09-19), as the one concrete instance of that
4033            // note's class that survived contact with source. Direction is
4034            // benign: retryable where terminal was intended, never the reverse.
4035            self.supervisor.record_rescan_removal(module_id);
4036            self.supervisor.retire(module_id);
4037        }
4038
4039        for module_id in configured.keys() {
4040            let Some((module, _, _, _)) = current.get(module_id) else {
4041                continue;
4042            };
4043            let configured_module = configured
4044                .get(module_id)
4045                .expect("configured module id came from configured map");
4046            if configuration_changes.contains(module_id) {
4047                module
4048                    .update_configuration(
4049                        configured_module.module_spec(),
4050                        configured_module.health,
4051                        configured_module.drain_timeout_ms,
4052                    )
4053                    .await
4054                    .map_err(|err| {
4055                        format!(
4056                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
4057                        )
4058                    })?;
4059            }
4060            if enabled_changes.contains(module_id) {
4061                module
4062                    .set_enabled(configured_module.enabled)
4063                    .await
4064                    .map_err(|err| {
4065                        format!(
4066                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
4067                            configured_module.enabled
4068                        )
4069                    })?;
4070            }
4071        }
4072
4073        for module_id in &added {
4074            let configured_module = configured
4075                .get(module_id)
4076                .expect("added module id came from configured map");
4077            supervisor
4078                .supervise_configured_with_health(
4079                    configured_module.module_spec(),
4080                    configured_module.enabled,
4081                    configured_module.health,
4082                    configured_module.drain_timeout_ms,
4083                    configured_module.restart,
4084                )
4085                .map_err(|err| {
4086                    format!("failed to add module_id '{module_id}' during rescan: {err}")
4087                })?;
4088        }
4089
4090        Ok(SupervisorRescanResult {
4091            added,
4092            removed,
4093            changed_pending_reload,
4094            enabled_changes: enabled_changes.iter().cloned().collect(),
4095            unchanged,
4096            preview: false,
4097            // Filled by the caller, which is the only layer that can see the
4098            // previous config to diff against.
4099            restart_required: Vec::new(),
4100            capability_warnings: Vec::new(),
4101        })
4102    }
4103
4104    async fn handle_supervisor_set_enabled(
4105        &self,
4106        frame: Frame,
4107        module_id: String,
4108        enabled: bool,
4109    ) -> Result<Vec<Frame>, RouterError> {
4110        let operation_lock = self.supervisor.operation_lock();
4111        let _operation_guard = operation_lock.lock().await;
4112        let Some(module) = self.supervisor.get(&module_id) else {
4113            return Ok(vec![control_error_frame(
4114                &frame,
4115                "unknown_module",
4116                format!("module_id '{module_id}' is not supervised"),
4117            )?]);
4118        };
4119
4120        let applied = match module.set_enabled(enabled).await {
4121            Ok(applied) => applied,
4122            Err(err) => {
4123                return Ok(vec![control_error_frame(
4124                    &frame,
4125                    "target_unavailable",
4126                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
4127                )?])
4128            }
4129        };
4130
4131        self.capability_evaluator.wake_deadline_loop();
4132        self.refresh_capability_requirements();
4133        let response = ClientControlResponse::SupervisorAck { module_id, applied };
4134        Ok(vec![control_response_body_frame(
4135            &frame,
4136            &response,
4137            "ClientControlResponse::SupervisorAck",
4138        )?])
4139    }
4140
4141    async fn handle_supervisor_health_probe(
4142        &self,
4143        frame: Frame,
4144        module_id: String,
4145    ) -> Result<Vec<Frame>, RouterError> {
4146        self.refresh_capability_requirements();
4147        let Some(registration) = self
4148            .registry
4149            .get_module(&module_id)
4150            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4151        else {
4152            return Ok(vec![control_error_frame(
4153                &frame,
4154                "unknown_module",
4155                format!("module_id '{module_id}' is not registered"),
4156            )?]);
4157        };
4158
4159        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
4160        // named for it. Making `module_registration_grants_op` return false
4161        // unconditionally reddens five tests, and every one is named for something
4162        // else -- capability relay, probe/bind demultiplexing, supervision-only
4163        // probing. They exercise a successful advertisement check on the way to their
4164        // own subject.
4165        //
4166        // Real protection, fragile in a specific way: narrowing any of those tests to
4167        // focus on its stated subject would silently remove coverage nobody knows
4168        // they are carrying. Recorded here rather than as a sixth test, because the
4169        // useful fact is WHICH tests hold the guard up -- a new test would add
4170        // coverage without telling the next person what the existing ones quietly do.
4171        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
4172        {
4173            return Ok(vec![control_error_frame(
4174                &frame,
4175                "health_not_advertised",
4176                format!("module_id '{module_id}' did not advertise health.check"),
4177            )?]);
4178        }
4179
4180        let deadline = Instant::now() + self.health_probe_timeout;
4181        let pending = match self.forwarding.begin_module_control_rpc_for(
4182            &module_id,
4183            MODULE_CONTROL_OP_HEALTH_CHECK,
4184            deadline,
4185        ) {
4186            Ok(pending) => pending,
4187            Err(err) => {
4188                return Ok(vec![control_error_frame(
4189                    &frame,
4190                    forwarding_error_code(&err),
4191                    err.to_string(),
4192                )?])
4193            }
4194        };
4195
4196        let PendingModuleControlRpc {
4197            endpoint,
4198            module_sink,
4199            negotiated_ver,
4200            corr: probe_corr,
4201            receiver,
4202        } = pending;
4203        let mut guard =
4204            ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
4205        let probe_body =
4206            serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
4207                RouterError::backend(
4208                    0,
4209                    frame.header.corr,
4210                    format!("failed to encode health.check request: {err}"),
4211                )
4212            })?;
4213        let probe_frame = Frame::build_with_version(
4214            negotiated_ver,
4215            FrameType::Request,
4216            control_flags(),
4217            0,
4218            0,
4219            probe_corr,
4220            probe_body,
4221        )
4222        .map_err(RouterError::FrameBuild)?;
4223
4224        if let Err(err) = module_sink.send(probe_frame).await {
4225            return Ok(vec![control_error_frame(
4226                &frame,
4227                "target_unavailable",
4228                err.to_string(),
4229            )?]);
4230        }
4231
4232        match timeout_at(deadline, receiver).await {
4233            Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
4234                guard.disarm();
4235                let Some(report) = response.health_report() else {
4236                    return Ok(vec![control_error_frame(
4237                        &frame,
4238                        "invalid_control_body",
4239                        "health.check RPC returned a non-health response",
4240                    )?]);
4241                };
4242                // Metrics go out whole here. The supervisor's cached snapshot
4243                // caps this blob (see truncate_health_metrics), and this path
4244                // exists precisely to answer without that cap -- so applying it
4245                // here would leave no way to see what the cached view drops.
4246                let HealthReport {
4247                    status,
4248                    detail,
4249                    metrics,
4250                } = report;
4251                let capability_detail = self
4252                    .capability_evaluator
4253                    .required_problem_detail(&module_id);
4254                let response = ClientControlResponse::SupervisorHealthProbe {
4255                    module_id,
4256                    status,
4257                    detail: append_capability_problem_detail(detail, capability_detail),
4258                    metrics,
4259                };
4260                Ok(vec![control_response_body_frame(
4261                    &frame,
4262                    &response,
4263                    "ClientControlResponse::SupervisorHealthProbe",
4264                )?])
4265            }
4266            Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
4267                guard.disarm();
4268                Ok(vec![control_error_body_frame(&frame, body)?])
4269            }
4270            Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
4271                guard.disarm();
4272                Ok(vec![control_error_frame(
4273                    &frame,
4274                    "target_unavailable",
4275                    message,
4276                )?])
4277            }
4278            Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
4279                guard.disarm();
4280                Ok(vec![control_error_frame(
4281                    &frame,
4282                    "invalid_control_body",
4283                    message,
4284                )?])
4285            }
4286            Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
4287                guard.disarm();
4288                Ok(vec![control_error_frame(
4289                    &frame,
4290                    "invalid_control_body",
4291                    format!("expected module-control op '{expected}', got '{actual}'"),
4292                )?])
4293            }
4294            Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
4295                guard.disarm();
4296                Ok(vec![control_error_frame(
4297                    &frame,
4298                    "module_timeout",
4299                    format!(
4300                        "module_id '{module_id}' answered health.check after {:?}",
4301                        self.health_probe_timeout
4302                    ),
4303                )?])
4304            }
4305            Ok(Err(_)) => Ok(vec![control_error_frame(
4306                &frame,
4307                "target_unavailable",
4308                "health.check waiter was canceled before the module responded",
4309            )?]),
4310            Err(_) => Ok(vec![control_error_frame(
4311                &frame,
4312                "module_timeout",
4313                format!(
4314                    "module_id '{module_id}' did not answer health.check within {:?}",
4315                    self.health_probe_timeout
4316                ),
4317            )?]),
4318        }
4319    }
4320
4321    fn supervisor_status(
4322        &self,
4323        module_id: &str,
4324        corr: u64,
4325    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
4326        self.supervisor
4327            .get(module_id)
4328            .map(|module| {
4329                let warming = module.is_warming_for_control("status").map_err(|err| {
4330                    RouterError::backend(
4331                        0,
4332                        corr,
4333                        format!(
4334                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
4335                        ),
4336                    )
4337                })?;
4338                module.status_for_control("status").map_err(|err| {
4339                    RouterError::backend(
4340                        0,
4341                        corr,
4342                        format!(
4343                            "failed to read supervisor status for module_id '{module_id}': {err}"
4344                        ),
4345                    )
4346                }).map(|status| (status, warming))
4347            })
4348            .transpose()
4349    }
4350
4351    fn guard_module_control_op(
4352        &self,
4353        frame: &Frame,
4354        module_id: &str,
4355        op: &str,
4356    ) -> Result<Option<Frame>, RouterError> {
4357        if self.module_grants_op(module_id, op, frame.header.corr)? {
4358            return Ok(None);
4359        }
4360
4361        Ok(Some(control_error_frame(
4362            frame,
4363            "op_not_allowed",
4364            format!("module_id '{module_id}' did not grant control op '{op}'"),
4365        )?))
4366    }
4367
4368    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
4369        let Some(registration) = self
4370            .registry
4371            .get_module(module_id)
4372            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
4373        else {
4374            return Ok(false);
4375        };
4376        Ok(module_registration_grants_op(&registration.control_ops, op))
4377    }
4378
4379    fn handle_status_update(
4380        &self,
4381        endpoint: ModuleEndpointId,
4382        frame: Frame,
4383    ) -> Result<Vec<Frame>, RouterError> {
4384        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
4385            Ok(update) => update,
4386            Err(err) => {
4387                // Forward-compat: a newer module may push a channel-0 op this subc
4388                // version doesn't know. The control contract says unknown push ops
4389                // are IGNORED, never answered with an error. Only a malformed body
4390                // for an op we DO know is a real error worth surfacing.
4391                if is_known_module_push_op(&frame.body) {
4392                    return Ok(vec![control_error_frame(
4393                        &frame,
4394                        "invalid_control_body",
4395                        format!("malformed module control push body: {err}"),
4396                    )?]);
4397                }
4398                return Ok(Vec::new());
4399            }
4400        };
4401
4402        match update {
4403            ModuleControlPush::RouteStatus {
4404                route_channel,
4405                route_epoch,
4406                status,
4407            } => {
4408                self.forwarding
4409                    .cache_status(endpoint, route_channel, route_epoch, status)
4410                    .map_err(RouterError::Forwarding)?;
4411            }
4412        }
4413        Ok(Vec::new())
4414    }
4415
4416    fn handle_route_poll(
4417        &self,
4418        ctx: &RouteCtx,
4419        frame: Frame,
4420        route_channel: u16,
4421        route_epoch: u32,
4422        kind: PollKind,
4423    ) -> Result<Vec<Frame>, RouterError> {
4424        let snapshot = self
4425            .forwarding
4426            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
4427            .map_err(RouterError::Forwarding)?;
4428        let response = match (kind, snapshot) {
4429            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
4430                ClientControlResponse::RoutePoll {
4431                    route_channel,
4432                    route_epoch,
4433                    status,
4434                    live: None,
4435                }
4436            }
4437            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
4438                route_channel,
4439                route_epoch,
4440                status: None,
4441                live: None,
4442            },
4443            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
4444                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
4445                // is what makes reporting `true` correct rather than a
4446                // confident guess. `process_live` returns None only when the
4447                // module id has no supervisor snapshot at all -- an
4448                // externally-started module the daemon did not spawn -- and
4449                // for those the supervisor has no opinion to offer, ever. It
4450                // is never None for a supervised module in an unknown state:
4451                // a supervised module always has a snapshot, and the answer
4452                // comes from `state == Running && process_alive`.
4453                //
4454                // The route is Bound, so the module completed a HELLO on a
4455                // live connection; "the process this route points at is
4456                // running" is therefore attested by the binding rather than
4457                // assumed. Reporting `false` for an unsupervised module would
4458                // be the actual lie -- it would tell a client its healthy
4459                // route is dead because the daemon does not manage the
4460                // process.
4461                //
4462                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
4463                // module whose liveness is genuinely unknown, e.g. a snapshot
4464                // that has not been populated yet -- THIS DEFAULT BECOMES
4465                // WRONG and must split: unsupervised stays true, unknown
4466                // becomes null so the client can tell the two apart. The
4467                // response field is already `Option<bool>`, so the wire can
4468                // carry that distinction today.
4469                let live = self
4470                    .process_liveness
4471                    .as_ref()
4472                    .and_then(|source| source.process_live(&module_id))
4473                    .unwrap_or(true);
4474                ClientControlResponse::RoutePoll {
4475                    route_channel,
4476                    route_epoch,
4477                    status: None,
4478                    live: Some(live),
4479                }
4480            }
4481            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
4482                route_channel,
4483                route_epoch,
4484                status: None,
4485                live: Some(false),
4486            },
4487        };
4488
4489        Ok(vec![control_response_body_frame(
4490            &frame,
4491            &response,
4492            "ClientControlResponse::RoutePoll",
4493        )?])
4494    }
4495
4496    pub(crate) fn observe_module_control_completion(
4497        &self,
4498        completion: ModuleControlRpcCompletion,
4499    ) -> bool {
4500        match completion {
4501            ModuleControlRpcCompletion::Unknown => false,
4502            ModuleControlRpcCompletion::Settled => true,
4503            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
4504                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
4505                info!(
4506                    module_id = %module_id,
4507                    latency_ms,
4508                    "late health.check answer proves the module is alive"
4509                );
4510                match self
4511                    .supervisor
4512                    .record_late_health_answer(&module_id, latency_ms)
4513                {
4514                    Ok(true) => {}
4515                    Ok(false) => debug!(
4516                        module_id = %module_id,
4517                        latency_ms,
4518                        "late health.check answer has no active supervisor snapshot"
4519                    ),
4520                    Err(err) => warn!(
4521                        module_id = %module_id,
4522                        latency_ms,
4523                        error = %err,
4524                        "failed to record late health.check answer"
4525                    ),
4526                }
4527                true
4528            }
4529        }
4530    }
4531
4532    /// Decide whether a failure while settling a relayed `route.bind` belongs to
4533    /// the module connection whose frame is being handled, or to the client that
4534    /// relay was opened for.
4535    ///
4536    /// This runs on the MODULE connection's frame handler, where returning `Err`
4537    /// ends that connection -- and a module connection carries every client's
4538    /// routes to that module, so ending it costs the whole fleet its tools.
4539    /// `ConnectionClosing` carries the id of the connection that is closing, and
4540    /// when that id is a CLIENT's, the condition is entirely about that one
4541    /// client's route.open. A client-scoped condition has no authority over a
4542    /// shared module connection, so it is logged and the single relay is dropped:
4543    /// the client is going away, and `complete_pending_relay` already removed the
4544    /// relay before failing, so there is nothing left to settle. Anything that
4545    /// relay still reserved is released by that client's own connection teardown,
4546    /// which is already under way -- that is what "closing" means.
4547    ///
4548    /// Every other failure is a statement about THIS connection and stays fatal:
4549    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
4550    /// id in `ConnectionClosing` all mean this connection cannot keep serving
4551    /// frames correctly.
4552    fn refuse_to_end_module_connection_for_a_client(
4553        &self,
4554        module_connection_id: ConnectionId,
4555        corr: u64,
4556        err: ForwardingError,
4557    ) -> Result<(), RouterError> {
4558        if let ForwardingError::ConnectionClosing { connection_id } = err {
4559            if connection_id != module_connection_id {
4560                warn!(
4561                    module_connection_id = module_connection_id.get(),
4562                    client_connection_id = connection_id.get(),
4563                    corr,
4564                    "dropping a route.bind response for a closing client; the module connection keeps serving"
4565                );
4566                return Ok(());
4567            }
4568        }
4569        Err(RouterError::Forwarding(err))
4570    }
4571
4572    fn handle_module_relay_response(
4573        &self,
4574        connection_id: ConnectionId,
4575        frame: Frame,
4576    ) -> Result<Vec<Frame>, RouterError> {
4577        let mut secondary_error = None;
4578        let outcome = match frame.header.ty {
4579            FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
4580                Ok(probe) if probe.op == "route.bind" => {
4581                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
4582                        Ok(ModuleControlResponse::RouteBindAck {}) => {
4583                            RouteBindRelayOutcome::Accepted
4584                        }
4585                        Ok(other) => {
4586                            let message =
4587                                format!("route.bind response carried unexpected body: {other:?}");
4588                            secondary_error = Some(control_error_frame(
4589                                &frame,
4590                                "invalid_control_body",
4591                                message.clone(),
4592                            )?);
4593                            RouteBindRelayOutcome::ModuleGone(message)
4594                        }
4595                        Err(err) => {
4596                            let message = format!("malformed route.bind response body: {err}");
4597                            secondary_error = Some(control_error_frame(
4598                                &frame,
4599                                "invalid_control_body",
4600                                message.clone(),
4601                            )?);
4602                            RouteBindRelayOutcome::ModuleGone(message)
4603                        }
4604                    }
4605                }
4606                Ok(probe) => {
4607                    let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
4608                    {
4609                        Ok(response) => ModuleControlRpcOutcome::Response(response),
4610                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
4611                            "malformed {} response body: {err}",
4612                            probe.op
4613                        )),
4614                    };
4615                    let completion = self
4616                        .forwarding
4617                        .complete_module_control_rpc(
4618                            connection_id,
4619                            frame.header.corr,
4620                            Some(&probe.op),
4621                            outcome,
4622                        )
4623                        .map_err(RouterError::Forwarding)?;
4624                    if !self.observe_module_control_completion(completion) {
4625                        debug!(
4626                            connection_id = connection_id.get(),
4627                            corr = frame.header.corr,
4628                            op = %probe.op,
4629                            "dropping late or unknown module-control RPC response"
4630                        );
4631                    }
4632                    return Ok(Vec::new());
4633                }
4634                Err(err) => {
4635                    if let Some(expected_op) = self
4636                        .forwarding
4637                        .pending_module_control_op(connection_id, frame.header.corr)
4638                        .map_err(RouterError::Forwarding)?
4639                    {
4640                        let completion = self
4641                            .forwarding
4642                            .complete_module_control_rpc(
4643                                connection_id,
4644                                frame.header.corr,
4645                                None,
4646                                ModuleControlRpcOutcome::MalformedResponse(format!(
4647                                    "malformed {expected_op} response body: {err}"
4648                                )),
4649                            )
4650                            .map_err(RouterError::Forwarding)?;
4651                        if !self.observe_module_control_completion(completion) {
4652                            debug!(
4653                                connection_id = connection_id.get(),
4654                                corr = frame.header.corr,
4655                                "dropping late malformed module-control RPC response"
4656                            );
4657                        }
4658                        return Ok(Vec::new());
4659                    }
4660                    let message = format!("malformed route.bind response body: {err}");
4661                    secondary_error = Some(control_error_frame(
4662                        &frame,
4663                        "invalid_control_body",
4664                        message.clone(),
4665                    )?);
4666                    RouteBindRelayOutcome::ModuleGone(message)
4667                }
4668            },
4669            FrameType::Error => {
4670                if self
4671                    .forwarding
4672                    .pending_module_control_op(connection_id, frame.header.corr)
4673                    .map_err(RouterError::Forwarding)?
4674                    .is_some()
4675                {
4676                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
4677                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
4678                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
4679                            "malformed module-control ERROR body: {err}"
4680                        )),
4681                    };
4682                    let completion = self
4683                        .forwarding
4684                        .complete_module_control_rpc(
4685                            connection_id,
4686                            frame.header.corr,
4687                            None,
4688                            outcome,
4689                        )
4690                        .map_err(RouterError::Forwarding)?;
4691                    if !self.observe_module_control_completion(completion) {
4692                        debug!(
4693                            connection_id = connection_id.get(),
4694                            corr = frame.header.corr,
4695                            "dropping late or unknown module-control RPC error"
4696                        );
4697                    }
4698                    return Ok(Vec::new());
4699                }
4700                match serde_json::from_slice::<ErrorBody>(&frame.body) {
4701                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
4702                    Err(err) => {
4703                        let message = format!("malformed route.bind ERROR body: {err}");
4704                        secondary_error = Some(control_error_frame(
4705                            &frame,
4706                            "invalid_control_body",
4707                            message.clone(),
4708                        )?);
4709                        RouteBindRelayOutcome::ModuleGone(message)
4710                    }
4711                }
4712            }
4713            ty => {
4714                return Ok(vec![control_error_frame(
4715                    &frame,
4716                    "unsupported_control_frame",
4717                    format!("unsupported module channel-0 frame {ty:?}"),
4718                )?])
4719            }
4720        };
4721
4722        let settled =
4723            self.forwarding
4724                .complete_pending_relay(connection_id, frame.header.corr, outcome);
4725        let completion = match settled {
4726            Ok(completion) => completion,
4727            Err(err) => {
4728                self.refuse_to_end_module_connection_for_a_client(
4729                    connection_id,
4730                    frame.header.corr,
4731                    err,
4732                )?;
4733                return Ok(secondary_error.into_iter().collect());
4734            }
4735        };
4736        if let Some(target) = completion.abandoned.as_ref() {
4737            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
4738        }
4739        if !completion.settled {
4740            debug!(
4741                connection_id = connection_id.get(),
4742                corr = frame.header.corr,
4743                frame_type = ?frame.header.ty,
4744                "dropping late or unknown route.bind relay response"
4745            );
4746        }
4747        Ok(secondary_error.into_iter().collect())
4748    }
4749
4750    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
4751        debug!(connection_id = connection_id.get(), "handling GOODBYE");
4752        let registrations = self
4753            .deregister_connection(connection_id)
4754            .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
4755        let released_routes = self
4756            .forwarding
4757            .cleanup_connection(connection_id)
4758            .map_err(RouterError::Forwarding)?;
4759        self.emit_route_goodbyes(released_routes);
4760        // Notify only after forwarding teardown completes (see cleanup_connection).
4761        if !registrations.is_empty() {
4762            crate::supervise::notify_registration_release();
4763        }
4764        Ok(Vec::new())
4765    }
4766}
4767
4768impl Default for ControlHandler {
4769    fn default() -> Self {
4770        Self::new(Arc::new(Registry::default()))
4771    }
4772}
4773
4774impl crate::supervise::SwapPromotionObserver for ControlHandler {
4775    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
4776        self.apply_registration_capabilities(registration);
4777    }
4778}
4779
4780fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
4781    CapabilityRequirementStatus {
4782        consumer: status.consumer,
4783        capability: status.capability,
4784        need: match status.need {
4785            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
4786            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
4787        },
4788        verdict: status.verdict.as_str().to_string(),
4789        episode_seq: status.episode_seq,
4790        config_satisfiable: status.config_satisfiable,
4791        runtime_available: status.runtime_available,
4792        detail: status.detail,
4793    }
4794}
4795
4796fn append_capability_problem_detail(
4797    detail: Option<String>,
4798    capability_detail: Option<String>,
4799) -> Option<String> {
4800    match (detail, capability_detail) {
4801        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
4802        (Some(detail), None) => Some(detail),
4803        (None, Some(capability_detail)) => Some(capability_detail),
4804        (None, None) => None,
4805    }
4806}
4807
4808fn subc_ops() -> Vec<String> {
4809    SUBC_CONTROL_OPS
4810        .iter()
4811        .map(|op| (*op).to_string())
4812        .collect()
4813}
4814
4815fn module_subc_ops() -> Vec<String> {
4816    SUBC_CONTROL_OPS
4817        .iter()
4818        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
4819        .map(|op| (*op).to_string())
4820        .collect()
4821}
4822
4823#[cfg(test)]
4824fn module_baseline_control_ops() -> Vec<String> {
4825    MODULE_BASELINE_CONTROL_OPS
4826        .iter()
4827        .map(|op| (*op).to_string())
4828        .collect()
4829}
4830
4831fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
4832    let mut seen = HashSet::new();
4833    let mut effective = Vec::new();
4834    for op in MODULE_BASELINE_CONTROL_OPS {
4835        if seen.insert((*op).to_string()) {
4836            effective.push((*op).to_string());
4837        }
4838    }
4839    for op in declared.unwrap_or_default() {
4840        if seen.insert(op.clone()) {
4841            effective.push(op);
4842        }
4843    }
4844    effective
4845}
4846
4847fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
4848    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
4849}
4850
4851fn target_module_id(target: &RouteTarget) -> &str {
4852    match target {
4853        RouteTarget::ToolProvider { module_id }
4854        | RouteTarget::ManagementSurface { module_id }
4855        | RouteTarget::InternalService { module_id, .. } => module_id,
4856    }
4857}
4858
4859fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
4860    roles.iter().any(|role| match (target, role) {
4861        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
4862        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
4863        (
4864            RouteTarget::InternalService { service_id, .. },
4865            ProviderRole::InternalService {
4866                service_id: provided,
4867                ..
4868            },
4869        ) => service_id == provided,
4870        _ => false,
4871    })
4872}
4873
4874fn is_routable_role(role: &ProviderRole) -> bool {
4875    matches!(
4876        role,
4877        ProviderRole::ToolProvider { .. }
4878            | ProviderRole::ManagementSurface { .. }
4879            | ProviderRole::InternalService { .. }
4880    )
4881}
4882
4883#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4884enum ControlRequestBodyError {
4885    UnknownOp,
4886    InvalidBody,
4887}
4888
4889#[derive(Debug, Deserialize)]
4890struct ControlOpProbe {
4891    op: String,
4892}
4893
4894/// Channel-0 push ops this subc version understands. A push whose `op` is not in
4895/// this set is treated as a forward-compat unknown and ignored rather than errored.
4896const MODULE_PUSH_OPS: &[&str] = &["route.status"];
4897
4898fn is_known_module_push_op(body: &[u8]) -> bool {
4899    serde_json::from_slice::<ControlOpProbe>(body)
4900        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
4901        .unwrap_or(false)
4902}
4903
4904fn is_known_module_request_op(body: &[u8]) -> bool {
4905    serde_json::from_slice::<ControlOpProbe>(body)
4906        .map(|probe| MODULE_TO_SUBC_CONTROL_OPS.contains(&probe.op.as_str()))
4907        .unwrap_or(false)
4908}
4909
4910fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
4911    debug!(
4912        op = %op,
4913        connection_id = connection_id.get(),
4914        corr,
4915        "control dispatch"
4916    );
4917}
4918
4919fn log_slow_control_dispatch(
4920    dispatch_started_at: Option<StdInstant>,
4921    op: &'static str,
4922    connection_id: ConnectionId,
4923    corr: u64,
4924) {
4925    let Some(dispatch_started_at) = dispatch_started_at else {
4926        return;
4927    };
4928    let elapsed = dispatch_started_at.elapsed();
4929    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
4930        warn!(
4931            op = %op,
4932            connection_id = connection_id.get(),
4933            corr,
4934            elapsed_ms = elapsed.as_millis() as u64,
4935            "slow control dispatch"
4936        );
4937    }
4938}
4939
4940fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
4941    match request {
4942        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
4943        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
4944        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
4945        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
4946        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
4947        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
4948        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
4949        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
4950        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
4951        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
4952        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
4953        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
4954        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
4955        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
4956        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
4957        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
4958        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
4959        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
4960        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
4961    }
4962}
4963
4964fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
4965    match request {
4966        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
4967        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
4968    }
4969}
4970
4971fn parse_client_control_request(
4972    body: &[u8],
4973) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
4974    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
4975        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
4976            Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
4977                ControlRequestBodyError::InvalidBody
4978            }
4979            Ok(_) => ControlRequestBodyError::UnknownOp,
4980            Err(_) => ControlRequestBodyError::InvalidBody,
4981        };
4982        (err, classification)
4983    })
4984}
4985
4986fn parse_module_control_request_from_module(
4987    body: &[u8],
4988) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
4989    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
4990        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
4991            Ok(probe) if MODULE_TO_SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
4992                ControlRequestBodyError::InvalidBody
4993            }
4994            Ok(_) => ControlRequestBodyError::UnknownOp,
4995            Err(_) => ControlRequestBodyError::InvalidBody,
4996        };
4997        (err, classification)
4998    })
4999}
5000
5001#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
5002enum ProviderRoleKind {
5003    ToolProvider,
5004    PipelineStage,
5005    ManagementSurface,
5006    InternalService,
5007}
5008
5009fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
5010    match role {
5011        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
5012        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
5013        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
5014        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
5015    }
5016}
5017
5018fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
5019    roles.iter().map(provider_role_kind).collect()
5020}
5021
5022/// Return whether a catalog change can create a newly violating live route.
5023/// Removing an attested claim is intentionally excluded: it makes fewer routes
5024/// forbidden and therefore must leave the existing route census untouched.
5025fn capability_census_trigger(
5026    old: Option<&CapabilityDeclarations>,
5027    new: Option<&CapabilityDeclarations>,
5028) -> bool {
5029    let old_provides = old
5030        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
5031        .unwrap_or_default();
5032    let old_denies = old
5033        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
5034        .unwrap_or_default();
5035    let new = new.cloned().unwrap_or(CapabilityDeclarations {
5036        provides: Vec::new(),
5037        requires: Vec::new(),
5038        must_never_reach: Vec::new(),
5039    });
5040
5041    new.provides
5042        .iter()
5043        .any(|capability| !old_provides.contains(capability))
5044        || new
5045            .must_never_reach
5046            .iter()
5047            .any(|capability| !old_denies.contains(capability))
5048}
5049
5050/// Find the first capability an attested opener denies that an attested target
5051/// claims. Both manifests are live registry records, never cached or client data.
5052fn denied_capability<'a>(
5053    opening_manifest: &'a ModuleManifest,
5054    target_manifest: &ModuleManifest,
5055) -> Option<&'a str> {
5056    let opening_capabilities = opening_manifest.capabilities.as_ref()?;
5057    let target_capabilities = target_manifest.capabilities.as_ref()?;
5058    opening_capabilities
5059        .must_never_reach
5060        .iter()
5061        .find(|denied| {
5062            target_capabilities
5063                .provides
5064                .iter()
5065                .any(|provided| provided == *denied)
5066        })
5067        .map(String::as_str)
5068}
5069
5070fn catalog_update_frozen_field_message(
5071    registered: &ModuleManifest,
5072    provides: &[ProviderRole],
5073) -> Option<String> {
5074    let old_has_provides = !registered.provides.is_empty();
5075    let new_has_provides = !provides.is_empty();
5076    if old_has_provides != new_has_provides {
5077        return Some(format!(
5078            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
5079            registered.module_id
5080        ));
5081    }
5082
5083    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
5084        return Some(format!(
5085            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
5086            registered.module_id
5087        ));
5088    }
5089
5090    let registered_concurrency = manifest_concurrency(registered);
5091    let mut candidate = registered.clone();
5092    candidate.provides = provides.to_vec();
5093    let candidate_concurrency = manifest_concurrency(&candidate);
5094    if candidate_concurrency != registered_concurrency {
5095        return Some(format!(
5096            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
5097            registered.module_id, registered_concurrency, candidate_concurrency
5098        ));
5099    }
5100
5101    // control_ops live beside the manifest in the HELLO body, not inside
5102    // ModuleManifest, so a provides-only catalog.update cannot change them.
5103    None
5104}
5105
5106fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
5107    manifest.provides.iter().any(is_routable_role)
5108}
5109
5110/// Returns the routable-provider concurrency subc should enforce for this manifest.
5111///
5112/// ToolProvider and ManagementSurface store their delivery concurrency directly.
5113/// InternalService has no role-specific concurrency field, so it retains the
5114/// existing ModuleManaged default for backward compatibility.
5115fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
5116    manifest
5117        .provides
5118        .iter()
5119        .find_map(|provider| match provider {
5120            ProviderRole::ToolProvider { concurrency, .. }
5121            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
5122            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
5123        })
5124        .unwrap_or(Concurrency::ModuleManaged)
5125}
5126
5127/// True when the manifest carries a ManagementSurface role whose concurrency
5128/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
5129/// bytes because the typed manifest deliberately erases that distinction: the
5130/// default exists for wire compatibility, and this probe exists so the default
5131/// stays observable. Any parse irregularity returns false -- the caller only
5132/// logs, and a malformed body already failed registration upstream.
5133fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
5134    let has_management_surface = manifest
5135        .provides
5136        .iter()
5137        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
5138    if !has_management_surface {
5139        return false;
5140    }
5141    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
5142        return false;
5143    };
5144    let Some(provides) = raw
5145        .get("manifest")
5146        .and_then(|manifest| manifest.get("provides"))
5147        .and_then(serde_json::Value::as_array)
5148    else {
5149        return false;
5150    };
5151    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
5152    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
5153    // against the management_surface_manifest_without_concurrency golden, not
5154    // recalled (the externally-tagged guess was this function's first bug).
5155    provides.iter().any(|role| {
5156        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
5157            && role.get("concurrency").is_none()
5158    })
5159}
5160
5161fn negotiate_version(peer_version: u8) -> Result<u8, String> {
5162    if peer_version != PROTOCOL_VERSION {
5163        return Err(format!(
5164            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
5165        ));
5166    }
5167    Ok(PROTOCOL_VERSION)
5168}
5169
5170fn pong(frame: &Frame) -> Result<Frame, RouterError> {
5171    Frame::build_with_version(
5172        response_version(frame),
5173        FrameType::Pong,
5174        frame.header.flags,
5175        0,
5176        0,
5177        frame.header.corr,
5178        Vec::new(),
5179    )
5180    .map_err(RouterError::FrameBuild)
5181}
5182
5183fn control_error_frame(
5184    frame: &Frame,
5185    code: &'static str,
5186    message: impl Into<String>,
5187) -> Result<Frame, RouterError> {
5188    control_error_body_frame(
5189        frame,
5190        ErrorBody {
5191            code: code.to_string(),
5192            message: message.into(),
5193            detail: None,
5194        },
5195    )
5196}
5197
5198fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
5199    let body = serde_json::to_vec(&error).map_err(|err| {
5200        RouterError::backend(
5201            0,
5202            frame.header.corr,
5203            format!("failed to encode control ERROR: {err}"),
5204        )
5205    })?;
5206
5207    Frame::build_with_version(
5208        response_version(frame),
5209        FrameType::Error,
5210        control_flags(),
5211        0,
5212        0,
5213        frame.header.corr,
5214        body,
5215    )
5216    .map_err(RouterError::FrameBuild)
5217}
5218
5219fn control_response_body_frame<T: Serialize>(
5220    frame: &Frame,
5221    reply: &T,
5222    label: &'static str,
5223) -> Result<Frame, RouterError> {
5224    let body = serde_json::to_vec(reply).map_err(|err| {
5225        RouterError::backend(
5226            0,
5227            frame.header.corr,
5228            format!("failed to encode {label}: {err}"),
5229        )
5230    })?;
5231
5232    Frame::build_with_version(
5233        response_version(frame),
5234        FrameType::Response,
5235        control_flags(),
5236        0,
5237        0,
5238        frame.header.corr,
5239        body,
5240    )
5241    .map_err(RouterError::FrameBuild)
5242}
5243
5244/// Map a forwarding failure to the wire code a client sees.
5245///
5246/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
5247/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
5248/// chosen here decides whether a caller retries or gives up.
5249///
5250/// That makes attribution the load-bearing property, not merely having a code. A
5251/// permanent fault published as a retryable one produces a fleet-wide retry storm
5252/// against something that can never recover; a transient fault published as
5253/// permanent gives up on work that would have succeeded. Both look correct in a
5254/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
5255/// the mapping per variant rather than merely asserting that some code exists.
5256///
5257/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
5258/// two codes on the same side of the boundary passes it. Measured rather than
5259/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
5260/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
5261/// a test named for something else that happens to assert the string.
5262///
5263/// That accidental coverage is deliberately left alone rather than promoted to a
5264/// named test, because it guards a property this function does not promise.
5265/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
5266/// specific code within a class, so identity is free to change and only the
5267/// partition is a contract. Splitting it out would assert a guarantee nothing
5268/// depends on — and a suite that promises more than the code does is the harder
5269/// thing to correct later, because the next reader cannot tell which assertions
5270/// are load-bearing.
5271///
5272/// Pin identity here the moment a consumer branches on a specific code.
5273fn forwarding_error_code(err: &ForwardingError) -> &'static str {
5274    match err {
5275        ForwardingError::NoModuleConnection => "target_unavailable",
5276        ForwardingError::ModuleReloading { .. } => "module_reloading",
5277        ForwardingError::ClientRouteChannelExhausted { .. }
5278        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
5279        ForwardingError::StaleModuleEndpoint
5280        | ForwardingError::UnknownReservation { .. }
5281        | ForwardingError::ConnectionClosing { .. }
5282        | ForwardingError::ClientEgressClosed { .. }
5283        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
5284        // Only a swap candidate's registration can produce this, and it means
5285        // exactly what a second active HELLO for a live id means.
5286        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
5287        ForwardingError::RelayCorrelationExhausted
5288        | ForwardingError::RouteOpenBuild(_)
5289        | ForwardingError::Poisoned => "forwarding_error",
5290    }
5291}
5292
5293fn response_version(frame: &Frame) -> u8 {
5294    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
5295        frame.header.ver
5296    } else {
5297        PROTOCOL_VERSION
5298    }
5299}
5300
5301fn control_flags() -> Flags {
5302    Flags::new(false, Priority::Passive, false)
5303}
5304
5305/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
5306/// channel. The target is the module (a client never saw the route), so this
5307/// takes the module path: delivered late rather than dropped when the module's
5308/// queue is momentarily full, and never closing its connection.
5309fn send_goodbye_target_best_effort(
5310    counters: &DaemonCounters,
5311    target: &GoodbyeTarget,
5312    context: &'static str,
5313) {
5314    let Ok(frame) = Frame::build_with_version(
5315        target.negotiated_ver,
5316        FrameType::Goodbye,
5317        control_flags(),
5318        target.channel,
5319        target.epoch,
5320        0,
5321        Vec::new(),
5322    ) else {
5323        return;
5324    };
5325    crate::forwarding::send_module_route_goodbye(
5326        counters,
5327        &target.sink,
5328        frame,
5329        target.module_id.as_deref(),
5330        context,
5331    );
5332}
5333
5334pub(crate) fn send_route_control_pushes(
5335    forwarding: &ForwardingTable,
5336    routes: Vec<EndpointRoute>,
5337    push: ClientControlPush,
5338) {
5339    let body = match serde_json::to_vec(&push) {
5340        Ok(body) => body,
5341        Err(err) => {
5342            warn!(error = %err, "failed to serialize route lifecycle control PUSH");
5343            return;
5344        }
5345    };
5346    let mut targets = Vec::new();
5347    for route in routes {
5348        let target = route.goodbye_target;
5349        if let Some(existing) = targets
5350            .iter()
5351            .find(|existing: &&GoodbyeTarget| existing.connection_id == target.connection_id)
5352        {
5353            debug_assert_eq!(
5354                existing.negotiated_ver, target.negotiated_ver,
5355                "one connection cannot negotiate multiple frame versions"
5356            );
5357            continue;
5358        }
5359        targets.push(target);
5360    }
5361    for target in targets {
5362        let frame = match Frame::build_with_version(
5363            target.negotiated_ver,
5364            FrameType::Push,
5365            control_flags(),
5366            0,
5367            0,
5368            0,
5369            body.clone(),
5370        ) {
5371            Ok(frame) => frame,
5372            Err(err) => {
5373                warn!(
5374                    route_channel = target.channel,
5375                    error = %err,
5376                    "failed to build route lifecycle control PUSH frame"
5377                );
5378                continue;
5379            }
5380        };
5381        if let Err(err) = target.sink.try_send(frame) {
5382            if target.close_on_delivery_failure() {
5383                warn!(
5384                    target_connection_id = target.connection_id.get(),
5385                    route_channel = target.channel,
5386                    error = %err,
5387                    "route lifecycle control PUSH was not delivered to client; closing target connection"
5388                );
5389                let _ = forwarding.escalate_client_delivery_failure(
5390                    target.connection_id,
5391                    target.channel,
5392                    target.epoch,
5393                    CloseReason::new(
5394                        "route_lifecycle_push_delivery_failed",
5395                        format!(
5396                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
5397                            target.channel
5398                        ),
5399                    ),
5400                    crate::forwarding::UndeliveredFrame {
5401                        module_id: target.module_id.as_deref(),
5402                        sink: &target.sink,
5403                    },
5404                );
5405            }
5406        }
5407    }
5408}
5409
5410#[cfg(test)]
5411mod tests {
5412    use std::{
5413        collections::BTreeMap,
5414        fmt,
5415        path::PathBuf,
5416        sync::{Arc, Mutex},
5417        time::Duration,
5418    };
5419
5420    use serde_json::{json, Value};
5421    use subc_protocol::{
5422        manifest::{
5423            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
5424            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
5425        },
5426        session::HealthStatus,
5427        FrameType,
5428    };
5429
5430    use super::*;
5431    use crate::{
5432        forwarding::{DataRoute, DataRouteState},
5433        registry::ChannelState,
5434        router::FrameSink,
5435        stderr_tail::DEFAULT_MAX_LINE_BYTES,
5436        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
5437        test_support::TestTempDir,
5438        RouteCtx, Router,
5439    };
5440    use tokio::{
5441        sync::mpsc,
5442        time::{sleep, Instant},
5443    };
5444    use tracing::{
5445        field::{Field, Visit},
5446        Event, Subscriber,
5447    };
5448    use tracing_subscriber::{layer::Context, prelude::*, Layer};
5449
5450    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
5451    ///
5452    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
5453    /// is only populated for `tests/*.rs` integration test binaries -- this file
5454    /// compiles as part of the library target, which gets neither. This test's
5455    /// own executable path is `<target-dir>/<profile>/deps/subc_core-<hash>`,
5456    /// and the sibling binary lives two directories up at
5457    /// `<target-dir>/<profile>/fake-aft-stub`.
5458    ///
5459    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
5460    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
5461    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
5462    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
5463    /// `NotFound`, which reads as a broken test rather than an unbuilt
5464    /// dependency -- so state the cause and the remedy instead. Deliberately a
5465    /// panic and not a silent skip: a test that quietly passes when it could not
5466    /// run is worse than one that fails, because it reports health it never
5467    /// verified.
5468    fn fake_aft_stub_path() -> PathBuf {
5469        let mut path = std::env::current_exe().expect("current_exe available in tests");
5470        path.pop(); // .../deps/
5471        path.pop(); // .../<profile>/
5472        path.push(if cfg!(windows) {
5473            "fake-aft-stub.exe"
5474        } else {
5475            "fake-aft-stub"
5476        });
5477        assert!(
5478            path.exists(),
5479            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
5480             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
5481            path.display()
5482        );
5483        path
5484    }
5485
5486    /// Whether clients retry `code` in place: the predicate itself, never a copy
5487    /// of its set. A copied list breaks silently when a code is added to or
5488    /// removed from the real one, and a stale copy here would let exactly the
5489    /// failure this test exists to catch pass.
5490    fn client_retries(code: &str) -> bool {
5491        subc_protocol::error_codes::is_retryable_route_open(code)
5492    }
5493
5494    /// A code is not a label — clients branch on it, so publishing the wrong KIND
5495    /// of failure is worse than publishing none. A permanent fault dressed as
5496    /// retryable makes every client in the fleet retry forever against something
5497    /// that cannot recover; a transient fault dressed as permanent abandons work
5498    /// that would have succeeded.
5499    ///
5500    /// Asserting "a code exists" cannot catch either, because the string is free
5501    /// to say anything. This enumerates every variant and pins which side of the
5502    /// retry boundary it lands on, so a new variant must be classified here
5503    /// deliberately rather than inheriting whichever arm it was appended to.
5504    #[test]
5505    fn retryability_of_forwarding_codes_matches_the_failure() {
5506        // Transient by nature: the target is booting, reloading, or its endpoint
5507        // was swapped mid-flight. Retrying is how these resolve.
5508        let transient = [
5509            ForwardingError::NoModuleConnection,
5510            ForwardingError::ModuleReloading {
5511                module_id: "m".into(),
5512            },
5513            ForwardingError::StaleModuleEndpoint,
5514            ForwardingError::UnknownReservation {
5515                client_channel: 1,
5516                module_channel: 1,
5517            },
5518            ForwardingError::ConnectionClosing {
5519                connection_id: ConnectionId::new(1),
5520            },
5521            ForwardingError::ClientEgressClosed {
5522                connection_id: ConnectionId::new(1),
5523            },
5524            ForwardingError::ModuleEgressUnavailable {
5525                connection_id: ConnectionId::new(1),
5526            },
5527        ];
5528        for err in transient {
5529            let code = forwarding_error_code(&err);
5530            assert!(
5531                client_retries(code),
5532                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
5533            );
5534        }
5535
5536        // Not fixed by retrying. Channel and correlation exhaustion need the
5537        // caller to close routes, and a poisoned lock is a daemon that cannot
5538        // recover at all — the worst thing to advertise as retryable, since every
5539        // client would storm a daemon that will never answer.
5540        let permanent = [
5541            ForwardingError::ClientRouteChannelExhausted {
5542                connection_id: ConnectionId::new(1),
5543            },
5544            ForwardingError::ModuleRouteChannelExhausted {
5545                endpoint: ModuleEndpointId {
5546                    connection_id: ConnectionId::new(1),
5547                    generation: 1,
5548                },
5549            },
5550            ForwardingError::RelayCorrelationExhausted,
5551            ForwardingError::RouteOpenBuild("x".into()),
5552            ForwardingError::Poisoned,
5553        ];
5554        for err in permanent {
5555            let code = forwarding_error_code(&err);
5556            assert!(
5557                !client_retries(code),
5558                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
5559            );
5560        }
5561    }
5562
5563    /// The principal is the daemon's answer to "who is calling", and modules
5564    /// branch on it: aft gates bash on it, cerebellum gates browser control,
5565    /// plexus gates connector invocation. So a stamp is an authorization input in
5566    /// another process, not a label — and both possible answers SUCCEED, which is
5567    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
5568    /// first-party capability to something that never proved it; a supervised one
5569    /// stamped `Direct` silently strips a module of capability it is entitled to.
5570    ///
5571    /// Neither shows up in a test that only checks the bind succeeded. Before this
5572    /// test the only coverage was accidental —
5573    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
5574    /// stamped principal on its way past, so narrowing that wire-shape test to its
5575    /// stated subject would have deleted the last assertion on this value. It
5576    /// still asserts the stamp, which is now redundancy rather than the only
5577    /// guard: both fail under the same mutation, and this one names the reason.
5578    /// SCOPE: this handler's supervisor has spawned nothing, so
5579    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
5580    /// is unreachable here. Both assertions below are refusals, and a mutant that
5581    /// refuses everything would satisfy them.
5582    ///
5583    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
5584    /// spawns a supervised consumer, reads its live nonce, and asserts the module
5585    /// observed `principal.kind == "reserved"` carrying that module_id — verified
5586    /// at source rather than assumed, since a citation is a claim about another
5587    /// file and ages like one. Recorded because a harness that structurally
5588    /// cannot reach an arm reports "none" for that arm identically to one that
5589    /// covers it and found nothing.
5590    #[tokio::test]
5591    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
5592        let handler = ControlHandler::default();
5593        let frame =
5594            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
5595
5596        // Absent consumer_identity is the ordinary case: a human at a terminal, or
5597        // any process holding the connection file. Nothing was proved, so nothing
5598        // may be granted beyond the unattested floor.
5599        let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
5600        assert_eq!(
5601            stamped,
5602            Principal::Direct,
5603            "a caller that proved nothing must not be stamped as a supervised module"
5604        );
5605
5606        // A claimed module_id with a nonce no supervised child was given is a
5607        // forgery attempt, not a weaker caller: it must be REFUSED rather than
5608        // quietly demoted to Direct, or an impersonation attempt looks identical
5609        // to an ordinary unattested connection.
5610        let forged = handler
5611            .route_open_principal(
5612                &frame,
5613                Some(ConsumerIdentity {
5614                    module_id: "aft".to_string(),
5615                    launch_nonce: "not-a-real-nonce".to_string(),
5616                }),
5617            )
5618            .unwrap();
5619        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
5620        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
5621    }
5622
5623    /// The test above hands `route_open_principal` an identity it built itself,
5624    /// which proves the stamping rule and nothing about where the identity comes
5625    /// from. The real producer is a wire body, and the two are joined by a serde
5626    /// field name that nothing else asserts.
5627    ///
5628    /// That join fails quietly in one specific way: an unrecognised key is simply
5629    /// absent after parsing, so a renamed or misspelled `consumer_identity`
5630    /// yields `None` and every supervised module silently drops to `Direct`.
5631    /// Capability-wise that is the safe direction, but it surfaces far from its
5632    /// cause — as a module mysteriously refused bash — and it would pass every
5633    /// test that builds its own input.
5634    ///
5635    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
5636    /// would break every client the moment the daemon gains a field, trading a
5637    /// quiet demotion for a hard refusal on additive change. Asserting the join
5638    /// instead means a rename breaks a test here rather than the fleet.
5639    #[test]
5640    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
5641        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
5642        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
5643        let ClientControlRequest::RouteOpen {
5644            consumer_identity, ..
5645        } = parsed
5646        else {
5647            panic!("route.open body must parse as RouteOpen");
5648        };
5649        assert_eq!(
5650            consumer_identity,
5651            Some(ConsumerIdentity {
5652                module_id: "aft".to_string(),
5653                launch_nonce: "n".to_string(),
5654            }),
5655            "the wire field name must reach the value route_open_principal reads"
5656        );
5657    }
5658
5659    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
5660        ModuleManifest::builder(module_id, "0.1.0")
5661            .protocol_ver(protocol_ver)
5662            .provides(vec![ProviderRole::ToolProvider {
5663                tools: vec![Tool {
5664                    name: "read".to_string(),
5665                    description: None,
5666                    execution_mode: ExecutionMode::Pure,
5667                    schema: json!({"type": "object"}),
5668                }],
5669                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
5670                concurrency: Concurrency::ModuleManaged,
5671                emits_push: true,
5672                sub_supervises: true,
5673            }])
5674            .build()
5675    }
5676
5677    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
5678        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
5679    }
5680
5681    fn hello_frame_with_control_ops(
5682        module_id: &str,
5683        protocol_ver: u8,
5684        corr: u64,
5685        control_ops: Option<Vec<String>>,
5686    ) -> Frame {
5687        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
5688    }
5689
5690    fn hello_frame_with_nonce(
5691        module_id: &str,
5692        protocol_ver: u8,
5693        corr: u64,
5694        launch_nonce: Option<&str>,
5695    ) -> Frame {
5696        hello_frame_full(
5697            module_id,
5698            protocol_ver,
5699            corr,
5700            None,
5701            launch_nonce.map(ToOwned::to_owned),
5702        )
5703    }
5704
5705    fn hello_frame_full(
5706        module_id: &str,
5707        protocol_ver: u8,
5708        corr: u64,
5709        control_ops: Option<Vec<String>>,
5710        launch_nonce: Option<String>,
5711    ) -> Frame {
5712        let body = serde_json::to_vec(&ModuleHelloBody {
5713            manifest: manifest(module_id, protocol_ver),
5714            protocol_ver,
5715            control_ops,
5716            launch_nonce,
5717        })
5718        .unwrap();
5719        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
5720    }
5721
5722    fn non_routable_hello_frame_with_control_ops(
5723        module_id: &str,
5724        corr: u64,
5725        control_ops: Option<Vec<String>>,
5726    ) -> Frame {
5727        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
5728        manifest.provides.clear();
5729        let body = serde_json::to_vec(&ModuleHelloBody {
5730            manifest,
5731            protocol_ver: PROTOCOL_VERSION,
5732            control_ops,
5733            launch_nonce: None,
5734        })
5735        .unwrap();
5736        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
5737    }
5738
5739    fn capability_grammar_hello_frame(
5740        capabilities: Value,
5741        runtime_computed: Option<Value>,
5742        corr: u64,
5743    ) -> Frame {
5744        let mut body = serde_json::to_value(ModuleHelloBody {
5745            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
5746            protocol_ver: PROTOCOL_VERSION,
5747            control_ops: None,
5748            launch_nonce: None,
5749        })
5750        .expect("HELLO body serializes");
5751        body["manifest"]["capabilities"] = capabilities;
5752        if let Some(runtime_computed) = runtime_computed {
5753            body["runtime_computed"] = runtime_computed;
5754        }
5755        Frame::build(
5756            FrameType::Hello,
5757            control_flags(),
5758            0,
5759            0,
5760            corr,
5761            serde_json::to_vec(&body).expect("HELLO body reserializes"),
5762        )
5763        .expect("HELLO frame builds")
5764    }
5765
5766    fn channel_request(channel: u16, corr: u64) -> Frame {
5767        Frame::build(
5768            FrameType::Request,
5769            Flags::new(true, Priority::Interactive, false),
5770            channel,
5771            0,
5772            corr,
5773            b"opaque".to_vec(),
5774        )
5775        .unwrap()
5776    }
5777
5778    fn route_ctx(
5779        connection_id: ConnectionId,
5780    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
5781        let (tx, rx) = mpsc::channel(8);
5782        (
5783            RouteCtx {
5784                connection_id,
5785                egress: FrameSink::new(tx),
5786            },
5787            rx,
5788        )
5789    }
5790
5791    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
5792        serde_json::from_slice(&frame.body).unwrap()
5793    }
5794
5795    /// Register a module over a connection that has a sink and return the
5796    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
5797    /// module's own sink rather than returning it as a reply, so the ack is
5798    /// taken off `rx` here and whatever the test reads next is what followed it.
5799    async fn hello_via_sink(
5800        handler: &ControlHandler,
5801        ctx: &RouteCtx,
5802        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
5803        hello: Frame,
5804    ) -> Frame {
5805        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
5806        assert!(
5807            replies.is_empty(),
5808            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
5809        );
5810        let ack = rx
5811            .try_recv()
5812            .expect("HELLO_ACK is queued on the module sink")
5813            .frame;
5814        assert_eq!(ack.header.ty, FrameType::HelloAck);
5815        ack
5816    }
5817
5818    fn parse_error(frame: &Frame) -> Value {
5819        serde_json::from_slice(&frame.body).unwrap()
5820    }
5821
5822    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
5823        serde_json::from_slice(&frame.body).unwrap()
5824    }
5825
5826    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
5827        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
5828            route_channel,
5829            route_epoch: 0,
5830            kind,
5831        })
5832        .unwrap();
5833        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5834    }
5835
5836    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
5837        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
5838            module_id: module_id.to_string(),
5839        })
5840        .unwrap();
5841        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5842    }
5843
5844    fn route_open_frame(corr: u64, module_id: &str, project_root: TestTempDir) -> Frame {
5845        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
5846    }
5847
5848    fn route_open_frame_with_consumer_capabilities(
5849        corr: u64,
5850        module_id: &str,
5851        project_root: TestTempDir,
5852        consumer_capabilities: Option<Vec<String>>,
5853    ) -> Frame {
5854        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
5855            target: RouteTarget::ToolProvider {
5856                module_id: module_id.to_string(),
5857            },
5858            identity: BindIdentity::new(
5859                project_root.path().to_path_buf(),
5860                "unit".to_string(),
5861                "session".to_string(),
5862            ),
5863            consumer_identity: None,
5864            consumer_capabilities,
5865            admission_facts: None,
5866        })
5867        .unwrap();
5868        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5869    }
5870
5871    fn route_open_frame_with_admission_facts(
5872        corr: u64,
5873        module_id: &str,
5874        project_root: TestTempDir,
5875        consumer_identity: Option<subc_control::ConsumerIdentity>,
5876        facts: Option<Value>,
5877    ) -> Frame {
5878        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
5879            target: RouteTarget::ToolProvider {
5880                module_id: module_id.to_string(),
5881            },
5882            identity: BindIdentity::new(
5883                project_root.path().to_path_buf(),
5884                "unit".to_string(),
5885                format!("session-{corr}"),
5886            ),
5887            consumer_identity,
5888            consumer_capabilities: None,
5889            admission_facts: facts,
5890        })
5891        .unwrap();
5892        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5893    }
5894
5895    #[derive(Clone, Default)]
5896    struct EventCapture {
5897        events: Arc<Mutex<Vec<CapturedEvent>>>,
5898    }
5899
5900    #[derive(Clone, Debug)]
5901    struct CapturedEvent {
5902        target: String,
5903        fields: BTreeMap<String, String>,
5904    }
5905
5906    impl EventCapture {
5907        fn events(&self) -> Vec<CapturedEvent> {
5908            self.events.lock().unwrap().clone()
5909        }
5910    }
5911
5912    impl<S> Layer<S> for EventCapture
5913    where
5914        S: Subscriber,
5915    {
5916        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
5917            let mut visitor = EventFieldVisitor::default();
5918            event.record(&mut visitor);
5919            self.events.lock().unwrap().push(CapturedEvent {
5920                target: event.metadata().target().to_string(),
5921                fields: visitor.fields,
5922            });
5923        }
5924    }
5925
5926    #[derive(Default)]
5927    struct EventFieldVisitor {
5928        fields: BTreeMap<String, String>,
5929    }
5930
5931    impl Visit for EventFieldVisitor {
5932        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
5933            self.fields
5934                .insert(field.name().to_string(), format!("{value:?}"));
5935        }
5936    }
5937
5938    fn health_response(corr: u64, status: HealthStatus) -> Frame {
5939        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
5940            status,
5941            detail: Some("warming".to_string()),
5942            metrics: Some(json!({"queue_depth": 3})),
5943        })
5944        .unwrap();
5945        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
5946    }
5947
5948    fn route_bind_ack(corr: u64) -> Frame {
5949        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
5950        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
5951    }
5952
5953    fn unique_project_root(label: &str) -> TestTempDir {
5954        TestTempDir::new(label)
5955    }
5956
5957    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
5958        match parse_route_poll(frame) {
5959            ClientControlResponse::RoutePoll {
5960                status: None,
5961                live: Some(live),
5962                ..
5963            } => assert_eq!(live, expected_live),
5964            other => panic!("unexpected route.poll response: {other:?}"),
5965        }
5966    }
5967
5968    fn bind_liveness_route(
5969        registry: &Registry,
5970        forwarding: &ForwardingTable,
5971        module_id: &str,
5972    ) -> (RouteCtx, u16, u32) {
5973        let module_connection = ConnectionId::new(101);
5974        let client_connection = ConnectionId::new(202);
5975        let registration = registry
5976            .register_with_control_ops(
5977                manifest(module_id, PROTOCOL_VERSION),
5978                PROTOCOL_VERSION,
5979                module_connection,
5980                module_baseline_control_ops(),
5981            )
5982            .unwrap();
5983        let (module_tx, _module_rx) = mpsc::channel(8);
5984        let endpoint = forwarding
5985            .register_module_connection(
5986                module_connection,
5987                module_id.to_string(),
5988                PROTOCOL_VERSION,
5989                manifest_concurrency(&registration.manifest),
5990                FrameSink::new(module_tx),
5991            )
5992            .unwrap();
5993        let (client_ctx, _client_rx) = route_ctx(client_connection);
5994        let pending = forwarding
5995            .begin_route_bind_relay_for_test(
5996                client_connection,
5997                client_ctx.egress.clone(),
5998                1,
5999                module_id,
6000            )
6001            .unwrap();
6002        assert_eq!(pending.endpoint, endpoint);
6003        let route_channel = pending.client_channel;
6004        let route_epoch = pending.client_epoch;
6005        forwarding
6006            .complete_pending_relay(
6007                module_connection,
6008                pending.corr,
6009                RouteBindRelayOutcome::Accepted,
6010            )
6011            .unwrap();
6012        (client_ctx, route_channel, route_epoch)
6013    }
6014
6015    struct FakeProcessLiveness {
6016        live: Option<bool>,
6017    }
6018
6019    impl ModuleProcessLiveness for FakeProcessLiveness {
6020        fn process_live(&self, _module_id: &str) -> Option<bool> {
6021            self.live
6022        }
6023    }
6024
6025    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6026    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
6027    {
6028        let registry = Arc::new(Registry::default());
6029        let supervisor_handle = SupervisorHandle::new();
6030        let supervisor = Supervisor::new(
6031            Arc::clone(&registry),
6032            RestartPolicy::new(1, Duration::from_millis(10)),
6033        )
6034        .with_handle(supervisor_handle.clone());
6035        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
6036        let module = supervisor
6037            .spawn(ModuleSpec {
6038                module_id: "stderr-tail-wire".to_string(),
6039                program: fake_aft_stub_path(),
6040                args: Vec::new(),
6041                env: vec![
6042                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
6043                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
6044                ],
6045                reserved: false,
6046                reserved_prefixes: Vec::new(),
6047                protocol: ModuleProtocol::Subc,
6048                overlap: Default::default(),
6049            })
6050            .unwrap();
6051
6052        let deadline = Instant::now() + Duration::from_secs(5);
6053        loop {
6054            let tail = module.stderr_tail(None, None);
6055            if tail
6056                .entries
6057                .iter()
6058                .any(|entry| matches!(entry, TailEntry::ProcessStart))
6059                && tail.entries.iter().any(|entry| {
6060                    matches!(
6061                        entry,
6062                        TailEntry::Line {
6063                            truncated: true,
6064                            ..
6065                        }
6066                    )
6067                })
6068            {
6069                break;
6070            }
6071            assert!(
6072                Instant::now() < deadline,
6073                "module did not produce a truncated line and restart boundary: {tail:?}"
6074            );
6075            sleep(Duration::from_millis(10)).await;
6076        }
6077
6078        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
6079        let request = ClientControlRequest::SupervisorStderrTail {
6080            module_id: "stderr-tail-wire".to_string(),
6081            max_lines: None,
6082            max_bytes: None,
6083        };
6084        let frame = Frame::build(
6085            FrameType::Request,
6086            control_flags(),
6087            0,
6088            0,
6089            1,
6090            serde_json::to_vec(&request).unwrap(),
6091        )
6092        .unwrap();
6093        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6094        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6095        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
6096            serde_json::from_slice(&responses[0].body).unwrap()
6097        else {
6098            panic!("expected supervisor.stderr_tail response");
6099        };
6100
6101        assert!(
6102            tail.entries
6103                .iter()
6104                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
6105            "the control response lost the restart boundary"
6106        );
6107        let Some(StderrTailEntry::Line { text, truncated }) = tail.entries.iter().find(|entry| {
6108            matches!(
6109                entry,
6110                StderrTailEntry::Line {
6111                    truncated: true,
6112                    ..
6113                }
6114            )
6115        }) else {
6116            panic!("the control response lost the truncated line");
6117        };
6118        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
6119        assert!(*truncated);
6120    }
6121
6122    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
6123    /// read done on the worker thread would stall every other task until it
6124    /// finished; the read must run off the worker so this test's own task keeps
6125    /// running while the read is paused.
6126    #[tokio::test(flavor = "current_thread")]
6127    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
6128        let dir = TestTempDir::new("terminals-off-worker");
6129        let journal_path = dir.join("terminals.jsonl");
6130        let registry = Arc::new(Registry::default());
6131        let supervisor_handle = SupervisorHandle::new();
6132        let supervisor =
6133            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
6134                .with_handle(supervisor_handle.clone())
6135                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
6136        let module = supervisor
6137            .spawn(ModuleSpec {
6138                module_id: "terminal-off-worker".to_string(),
6139                program: fake_aft_stub_path(),
6140                args: Vec::new(),
6141                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6142                reserved: false,
6143                reserved_prefixes: Vec::new(),
6144                protocol: ModuleProtocol::Subc,
6145                overlap: Default::default(),
6146            })
6147            .unwrap();
6148        let deadline = Instant::now() + Duration::from_secs(5);
6149        while module.terminal_history().entries.len() != 2 {
6150            assert!(Instant::now() < deadline, "module did not record two exits");
6151            sleep(Duration::from_millis(10)).await;
6152        }
6153
6154        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
6155        let handler =
6156            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
6157        let frame = Frame::build(
6158            FrameType::Request,
6159            control_flags(),
6160            0,
6161            0,
6162            1,
6163            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
6164                module_id: "terminal-off-worker".to_string(),
6165            })
6166            .unwrap(),
6167        )
6168        .unwrap();
6169        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6170        let spawned_at = std::time::Instant::now();
6171        let read = tokio::spawn({
6172            let handler = Arc::clone(&handler);
6173            async move { handler.handle_control_frame(&ctx, frame).await }
6174        });
6175        // Waiting for the pause from a blocking thread keeps this task pending,
6176        // so the runtime's single worker is free to run the read task.
6177        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
6178            .await
6179            .unwrap()
6180            .expect("the history read reached its pause");
6181        let elapsed = spawned_at.elapsed();
6182        assert!(
6183            elapsed < Duration::from_secs(2) && !read.is_finished(),
6184            "this task could not run while the history read was paused \
6185             (resumed after {elapsed:?}, read finished: {})",
6186            read.is_finished()
6187        );
6188
6189        drop(release);
6190        let responses = read.await.unwrap().unwrap();
6191        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6192        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
6193            panic!("expected supervisor.terminals response");
6194        };
6195        assert_eq!(terminals.entries.len(), 2);
6196        assert_eq!(terminals.journal_skipped_lines, 0);
6197        assert_eq!(terminals.journal_read_errors, 0);
6198    }
6199
6200    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6201    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
6202        let registry = Arc::new(Registry::default());
6203        let supervisor_handle = SupervisorHandle::new();
6204        let supervisor =
6205            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
6206                .with_handle(supervisor_handle.clone());
6207        let module = supervisor
6208            .spawn(ModuleSpec {
6209                module_id: "terminal-golden".to_string(),
6210                program: fake_aft_stub_path(),
6211                args: Vec::new(),
6212                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6213                reserved: false,
6214                reserved_prefixes: Vec::new(),
6215                protocol: ModuleProtocol::Subc,
6216                overlap: Default::default(),
6217            })
6218            .unwrap();
6219
6220        let deadline = Instant::now() + Duration::from_secs(5);
6221        while module.terminal_history().entries.len() != 2 {
6222            assert!(
6223                Instant::now() < deadline,
6224                "module did not retain two terminal exits: {:?}",
6225                module.terminal_history()
6226            );
6227            sleep(Duration::from_millis(10)).await;
6228        }
6229
6230        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
6231        let request = ClientControlRequest::SupervisorTerminals {
6232            module_id: "terminal-golden".to_string(),
6233        };
6234        let frame = Frame::build(
6235            FrameType::Request,
6236            control_flags(),
6237            0,
6238            0,
6239            1,
6240            serde_json::to_vec(&request).unwrap(),
6241        )
6242        .unwrap();
6243        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6244        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6245        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6246        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
6247            panic!("expected supervisor.terminals response");
6248        };
6249        assert_eq!(terminals.entries.len(), 2);
6250        assert_eq!(terminals.dropped, 0);
6251
6252        let mut rendered = serde_json::to_value(response).unwrap();
6253        // Wall-clock fields are the observation contract, but not stable fixture
6254        // bytes; normalize only them after the real handler has shaped the response.
6255        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
6256        for (index, entry) in rendered["entries"]
6257            .as_array_mut()
6258            .expect("terminal response entries array")
6259            .iter_mut()
6260            .enumerate()
6261        {
6262            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
6263        }
6264
6265        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
6266            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
6267        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
6268        if std::env::var_os("UPDATE_GOLDEN").is_some() {
6269            std::fs::write(&golden_path, &serialized).unwrap();
6270        }
6271        let expected: Value =
6272            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
6273        assert_eq!(rendered, expected);
6274    }
6275
6276    #[test]
6277    fn hello_registers_manifest_and_returns_ack() {
6278        let registry = Arc::new(Registry::default());
6279        let handler = ControlHandler::new(Arc::clone(&registry));
6280        let conn = ConnectionId::new(1);
6281
6282        let responses = handler
6283            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
6284            .unwrap();
6285
6286        assert_eq!(responses.len(), 1);
6287        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6288        assert_eq!(responses[0].header.channel, 0);
6289        assert_eq!(responses[0].header.corr, 7);
6290        let ack = parse_ack(&responses[0]);
6291        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
6292        assert!(ack
6293            .subc_capabilities
6294            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
6295        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
6296        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
6297        assert!(ack
6298            .subc_ops
6299            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
6300        assert!(ack
6301            .subc_ops
6302            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
6303
6304        let registration = registry.get_module("aft").unwrap().unwrap();
6305        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
6306        assert_eq!(registration.state, ChannelState::Active);
6307        assert_eq!(registration.connection_id, conn);
6308        assert_eq!(registration.control_ops, module_baseline_control_ops());
6309    }
6310
6311    #[test]
6312    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
6313        let invalid_identifiers = [
6314            ("case_change", "credentials-Provider/v1"),
6315            ("leading_zero", "credentials-provider/v01"),
6316            ("trailing_hyphen", "credentials-provider-/v1"),
6317            ("consecutive_hyphens", "credentials--provider/v1"),
6318            ("uppercase", "Credentials-provider/v1"),
6319            ("missing_v", "credentials-provider/1"),
6320            ("whitespace", "credentials provider/v1"),
6321            ("zero_version", "credentials-provider/v0"),
6322            ("out_of_range_version", "credentials-provider/v4294967296"),
6323            (
6324                "overlength_name",
6325                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
6326            ),
6327        ];
6328        let mut cases = invalid_identifiers
6329            .into_iter()
6330            .map(|(name, identifier)| {
6331                (
6332                    format!("identifier_{name}"),
6333                    "capabilities.provides[0]".to_string(),
6334                    identifier.to_string(),
6335                    json!({ "provides": [identifier] }),
6336                    None,
6337                )
6338            })
6339            .collect::<Vec<_>>();
6340        cases.extend([
6341            (
6342                "unknown_need".to_string(),
6343                "capabilities.requires[0].need".to_string(),
6344                "deferred".to_string(),
6345                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
6346                None,
6347            ),
6348            (
6349                "duplicate_provides".to_string(),
6350                "capabilities.provides[1]".to_string(),
6351                "credentials-provider/v1".to_string(),
6352                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
6353                None,
6354            ),
6355            (
6356                "duplicate_must_never_reach".to_string(),
6357                "capabilities.must_never_reach[1]".to_string(),
6358                "credentials-provider/v1".to_string(),
6359                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
6360                None,
6361            ),
6362            (
6363                "duplicate_requires_same_need".to_string(),
6364                "capabilities.requires[1]".to_string(),
6365                "credentials-provider/v1".to_string(),
6366                json!({ "requires": [
6367                    { "capability": "credentials-provider/v1", "need": "required" },
6368                    { "capability": "credentials-provider/v1", "need": "required" }
6369                ] }),
6370                None,
6371            ),
6372            (
6373                "duplicate_requires_conflicting_need".to_string(),
6374                "capabilities.requires[1]".to_string(),
6375                "credentials-provider/v1".to_string(),
6376                json!({ "requires": [
6377                    { "capability": "credentials-provider/v1", "need": "required" },
6378                    { "capability": "credentials-provider/v1", "need": "optional" }
6379                ] }),
6380                None,
6381            ),
6382            (
6383                "capabilities_root_pointer".to_string(),
6384                "runtime_computed[0]".to_string(),
6385                "/capabilities".to_string(),
6386                json!({}),
6387                Some(json!(["/capabilities"])),
6388            ),
6389            (
6390                "capabilities_descendant_pointer".to_string(),
6391                "runtime_computed[0]".to_string(),
6392                "/capabilities/provides".to_string(),
6393                json!({}),
6394                Some(json!(["/capabilities/provides"])),
6395            ),
6396            (
6397                "malformed_pointer_without_leading_slash".to_string(),
6398                "runtime_computed[0]".to_string(),
6399                "capabilities".to_string(),
6400                json!({}),
6401                Some(json!(["capabilities"])),
6402            ),
6403            (
6404                "malformed_pointer_escape".to_string(),
6405                "runtime_computed[0]".to_string(),
6406                "/roles/~2/tools".to_string(),
6407                json!({}),
6408                Some(json!(["/roles/~2/tools"])),
6409            ),
6410            (
6411                "unknown_capabilities_field".to_string(),
6412                "capabilities.future".to_string(),
6413                "<array>".to_string(),
6414                json!({ "future": [] }),
6415                None,
6416            ),
6417        ]);
6418
6419        for (index, (name, field, value, capabilities, runtime_computed)) in
6420            cases.into_iter().enumerate()
6421        {
6422            let registry = Arc::new(Registry::default());
6423            let handler = ControlHandler::new(Arc::clone(&registry));
6424            let response = handler
6425                .handle_control(
6426                    ConnectionId::new((index + 1) as u64),
6427                    capability_grammar_hello_frame(
6428                        capabilities,
6429                        runtime_computed,
6430                        index as u64 + 1,
6431                    ),
6432                )
6433                .expect("invalid HELLO returns a refusal");
6434
6435            assert_eq!(response.len(), 1, "{name} must emit one refusal");
6436            let error = parse_error(&response[0]);
6437            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
6438            let message = error["message"]
6439                .as_str()
6440                .expect("error message is a string");
6441            assert!(
6442                message.contains(&field),
6443                "{name}: field missing from {message}"
6444            );
6445            assert!(
6446                message.contains(&value),
6447                "{name}: value missing from {message}"
6448            );
6449            assert_eq!(
6450                registry
6451                    .active_registration_count()
6452                    .expect("registry reads"),
6453                0,
6454                "{name}: refused HELLO must not create a catalog entry"
6455            );
6456        }
6457    }
6458
6459    #[test]
6460    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
6461        let registry = Arc::new(Registry::default());
6462        let handler = ControlHandler::new(Arc::clone(&registry));
6463        let capabilities = json!({
6464            "provides": ["credentials-provider/v1"],
6465            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
6466            "must_never_reach": ["federation-transport/v1"]
6467        });
6468        let response = handler
6469            .handle_control(
6470                ConnectionId::new(99),
6471                capability_grammar_hello_frame(
6472                    capabilities.clone(),
6473                    Some(json!(["/roles/0/tools"])),
6474                    99,
6475                ),
6476            )
6477            .expect("valid HELLO registers");
6478        assert_eq!(response[0].header.ty, FrameType::HelloAck);
6479
6480        let request = Frame::build(
6481            FrameType::Request,
6482            control_flags(),
6483            0,
6484            0,
6485            100,
6486            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6487                .expect("catalog request serializes"),
6488        )
6489        .expect("catalog request frame builds");
6490        let response = handler
6491            .handle_catalog_list(request, None)
6492            .expect("catalog list succeeds");
6493        let ClientControlResponse::CatalogList { modules, .. } =
6494            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
6495        else {
6496            panic!("catalog request must return catalog.list");
6497        };
6498        assert_eq!(modules.len(), 1);
6499        assert_eq!(
6500            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
6501            capabilities
6502        );
6503    }
6504
6505    #[test]
6506    fn catalog_list_mirrors_management_operation_description() {
6507        let registry = Arc::new(Registry::default());
6508        let handler = ControlHandler::new(Arc::clone(&registry));
6509        let description = "List managed records and return their identifiers and metadata.";
6510        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
6511        manifest.provides = vec![ProviderRole::ManagementSurface {
6512            operations: vec![ManagementOperation {
6513                name: "records.list".to_string(),
6514                kind: ManagementOperationKind::Query,
6515                description: Some(description.to_string()),
6516            }],
6517            config_schema: json!({"type": "object"}),
6518            observability: vec![ObservabilitySurface {
6519                name: "records.stats".to_string(),
6520                kind: ObservabilityKind::Snapshot,
6521            }],
6522            identity_scope: vec![IdentityScope::Project],
6523            concurrency: Concurrency::ModuleManaged,
6524        }];
6525        registry
6526            .register_with_control_ops(
6527                manifest,
6528                PROTOCOL_VERSION,
6529                ConnectionId::new(99),
6530                Vec::new(),
6531            )
6532            .expect("described management manifest registers");
6533
6534        let request = Frame::build(
6535            FrameType::Request,
6536            control_flags(),
6537            0,
6538            0,
6539            100,
6540            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6541                .expect("catalog request serializes"),
6542        )
6543        .expect("catalog request frame builds");
6544        let response = handler
6545            .handle_catalog_list(request, None)
6546            .expect("catalog list succeeds");
6547        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
6548        assert_eq!(
6549            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
6550            "catalog.list must preserve the declared operation description verbatim"
6551        );
6552    }
6553
6554    #[test]
6555    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
6556        let registry = Arc::new(Registry::default());
6557        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
6558            [("vault".to_string(), true), ("squatter".to_string(), true)],
6559            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
6560        );
6561        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
6562        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
6563            provides: vec!["credentials-provider/v1".to_string()],
6564            requires: Vec::new(),
6565            must_never_reach: Vec::new(),
6566        });
6567        let frame = Frame::build(
6568            FrameType::Hello,
6569            control_flags(),
6570            0,
6571            0,
6572            77,
6573            serde_json::to_vec(&ModuleHelloBody {
6574                manifest: squatter,
6575                protocol_ver: PROTOCOL_VERSION,
6576                control_ops: None,
6577                launch_nonce: None,
6578            })
6579            .expect("HELLO serializes"),
6580        )
6581        .expect("HELLO frame builds");
6582        let response = handler
6583            .handle_control(ConnectionId::new(77), frame)
6584            .expect("reserved claim receives a typed refusal");
6585        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
6586        assert_eq!(
6587            registry
6588                .active_registration_count()
6589                .expect("registry reads"),
6590            0,
6591            "a reserved capability refusal must not leave a catalog entry"
6592        );
6593    }
6594
6595    #[test]
6596    fn server_describe_surfaces_required_capability_verdict_fields() {
6597        let registry = Arc::new(Registry::default());
6598        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
6599            [
6600                ("consumer".to_string(), true),
6601                ("provider".to_string(), false),
6602            ],
6603            BTreeMap::new(),
6604        );
6605        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
6606        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
6607            provides: Vec::new(),
6608            requires: vec![subc_protocol::manifest::CapabilityRequirement {
6609                capability: "credentials-provider/v1".to_string(),
6610                need: subc_protocol::manifest::CapabilityNeed::Required,
6611            }],
6612            must_never_reach: Vec::new(),
6613        });
6614        let hello = Frame::build(
6615            FrameType::Hello,
6616            control_flags(),
6617            0,
6618            0,
6619            78,
6620            serde_json::to_vec(&ModuleHelloBody {
6621                manifest: consumer,
6622                protocol_ver: PROTOCOL_VERSION,
6623                control_ops: None,
6624                launch_nonce: None,
6625            })
6626            .expect("HELLO serializes"),
6627        )
6628        .expect("HELLO frame builds");
6629        handler
6630            .handle_control(ConnectionId::new(78), hello)
6631            .expect("consumer registers");
6632        let describe = Frame::build(
6633            FrameType::Request,
6634            control_flags(),
6635            0,
6636            0,
6637            79,
6638            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
6639                .expect("request serializes"),
6640        )
6641        .expect("describe frame builds");
6642        let response = handler
6643            .handle_server_describe(describe)
6644            .expect("server.describe succeeds");
6645        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
6646        let requirement = &rendered["capability_requirements"][0];
6647        assert_eq!(requirement["consumer"], "consumer");
6648        assert_eq!(requirement["verdict"], "never_provided");
6649        assert_eq!(requirement["episode_seq"], 1);
6650        assert_eq!(requirement["config_satisfiable"], false);
6651        assert_eq!(requirement["runtime_available"], false);
6652        assert!(requirement["detail"]
6653            .as_str()
6654            .expect("detail string")
6655            .contains("credentials-provider/v1"));
6656    }
6657
6658    #[test]
6659    fn catalog_list_omits_capabilities_for_legacy_manifest() {
6660        let registry = Arc::new(Registry::default());
6661        let handler = ControlHandler::new(Arc::clone(&registry));
6662        let hello = handler
6663            .handle_control(
6664                ConnectionId::new(101),
6665                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
6666            )
6667            .expect("legacy HELLO registers");
6668        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
6669
6670        let request = Frame::build(
6671            FrameType::Request,
6672            control_flags(),
6673            0,
6674            0,
6675            102,
6676            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6677                .expect("catalog request serializes"),
6678        )
6679        .expect("catalog request frame builds");
6680        let response = handler
6681            .handle_catalog_list(request, None)
6682            .expect("catalog list succeeds");
6683        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
6684        assert!(
6685            body["modules"][0].get("capabilities").is_none(),
6686            "legacy manifest must retain an absent capabilities field on catalog.list"
6687        );
6688    }
6689
6690    #[test]
6691    fn hello_ack_omits_storage_when_no_storage_config() {
6692        let registry = Arc::new(Registry::default());
6693        let handler = ControlHandler::new(Arc::clone(&registry));
6694        let responses = handler
6695            .handle_control(
6696                ConnectionId::new(1),
6697                hello_frame("aft", PROTOCOL_VERSION, 7),
6698            )
6699            .unwrap();
6700        let ack = parse_ack(&responses[0]);
6701        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
6702        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
6703    }
6704
6705    #[tokio::test]
6706    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
6707        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
6708        let registry = Arc::new(Registry::default());
6709        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
6710        let responses = handler
6711            .handle_control(
6712                ConnectionId::new(1),
6713                hello_frame("aft", PROTOCOL_VERSION, 7),
6714            )
6715            .unwrap();
6716        let ack = parse_ack(&responses[0]);
6717        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
6718
6719        let described = handler
6720            .handle_control_frame(
6721                &route_ctx(ConnectionId::new(2)).0,
6722                Frame::build(
6723                    FrameType::Request,
6724                    control_flags(),
6725                    0,
6726                    0,
6727                    9,
6728                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
6729                )
6730                .unwrap(),
6731            )
6732            .await
6733            .unwrap();
6734        let ClientControlResponse::ServerDescribe { machine_id, .. } =
6735            serde_json::from_slice(&described[0].body).unwrap()
6736        else {
6737            panic!("server.describe answered with another shape");
6738        };
6739        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
6740    }
6741
6742    #[test]
6743    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
6744        // With a central sqlite storage policy, each registering module gets its
6745        // own resolved descriptor in HELLO_ACK, keyed by its module id.
6746        let registry = Arc::new(Registry::default());
6747        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
6748            crate::daemon_config::StorageConfig::Sqlite {
6749                data_home: std::path::PathBuf::from("/data"),
6750            },
6751        ));
6752
6753        let responses = handler
6754            .handle_control(
6755                ConnectionId::new(1),
6756                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
6757            )
6758            .unwrap();
6759        let ack = parse_ack(&responses[0]);
6760        assert_eq!(
6761            ack.storage,
6762            Some(serde_json::json!({
6763                "module_id": "alfonso-routing",
6764                "storage_namespace": "default",
6765                "isolation": { "kind": "module" },
6766                "backend": {
6767                    "backend": "sqlite",
6768                    "path": "/data/cortexkit/alfonso-routing/store.db"
6769                }
6770            })),
6771            "the delivered descriptor is the module's own sqlite store path"
6772        );
6773    }
6774
6775    #[test]
6776    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
6777        let registry = Arc::new(Registry::default());
6778        let handler = ControlHandler::new(Arc::clone(&registry));
6779        let conn = ConnectionId::new(1);
6780        let responses = handler
6781            .handle_control(
6782                conn,
6783                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
6784            )
6785            .unwrap();
6786        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6787        let registration = registry.get_module("aft").unwrap().unwrap();
6788        assert_eq!(registration.control_ops, module_baseline_control_ops());
6789
6790        let frame =
6791            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
6792        assert!(handler
6793            .guard_module_control_op(&frame, "aft", "route.bind")
6794            .unwrap()
6795            .is_none());
6796        let error = handler
6797            .guard_module_control_op(&frame, "aft", "test.synthetic")
6798            .unwrap()
6799            .expect("synthetic ungranted op should be rejected");
6800        assert_eq!(error.header.ty, FrameType::Error);
6801        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
6802    }
6803
6804    #[test]
6805    fn hello_control_ops_some_adds_optional_grants() {
6806        let registry = Arc::new(Registry::default());
6807        let handler = ControlHandler::new(Arc::clone(&registry));
6808        handler
6809            .handle_control(
6810                ConnectionId::new(1),
6811                hello_frame_with_control_ops(
6812                    "aft",
6813                    PROTOCOL_VERSION,
6814                    7,
6815                    Some(vec![
6816                        "future.synthetic".to_string(),
6817                        "route.bind".to_string(),
6818                    ]),
6819                ),
6820            )
6821            .unwrap();
6822        let registration = registry.get_module("aft").unwrap().unwrap();
6823        assert_eq!(
6824            registration.control_ops,
6825            vec![
6826                "route.bind".to_string(),
6827                "route.status".to_string(),
6828                "future.synthetic".to_string(),
6829            ]
6830        );
6831        let frame =
6832            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
6833        assert!(handler
6834            .guard_module_control_op(&frame, "aft", "future.synthetic")
6835            .unwrap()
6836            .is_none());
6837    }
6838
6839    #[tokio::test]
6840    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
6841        let registry = Arc::new(Registry::default());
6842        let forwarding = Arc::new(ForwardingTable::default());
6843        let handler =
6844            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
6845        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
6846        hello_via_sink(
6847            &handler,
6848            &module_ctx,
6849            &mut module_rx,
6850            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
6851        )
6852        .await;
6853
6854        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
6855        let responses = handler
6856            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
6857            .await
6858            .unwrap();
6859        assert_eq!(responses.len(), 1);
6860        assert_eq!(responses[0].header.ty, FrameType::Error);
6861        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
6862        assert!(module_rx.try_recv().is_err());
6863    }
6864
6865    #[tokio::test]
6866    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
6867        let registry = Arc::new(Registry::default());
6868        let forwarding = Arc::new(ForwardingTable::default());
6869        let handler =
6870            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
6871        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
6872        hello_via_sink(
6873            &handler,
6874            &module_ctx,
6875            &mut module_rx,
6876            hello_frame_with_control_ops(
6877                "aft",
6878                PROTOCOL_VERSION,
6879                7,
6880                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
6881            ),
6882        )
6883        .await;
6884
6885        let project_root = unique_project_root("demux");
6886        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
6887        let route_handler = handler.clone();
6888        let route_task = tokio::spawn(async move {
6889            route_handler
6890                .handle_control_frame(
6891                    &route_client_ctx,
6892                    route_open_frame(100, "aft", project_root),
6893                )
6894                .await
6895                .unwrap()
6896        });
6897        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
6898            .await
6899            .unwrap()
6900            .unwrap();
6901        assert!(matches!(
6902            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
6903            ModuleControlRequest::RouteBind { .. }
6904        ));
6905
6906        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
6907        let health_handler = handler.clone();
6908        let health_task = tokio::spawn(async move {
6909            health_handler
6910                .handle_control_frame(
6911                    &health_client_ctx,
6912                    supervisor_health_probe_frame(101, "aft"),
6913                )
6914                .await
6915                .unwrap()
6916        });
6917        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
6918            .await
6919            .unwrap()
6920            .unwrap();
6921        assert_eq!(
6922            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
6923            ModuleControlRequest::HealthCheck {}
6924        );
6925
6926        handler
6927            .handle_control_frame(
6928                &module_ctx,
6929                health_response(health_frame.header.corr, HealthStatus::Degraded),
6930            )
6931            .await
6932            .unwrap();
6933        let health_response = health_task.await.unwrap();
6934        assert_eq!(health_response.len(), 1);
6935        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
6936            ClientControlResponse::SupervisorHealthProbe {
6937                module_id,
6938                status,
6939                detail,
6940                metrics,
6941            } => {
6942                assert_eq!(module_id, "aft");
6943                assert_eq!(status, HealthStatus::Degraded);
6944                assert_eq!(detail.as_deref(), Some("warming"));
6945                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
6946            }
6947            other => panic!("unexpected health response: {other:?}"),
6948        }
6949
6950        handler
6951            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
6952            .await
6953            .unwrap();
6954        let route_response = route_task.await.unwrap();
6955        assert!(route_response.is_empty());
6956        let published = route_client_rx.recv().await.unwrap();
6957        assert!(matches!(
6958            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
6959            ClientControlResponse::RouteOpen { .. }
6960        ));
6961    }
6962
6963    /// Start one `route.open` on `client_connection` and return its still-running
6964    /// handler task together with the `route.bind` the module received for it.
6965    /// The handler blocks until the module answers, so it has to run as a task
6966    /// while the test drives the module side.
6967    async fn relay_route_open(
6968        handler: &ControlHandler,
6969        client_connection: ConnectionId,
6970        client_egress: &FrameSink,
6971        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
6972        corr: u64,
6973        module_id: &str,
6974        project_root_label: &str,
6975    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
6976        let ctx = RouteCtx {
6977            connection_id: client_connection,
6978            egress: client_egress.clone(),
6979        };
6980        let handler = handler.clone();
6981        let project_root = unique_project_root(project_root_label);
6982        let module_id = module_id.to_string();
6983        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
6984        let task = tokio::spawn(async move {
6985            let _guard = tracing::dispatcher::set_default(&dispatch);
6986            handler
6987                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
6988                .await
6989                .unwrap()
6990        });
6991        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
6992            .await
6993            .expect("module receives the relayed route.bind")
6994            .expect("module egress is open");
6995        (task, bind.frame)
6996    }
6997
6998    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
6999        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
7000            ModuleControlRequest::RouteBind {
7001                route_channel,
7002                epoch,
7003                ..
7004            } => (route_channel, epoch),
7005            other => panic!("expected a route.bind request, got {other:?}"),
7006        }
7007    }
7008
7009    fn published_route(frame: &Frame) -> (u16, u32) {
7010        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
7011            ClientControlResponse::RouteOpen {
7012                route_channel,
7013                route_epoch,
7014            } => (route_channel, route_epoch),
7015            other => panic!("expected a route.open response, got {other:?}"),
7016        }
7017    }
7018
7019    /// Reproduction of a production outage. A client had `route.open`s in
7020    /// flight to a module and was already marked closing -- its egress had refused a
7021    /// module frame, so the daemon asked its connection to end -- while its sink
7022    /// was still open. When the module acked those binds, the daemon refused to
7023    /// commit a route for a closing client, and that refusal was returned from
7024    /// the MODULE connection's frame handler, where a router error that has no
7025    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
7026    /// the supervisor correctly did not respawn a clean exit, and every seat lost
7027    /// its tools for hours -- one client's teardown took down a connection
7028    /// carrying ~170 other routes.
7029    ///
7030    /// The window is opened here by calling the production path that opens it
7031    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
7032    /// state that matters is "in `closing_connections`, sink still open, relay
7033    /// still pending", and it lasts only from the close request until the
7034    /// connection loop reacts to it; a socket-level test can flood a client into
7035    /// that escalation but cannot pin the module's ack inside the window. Closing
7036    /// the socket instead takes the other path entirely -- connection teardown
7037    /// removes the pending relay under the same lock, so the ack finds nothing.
7038    #[tokio::test]
7039    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
7040        let registry = Arc::new(Registry::default());
7041        let forwarding = Arc::new(ForwardingTable::default());
7042        let handler =
7043            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
7044
7045        let module_connection = ConnectionId::new(30);
7046        let (module_ctx, mut module_rx) = route_ctx(module_connection);
7047        hello_via_sink(
7048            &handler,
7049            &module_ctx,
7050            &mut module_rx,
7051            hello_frame("aft", PROTOCOL_VERSION, 7),
7052        )
7053        .await;
7054
7055        let dying_client = ConnectionId::new(31);
7056        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
7057
7058        // A published route on the dying client. The escalation below only marks
7059        // a connection closing for a route it has already published.
7060        let (first_task, first_bind) = relay_route_open(
7061            &handler,
7062            dying_client,
7063            &dying_ctx.egress,
7064            &mut module_rx,
7065            100,
7066            "aft",
7067            "closing-first",
7068        )
7069        .await;
7070        handler
7071            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
7072            .await
7073            .unwrap();
7074        assert!(first_task.await.unwrap().is_empty());
7075        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
7076
7077        // A second route.open from the same client, relayed and awaiting its ack.
7078        let (second_task, second_bind) = relay_route_open(
7079            &handler,
7080            dying_client,
7081            &dying_ctx.egress,
7082            &mut module_rx,
7083            101,
7084            "aft",
7085            "closing-second",
7086        )
7087        .await;
7088        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
7089
7090        // The window: the client is closing, its sink is still open, and its
7091        // second bind is still pending.
7092        assert!(forwarding
7093            .escalate_client_delivery_failure(
7094                dying_client,
7095                first_channel,
7096                first_epoch,
7097                CloseReason::new(
7098                    "module_to_client_delivery_failed",
7099                    "client egress refused a module frame",
7100                ),
7101                crate::forwarding::UndeliveredFrame {
7102                    module_id: None,
7103                    sink: &dying_ctx.egress,
7104                },
7105            )
7106            .unwrap());
7107        assert!(!dying_ctx.egress.is_closed());
7108
7109        // The frame that used to end the module connection.
7110        let ack = handler
7111            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
7112            .await;
7113        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
7114        if module_loop_error.is_some() {
7115            // What the server's connection loop does with a router error that has
7116            // no ERROR-frame translation: end the connection, which releases the
7117            // module's registration and every route on it.
7118            handler.cleanup_connection(module_connection).unwrap();
7119        }
7120        // Read the module's next frame before opening the co-tenant's route, so
7121        // the GOODBYE assertion below is about THIS ack and not about later
7122        // traffic. `None` means the module was told nothing.
7123        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7124            .await
7125            .ok()
7126            .flatten();
7127
7128        // 1. The module connection is still registered.
7129        assert!(
7130            registry
7131                .get_module_by_connection(module_connection)
7132                .unwrap()
7133                .is_some(),
7134            "one client's closing connection ended the shared module connection: \
7135             {module_loop_error:?}"
7136        );
7137        // ...and still serving: another client can open and use a route on it.
7138        let cotenant = ConnectionId::new(32);
7139        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
7140        let (cotenant_task, cotenant_bind) = relay_route_open(
7141            &handler,
7142            cotenant,
7143            &cotenant_ctx.egress,
7144            &mut module_rx,
7145            102,
7146            "aft",
7147            "closing-cotenant",
7148        )
7149        .await;
7150        handler
7151            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
7152            .await
7153            .unwrap();
7154        assert!(cotenant_task.await.unwrap().is_empty());
7155        let (cotenant_channel, cotenant_epoch) =
7156            published_route(&cotenant_rx.recv().await.unwrap());
7157        assert!(matches!(
7158            forwarding
7159                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
7160                .unwrap(),
7161            DataRoute::Client(DataRouteState::Bound(_))
7162        ));
7163
7164        // 2. The module was told to drop the binding it created for the route
7165        //    that will never be published.
7166        let goodbye = post_ack_module_frame
7167            .expect("module receives a GOODBYE for the abandoned route channel");
7168        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
7169        assert_eq!(goodbye.header.channel, abandoned_channel);
7170        assert_eq!(goodbye.header.epoch, abandoned_epoch);
7171
7172        // 3. The dying client received nothing: no route was ever published to
7173        //    it. Its route.open is answered as unavailable, which the connection
7174        //    loop would write to a socket that is already going away.
7175        assert!(dying_rx.try_recv().is_err());
7176        let second_response = second_task.await.unwrap();
7177        assert_eq!(second_response.len(), 1);
7178        assert_eq!(
7179            parse_error(&second_response[0])["code"],
7180            "target_unavailable"
7181        );
7182    }
7183
7184    /// The fence at the module-loop boundary, stated as its own contract: which
7185    /// forwarding failures are allowed to end the module connection that is being
7186    /// served. A `ConnectionClosing` naming some client is about that client, and
7187    /// a module connection is shared; the same error naming the module's own
7188    /// connection is about this connection and must stay fatal, as must failures
7189    /// that are about the forwarding table itself.
7190    #[test]
7191    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
7192        let handler = ControlHandler::default();
7193        let module_connection = ConnectionId::new(30);
7194        let client_connection = ConnectionId::new(31);
7195
7196        handler
7197            .refuse_to_end_module_connection_for_a_client(
7198                module_connection,
7199                77,
7200                ForwardingError::ConnectionClosing {
7201                    connection_id: client_connection,
7202                },
7203            )
7204            .expect("a closing client must never end the module connection");
7205
7206        assert!(matches!(
7207            handler.refuse_to_end_module_connection_for_a_client(
7208                module_connection,
7209                78,
7210                ForwardingError::ConnectionClosing {
7211                    connection_id: module_connection,
7212                },
7213            ),
7214            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
7215                connection_id
7216            })) if connection_id == module_connection
7217        ));
7218        assert!(matches!(
7219            handler.refuse_to_end_module_connection_for_a_client(
7220                module_connection,
7221                79,
7222                ForwardingError::Poisoned,
7223            ),
7224            Err(RouterError::Forwarding(ForwardingError::Poisoned))
7225        ));
7226        assert!(matches!(
7227            handler.refuse_to_end_module_connection_for_a_client(
7228                module_connection,
7229                80,
7230                ForwardingError::StaleModuleEndpoint,
7231            ),
7232            Err(RouterError::Forwarding(
7233                ForwardingError::StaleModuleEndpoint
7234            ))
7235        ));
7236    }
7237
7238    /// The spawn-attestation guard is what stops a connected module from claiming
7239    /// another module's identity and being stamped `Reserved` for it. Every other
7240    /// test that supplies a consumer_identity supplies a CORRECT one, because a
7241    /// correct one is what the rest of the flow needs -- so the guard's rejection
7242    /// branch was never the subject of an assertion, only its acceptance branch.
7243    ///
7244    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
7245    /// whole subc-core library suite green; only the forwarding integration tests
7246    /// notice, and they notice for unrelated reasons. This test exists so the
7247    /// refusal itself is asserted where the guard lives: it fails if the identity
7248    /// check stops refusing, which is the direction that matters, since a guard
7249    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
7250    #[tokio::test]
7251    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
7252        let registry = Arc::new(Registry::default());
7253        let forwarding = Arc::new(ForwardingTable::default());
7254        let supervisor = SupervisorHandle::new();
7255        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7256        let handler =
7257            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
7258                .with_supervisor(supervisor);
7259
7260        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
7261        hello_via_sink(
7262            &handler,
7263            &target_ctx,
7264            &mut target_rx,
7265            hello_frame("target", PROTOCOL_VERSION, 1),
7266        )
7267        .await;
7268
7269        // A real supervised module id presenting the wrong nonce. This is the
7270        // impersonation case: the attacker knows a privileged module_id, which is
7271        // public, and guesses at the nonce, which is not.
7272        let wrong_nonce = handler
7273            .handle_control_frame(
7274                &route_ctx(ConnectionId::new(91)).0,
7275                route_open_frame_with_admission_facts(
7276                    20,
7277                    "target",
7278                    unique_project_root("admission-facts"),
7279                    Some(subc_control::ConsumerIdentity {
7280                        module_id: "fed".to_string(),
7281                        launch_nonce: "not-the-real-nonce".to_string(),
7282                    }),
7283                    None,
7284                ),
7285            )
7286            .await
7287            .unwrap();
7288        assert_eq!(
7289            parse_error(&wrong_nonce[0])["code"],
7290            "bad_consumer_identity",
7291            "a mismatched launch nonce must be refused, not stamped Reserved"
7292        );
7293
7294        // A module id the supervisor never spawned at all, so no nonce exists to
7295        // compare against. An implementation that treats "no record" as "nothing
7296        // to check" fails open here while passing the case above.
7297        let never_spawned = handler
7298            .handle_control_frame(
7299                &route_ctx(ConnectionId::new(92)).0,
7300                route_open_frame_with_admission_facts(
7301                    21,
7302                    "target",
7303                    unique_project_root("admission-facts"),
7304                    Some(subc_control::ConsumerIdentity {
7305                        module_id: "never-spawned".to_string(),
7306                        launch_nonce: "any-nonce".to_string(),
7307                    }),
7308                    None,
7309                ),
7310            )
7311            .await
7312            .unwrap();
7313        assert_eq!(
7314            parse_error(&never_spawned[0])["code"],
7315            "bad_consumer_identity",
7316            "an unspawned module_id must be refused rather than accepted for lack of a record"
7317        );
7318    }
7319
7320    /// The refusal test above proves the guard says NO. Nothing proved it can say
7321    /// YES, and the difference is not academic: replacing the whole authorization
7322    /// with `false` -- admitting no consumer identity at all, revoking Reserved
7323    /// standing for every supervised module in the fleet -- leaves 110 of the 111
7324    /// library tests GREEN. The one that notices does so by HANGING, because it
7325    /// waits for a bind that can no longer happen.
7326    ///
7327    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
7328    /// or flaky test, invites a RETRY rather than an investigation, and the retry
7329    /// hangs too and gets blamed on the runner. So a total revocation of the
7330    /// daemon's trust grant would have shipped behind a symptom nobody attributes
7331    /// to code.
7332    ///
7333    /// The bias is structural rather than accidental. A REFUSAL looks like a
7334    /// failure someone writes a test for; a GRANT looks like the happy path. Every
7335    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
7336    /// suite for that reason, and this one is the purest case in the daemon.
7337    ///
7338    /// This test asserts the EFFECT rather than the absence of an error: the module
7339    /// receives a RouteBind and it carries `Reserved` naming the attested module.
7340    /// A guard that admitted nobody would produce no bind at all; one that admitted
7341    /// everybody would stamp the wrong principal, which the refusal test catches.
7342    #[tokio::test]
7343    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
7344        let registry = Arc::new(Registry::default());
7345        let forwarding = Arc::new(ForwardingTable::default());
7346        let supervisor = SupervisorHandle::new();
7347        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7348        let handler =
7349            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
7350                .with_supervisor(supervisor);
7351
7352        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
7353        hello_via_sink(
7354            &handler,
7355            &target_ctx,
7356            &mut target_rx,
7357            hello_frame("target", PROTOCOL_VERSION, 1),
7358        )
7359        .await;
7360
7361        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
7362        let route_handler = handler.clone();
7363        let route_task = tokio::spawn(async move {
7364            route_handler
7365                .handle_control_frame(
7366                    &client_ctx,
7367                    route_open_frame_with_admission_facts(
7368                        30,
7369                        "target",
7370                        unique_project_root("admission-facts"),
7371                        Some(subc_control::ConsumerIdentity {
7372                            module_id: "fed".to_string(),
7373                            launch_nonce: "fed-nonce".to_string(),
7374                        }),
7375                        None,
7376                    ),
7377                )
7378                .await
7379                .unwrap()
7380        });
7381
7382        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
7383        // the very mutation it exists to catch -- a guard that admits nobody -- no
7384        // bind is ever sent, so it HUNG rather than failing. That reproduces the
7385        // exact defect being fixed: a total revocation detected only as a stalled
7386        // suite, which reads as flakiness and invites a retry. An acceptance test
7387        // that waits for an effect must bound the wait, or a red becomes a hang.
7388        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7389            .await
7390            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
7391            .expect("module control channel closed before route.bind");
7392        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
7393        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
7394            panic!("expected route.bind")
7395        };
7396        assert_eq!(
7397            principal,
7398            Some(Principal::Reserved {
7399                module_id: "fed".to_string()
7400            }),
7401            "a correctly attested consumer must be stamped Reserved for its own id"
7402        );
7403
7404        handler
7405            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
7406            .await
7407            .unwrap();
7408        assert!(route_task.await.unwrap().is_empty());
7409        assert!(
7410            matches!(
7411                serde_json::from_slice::<ClientControlResponse>(
7412                    &client_rx.recv().await.unwrap().body
7413                )
7414                .unwrap(),
7415                ClientControlResponse::RouteOpen { .. }
7416            ),
7417            "the route must actually open, not merely avoid an error"
7418        );
7419    }
7420
7421    #[tokio::test(start_paused = true)]
7422    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
7423        let registry = Arc::new(Registry::default());
7424        let forwarding = Arc::new(ForwardingTable::default());
7425        let supervisor = SupervisorHandle::new();
7426        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7427        let handler =
7428            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
7429                .with_supervisor(supervisor);
7430
7431        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
7432        hello_via_sink(
7433            &handler,
7434            &target_ctx,
7435            &mut target_rx,
7436            hello_frame("target", PROTOCOL_VERSION, 1),
7437        )
7438        .await;
7439
7440        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
7441        let direct_handler = handler.clone();
7442        let direct_open = tokio::spawn(async move {
7443            direct_handler
7444                .handle_control_frame(
7445                    &direct_ctx,
7446                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
7447                )
7448                .await
7449                .unwrap()
7450        });
7451        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7452            .await
7453            .expect("no direct route.bind within 5s")
7454            .expect("target control channel closed before direct route.bind");
7455        handler
7456            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
7457            .await
7458            .unwrap();
7459        assert!(direct_open.await.unwrap().is_empty());
7460        let _ = direct_rx.recv().await.unwrap();
7461
7462        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
7463        let reserved_handler = handler.clone();
7464        let reserved_open = tokio::spawn(async move {
7465            reserved_handler
7466                .handle_control_frame(
7467                    &reserved_ctx,
7468                    route_open_frame_with_admission_facts(
7469                        3,
7470                        "target",
7471                        unique_project_root("admission-facts"),
7472                        Some(ConsumerIdentity {
7473                            module_id: "fed".to_string(),
7474                            launch_nonce: "fed-nonce".to_string(),
7475                        }),
7476                        None,
7477                    ),
7478                )
7479                .await
7480                .unwrap()
7481        });
7482        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7483            .await
7484            .expect("no reserved route.bind within 5s")
7485            .expect("target control channel closed before reserved route.bind");
7486        handler
7487            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
7488            .await
7489            .unwrap();
7490        assert!(reserved_open.await.unwrap().is_empty());
7491        let _ = reserved_rx.recv().await.unwrap();
7492
7493        forwarding
7494            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
7495            .unwrap();
7496        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
7497        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
7498            module_id: Some("target".to_string()),
7499        })
7500        .unwrap();
7501        let census_frame =
7502            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
7503        let response = handler
7504            .handle_control_frame(&census_ctx, census_frame)
7505            .await
7506            .unwrap()
7507            .pop()
7508            .unwrap();
7509        let actual: Value = serde_json::from_slice(&response.body).unwrap();
7510        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
7511        assert!(matches!(
7512            decoded,
7513            ClientControlResponse::SupervisorRoutes { .. }
7514        ));
7515        let routes = actual["modules"][0]["routes"].as_array().unwrap();
7516        assert_eq!(routes.len(), 2);
7517        assert!(routes.iter().all(|route| route["draining"] == true));
7518        // The census carries WHY: the reason the drain was begun with, in the
7519        // route.closing vocabulary, on every draining route this drain marked.
7520        assert!(
7521            routes.iter().all(|route| route["drain_reason"] == "reload"),
7522            "draining routes must name the drain's reason: {routes:?}"
7523        );
7524        assert!(routes.iter().any(|route| {
7525            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
7526        }));
7527        assert!(routes.iter().any(|route| {
7528            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
7529        }));
7530
7531        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7532            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
7533        if std::env::var_os("UPDATE_GOLDEN").is_some() {
7534            std::fs::write(
7535                &golden_path,
7536                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
7537            )
7538            .unwrap();
7539        }
7540        let expected: Value =
7541            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
7542        assert_eq!(actual, expected);
7543    }
7544
7545    async fn query_live_roots(
7546        handler: &ControlHandler,
7547        module_ctx: &RouteCtx,
7548    ) -> ModuleControlResponseToModule {
7549        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
7550        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
7551        let response = handler
7552            .handle_control_frame(module_ctx, frame)
7553            .await
7554            .unwrap()
7555            .pop()
7556            .unwrap();
7557        serde_json::from_slice(&response.body).unwrap()
7558    }
7559
7560    #[tokio::test(start_paused = true)]
7561    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
7562        let registry = Arc::new(Registry::default());
7563        let forwarding = Arc::new(ForwardingTable::default());
7564        let handler = ControlHandler::with_forwarding(registry, forwarding);
7565        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
7566        hello_via_sink(
7567            &handler,
7568            &target_ctx,
7569            &mut target_rx,
7570            hello_frame("target", PROTOCOL_VERSION, 1),
7571        )
7572        .await;
7573        let root = unique_project_root("live-roots-known");
7574        let path = ProjectRootId::from_path_allowing_missing(root.path())
7575            .unwrap()
7576            .as_path()
7577            .to_path_buf();
7578        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
7579        let open_handler = handler.clone();
7580        let opened = tokio::spawn(async move {
7581            open_handler
7582                .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
7583                .await
7584                .unwrap()
7585        });
7586        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7587            .await
7588            .unwrap()
7589            .unwrap();
7590        handler
7591            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
7592            .await
7593            .unwrap();
7594        assert!(opened.await.unwrap().is_empty());
7595        let _ = client_rx.recv().await.unwrap();
7596
7597        let root = unique_project_root("live-roots-pending");
7598        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
7599            .unwrap()
7600            .as_path()
7601            .to_path_buf();
7602        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
7603        let open_handler = handler.clone();
7604        let pending = tokio::spawn(async move {
7605            open_handler
7606                .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
7607                .await
7608                .unwrap()
7609        });
7610        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7611            .await
7612            .unwrap()
7613            .unwrap();
7614        let actual = query_live_roots(&handler, &target_ctx).await;
7615        let ModuleControlResponseToModule::LiveRoots {
7616            roots,
7617            unknown_root_bindings,
7618            total_bindings,
7619        } = actual
7620        else {
7621            panic!("expected live roots")
7622        };
7623        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
7624        assert_eq!(unknown_root_bindings, 0);
7625        assert_eq!(
7626            roots.len(),
7627            2,
7628            "root-known arm must retain each canonical root"
7629        );
7630        assert_eq!(
7631            total_bindings,
7632            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7633        );
7634        let counts = roots
7635            .iter()
7636            .map(|root| (root.project_root.clone(), root.bound, root.pending))
7637            .collect::<Vec<_>>();
7638        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
7639        expected.sort_by(|a, b| a.0.cmp(&b.0));
7640        assert_eq!(
7641            counts, expected,
7642            "roots must sort by path and count pending separately"
7643        );
7644        handler
7645            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
7646            .await
7647            .unwrap();
7648        assert!(pending.await.unwrap().is_empty());
7649    }
7650
7651    #[tokio::test(start_paused = true)]
7652    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
7653        let forwarding = Arc::new(ForwardingTable::default());
7654        let handler =
7655            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
7656        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
7657        hello_via_sink(
7658            &handler,
7659            &target_ctx,
7660            &mut target_rx,
7661            hello_frame("target", PROTOCOL_VERSION, 1),
7662        )
7663        .await;
7664        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
7665        let pending = forwarding
7666            .begin_route_bind_relay_for_test(
7667                client_ctx.connection_id,
7668                client_ctx.egress.clone(),
7669                2,
7670                "target",
7671            )
7672            .unwrap();
7673        forwarding
7674            .complete_pending_relay(
7675                target_ctx.connection_id,
7676                pending.corr,
7677                RouteBindRelayOutcome::Accepted,
7678            )
7679            .unwrap();
7680        let actual = query_live_roots(&handler, &target_ctx).await;
7681        let ModuleControlResponseToModule::LiveRoots {
7682            roots,
7683            unknown_root_bindings,
7684            total_bindings,
7685        } = actual
7686        else {
7687            panic!("expected live roots")
7688        };
7689        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
7690        assert_eq!(
7691            unknown_root_bindings, 1,
7692            "unknown-root arm must not read as no bindings"
7693        );
7694        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
7695        assert_eq!(
7696            total_bindings,
7697            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7698        );
7699    }
7700
7701    /// A module reads HELLO_ACK as its first frame and exits on anything else,
7702    /// so the ack has to be on its outbound queue before the module is
7703    /// routable. The connection loop writes a handler's replies only after the
7704    /// handler returns; this test stops in exactly that gap, runs a real
7705    /// route.open from another connection, and only then writes whatever the
7706    /// HELLO handler returned, the way the loop would. If the ack were still a
7707    /// reply, the route.bind request would reach the module first.
7708    #[tokio::test(start_paused = true)]
7709    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
7710        let forwarding = Arc::new(ForwardingTable::default());
7711        let handler =
7712            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
7713        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
7714        let replies = handler
7715            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
7716            .await
7717            .unwrap();
7718        let queued_by_hello = module_rx.len();
7719
7720        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
7721        let open_handler = handler.clone();
7722        let open = tokio::spawn(async move {
7723            open_handler
7724                .handle_control_frame(
7725                    &client_ctx,
7726                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
7727                )
7728                .await
7729                .unwrap()
7730        });
7731        // Let the route.open run until its route.bind is on the module's queue.
7732        let mut spins = 0;
7733        while module_rx.len() == queued_by_hello {
7734            spins += 1;
7735            assert!(spins < 10_000, "route.open never queued a route.bind");
7736            tokio::task::yield_now().await;
7737        }
7738
7739        // Now the connection loop's half: write the HELLO handler's replies.
7740        for reply in replies {
7741            module_ctx.egress.send(reply).await.unwrap();
7742        }
7743
7744        let first = module_rx.recv().await.unwrap().frame;
7745        assert_eq!(
7746            first.header.ty,
7747            FrameType::HelloAck,
7748            "the first frame a registering module reads must be its HELLO_ACK"
7749        );
7750        assert_eq!(first.header.corr, 7);
7751        let second = module_rx.recv().await.unwrap().frame;
7752        assert_eq!(second.header.ty, FrameType::Request);
7753        assert!(
7754            matches!(
7755                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
7756                ModuleControlRequest::RouteBind { .. }
7757            ),
7758            "the route.bind follows the ack"
7759        );
7760        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
7761
7762        handler
7763            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
7764            .await
7765            .unwrap();
7766        assert!(open.await.unwrap().is_empty());
7767        let _ = client_rx.recv().await.unwrap();
7768    }
7769
7770    #[tokio::test(start_paused = true)]
7771    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
7772        let handler = ControlHandler::with_forwarding(
7773            Arc::new(Registry::default()),
7774            Arc::new(ForwardingTable::default()),
7775        );
7776        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
7777        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
7778        hello_via_sink(
7779            &handler,
7780            &first_ctx,
7781            &mut first_rx,
7782            hello_frame("first", PROTOCOL_VERSION, 1),
7783        )
7784        .await;
7785        hello_via_sink(
7786            &handler,
7787            &second_ctx,
7788            &mut second_rx,
7789            hello_frame("second", PROTOCOL_VERSION, 2),
7790        )
7791        .await;
7792        let root = unique_project_root("second-only");
7793        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
7794        let cloned = handler.clone();
7795        let open = tokio::spawn(async move {
7796            cloned
7797                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
7798                .await
7799                .unwrap()
7800        });
7801        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
7802            .await
7803            .unwrap()
7804            .unwrap();
7805        let first = query_live_roots(&handler, &first_ctx).await;
7806        let second = query_live_roots(&handler, &second_ctx).await;
7807        assert!(
7808            matches!(
7809                first,
7810                ModuleControlResponseToModule::LiveRoots {
7811                    total_bindings: 0,
7812                    ..
7813                }
7814            ),
7815            "cross-module scope must not expose another module's roots"
7816        );
7817        assert!(
7818            matches!(
7819                second,
7820                ModuleControlResponseToModule::LiveRoots {
7821                    total_bindings: 1,
7822                    ..
7823                }
7824            ),
7825            "second module must see its pending route"
7826        );
7827        handler
7828            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
7829            .await
7830            .unwrap();
7831        assert!(open.await.unwrap().is_empty());
7832    }
7833
7834    #[tokio::test(start_paused = true)]
7835    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
7836        let handler = ControlHandler::with_forwarding(
7837            Arc::new(Registry::default()),
7838            Arc::new(ForwardingTable::default()),
7839        );
7840        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
7841        hello_via_sink(
7842            &handler,
7843            &target_ctx,
7844            &mut target_rx,
7845            hello_frame("target", PROTOCOL_VERSION, 1),
7846        )
7847        .await;
7848        let actual = query_live_roots(&handler, &target_ctx).await;
7849        let ModuleControlResponseToModule::LiveRoots {
7850            roots,
7851            unknown_root_bindings,
7852            total_bindings,
7853        } = actual
7854        else {
7855            panic!("expected live roots")
7856        };
7857        assert!(roots.is_empty());
7858        assert_eq!(unknown_root_bindings, 0);
7859        assert_eq!(total_bindings, 0);
7860        assert_eq!(
7861            total_bindings,
7862            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7863        );
7864    }
7865
7866    /// Read the vendored fed corpus rather than hand-building a package.
7867    ///
7868    /// A hand-built object encodes what the test author believed the carrier
7869    /// emits. These vectors are what it actually emits, and one of them exists
7870    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
7871    /// ignores additive unknown fields at the traversal emit terminus."
7872    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
7873        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7874            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
7875        let text = std::fs::read_to_string(&path)
7876            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
7877        let vectors: Vec<(String, Value)> = text
7878            .lines()
7879            .filter(|line| !line.trim().is_empty())
7880            .map(|line| {
7881                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
7882                let id = entry["corpus_id"]
7883                    .as_str()
7884                    .expect("every vector carries a corpus_id")
7885                    .to_string();
7886                (id, entry["package"].clone())
7887            })
7888            .collect();
7889        // Pin the count: a corpus that silently shrinks would take its coverage
7890        // with it, and a suite reading N-1 vectors reports the same clean pass
7891        // as one reading N.
7892        assert_eq!(
7893            vectors.len(),
7894            3,
7895            "vendored fed corpus changed size; re-sync from subc-federation"
7896        );
7897
7898        // Pin what makes the corpus DISCRIMINATING, not just present.
7899        //
7900        // The relay test below takes its expected value from the corpus, so the
7901        // corpus supplies the test's power to detect a lossy relay rather than
7902        // its correctness. A relay that dropped unrecognised fields would still
7903        // be caught -- but only by a package carrying fields it does not know.
7904        // Shrink every package to the handful of keys any implementation would
7905        // recognise and the test keeps passing over an input that can no longer
7906        // fail, which is the same clean green as a corpus that shrank away.
7907        //
7908        // So assert the precondition rather than duplicating the packages here:
7909        // at least one vector must carry a field beyond the small common set.
7910        // That is one claim to maintain instead of nine, and it fails loudly if
7911        // a re-sync ever flattens the corpus.
7912        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
7913        let richest = vectors
7914            .iter()
7915            .filter_map(|(_, package)| package.as_object())
7916            .map(|object| {
7917                object
7918                    .keys()
7919                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
7920                    .count()
7921            })
7922            .max()
7923            .unwrap_or(0);
7924        assert!(
7925            richest >= 2,
7926            "vendored corpus no longer carries a package with unmodelled fields, \
7927             so the relay test can no longer distinguish a verbatim relay from a lossy one"
7928        );
7929
7930        vectors
7931    }
7932
7933    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
7934    ///
7935    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
7936    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
7937    /// three-key object, so a relay that quietly dropped fields it did not
7938    /// recognise would satisfy it. These vectors carry nine keys including ones
7939    /// this crate has no type for, so a typed relay fails here and only here.
7940    #[tokio::test]
7941    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
7942        for (corpus_id, package) in fed_admission_facts_vectors() {
7943            let registry = Arc::new(Registry::default());
7944            let forwarding = Arc::new(ForwardingTable::default());
7945            let supervisor = SupervisorHandle::new();
7946            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7947            let handler =
7948                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
7949                    .with_supervisor(supervisor)
7950                    .with_admission_facts_config(
7951                        Some("fed".to_string()),
7952                        Some(vec!["target".to_string()]),
7953                    );
7954
7955            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
7956            hello_via_sink(
7957                &handler,
7958                &target_ctx,
7959                &mut target_rx,
7960                hello_frame("target", PROTOCOL_VERSION, 1),
7961            )
7962            .await;
7963
7964            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
7965            let route_handler = handler.clone();
7966            let expected = package.clone();
7967            let route_task = tokio::spawn(async move {
7968                route_handler
7969                    .handle_control_frame(
7970                        &client_ctx,
7971                        route_open_frame_with_admission_facts(
7972                            20,
7973                            "target",
7974                            unique_project_root("admission-facts"),
7975                            Some(subc_control::ConsumerIdentity {
7976                                module_id: "fed".to_string(),
7977                                launch_nonce: "fed-nonce".to_string(),
7978                            }),
7979                            Some(package),
7980                        ),
7981                    )
7982                    .await
7983                    .unwrap()
7984            });
7985
7986            let bind_frame = target_rx.recv().await.unwrap();
7987            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
7988            let ModuleControlRequest::RouteBind {
7989                admission_facts, ..
7990            } = bind
7991            else {
7992                panic!("{corpus_id}: expected route.bind")
7993            };
7994            assert_eq!(
7995                admission_facts,
7996                Some(expected),
7997                "{corpus_id}: relay must not add, drop or reshape any field"
7998            );
7999
8000            handler
8001                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8002                .await
8003                .unwrap();
8004            route_task.await.unwrap();
8005        }
8006    }
8007
8008    #[tokio::test]
8009    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
8010        let registry = Arc::new(Registry::default());
8011        let forwarding = Arc::new(ForwardingTable::default());
8012        let supervisor = SupervisorHandle::new();
8013        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8014        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
8015        let handler =
8016            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8017                .with_supervisor(supervisor)
8018                .with_admission_facts_config(
8019                    Some("fed".to_string()),
8020                    Some(vec!["target".to_string()]),
8021                );
8022
8023        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
8024        hello_via_sink(
8025            &handler,
8026            &target_ctx,
8027            &mut target_rx,
8028            hello_frame("target", PROTOCOL_VERSION, 1),
8029        )
8030        .await;
8031        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
8032        hello_via_sink(
8033            &handler,
8034            &other_ctx,
8035            &mut other_rx,
8036            hello_frame("other", PROTOCOL_VERSION, 2),
8037        )
8038        .await;
8039
8040        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
8041        let expected_facts = facts.clone();
8042        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
8043        let route_handler = handler.clone();
8044        let route_task = tokio::spawn(async move {
8045            route_handler
8046                .handle_control_frame(
8047                    &client_ctx,
8048                    route_open_frame_with_admission_facts(
8049                        10,
8050                        "target",
8051                        unique_project_root("admission-facts"),
8052                        Some(subc_control::ConsumerIdentity {
8053                            module_id: "fed".to_string(),
8054                            launch_nonce: "fed-nonce".to_string(),
8055                        }),
8056                        Some(facts.clone()),
8057                    ),
8058                )
8059                .await
8060                .unwrap()
8061        });
8062        let bind_frame = target_rx.recv().await.unwrap();
8063        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8064        let ModuleControlRequest::RouteBind {
8065            admission_facts, ..
8066        } = bind
8067        else {
8068            panic!("expected route.bind")
8069        };
8070        assert_eq!(admission_facts, Some(expected_facts));
8071        handler
8072            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8073            .await
8074            .unwrap();
8075        assert!(route_task.await.unwrap().is_empty());
8076        assert!(matches!(
8077            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
8078                .unwrap(),
8079            ClientControlResponse::RouteOpen { .. }
8080        ));
8081
8082        let direct = handler
8083            .handle_control_frame(
8084                &route_ctx(ConnectionId::new(73)).0,
8085                route_open_frame_with_admission_facts(
8086                    11,
8087                    "target",
8088                    unique_project_root("admission-facts"),
8089                    None,
8090                    Some(json!({"x": 1})),
8091                ),
8092            )
8093            .await
8094            .unwrap();
8095        assert_eq!(
8096            parse_error(&direct[0])["code"],
8097            "admission_facts_not_permitted"
8098        );
8099
8100        let different_reserved = handler
8101            .handle_control_frame(
8102                &route_ctx(ConnectionId::new(77)).0,
8103                route_open_frame_with_admission_facts(
8104                    15,
8105                    "target",
8106                    unique_project_root("admission-facts"),
8107                    Some(subc_control::ConsumerIdentity {
8108                        module_id: "other".to_string(),
8109                        launch_nonce: "other-nonce".to_string(),
8110                    }),
8111                    Some(json!({"x": 1})),
8112                ),
8113            )
8114            .await
8115            .unwrap();
8116        assert_eq!(
8117            parse_error(&different_reserved[0])["code"],
8118            "admission_facts_not_permitted"
8119        );
8120
8121        let other_target = handler
8122            .handle_control_frame(
8123                &route_ctx(ConnectionId::new(74)).0,
8124                route_open_frame_with_admission_facts(
8125                    12,
8126                    "other",
8127                    unique_project_root("admission-facts"),
8128                    Some(subc_control::ConsumerIdentity {
8129                        module_id: "fed".to_string(),
8130                        launch_nonce: "fed-nonce".to_string(),
8131                    }),
8132                    Some(json!({"x": 1})),
8133                ),
8134            )
8135            .await
8136            .unwrap();
8137        assert_eq!(
8138            parse_error(&other_target[0])["code"],
8139            "admission_facts_target_not_allowed"
8140        );
8141
8142        let nonexistent = handler
8143            .handle_control_frame(
8144                &route_ctx(ConnectionId::new(75)).0,
8145                route_open_frame_with_admission_facts(
8146                    13,
8147                    "missing",
8148                    unique_project_root("admission-facts"),
8149                    None,
8150                    Some(json!({"x": 1})),
8151                ),
8152            )
8153            .await
8154            .unwrap();
8155        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
8156
8157        let described = handler
8158            .handle_control_frame(
8159                &route_ctx(ConnectionId::new(76)).0,
8160                Frame::build(
8161                    FrameType::Request,
8162                    control_flags(),
8163                    0,
8164                    0,
8165                    14,
8166                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8167                )
8168                .unwrap(),
8169            )
8170            .await
8171            .unwrap();
8172        let ClientControlResponse::ServerDescribe { capabilities, .. } =
8173            serde_json::from_slice(&described[0].body).unwrap()
8174        else {
8175            panic!("expected server.describe response")
8176        };
8177        assert!(capabilities
8178            .iter()
8179            .any(|cap| cap == "admission_facts_relay_v1"));
8180    }
8181
8182    #[tokio::test]
8183    async fn admission_facts_without_configured_carrier_are_rejected() {
8184        let registry = Arc::new(Registry::default());
8185        let forwarding = Arc::new(ForwardingTable::default());
8186        let handler = ControlHandler::with_forwarding(registry, forwarding);
8187        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
8188        hello_via_sink(
8189            &handler,
8190            &target_ctx,
8191            &mut target_rx,
8192            hello_frame("target", PROTOCOL_VERSION, 1),
8193        )
8194        .await;
8195
8196        let responses = handler
8197            .handle_control_frame(
8198                &route_ctx(ConnectionId::new(79)).0,
8199                route_open_frame_with_admission_facts(
8200                    16,
8201                    "target",
8202                    unique_project_root("admission-facts"),
8203                    None,
8204                    Some(json!({"x": 1})),
8205                ),
8206            )
8207            .await
8208            .unwrap();
8209        assert_eq!(
8210            parse_error(&responses[0])["code"],
8211            "admission_facts_not_permitted"
8212        );
8213    }
8214
8215    #[tokio::test]
8216    async fn route_open_relays_consumer_capabilities_verbatim() {
8217        let registry = Arc::new(Registry::default());
8218        let forwarding = Arc::new(ForwardingTable::default());
8219        let handler =
8220            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
8221        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
8222        hello_via_sink(
8223            &handler,
8224            &module_ctx,
8225            &mut module_rx,
8226            hello_frame("aft", PROTOCOL_VERSION, 7),
8227        )
8228        .await;
8229
8230        let expected = vec!["elicitation".to_string(), "roots".to_string()];
8231        let expected_for_request = expected.clone();
8232        let project_root = unique_project_root("consumer-capabilities-present");
8233        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
8234        let route_handler = handler.clone();
8235        let route_task = tokio::spawn(async move {
8236            route_handler
8237                .handle_control_frame(
8238                    &client_ctx,
8239                    route_open_frame_with_consumer_capabilities(
8240                        401,
8241                        "aft",
8242                        project_root,
8243                        Some(expected_for_request),
8244                    ),
8245                )
8246                .await
8247                .unwrap()
8248        });
8249        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8250            .await
8251            .unwrap()
8252            .unwrap();
8253        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8254        let ModuleControlRequest::RouteBind {
8255            consumer_capabilities,
8256            ..
8257        } = bind
8258        else {
8259            panic!("expected route.bind request, got {bind:?}");
8260        };
8261        assert_eq!(consumer_capabilities, Some(expected.clone()));
8262
8263        handler
8264            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8265            .await
8266            .unwrap();
8267        let route_response = route_task.await.unwrap();
8268        assert!(route_response.is_empty());
8269        let published = client_rx.recv().await.unwrap();
8270        assert!(matches!(
8271            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8272            ClientControlResponse::RouteOpen { .. }
8273        ));
8274    }
8275
8276    #[tokio::test]
8277    async fn route_open_without_consumer_capabilities_relays_none() {
8278        let registry = Arc::new(Registry::default());
8279        let forwarding = Arc::new(ForwardingTable::default());
8280        let handler =
8281            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
8282        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
8283        hello_via_sink(
8284            &handler,
8285            &module_ctx,
8286            &mut module_rx,
8287            hello_frame("aft", PROTOCOL_VERSION, 7),
8288        )
8289        .await;
8290
8291        let project_root = unique_project_root("consumer-capabilities-absent");
8292        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
8293        let route_handler = handler.clone();
8294        let route_task = tokio::spawn(async move {
8295            route_handler
8296                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
8297                .await
8298                .unwrap()
8299        });
8300        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8301            .await
8302            .unwrap()
8303            .unwrap();
8304        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8305        let ModuleControlRequest::RouteBind {
8306            consumer_capabilities,
8307            ..
8308        } = bind
8309        else {
8310            panic!("expected route.bind request, got {bind:?}");
8311        };
8312        assert_eq!(consumer_capabilities, None);
8313
8314        handler
8315            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8316            .await
8317            .unwrap();
8318        let route_response = route_task.await.unwrap();
8319        assert!(route_response.is_empty());
8320        let published = client_rx.recv().await.unwrap();
8321        assert!(matches!(
8322            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8323            ClientControlResponse::RouteOpen { .. }
8324        ));
8325    }
8326
8327    #[tokio::test]
8328    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
8329        let registry = Arc::new(Registry::default());
8330        let forwarding = Arc::new(ForwardingTable::default());
8331        let handler =
8332            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8333                .with_health_probe_timeout(Duration::from_secs(5));
8334        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
8335        hello_via_sink(
8336            &handler,
8337            &module_ctx,
8338            &mut module_rx,
8339            non_routable_hello_frame_with_control_ops(
8340                "mcp",
8341                300,
8342                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8343            ),
8344        )
8345        .await;
8346        assert!(registry
8347            .get_module("mcp")
8348            .unwrap()
8349            .unwrap()
8350            .manifest
8351            .provides
8352            .is_empty());
8353
8354        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
8355        let route_response = handler
8356            .handle_control_frame(
8357                &route_client_ctx,
8358                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
8359            )
8360            .await
8361            .unwrap();
8362        assert_eq!(route_response[0].header.ty, FrameType::Error);
8363        assert_eq!(
8364            parse_error(&route_response[0])["code"],
8365            "target_unavailable"
8366        );
8367        assert!(parse_error(&route_response[0])["message"]
8368            .as_str()
8369            .unwrap()
8370            .contains("does not provide the requested target"));
8371        assert!(module_rx.try_recv().is_err());
8372
8373        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
8374        let health_handler = handler.clone();
8375        let health_task = tokio::spawn(async move {
8376            health_handler
8377                .handle_control_frame(
8378                    &health_client_ctx,
8379                    supervisor_health_probe_frame(302, "mcp"),
8380                )
8381                .await
8382                .unwrap()
8383        });
8384        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8385            .await
8386            .unwrap()
8387            .unwrap();
8388        assert_eq!(
8389            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
8390            ModuleControlRequest::HealthCheck {}
8391        );
8392        handler
8393            .handle_control_frame(
8394                &module_ctx,
8395                health_response(health_frame.header.corr, HealthStatus::Ok),
8396            )
8397            .await
8398            .unwrap();
8399        let health_response = health_task.await.unwrap();
8400        assert_eq!(health_response[0].header.ty, FrameType::Response);
8401        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
8402            ClientControlResponse::SupervisorHealthProbe {
8403                module_id, status, ..
8404            } => {
8405                assert_eq!(module_id, "mcp");
8406                assert_eq!(status, HealthStatus::Ok);
8407            }
8408            other => panic!("unexpected health response: {other:?}"),
8409        }
8410
8411        // Exercise the forwarding cleanup path directly while leaving the registry
8412        // advertisement in place. If cleanup leaves a stale control sink behind,
8413        // the next probe will enqueue onto it and wait for the long probe timeout
8414        // instead of returning an immediate no-connection error.
8415        forwarding
8416            .cleanup_connection(module_ctx.connection_id)
8417            .unwrap();
8418        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
8419        let cleanup_response = tokio::time::timeout(
8420            Duration::from_millis(200),
8421            handler.handle_control_frame(
8422                &cleanup_probe_ctx,
8423                supervisor_health_probe_frame(303, "mcp"),
8424            ),
8425        )
8426        .await
8427        .expect("probe should fail immediately when the control lane is gone")
8428        .unwrap();
8429        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
8430        assert_eq!(
8431            parse_error(&cleanup_response[0])["code"],
8432            "target_unavailable"
8433        );
8434        assert!(parse_error(&cleanup_response[0])["message"]
8435            .as_str()
8436            .unwrap()
8437            .contains("no module connection"));
8438
8439        handler
8440            .cleanup_connection(module_ctx.connection_id)
8441            .unwrap();
8442    }
8443
8444    #[tokio::test]
8445    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
8446        let registry = Arc::new(Registry::default());
8447        let supervisor_handle = SupervisorHandle::new();
8448        let supervisor =
8449            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
8450                .with_handle(supervisor_handle.clone())
8451                .with_connection_file_path(
8452                    std::env::temp_dir()
8453                        .join(format!("subc-route-open-warming-{}", std::process::id())),
8454                );
8455        let module = supervisor
8456            .supervise_configured(
8457                ModuleSpec {
8458                    module_id: "warming".to_string(),
8459                    program: fake_aft_stub_path(),
8460                    args: Vec::new(),
8461                    env: Vec::new(),
8462                    reserved: false,
8463                    reserved_prefixes: Vec::new(),
8464                    protocol: ModuleProtocol::Subc,
8465                    overlap: Default::default(),
8466                },
8467                true,
8468            )
8469            .unwrap();
8470        assert_eq!(module.state().unwrap(), ModuleState::Running);
8471
8472        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8473        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
8474        let response = handler
8475            .handle_control_frame(
8476                &ctx,
8477                route_open_frame(304, "warming", unique_project_root("warming")),
8478            )
8479            .await
8480            .unwrap();
8481        module.stop().await.unwrap();
8482
8483        assert_eq!(response[0].header.ty, FrameType::Error);
8484        let error = parse_error(&response[0]);
8485        assert_eq!(error["code"], "module_warming");
8486        assert!(error["message"]
8487            .as_str()
8488            .unwrap()
8489            .contains("state=running, enabled=true, live=false"));
8490    }
8491
8492    /// One wire code has several senders, so the refusal line names the check
8493    /// that refused. This drives the shared refusal path (every non-supervised
8494    /// refusal goes through `route_open_refusal_frame`) with an unregistered
8495    /// target and requires the branch label on the event.
8496    #[tokio::test]
8497    async fn route_open_refusal_names_the_check_that_refused() {
8498        let handler = ControlHandler::new(Arc::new(Registry::default()));
8499        let capture = EventCapture::default();
8500        let _subscriber =
8501            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8502        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
8503        let response = handler
8504            .handle_control_frame(
8505                &ctx,
8506                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
8507            )
8508            .await
8509            .unwrap();
8510
8511        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
8512        let event = capture
8513            .events()
8514            .into_iter()
8515            .find(|event| {
8516                event.target == "control"
8517                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
8518            })
8519            .expect("route.open refusal event");
8520        assert_eq!(
8521            event.fields.get("reason"),
8522            Some(&"\"not_registered\"".to_string())
8523        );
8524    }
8525
8526    #[tokio::test]
8527    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
8528        let registry = Arc::new(Registry::default());
8529        let supervisor_handle = SupervisorHandle::new();
8530        let supervisor =
8531            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
8532                .with_handle(supervisor_handle.clone())
8533                .with_connection_file_path(std::env::temp_dir().join(format!(
8534                    "subc-route-open-refusal-info-{}",
8535                    std::process::id()
8536                )));
8537        let module = supervisor
8538            .supervise_configured(
8539                ModuleSpec {
8540                    module_id: "warming".to_string(),
8541                    program: fake_aft_stub_path(),
8542                    args: Vec::new(),
8543                    env: Vec::new(),
8544                    reserved: false,
8545                    reserved_prefixes: Vec::new(),
8546                    protocol: ModuleProtocol::Subc,
8547                    overlap: Default::default(),
8548                },
8549                true,
8550            )
8551            .unwrap();
8552        assert_eq!(module.state().unwrap(), ModuleState::Running);
8553
8554        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8555        assert!(handler
8556            .counters()
8557            .snapshot()
8558            .get("route_open_refused_by_code")
8559            .is_none());
8560        let capture = EventCapture::default();
8561        let _subscriber =
8562            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8563        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
8564        let response = handler
8565            .handle_control_frame(
8566                &ctx,
8567                route_open_frame(394, "warming", unique_project_root("refusal-info")),
8568            )
8569            .await
8570            .unwrap();
8571        module.stop().await.unwrap();
8572
8573        assert_eq!(parse_error(&response[0])["code"], "module_warming");
8574        let event = capture
8575            .events()
8576            .into_iter()
8577            .find(|event| {
8578                event.target == "control"
8579                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
8580            })
8581            .expect("route.open refusal event");
8582        assert_eq!(
8583            event.fields.get("module_id"),
8584            Some(&"\"warming\"".to_string())
8585        );
8586        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
8587        assert_eq!(
8588            event.fields.get("reason"),
8589            Some(&"\"supervised_not_registered\"".to_string())
8590        );
8591        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
8592        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
8593        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
8594        assert_eq!(
8595            handler.counters().snapshot()["route_open_refused_by_code"],
8596            json!({ "module_warming": 1 })
8597        );
8598    }
8599
8600    #[tokio::test(flavor = "current_thread")]
8601    async fn route_open_unknown_module_escapes_target_module_id() {
8602        let handler = ControlHandler::new(Arc::new(Registry::default()));
8603        let capture = EventCapture::default();
8604        let _subscriber =
8605            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8606        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
8607        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
8608        let response = handler
8609            .handle_control_frame(
8610                &ctx,
8611                route_open_frame(
8612                    395,
8613                    hostile_module_id,
8614                    unique_project_root("hostile-target-module-id"),
8615                ),
8616            )
8617            .await
8618            .unwrap();
8619
8620        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
8621        let event = capture
8622            .events()
8623            .into_iter()
8624            .find(|event| {
8625                event.target == "control"
8626                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
8627            })
8628            .expect("route.open unknown-module refusal event");
8629        let logged = event.fields.get("module_id").expect("module_id field");
8630        assert!(!logged.bytes().any(|byte| byte < 0x20));
8631        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
8632    }
8633
8634    #[tokio::test(flavor = "current_thread")]
8635    async fn route_open_module_rejection_uses_daemon_counter_key() {
8636        let registry = Arc::new(Registry::default());
8637        let forwarding = Arc::new(ForwardingTable::default());
8638        let handler =
8639            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
8640        let module_connection = ConnectionId::new(95);
8641        let (module_ctx, mut module_rx) = route_ctx(module_connection);
8642        hello_via_sink(
8643            &handler,
8644            &module_ctx,
8645            &mut module_rx,
8646            hello_frame("aft", PROTOCOL_VERSION, 395),
8647        )
8648        .await;
8649
8650        let client_connection = ConnectionId::new(96);
8651        let (client_ctx, _client_rx) = route_ctx(client_connection);
8652        let capture = EventCapture::default();
8653        let _subscriber =
8654            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8655        let (route_task, bind) = relay_route_open(
8656            &handler,
8657            client_connection,
8658            &client_ctx.egress,
8659            &mut module_rx,
8660            396,
8661            "aft",
8662            "hostile-module-code",
8663        )
8664        .await;
8665        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
8666        let rejection = Frame::build(
8667            FrameType::Error,
8668            control_flags(),
8669            0,
8670            0,
8671            bind.header.corr,
8672            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
8673        )
8674        .unwrap();
8675        handler
8676            .handle_control_frame(&module_ctx, rejection)
8677            .await
8678            .unwrap();
8679
8680        let response = route_task.await.unwrap();
8681        assert_eq!(parse_error(&response[0])["code"], hostile_code);
8682        let counters = handler.counters().snapshot();
8683        assert_eq!(
8684            counters["route_open_refused_by_code"],
8685            json!({ "module_rejected": 1 })
8686        );
8687        assert!(counters["route_open_refused_by_code"]
8688            .get(hostile_code)
8689            .is_none());
8690
8691        let event = capture
8692            .events()
8693            .into_iter()
8694            .find(|event| {
8695                event.target == "control"
8696                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
8697            })
8698            .expect("route.open module-rejection refusal event");
8699        let logged = event.fields.get("module_code").expect("module_code field");
8700        assert!(!logged.bytes().any(|byte| byte < 0x20));
8701        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
8702    }
8703
8704    #[tokio::test]
8705    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
8706        let registry = Arc::new(Registry::default());
8707        let supervisor_handle = SupervisorHandle::new();
8708        let missing_program = std::env::temp_dir().join(format!(
8709            "subc-route-open-missing-program-{}",
8710            std::process::id()
8711        ));
8712        let supervisor =
8713            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
8714                .with_handle(supervisor_handle.clone());
8715        let module = supervisor
8716            .supervise_configured(
8717                ModuleSpec {
8718                    module_id: "failed".to_string(),
8719                    program: missing_program,
8720                    args: Vec::new(),
8721                    env: Vec::new(),
8722                    reserved: false,
8723                    reserved_prefixes: Vec::new(),
8724                    protocol: ModuleProtocol::Subc,
8725                    overlap: Default::default(),
8726                },
8727                true,
8728            )
8729            .unwrap();
8730        assert_eq!(module.state().unwrap(), ModuleState::Failed);
8731
8732        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8733        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
8734        let response = handler
8735            .handle_control_frame(
8736                &ctx,
8737                route_open_frame(305, "failed", unique_project_root("failed")),
8738            )
8739            .await
8740            .unwrap();
8741
8742        assert_eq!(response[0].header.ty, FrameType::Error);
8743        let error = parse_error(&response[0]);
8744        assert_eq!(error["code"], "target_unavailable");
8745        assert!(error["message"]
8746            .as_str()
8747            .unwrap()
8748            .contains("state=failed, enabled=true, live=false"));
8749    }
8750
8751    #[tokio::test]
8752    async fn route_open_role_mismatch_remains_target_unavailable() {
8753        let registry = Arc::new(Registry::default());
8754        let handler = ControlHandler::new(Arc::clone(&registry));
8755        handler
8756            .handle_control(
8757                ConnectionId::new(41),
8758                non_routable_hello_frame_with_control_ops("health-only", 306, None),
8759            )
8760            .unwrap();
8761
8762        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
8763        let response = handler
8764            .handle_control_frame(
8765                &ctx,
8766                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
8767            )
8768            .await
8769            .unwrap();
8770
8771        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8772        assert!(parse_error(&response[0])["message"]
8773            .as_str()
8774            .unwrap()
8775            .contains("does not provide the requested target"));
8776    }
8777
8778    #[tokio::test]
8779    async fn route_open_inactive_registration_remains_target_unavailable() {
8780        let registry = Arc::new(Registry::default());
8781        let handler = ControlHandler::new(Arc::clone(&registry));
8782        handler
8783            .handle_control(
8784                ConnectionId::new(43),
8785                hello_frame("inactive", PROTOCOL_VERSION, 308),
8786            )
8787            .unwrap();
8788        assert!(registry
8789            .set_module_state_for_test("inactive", ChannelState::Closed)
8790            .unwrap());
8791
8792        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
8793        let response = handler
8794            .handle_control_frame(
8795                &ctx,
8796                route_open_frame(309, "inactive", unique_project_root("inactive")),
8797            )
8798            .await
8799            .unwrap();
8800
8801        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8802        assert!(parse_error(&response[0])["message"]
8803            .as_str()
8804            .unwrap()
8805            .contains("is not active"));
8806    }
8807
8808    #[tokio::test]
8809    async fn late_health_reply_is_recorded_through_the_module_response_path() {
8810        let registry = Arc::new(Registry::default());
8811        let forwarding = Arc::new(ForwardingTable::default());
8812        let supervisor_handle = SupervisorHandle::new();
8813        let supervisor = Supervisor::new(Arc::clone(&registry), crate::RestartPolicy::default())
8814            .with_forwarding(Arc::clone(&forwarding))
8815            .with_handle(supervisor_handle.clone());
8816        let module = supervisor
8817            .supervise_configured(
8818                crate::ModuleSpec {
8819                    module_id: "late-health-response".to_string(),
8820                    program: PathBuf::from("disabled-module"),
8821                    args: Vec::new(),
8822                    env: Vec::new(),
8823                    reserved: false,
8824                    reserved_prefixes: Vec::new(),
8825                    protocol: ModuleProtocol::Subc,
8826                    overlap: Default::default(),
8827                },
8828                false,
8829            )
8830            .unwrap();
8831        let handler =
8832            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8833                .with_supervisor(supervisor_handle);
8834        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
8835        handler
8836            .handle_control_frame(
8837                &module_ctx,
8838                hello_frame_with_control_ops(
8839                    "late-health-response",
8840                    PROTOCOL_VERSION,
8841                    7,
8842                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8843                ),
8844            )
8845            .await
8846            .unwrap();
8847        let probe_started_at = Instant::now() - Duration::from_millis(80);
8848        let pending = forwarding
8849            .begin_health_probe_rpc_for(
8850                "late-health-response",
8851                MODULE_CONTROL_OP_HEALTH_CHECK,
8852                probe_started_at,
8853                Instant::now() - Duration::from_millis(1),
8854            )
8855            .unwrap();
8856        assert!(forwarding
8857            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
8858            .unwrap());
8859
8860        let responses = handler
8861            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
8862            .await
8863            .unwrap();
8864
8865        assert!(responses.is_empty());
8866        let health = module.status().unwrap().health;
8867        assert_eq!(health.late_answer_count, 1);
8868        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
8869    }
8870
8871    #[tokio::test]
8872    async fn health_probe_timeout_and_module_death_are_typed() {
8873        let registry = Arc::new(Registry::default());
8874        let forwarding = Arc::new(ForwardingTable::default());
8875        let handler =
8876            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8877                .with_health_probe_timeout(Duration::from_millis(50));
8878        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
8879        hello_via_sink(
8880            &handler,
8881            &module_ctx,
8882            &mut module_rx,
8883            hello_frame_with_control_ops(
8884                "aft",
8885                PROTOCOL_VERSION,
8886                7,
8887                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8888            ),
8889        )
8890        .await;
8891
8892        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
8893        let responses = handler
8894            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
8895            .await
8896            .unwrap();
8897        assert_eq!(responses[0].header.ty, FrameType::Error);
8898        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
8899        let _ = module_rx.try_recv();
8900
8901        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
8902        let health_handler = handler.clone();
8903        let death_task = tokio::spawn(async move {
8904            health_handler
8905                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
8906                .await
8907                .unwrap()
8908        });
8909        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8910            .await
8911            .unwrap()
8912            .unwrap();
8913        handler
8914            .cleanup_connection(module_ctx.connection_id)
8915            .unwrap();
8916        let responses = death_task.await.unwrap();
8917        assert_eq!(responses[0].header.ty, FrameType::Error);
8918        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
8919    }
8920
8921    #[test]
8922    fn hello_requires_exact_protocol_version() {
8923        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
8924            let registry = Arc::new(Registry::default());
8925            let handler = ControlHandler::new(Arc::clone(&registry));
8926            let responses = handler
8927                .handle_control(
8928                    ConnectionId::new(connection),
8929                    hello_frame("aft", offered, 9),
8930                )
8931                .unwrap();
8932
8933            assert_eq!(responses.len(), 1);
8934            assert_eq!(responses[0].header.ty, FrameType::Error);
8935            let error = parse_error(&responses[0]);
8936            assert_eq!(error["code"], "version_unsupported");
8937            assert!(registry.get_module("aft").unwrap().is_none());
8938            assert_eq!(registry.active_registration_count().unwrap(), 0);
8939        }
8940    }
8941
8942    #[test]
8943    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
8944        let registry = Arc::new(Registry::default());
8945        let forwarding = Arc::new(ForwardingTable::default());
8946        let handler =
8947            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
8948        let module_connection = ConnectionId::new(301);
8949        let registration = registry
8950            .register_with_control_ops(
8951                manifest("aft-push", PROTOCOL_VERSION),
8952                PROTOCOL_VERSION,
8953                module_connection,
8954                module_baseline_control_ops(),
8955            )
8956            .unwrap();
8957        let (module_tx, _module_rx) = mpsc::channel(8);
8958        let endpoint = forwarding
8959            .register_module_connection(
8960                module_connection,
8961                "aft-push".to_string(),
8962                PROTOCOL_VERSION,
8963                manifest_concurrency(&registration.manifest),
8964                FrameSink::new(module_tx),
8965            )
8966            .unwrap();
8967
8968        // A push op this version does not know is ignored (forward-compat), not errored.
8969        let unknown = Frame::build(
8970            FrameType::Push,
8971            control_flags(),
8972            0,
8973            0,
8974            5,
8975            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
8976        )
8977        .unwrap();
8978        let out = handler.handle_status_update(endpoint, unknown).unwrap();
8979        assert!(
8980            out.is_empty(),
8981            "unknown push op must be ignored, got {out:?}"
8982        );
8983
8984        // A malformed body for a KNOWN op is a real error worth surfacing.
8985        let malformed = Frame::build(
8986            FrameType::Push,
8987            control_flags(),
8988            0,
8989            0,
8990            6,
8991            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
8992        )
8993        .unwrap();
8994        let out = handler.handle_status_update(endpoint, malformed).unwrap();
8995        assert_eq!(out.len(), 1);
8996        assert_eq!(out[0].header.ty, FrameType::Error);
8997        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
8998    }
8999
9000    #[test]
9001    fn hello_rejected_when_connection_already_owns_client_routes() {
9002        let registry = Arc::new(Registry::default());
9003        let forwarding = Arc::new(ForwardingTable::default());
9004        let handler =
9005            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9006        // Commits a client route on connection 202 (bound to a module on conn 101).
9007        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
9008        let client_connection = ConnectionId::new(202);
9009
9010        // That same connection now tries to register as a module: rejected, so one
9011        // connection never holds both client-route and module-endpoint state.
9012        let responses = handler
9013            .handle_control(
9014                client_connection,
9015                hello_frame("aft-second", PROTOCOL_VERSION, 9),
9016            )
9017            .unwrap();
9018        assert_eq!(responses[0].header.ty, FrameType::Error);
9019        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
9020        assert!(registry.get_module("aft-second").unwrap().is_none());
9021    }
9022
9023    #[test]
9024    fn reserved_module_hello_requires_matching_launch_nonce() {
9025        let registry = Arc::new(Registry::default());
9026        let supervisor = SupervisorHandle::new();
9027        // The supervisor recorded the nonce it injected when it spawned the reserved
9028        // module; the HELLO verifier checks against the same shared handle.
9029        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9030        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
9031
9032        // A HELLO with NO nonce is rejected.
9033        let no_nonce = handler
9034            .handle_control(
9035                ConnectionId::new(1),
9036                hello_frame("vault", PROTOCOL_VERSION, 1),
9037            )
9038            .unwrap();
9039        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
9040        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
9041        assert!(registry.get_module("vault").unwrap().is_none());
9042
9043        // A HELLO with the WRONG nonce is rejected.
9044        let wrong = handler
9045            .handle_control(
9046                ConnectionId::new(2),
9047                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
9048            )
9049            .unwrap();
9050        assert_eq!(wrong[0].header.ty, FrameType::Error);
9051        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
9052        assert!(registry.get_module("vault").unwrap().is_none());
9053
9054        // A HELLO with the CORRECT nonce registers.
9055        let ok = handler
9056            .handle_control(
9057                ConnectionId::new(3),
9058                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
9059            )
9060            .unwrap();
9061        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
9062        assert!(registry.get_module("vault").unwrap().is_some());
9063    }
9064
9065    #[test]
9066    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
9067        let registry = Arc::new(Registry::default());
9068        let supervisor = SupervisorHandle::new();
9069        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
9070        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
9071        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
9072
9073        let squat = handler
9074            .handle_control(
9075                ConnectionId::new(1),
9076                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
9077            )
9078            .unwrap();
9079        assert_eq!(squat[0].header.ty, FrameType::Error);
9080        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
9081        assert!(parse_error(&squat[0])["message"]
9082            .as_str()
9083            .unwrap()
9084            .contains("fed:"));
9085
9086        let accepted_peer = handler
9087            .handle_control(
9088                ConnectionId::new(2),
9089                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
9090            )
9091            .unwrap();
9092        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
9093
9094        let accepted_short = handler
9095            .handle_control(
9096                ConnectionId::new(3),
9097                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
9098            )
9099            .unwrap();
9100        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
9101
9102        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
9103            let response = handler
9104                .handle_control(
9105                    ConnectionId::new(conn),
9106                    hello_frame(module_id, PROTOCOL_VERSION, conn),
9107                )
9108                .unwrap();
9109            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
9110        }
9111    }
9112
9113    #[test]
9114    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
9115        let registry = Arc::new(Registry::default());
9116        let supervisor = SupervisorHandle::new();
9117        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
9118        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
9119        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
9120        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
9121
9122        let owner_nonce = handler
9123            .handle_control(
9124                ConnectionId::new(1),
9125                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
9126            )
9127            .unwrap();
9128        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
9129        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
9130        assert!(registry.get_module("fed:special").unwrap().is_none());
9131
9132        let exact_nonce = handler
9133            .handle_control(
9134                ConnectionId::new(2),
9135                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
9136            )
9137            .unwrap();
9138        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
9139        assert!(registry.get_module("fed:special").unwrap().is_some());
9140    }
9141
9142    #[test]
9143    fn non_reserved_module_ignores_launch_nonce() {
9144        let registry = Arc::new(Registry::default());
9145        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
9146        // registration succeeds whether a spawned process echoes a nonce or not.
9147        let handler = ControlHandler::new(Arc::clone(&registry));
9148        let no_nonce = handler
9149            .handle_control(
9150                ConnectionId::new(1),
9151                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
9152            )
9153            .unwrap();
9154        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
9155        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
9156
9157        let echoed_nonce = handler
9158            .handle_control(
9159                ConnectionId::new(2),
9160                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
9161            )
9162            .unwrap();
9163        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
9164        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
9165    }
9166
9167    #[test]
9168    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
9169        let handler = ControlHandler::default();
9170        let conn = ConnectionId::new(1);
9171        let malformed = Frame::build(
9172            FrameType::Hello,
9173            control_flags(),
9174            0,
9175            0,
9176            3,
9177            b"{not json".to_vec(),
9178        )
9179        .unwrap();
9180
9181        let error = handler.handle_control(conn, malformed).unwrap();
9182        assert_eq!(error[0].header.ty, FrameType::Error);
9183        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
9184
9185        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
9186        let pong = handler.handle_control(conn, ping).unwrap();
9187        assert_eq!(pong[0].header.ty, FrameType::Pong);
9188        assert_eq!(pong[0].header.corr, 4);
9189    }
9190
9191    #[test]
9192    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
9193        let registry = Arc::new(Registry::default());
9194        let handler = ControlHandler::new(Arc::clone(&registry));
9195
9196        handler
9197            .handle_control(
9198                ConnectionId::new(1),
9199                hello_frame("aft", PROTOCOL_VERSION, 1),
9200            )
9201            .unwrap();
9202        let duplicate = handler
9203            .handle_control(
9204                ConnectionId::new(2),
9205                hello_frame("aft", PROTOCOL_VERSION, 2),
9206            )
9207            .unwrap();
9208
9209        assert_eq!(duplicate[0].header.ty, FrameType::Error);
9210        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
9211        let registration = registry.get_module("aft").unwrap().unwrap();
9212        assert_eq!(registration.connection_id, ConnectionId::new(1));
9213    }
9214
9215    #[test]
9216    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
9217        let registry = Arc::new(Registry::default());
9218        let forwarding = Arc::new(ForwardingTable::default());
9219        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
9220        let handler =
9221            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9222                .with_process_liveness(process_liveness);
9223        let (ctx, route_channel, route_epoch) =
9224            bind_liveness_route(&registry, &forwarding, "aft-dead");
9225        let responses = handler
9226            .handle_route_poll(
9227                &ctx,
9228                route_poll_frame(41, PollKind::Liveness, route_channel),
9229                route_channel,
9230                route_epoch,
9231                PollKind::Liveness,
9232            )
9233            .unwrap();
9234
9235        assert_eq!(responses.len(), 1);
9236        assert_eq!(responses[0].header.ty, FrameType::Response);
9237        assert_route_poll_liveness(&responses[0], false);
9238    }
9239
9240    #[test]
9241    fn liveness_poll_without_process_source_uses_bound_route() {
9242        let registry = Arc::new(Registry::default());
9243        let forwarding = Arc::new(ForwardingTable::default());
9244        let handler =
9245            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9246        let (ctx, route_channel, route_epoch) =
9247            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
9248        let responses = handler
9249            .handle_route_poll(
9250                &ctx,
9251                route_poll_frame(42, PollKind::Liveness, route_channel),
9252                route_channel,
9253                route_epoch,
9254                PollKind::Liveness,
9255            )
9256            .unwrap();
9257
9258        assert_route_poll_liveness(&responses[0], true);
9259    }
9260
9261    #[test]
9262    fn liveness_poll_untracked_process_source_uses_bound_route() {
9263        let registry = Arc::new(Registry::default());
9264        let forwarding = Arc::new(ForwardingTable::default());
9265        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
9266        let handler =
9267            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9268                .with_process_liveness(process_liveness);
9269        let (ctx, route_channel, route_epoch) =
9270            bind_liveness_route(&registry, &forwarding, "aft-untracked");
9271        let responses = handler
9272            .handle_route_poll(
9273                &ctx,
9274                route_poll_frame(43, PollKind::Liveness, route_channel),
9275                route_channel,
9276                route_epoch,
9277                PollKind::Liveness,
9278            )
9279            .unwrap();
9280
9281        assert_route_poll_liveness(&responses[0], true);
9282    }
9283
9284    #[tokio::test]
9285    async fn unknown_op_returns_unknown_control_op() {
9286        let handler = ControlHandler::default();
9287        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
9288        let request = Frame::build(
9289            FrameType::Request,
9290            control_flags(),
9291            0,
9292            0,
9293            55,
9294            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
9295        )
9296        .unwrap();
9297
9298        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9299
9300        assert_eq!(response.len(), 1);
9301        assert_eq!(response[0].header.ty, FrameType::Error);
9302        assert_eq!(response[0].header.corr, 55);
9303        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
9304    }
9305
9306    #[tokio::test]
9307    async fn supervisor_provenance_rejects_unknown_exact_module() {
9308        let handler = ControlHandler::default();
9309        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
9310        let request = Frame::build(
9311            FrameType::Request,
9312            control_flags(),
9313            0,
9314            0,
9315            57,
9316            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
9317        )
9318        .unwrap();
9319
9320        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9321
9322        assert_eq!(response.len(), 1);
9323        assert_eq!(response[0].header.ty, FrameType::Error);
9324        assert_eq!(response[0].header.corr, 57);
9325        let error = parse_error(&response[0]);
9326        assert_eq!(error["code"], "unknown_module");
9327        assert_eq!(error["message"], "module_id 'missing' is not supervised");
9328    }
9329
9330    #[test]
9331    fn provenance_probe_override_keeps_handler_tests_deterministic() {
9332        let expected = subc_control::RunningImageAgreement::Unavailable {
9333            reason: subc_control::RunningImageUnavailableReason::HashFailed,
9334        };
9335        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
9336        assert_eq!(handler.provenance_probe_override, Some(expected));
9337    }
9338
9339    #[tokio::test]
9340    async fn malformed_control_bodies_return_invalid_control_body() {
9341        let handler = ControlHandler::default();
9342        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
9343
9344        for (corr, body) in [
9345            (56, br#"{"route_channel":1}"#.as_slice()),
9346            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
9347            (
9348                58,
9349                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
9350            ),
9351        ] {
9352            let request = Frame::build(
9353                FrameType::Request,
9354                control_flags(),
9355                0,
9356                0,
9357                corr,
9358                body.to_vec(),
9359            )
9360            .unwrap();
9361            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9362
9363            assert_eq!(response.len(), 1);
9364            assert_eq!(response[0].header.ty, FrameType::Error);
9365            assert_eq!(response[0].header.corr, corr);
9366            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
9367        }
9368    }
9369
9370    #[tokio::test]
9371    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
9372        let registry = Arc::new(Registry::default());
9373        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
9374        let router = Router::with_control_handler(Arc::clone(&control));
9375        let connection = router.begin_connection();
9376        let (ctx, mut rx) = route_ctx(connection.id());
9377
9378        router
9379            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
9380            .await
9381            .unwrap();
9382        let response = rx.recv().await.unwrap();
9383        let ack = parse_ack(&response);
9384        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9385        let channel = 1;
9386
9387        let goodbye =
9388            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
9389        router.route_for_connection(&ctx, goodbye).await.unwrap();
9390        assert!(rx.try_recv().is_err());
9391        assert!(registry.get_module("aft").unwrap().is_none());
9392
9393        router
9394            .route_for_connection(&ctx, channel_request(channel, 13))
9395            .await
9396            .unwrap();
9397        let error_frame = rx.recv().await.unwrap();
9398        assert_eq!(error_frame.header.ty, FrameType::Error);
9399        assert_eq!(error_frame.header.channel, channel);
9400    }
9401
9402    #[tokio::test]
9403    async fn dropping_router_connection_releases_registration() {
9404        let registry = Arc::new(Registry::default());
9405        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
9406        let router = Router::with_control_handler(control);
9407        let connection = router.begin_connection();
9408        let (ctx, mut rx) = route_ctx(connection.id());
9409
9410        router
9411            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
9412            .await
9413            .unwrap();
9414        let response = rx.recv().await.unwrap();
9415        let ack = parse_ack(&response);
9416        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9417        assert!(registry.get_module("aft").unwrap().is_some());
9418
9419        drop(connection);
9420
9421        assert!(registry.get_module("aft").unwrap().is_none());
9422        assert_eq!(registry.active_registration_count().unwrap(), 0);
9423    }
9424
9425    fn capability_manifest(
9426        module_id: &str,
9427        provides: &[&str],
9428        must_never_reach: &[&str],
9429    ) -> ModuleManifest {
9430        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
9431        manifest.capabilities = Some(CapabilityDeclarations {
9432            provides: provides
9433                .iter()
9434                .map(|capability| (*capability).to_string())
9435                .collect(),
9436            requires: Vec::new(),
9437            must_never_reach: must_never_reach
9438                .iter()
9439                .map(|capability| (*capability).to_string())
9440                .collect(),
9441        });
9442        manifest
9443    }
9444
9445    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
9446        Frame::build(
9447            FrameType::Hello,
9448            control_flags(),
9449            0,
9450            0,
9451            corr,
9452            serde_json::to_vec(&ModuleHelloBody {
9453                protocol_ver: manifest.protocol_ver,
9454                manifest,
9455                control_ops: None,
9456                launch_nonce: None,
9457            })
9458            .expect("capability test HELLO serializes"),
9459        )
9460        .expect("capability test HELLO frame builds")
9461    }
9462
9463    fn catalog_update_with_capabilities_frame(
9464        corr: u64,
9465        capabilities: CapabilityDeclarations,
9466    ) -> Frame {
9467        Frame::build(
9468            FrameType::Request,
9469            control_flags(),
9470            0,
9471            0,
9472            corr,
9473            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
9474                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
9475                capabilities: Some(capabilities),
9476                ready: None,
9477            })
9478            .expect("capability catalog.update serializes"),
9479        )
9480        .expect("capability catalog.update frame builds")
9481    }
9482
9483    async fn register_capability_manifest(
9484        handler: &ControlHandler,
9485        ctx: &RouteCtx,
9486        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9487        manifest: ModuleManifest,
9488        corr: u64,
9489    ) {
9490        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
9491    }
9492
9493    async fn open_route_for_capability_test(
9494        handler: &ControlHandler,
9495        target_ctx: &RouteCtx,
9496        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9497        client_connection_id: u64,
9498        corr: u64,
9499        target_module_id: &str,
9500        consumer_identity: Option<ConsumerIdentity>,
9501    ) -> (
9502        mpsc::Receiver<crate::router::OutboundFrame>,
9503        ModuleControlRequest,
9504    ) {
9505        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
9506        let route_handler = handler.clone();
9507        let target_module_id = target_module_id.to_string();
9508        let route_task = tokio::spawn(async move {
9509            route_handler
9510                .handle_control_frame(
9511                    &client_ctx,
9512                    route_open_frame_with_admission_facts(
9513                        corr,
9514                        &target_module_id,
9515                        unique_project_root("admission-facts"),
9516                        consumer_identity,
9517                        None,
9518                    ),
9519                )
9520                .await
9521                .expect("capability test route.open succeeds")
9522        });
9523        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
9524            .await
9525            .expect("capability test route.open must reach route.bind")
9526            .expect("target control receiver stays open");
9527        let bind_request: ModuleControlRequest =
9528            serde_json::from_slice(&bind.body).expect("route.bind decodes");
9529        handler
9530            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
9531            .await
9532            .expect("capability test route.bind ACK succeeds");
9533        assert!(route_task.await.expect("route.open task joins").is_empty());
9534        let opened = client_rx
9535            .recv()
9536            .await
9537            .expect("successful route.open publishes a response");
9538        assert!(matches!(
9539            serde_json::from_slice::<ClientControlResponse>(&opened.body),
9540            Ok(ClientControlResponse::RouteOpen { .. })
9541        ));
9542        (client_rx, bind_request)
9543    }
9544
9545    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
9546        assert_eq!(frame.header.ty, FrameType::Push);
9547        assert_eq!(frame.header.channel, 0);
9548        assert_eq!(
9549            serde_json::from_slice::<ClientControlPush>(&frame.body)
9550                .expect("route.closed control push decodes"),
9551            ClientControlPush::RouteClosed {
9552                module_id: target_module_id.to_string(),
9553                reason: RouteCloseReason::CapabilityDenied,
9554                drained: false,
9555                abandoned: 0,
9556                excluded_subscriptions: 0,
9557                terminal: Some(false),
9558            }
9559        );
9560    }
9561
9562    #[tokio::test]
9563    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
9564        let registry = Arc::new(Registry::default());
9565        let forwarding = Arc::new(ForwardingTable::default());
9566        let supervisor = SupervisorHandle::new();
9567        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9568        let handler =
9569            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9570                .with_supervisor(supervisor);
9571        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
9572        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
9573        register_capability_manifest(
9574            &handler,
9575            &target_ctx,
9576            &mut target_rx,
9577            capability_manifest("target", &["credentials-provider/v1"], &[]),
9578            1,
9579        )
9580        .await;
9581        register_capability_manifest(
9582            &handler,
9583            &opener_ctx,
9584            &mut opener_rx,
9585            capability_manifest("opener", &[], &["credentials-provider/v1"]),
9586            2,
9587        )
9588        .await;
9589
9590        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
9591        let replies = handler
9592            .handle_control_frame(
9593                &client_ctx,
9594                route_open_frame_with_admission_facts(
9595                    3,
9596                    "target",
9597                    unique_project_root("admission-facts"),
9598                    Some(ConsumerIdentity {
9599                        module_id: "opener".to_string(),
9600                        launch_nonce: "opener-nonce".to_string(),
9601                    }),
9602                    None,
9603                ),
9604            )
9605            .await
9606            .expect("denied route.open returns a typed frame");
9607        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
9608        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9609        assert!(
9610            target_rx.try_recv().is_err(),
9611            "forbidden route.open must not relay route.bind"
9612        );
9613    }
9614
9615    #[tokio::test]
9616    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
9617        let registry = Arc::new(Registry::default());
9618        let forwarding = Arc::new(ForwardingTable::default());
9619        let supervisor = SupervisorHandle::new();
9620        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9621        let handler =
9622            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9623                .with_supervisor(supervisor);
9624        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
9625        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
9626        register_capability_manifest(
9627            &handler,
9628            &target_ctx,
9629            &mut target_rx,
9630            capability_manifest("target", &["credentials-provider/v1"], &[]),
9631            1,
9632        )
9633        .await;
9634        register_capability_manifest(
9635            &handler,
9636            &old_opener_ctx,
9637            &mut old_opener_rx,
9638            capability_manifest("opener", &[], &[]),
9639            2,
9640        )
9641        .await;
9642        let (mut client_rx, _) = open_route_for_capability_test(
9643            &handler,
9644            &target_ctx,
9645            &mut target_rx,
9646            712,
9647            3,
9648            "target",
9649            Some(ConsumerIdentity {
9650                module_id: "opener".to_string(),
9651                launch_nonce: "opener-nonce".to_string(),
9652            }),
9653        )
9654        .await;
9655        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9656
9657        handler
9658            .cleanup_connection(old_opener_ctx.connection_id)
9659            .expect("old opener registration cleans up");
9660        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
9661        register_capability_manifest(
9662            &handler,
9663            &new_opener_ctx,
9664            &mut new_opener_rx,
9665            capability_manifest("opener", &[], &["credentials-provider/v1"]),
9666            4,
9667        )
9668        .await;
9669
9670        assert_capability_denied_push(
9671            client_rx
9672                .try_recv()
9673                .expect("HELLO deny addition must emit route.closed")
9674                .frame,
9675            "target",
9676        );
9677        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9678        assert!(matches!(
9679            target_rx.try_recv(),
9680            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
9681        ));
9682    }
9683
9684    #[tokio::test]
9685    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
9686        let registry = Arc::new(Registry::default());
9687        let forwarding = Arc::new(ForwardingTable::default());
9688        let supervisor = SupervisorHandle::new();
9689        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9690        let handler =
9691            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9692                .with_supervisor(supervisor);
9693        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
9694        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
9695        register_capability_manifest(
9696            &handler,
9697            &target_ctx,
9698            &mut target_rx,
9699            capability_manifest("target", &[], &[]),
9700            1,
9701        )
9702        .await;
9703        register_capability_manifest(
9704            &handler,
9705            &opener_ctx,
9706            &mut opener_rx,
9707            capability_manifest("opener", &[], &["credentials-provider/v1"]),
9708            2,
9709        )
9710        .await;
9711        let (mut client_rx, _) = open_route_for_capability_test(
9712            &handler,
9713            &target_ctx,
9714            &mut target_rx,
9715            722,
9716            3,
9717            "target",
9718            Some(ConsumerIdentity {
9719                module_id: "opener".to_string(),
9720                launch_nonce: "opener-nonce".to_string(),
9721            }),
9722        )
9723        .await;
9724        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9725
9726        let replies = handler
9727            .handle_control_frame(
9728                &target_ctx,
9729                catalog_update_with_capabilities_frame(
9730                    4,
9731                    CapabilityDeclarations {
9732                        provides: vec!["credentials-provider/v1".to_string()],
9733                        requires: Vec::new(),
9734                        must_never_reach: Vec::new(),
9735                    },
9736                ),
9737            )
9738            .await
9739            .expect("claim catalog.update succeeds");
9740        assert!(matches!(
9741            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
9742            Ok(ModuleControlResponseToModule::CatalogUpdate {})
9743        ));
9744        assert_capability_denied_push(
9745            client_rx
9746                .try_recv()
9747                .expect("claim addition must emit route.closed")
9748                .frame,
9749            "target",
9750        );
9751        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9752        assert!(matches!(
9753            target_rx.try_recv(),
9754            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
9755        ));
9756    }
9757
9758    #[tokio::test]
9759    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
9760        let registry = Arc::new(Registry::default());
9761        let forwarding = Arc::new(ForwardingTable::default());
9762        let supervisor = SupervisorHandle::new();
9763        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9764        let handler =
9765            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9766                .with_supervisor(supervisor);
9767        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
9768        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
9769        register_capability_manifest(
9770            &handler,
9771            &target_ctx,
9772            &mut target_rx,
9773            capability_manifest("target", &["credentials-provider/v1"], &[]),
9774            1,
9775        )
9776        .await;
9777        register_capability_manifest(
9778            &handler,
9779            &opener_ctx,
9780            &mut opener_rx,
9781            capability_manifest("opener", &[], &[]),
9782            2,
9783        )
9784        .await;
9785        let (mut client_rx, _) = open_route_for_capability_test(
9786            &handler,
9787            &target_ctx,
9788            &mut target_rx,
9789            732,
9790            3,
9791            "target",
9792            Some(ConsumerIdentity {
9793                module_id: "opener".to_string(),
9794                launch_nonce: "opener-nonce".to_string(),
9795            }),
9796        )
9797        .await;
9798        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9799
9800        handler
9801            .handle_control_frame(
9802                &target_ctx,
9803                catalog_update_with_capabilities_frame(
9804                    4,
9805                    CapabilityDeclarations {
9806                        provides: Vec::new(),
9807                        requires: Vec::new(),
9808                        must_never_reach: Vec::new(),
9809                    },
9810                ),
9811            )
9812            .await
9813            .expect("claim removal catalog.update succeeds");
9814        assert_eq!(
9815            forwarding.active_binding_count().unwrap(),
9816            1,
9817            "removing an attested target claim must leave the route census unchanged"
9818        );
9819        assert!(
9820            client_rx.try_recv().is_err(),
9821            "claim removal must not emit route.closed capability_denied"
9822        );
9823        assert!(
9824            target_rx.try_recv().is_err(),
9825            "claim removal must not send the target a route GOODBYE"
9826        );
9827    }
9828
9829    /// A direct client may open a route to a denied capability provider; this
9830    /// policy applies only to attested supervised module origins, not to direct clients.
9831    #[tokio::test]
9832    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
9833        let registry = Arc::new(Registry::default());
9834        let forwarding = Arc::new(ForwardingTable::default());
9835        let supervisor = SupervisorHandle::new();
9836        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9837        let handler =
9838            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9839                .with_supervisor(supervisor);
9840        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
9841        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
9842        register_capability_manifest(
9843            &handler,
9844            &target_ctx,
9845            &mut target_rx,
9846            capability_manifest("target", &["credentials-provider/v1"], &[]),
9847            1,
9848        )
9849        .await;
9850        register_capability_manifest(
9851            &handler,
9852            &opener_ctx,
9853            &mut opener_rx,
9854            capability_manifest("opener", &[], &["credentials-provider/v1"]),
9855            2,
9856        )
9857        .await;
9858
9859        let (_client_rx, bind) = open_route_for_capability_test(
9860            &handler,
9861            &target_ctx,
9862            &mut target_rx,
9863            742,
9864            3,
9865            "target",
9866            None,
9867        )
9868        .await;
9869        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
9870            panic!("direct scope-honesty route must bind");
9871        };
9872        assert_eq!(principal, Some(Principal::Direct));
9873        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9874    }
9875
9876    /// A module that denies a capability receives no self-route exemption when it
9877    /// also attestedly provides that capability.
9878    #[tokio::test]
9879    async fn must_never_reach_self_route_is_capability_forbidden() {
9880        let registry = Arc::new(Registry::default());
9881        let forwarding = Arc::new(ForwardingTable::default());
9882        let supervisor = SupervisorHandle::new();
9883        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
9884        let handler =
9885            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9886                .with_supervisor(supervisor);
9887        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
9888        register_capability_manifest(
9889            &handler,
9890            &self_ctx,
9891            &mut self_rx,
9892            capability_manifest(
9893                "self-provider",
9894                &["credentials-provider/v1"],
9895                &["credentials-provider/v1"],
9896            ),
9897            1,
9898        )
9899        .await;
9900
9901        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
9902        let replies = handler
9903            .handle_control_frame(
9904                &client_ctx,
9905                route_open_frame_with_admission_facts(
9906                    2,
9907                    "self-provider",
9908                    unique_project_root("admission-facts"),
9909                    Some(ConsumerIdentity {
9910                        module_id: "self-provider".to_string(),
9911                        launch_nonce: "self-nonce".to_string(),
9912                    }),
9913                    None,
9914                ),
9915            )
9916            .await
9917            .expect("self-route refusal returns a typed frame");
9918        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
9919        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9920        assert!(
9921            self_rx.try_recv().is_err(),
9922            "self denial must not relay route.bind"
9923        );
9924    }
9925
9926    #[test]
9927    fn unsupported_channel_zero_frame_returns_error() {
9928        let handler = ControlHandler::default();
9929        let request = Frame::build(
9930            FrameType::Request,
9931            control_flags(),
9932            0,
9933            0,
9934            21,
9935            b"opaque".to_vec(),
9936        )
9937        .unwrap();
9938
9939        let response = handler
9940            .handle_control(ConnectionId::new(1), request)
9941            .unwrap();
9942
9943        assert_eq!(response[0].header.ty, FrameType::Error);
9944        assert_eq!(
9945            parse_error(&response[0])["code"],
9946            "unsupported_control_frame"
9947        );
9948    }
9949
9950    /// Blue/green swap at the control-plane boundary. The supervisor that opens
9951    /// a swap is not wired yet, so the candidate is registered here directly
9952    /// into the registry and forwarding candidate slots, the way the swap's
9953    /// HELLO admission will.
9954    mod swap {
9955        use super::*;
9956
9957        const INCUMBENT: ConnectionId = ConnectionId::new(30);
9958        const CANDIDATE: ConnectionId = ConnectionId::new(40);
9959
9960        struct Swap {
9961            registry: Arc<Registry>,
9962            forwarding: Arc<ForwardingTable>,
9963            handler: ControlHandler,
9964            incumbent_ctx: RouteCtx,
9965            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
9966            candidate_ctx: RouteCtx,
9967            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
9968        }
9969
9970        async fn swap_with_incumbent() -> Swap {
9971            let registry = Arc::new(Registry::default());
9972            let forwarding = Arc::new(ForwardingTable::default());
9973            let handler =
9974                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9975            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
9976            hello_via_sink(
9977                &handler,
9978                &incumbent_ctx,
9979                &mut incumbent_rx,
9980                hello_frame("aft", PROTOCOL_VERSION, 7),
9981            )
9982            .await;
9983            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
9984            Swap {
9985                registry,
9986                forwarding,
9987                handler,
9988                incumbent_ctx,
9989                incumbent_rx,
9990                candidate_ctx,
9991                candidate_rx,
9992            }
9993        }
9994
9995        fn register_candidate(swap: &Swap, ready: Option<bool>) {
9996            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
9997            candidate_manifest.ready = ready;
9998            let registration = swap
9999                .registry
10000                .register_candidate_with_control_ops(
10001                    candidate_manifest,
10002                    PROTOCOL_VERSION,
10003                    CANDIDATE,
10004                    module_baseline_control_ops(),
10005                )
10006                .unwrap();
10007            swap.forwarding
10008                .register_candidate_module_connection(
10009                    CANDIDATE,
10010                    "aft".to_string(),
10011                    PROTOCOL_VERSION,
10012                    manifest_concurrency(&registration.manifest),
10013                    swap.candidate_ctx.egress.clone(),
10014                )
10015                .unwrap();
10016        }
10017
10018        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
10019            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
10020            swap.registry.promote_candidate("aft").unwrap().unwrap();
10021            cutover.incumbent.unwrap()
10022        }
10023
10024        fn keyed_total(counters: &Value, key: &str) -> u64 {
10025            counters[key]
10026                .as_object()
10027                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
10028                .unwrap_or(0)
10029        }
10030
10031        /// An ack from the incumbent for a bind it was sent before cutover,
10032        /// arriving before the incumbent is drained. The incumbent is the live
10033        /// connection carrying every other client's routes, so the ack must
10034        /// not end it: the waiting client is told to retry, the reservation is
10035        /// given back, and the incumbent is told to drop just that binding.
10036        #[tokio::test]
10037        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
10038            let mut swap = swap_with_incumbent().await;
10039            let handler = swap.handler.clone();
10040
10041            // A co-tenant route, bound on the incumbent before the swap.
10042            let cotenant = ConnectionId::new(31);
10043            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
10044            let (cotenant_task, cotenant_bind) = relay_route_open(
10045                &handler,
10046                cotenant,
10047                &cotenant_ctx.egress,
10048                &mut swap.incumbent_rx,
10049                100,
10050                "aft",
10051                "swap-cotenant",
10052            )
10053            .await;
10054            handler
10055                .handle_control_frame(
10056                    &swap.incumbent_ctx,
10057                    route_bind_ack(cotenant_bind.header.corr),
10058                )
10059                .await
10060                .unwrap();
10061            assert!(cotenant_task.await.unwrap().is_empty());
10062            let (cotenant_channel, cotenant_epoch) =
10063                published_route(&cotenant_rx.recv().await.unwrap());
10064
10065            // A second route.open, relayed to the incumbent and not yet acked.
10066            let caller = ConnectionId::new(32);
10067            let (caller_ctx, mut caller_rx) = route_ctx(caller);
10068            let (caller_task, caller_bind) = relay_route_open(
10069                &handler,
10070                caller,
10071                &caller_ctx.egress,
10072                &mut swap.incumbent_rx,
10073                101,
10074                "aft",
10075                "swap-caller",
10076            )
10077            .await;
10078            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
10079
10080            register_candidate(&swap, None);
10081            cutover(&swap);
10082
10083            // The incumbent acks after promotion and before any drain.
10084            let ack = handler
10085                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
10086                .await;
10087            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
10088            if module_loop_error.is_some() {
10089                // What the connection loop does with an untranslated router
10090                // error: end the connection, releasing every route on it.
10091                handler.cleanup_connection(INCUMBENT).unwrap();
10092            }
10093
10094            // 1. The incumbent's other routes survive.
10095            assert!(
10096                cotenant_rx.try_recv().is_err(),
10097                "the co-tenant route on the incumbent was torn down by one late ack: \
10098                 {module_loop_error:?}"
10099            );
10100            assert!(matches!(
10101                swap.forwarding
10102                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
10103                    .unwrap(),
10104                DataRoute::Client(DataRouteState::Bound(_))
10105            ));
10106            assert_eq!(module_loop_error, None);
10107            assert!(swap
10108                .registry
10109                .get_module_by_connection(INCUMBENT)
10110                .unwrap()
10111                .is_some());
10112
10113            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
10114            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
10115                .await
10116                .expect("the incumbent is told to drop the abandoned binding")
10117                .unwrap()
10118                .frame;
10119            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10120            assert_eq!(goodbye.header.channel, abandoned_channel);
10121            assert_eq!(goodbye.header.epoch, abandoned_epoch);
10122            assert!(swap.incumbent_rx.try_recv().is_err());
10123
10124            // 3. The waiting client gets a retryable refusal and no route.
10125            let response = caller_task.await.unwrap();
10126            assert_eq!(response.len(), 1);
10127            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
10128            assert!(caller_rx.try_recv().is_err());
10129
10130            // 4. The reservation pair is given back, and the pending bind
10131            //    settled exactly once: one accepted open (the co-tenant) and one
10132            //    refused open (the caller), nothing counted twice.
10133            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
10134            let counters = handler.counters().snapshot();
10135            assert_eq!(
10136                keyed_total(&counters, "route_open_accepted_by_principal"),
10137                1
10138            );
10139            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
10140            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
10141        }
10142
10143        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
10144        /// id would resolve to the promoted candidate and every new route.open
10145        /// would be refused as reloading, leaving neither process routable.
10146        #[tokio::test]
10147        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
10148            let mut swap = swap_with_incumbent().await;
10149            register_candidate(&swap, None);
10150            let incumbent = cutover(&swap);
10151            swap.forwarding
10152                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
10153                .unwrap()
10154                .expect("the incumbent is still registered");
10155
10156            let client = ConnectionId::new(33);
10157            let (client_ctx, mut client_rx) = route_ctx(client);
10158            let route_handler = swap.handler.clone();
10159            let open_ctx = RouteCtx {
10160                connection_id: client,
10161                egress: client_ctx.egress.clone(),
10162            };
10163            let mut route_task = tokio::spawn(async move {
10164                route_handler
10165                    .handle_control_frame(
10166                        &open_ctx,
10167                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
10168                    )
10169                    .await
10170                    .unwrap()
10171            });
10172            let bind = tokio::select! {
10173                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
10174                response = &mut route_task => {
10175                    let response = response.unwrap();
10176                    panic!(
10177                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
10178                        parse_error(&response[0])["code"]
10179                    );
10180                }
10181            };
10182            swap.handler
10183                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
10184                .await
10185                .unwrap();
10186            assert!(route_task.await.unwrap().is_empty());
10187            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
10188            match swap
10189                .forwarding
10190                .lookup_data_route(client, channel, epoch)
10191                .unwrap()
10192            {
10193                DataRoute::Client(DataRouteState::Bound(route)) => {
10194                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
10195                }
10196                other => panic!("expected a bound route on the candidate, got {other:?}"),
10197            }
10198            assert!(swap.incumbent_rx.try_recv().is_err());
10199        }
10200
10201        /// A candidate declares itself ready with `catalog.update` on its own
10202        /// connection. If the connection-keyed registry lookups searched only the
10203        /// active slot, this would answer `not_registered` and the candidate
10204        /// would never become ready.
10205        #[tokio::test]
10206        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
10207            let swap = swap_with_incumbent().await;
10208            register_candidate(&swap, Some(false));
10209            let update = Frame::build(
10210                FrameType::Request,
10211                control_flags(),
10212                0,
10213                0,
10214                55,
10215                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
10216                    provides: manifest("aft", PROTOCOL_VERSION).provides,
10217                    capabilities: None,
10218                    ready: Some(true),
10219                })
10220                .unwrap(),
10221            )
10222            .unwrap();
10223
10224            let replies = swap
10225                .handler
10226                .handle_control_frame(&swap.candidate_ctx, update)
10227                .await
10228                .unwrap();
10229
10230            assert_eq!(replies.len(), 1);
10231            assert_eq!(
10232                replies[0].header.ty,
10233                FrameType::Response,
10234                "candidate catalog.update was refused: {:?}",
10235                serde_json::from_slice::<Value>(&replies[0].body).ok()
10236            );
10237            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
10238            assert_eq!(
10239                swap.registry
10240                    .get_module("aft")
10241                    .unwrap()
10242                    .unwrap()
10243                    .connection_id,
10244                INCUMBENT
10245            );
10246        }
10247    }
10248
10249    /// The HELLO gate while the supervisor has a swap open: only the nonce it
10250    /// minted for the candidate admits a second process, into the candidate
10251    /// slot, and that check runs ahead of the reserved-module gate.
10252    mod swap_admission {
10253        use super::*;
10254
10255        const INCUMBENT_NONCE: &str = "incumbent-nonce";
10256        const CANDIDATE_NONCE: &str = "candidate-nonce";
10257
10258        fn handler_with_incumbent(
10259            module_id: &str,
10260            reserved: bool,
10261        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
10262            let registry = Arc::new(Registry::default());
10263            let supervisor = SupervisorHandle::new();
10264            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
10265            if reserved {
10266                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
10267            }
10268            let handler =
10269                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
10270            let incumbent = handler
10271                .handle_control(
10272                    ConnectionId::new(1),
10273                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
10274                )
10275                .unwrap();
10276            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
10277            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
10278            (registry, supervisor, handler)
10279        }
10280
10281        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
10282        /// admits every nonce, so while a swap is open the swap gate is the only
10283        /// thing between a key-holder and the candidate slot. A nonce the
10284        /// supervisor did not mint, or none at all, is refused, and neither the
10285        /// incumbent's registration nor the candidate slot moves.
10286        #[test]
10287        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
10288            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
10289
10290            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
10291                let replies = handler
10292                    .handle_control(
10293                        ConnectionId::new(connection),
10294                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
10295                    )
10296                    .unwrap();
10297                assert_eq!(replies[0].header.ty, FrameType::Error);
10298                assert_eq!(
10299                    parse_error(&replies[0])["code"],
10300                    "swap_token_invalid",
10301                    "nonce {nonce:?}"
10302                );
10303            }
10304            assert!(registry.get_candidate("aft").unwrap().is_none());
10305            assert_eq!(
10306                registry.get_module("aft").unwrap().unwrap().connection_id,
10307                ConnectionId::new(1)
10308            );
10309
10310            // Control: the minted token is admitted, into the candidate slot,
10311            // and only once.
10312            let admitted = handler
10313                .handle_control(
10314                    ConnectionId::new(4),
10315                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
10316                )
10317                .unwrap();
10318            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
10319            assert_eq!(
10320                registry
10321                    .get_candidate("aft")
10322                    .unwrap()
10323                    .unwrap()
10324                    .connection_id,
10325                ConnectionId::new(4)
10326            );
10327            assert_eq!(
10328                registry.get_module("aft").unwrap().unwrap().connection_id,
10329                ConnectionId::new(1),
10330                "the candidate must not take the active slot"
10331            );
10332            let replayed = handler
10333                .handle_control(
10334                    ConnectionId::new(5),
10335                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
10336                )
10337                .unwrap();
10338            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
10339
10340            // The case only this gate covers: the incumbent has died mid-swap,
10341            // so its duplicate refusal is gone too, and without the gate a
10342            // key-holder would take the id's ACTIVE slot.
10343            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
10344            let squatter = handler
10345                .handle_control(
10346                    ConnectionId::new(6),
10347                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
10348                )
10349                .unwrap();
10350            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
10351            assert!(
10352                registry.get_module("aft").unwrap().is_none(),
10353                "a squatter took the active slot of an id being swapped"
10354            );
10355        }
10356
10357        /// Design mutation arm (iii). A reserved module's candidate presents a
10358        /// nonce the reserved gate has never seen (that gate holds the
10359        /// incumbent's), so the swap gate must run first or the candidate is
10360        /// refused `reserved_module` and a reserved module can never be swapped.
10361        #[test]
10362        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
10363            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
10364
10365            let replies = handler
10366                .handle_control(
10367                    ConnectionId::new(2),
10368                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
10369                )
10370                .unwrap();
10371
10372            assert_eq!(
10373                replies[0].header.ty,
10374                FrameType::HelloAck,
10375                "reserved candidate refused: {:?}",
10376                serde_json::from_slice::<Value>(&replies[0].body).ok()
10377            );
10378            assert_eq!(
10379                registry
10380                    .get_candidate("vault")
10381                    .unwrap()
10382                    .unwrap()
10383                    .connection_id,
10384                ConnectionId::new(2)
10385            );
10386        }
10387
10388        /// With no swap open the gate is inert: the incumbent's reserved gate
10389        /// and duplicate refusal behave exactly as before.
10390        #[test]
10391        fn without_an_open_swap_the_ordinary_gates_decide() {
10392            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
10393            supervisor.close_swap("vault");
10394
10395            let candidate = handler
10396                .handle_control(
10397                    ConnectionId::new(2),
10398                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
10399                )
10400                .unwrap();
10401            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
10402            let duplicate = handler
10403                .handle_control(
10404                    ConnectionId::new(3),
10405                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
10406                )
10407                .unwrap();
10408            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10409            assert!(registry.get_candidate("vault").unwrap().is_none());
10410        }
10411    }
10412}
10413
10414#[cfg(test)]
10415mod concurrency_default_exposure_tests {
10416    use super::*;
10417
10418    fn hello_body(role_json: &str) -> Vec<u8> {
10419        format!(
10420            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
10421        )
10422        .into_bytes()
10423    }
10424
10425    fn manifest_from(body: &[u8]) -> ModuleManifest {
10426        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
10427        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
10428            .expect("manifest parses")
10429    }
10430
10431    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
10432
10433    #[test]
10434    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
10435        let body = hello_body(&format!(
10436            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
10437        ));
10438        let manifest = manifest_from(&body);
10439        // Precondition: serde really resolved it to the default, so the typed
10440        // manifest alone cannot answer the question this probe exists for.
10441        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
10442        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
10443    }
10444
10445    #[test]
10446    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
10447        let body = hello_body(&format!(
10448            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
10449        ));
10450        let manifest = manifest_from(&body);
10451        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
10452        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
10453    }
10454
10455    #[test]
10456    fn non_management_roles_are_never_reported() {
10457        let body = hello_body(
10458            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
10459        );
10460        let manifest = manifest_from(&body);
10461        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
10462    }
10463}