Skip to main content

subc_daemon/
control.rs

1use std::{
2    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3    fmt,
4    path::{Path, PathBuf},
5    sync::{Arc, Mutex},
6    time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14    ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20    error_codes,
21    manifest::{
22        validate_hello_capability_grammar, validate_hello_self_signal_declarations,
23        CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
24        ProviderRole,
25    },
26    session::{
27        HealthReport, ModuleControlPush, ModuleControlRequest, ModuleControlRequestFromModule,
28        ModuleControlResponse, ModuleControlResponseToModule, MODULE_CONTROL_OP_HEALTH_CHECK,
29        MODULE_TO_SUBC_OP_CATALOG_UPDATE,
30    },
31    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
32    Priority, RouteTarget, PROTOCOL_VERSION,
33};
34use tokio::time::{timeout_at, Instant};
35use tracing::{debug, info, warn};
36
37use crate::{
38    capability_requirements::{
39        log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
40        CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
41        RuntimeModule,
42    },
43    daemon_config::RestartRequiredSection,
44    forwarding::{
45        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
46        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
47        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
48    },
49    observability::{
50        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
51    },
52    provenance::{
53        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
54    },
55    registry::{ChannelState, ConnectionId, Registry, RegistryError},
56    router::{RouteCtx, RouterError},
57    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
58    stderr_tail::{CaptureState, TailEntry},
59    supervise::{
60        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
61        SupervisorHandle, SwapHelloAdmission,
62    },
63    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
64};
65
66/// Lowest envelope version this subc build will negotiate.
67///
68/// Module HELLO negotiation is exact: peers must use the daemon's locked
69/// protocol version. Older and newer peers receive `version_unsupported` and
70/// are not registered.
71pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
72
73const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
74const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
75const CAP_PING_PONG: &str = "ping_pong_v1";
76const CAP_SESSION_ATTACH: &str = "session_attach_v1";
77const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
78
79const SUBC_CONTROL_OPS: &[&str] = &[
80    ops::SERVER_DESCRIBE,
81    ops::CATALOG_LIST,
82    ops::ROUTE_OPEN,
83    ops::ROUTE_POLL,
84    ops::ROUTE_CLOSING,
85    ops::ROUTE_CLOSED,
86    ops::SUPERVISOR_LIST,
87    ops::SUPERVISOR_RESTART,
88    ops::SUPERVISOR_SWAP,
89    ops::SUPERVISOR_RELOAD,
90    ops::SUPERVISOR_RESCAN,
91    ops::SUPERVISOR_RELEASE_RESERVED,
92    ops::SUPERVISOR_SET_ENABLED,
93    ops::SUPERVISOR_HEALTH_PROBE,
94    ops::SUPERVISOR_HEALTH,
95    ops::SUPERVISOR_STDERR_TAIL,
96    ops::SUPERVISOR_TERMINALS,
97    ops::SUPERVISOR_ROUTES,
98    ops::SUPERVISOR_PROVENANCE,
99    ops::SUPERVISOR_SPAWN_SNAPSHOT,
100    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
101];
102
103const MODULE_TO_SUBC_CONTROL_OPS: &[&str] =
104    &[MODULE_TO_SUBC_OP_CATALOG_UPDATE, "supervisor.live_roots"];
105
106const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
107
108/// How long subc waits for a module to ack a relayed route.bind before returning
109/// `module_timeout`. The ack waits on the module's own configure, which for AFT
110/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
111/// and DB-open work — on a cold page cache or a large repo that legitimately
112/// exceeds a couple of seconds. The default is generous because rejecting a VALID
113/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
114/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
115pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
116
117/// How many CONSECUTIVE full-budget relay timeouts against one target module
118/// open that module's bind-relay breaker.
119///
120/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
121/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
122/// so three consecutive full-budget timeouts take ~36s to observe: every client
123/// whose open contributed to opening the breaker had already given up on its
124/// own. That is what makes opening the breaker unable to turn a call that would
125/// have succeeded into a refusal — it can only make an already-failing module
126/// fail faster.
127///
128/// Two would be reachable inside one default deadline. One would convict a
129/// module on a single cold-cache bind, which is exactly the valid-but-slow case
130/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
131pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
132
133/// How long a module's bind-relay breaker stays open before exactly one
134/// `route.open` is let through as a probe.
135///
136/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
137/// re-pays a full-budget stall almost continuously, and the breaker stops being
138/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
139/// deadline: a client that starts retrying after the module recovers has to get
140/// a probe opportunity inside its own deadline, or the breaker converts a
141/// recovered module into a failed call — the failure it exists to prevent,
142/// pointed the other way.
143///
144/// 20s sits between those with room on both sides, and it caps what a wedged
145/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
146/// rather than one per `route.open` per connection. The stall that motivated
147/// this, with its measurements, is written up in
148/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
149/// waited the whole budget out.
150pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
151
152const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
153const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
154
155fn reload_verdict(
156    configured: &Path,
157    spawned_from: Option<&Path>,
158    image: subc_control::RunningImageAgreement,
159) -> PendingReloadVerdict {
160    let path = match spawned_from {
161        Some(spawned_from) if configured == spawned_from => ReloadPathAgreement::Match,
162        Some(spawned_from) => ReloadPathAgreement::Mismatch {
163            configured: configured.to_path_buf(),
164            spawned_from: spawned_from.to_path_buf(),
165        },
166        None => ReloadPathAgreement::Unavailable {
167            reason: if matches!(
168                image,
169                subc_control::RunningImageAgreement::Unavailable {
170                    reason: subc_control::RunningImageUnavailableReason::NotRunning
171                }
172            ) {
173                ReloadPathUnavailableReason::NotRunning
174            } else {
175                ReloadPathUnavailableReason::SpawnedPathUnavailable
176            },
177        },
178    };
179    PendingReloadVerdict { path, image }
180}
181
182#[derive(Clone)]
183struct DaemonProvenanceFacts {
184    build: DaemonBuildProvenance,
185    pid: Option<u32>,
186    started_at_ms: Option<u64>,
187    start_clock: Option<crate::clock::StartClock>,
188    executable_path: Option<PathBuf>,
189    executable_identity: Option<SpawnedFileIdentity>,
190    process_start_time: Option<u64>,
191    probe: ExecutableIdentityProbe,
192}
193
194impl Default for DaemonProvenanceFacts {
195    fn default() -> Self {
196        Self {
197            build: DaemonBuildProvenance {
198                build_git_sha: None,
199                build_lock_digest: None,
200            },
201            pid: None,
202            started_at_ms: None,
203            start_clock: None,
204            executable_path: None,
205            executable_identity: None,
206            process_start_time: None,
207            probe: ExecutableIdentityProbe::default(),
208        }
209    }
210}
211
212#[derive(Debug, Clone)]
213struct SupervisorRescanContext {
214    supervisor: Supervisor,
215    config_path: PathBuf,
216    configured_port: Option<u16>,
217    storage_config: Option<crate::daemon_config::StorageConfig>,
218    admission_facts_carrier_module_id: Option<String>,
219    admission_facts_targets: Option<Vec<String>>,
220}
221
222/// Real channel-0 control handler for subc itself.
223#[derive(Clone)]
224pub struct ControlHandler {
225    registry: Arc<Registry>,
226    forwarding: Arc<ForwardingTable>,
227    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
228    supervisor: SupervisorHandle,
229    subc_capabilities: Arc<[String]>,
230    /// Daemon-wide route.bind relay budget. Used as the fallback when the
231    /// target module has no per-module override in
232    /// `route_bind_relay_timeouts`.
233    route_bind_relay_timeout: Duration,
234    /// Per-module route.bind relay budget overrides, keyed by module id. When
235    /// `handle_route_open` resolves the deadline for a target module, a
236    /// per-module entry wins over the daemon-wide value above.
237    route_bind_relay_timeouts: BTreeMap<String, Duration>,
238    /// Per-target-module bind-relay breaker state. Shared with the forwarding
239    /// table, which is where a new module connection resets it.
240    route_bind_breakers: RouteBindBreakers,
241    /// Live relay admissions keyed by target module. Shared through the
242    /// forwarding table so cloned or separately built handlers enforce one cap.
243    route_bind_concurrency: RouteBindConcurrency,
244    /// Consecutive relay timeouts that open a module's breaker.
245    route_bind_breaker_threshold: u32,
246    /// How long a breaker stays open before one probe is admitted.
247    route_bind_breaker_cooldown: Duration,
248    health_probe_timeout: Duration,
249    /// Central storage policy. When set, each registering module receives its
250    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
251    storage_config: Option<crate::daemon_config::StorageConfig>,
252    /// The machine id established at boot, served on every HELLO_ACK and on
253    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
254    /// changes the file, never this value. `None` serves no id.
255    machine_id: Option<crate::machine_id::MachineId>,
256    admission_facts_carrier_module_id: Option<String>,
257    admission_facts_targets: Option<Vec<String>>,
258    rescan: Option<SupervisorRescanContext>,
259    connected_clients: ConnectedClients,
260    counters: DaemonCounters,
261    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
262    daemon_provenance: DaemonProvenanceFacts,
263    #[cfg(test)]
264    control_dispatch_delay: Option<Duration>,
265    #[cfg(test)]
266    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
267}
268
269impl fmt::Debug for ControlHandler {
270    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
271        f.debug_struct("ControlHandler")
272            .field("registry", &self.registry)
273            .field("forwarding", &self.forwarding)
274            .field("process_liveness", &self.process_liveness.is_some())
275            .field("supervisor", &self.supervisor)
276            .field("subc_capabilities", &self.subc_capabilities)
277            .finish()
278    }
279}
280
281struct RouteOpenRequest {
282    target: RouteTarget,
283    identity: BindIdentity,
284    consumer_identity: Option<ConsumerIdentity>,
285    consumer_capabilities: Option<Vec<String>>,
286    admission_facts: Option<serde_json::Value>,
287}
288
289struct RouteBindReservationGuard {
290    forwarding: Arc<ForwardingTable>,
291    endpoint: ModuleEndpointId,
292    relay_corr: u64,
293    armed: bool,
294}
295
296struct ModuleControlRpcGuard {
297    forwarding: Arc<ForwardingTable>,
298    endpoint: ModuleEndpointId,
299    corr: u64,
300    armed: bool,
301}
302
303impl ModuleControlRpcGuard {
304    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
305        Self {
306            forwarding,
307            endpoint,
308            corr,
309            armed: true,
310        }
311    }
312
313    fn disarm(&mut self) {
314        self.armed = false;
315    }
316}
317
318impl Drop for ModuleControlRpcGuard {
319    fn drop(&mut self) {
320        if self.armed {
321            let _ = self
322                .forwarding
323                .cancel_module_control_rpc(self.endpoint, self.corr);
324        }
325    }
326}
327
328impl RouteBindReservationGuard {
329    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
330        Self {
331            forwarding,
332            endpoint,
333            relay_corr,
334            armed: true,
335        }
336    }
337
338    fn release_and_disarm(&mut self) {
339        if !self.armed {
340            return;
341        }
342        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
343            self.endpoint,
344            self.relay_corr,
345            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
346        ) {
347            send_goodbye_target_best_effort(
348                &self.forwarding.counters(),
349                &target,
350                "canceled route.bind",
351            );
352        }
353        self.armed = false;
354    }
355
356    fn disarm(&mut self) {
357        self.armed = false;
358    }
359}
360
361impl Drop for RouteBindReservationGuard {
362    fn drop(&mut self) {
363        self.release_and_disarm();
364    }
365}
366
367/// Per-target-module circuit breaker around the `route.bind` relay.
368///
369/// The connection reader is serial per connection, so a module whose `on_bind`
370/// sits on the ack blocks every LATER frame on the connections that call it,
371/// including calls to unrelated modules. This does not make any module's bind
372/// fast; it stops the daemon paying the full budget again and again for a
373/// condition it has already observed.
374///
375/// State is keyed by TARGET MODULE and shared by every connection: a wedged
376/// module wedges everyone, so what one connection learned should protect the
377/// rest.
378///
379/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
380/// relay to that module has actually timed out, and is removed again when a
381/// relay is accepted or the module reconnects, so it cannot grow with traffic
382/// or with modules that behave.
383///
384/// # Why a `std` mutex here is not the head-of-line defect again
385///
386/// Acquisition never awaits. The critical section is a hash lookup plus a few
387/// integer updates, with no I/O and no `.await` inside it, so a reader task
388/// cannot be descheduled behind it the way it can behind
389/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
390/// primitive, held for the same kind of work, as the refusal counter this very
391/// path already increments.
392///
393/// It is also NOT on the data-plane splice path: only `route.open` and module
394/// registration touch it, so bound-route frames gain no state check and no
395/// contention.
396#[derive(Debug, Clone, Default)]
397pub(crate) struct RouteBindBreakers {
398    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
399}
400
401#[derive(Debug, Clone, Default)]
402pub(crate) struct RouteBindConcurrency {
403    modules: Arc<Mutex<HashMap<String, usize>>>,
404}
405
406struct RouteBindConcurrencyGuard {
407    concurrency: RouteBindConcurrency,
408    module_id: String,
409}
410
411impl RouteBindConcurrency {
412    /// Admit without waiting. Waiting here would move the bind stall from the
413    /// module reply to a semaphore and restore reader head-of-line blocking.
414    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
415        let mut modules = self
416            .modules
417            .lock()
418            .expect("route.bind concurrency mutex poisoned");
419        let in_flight = modules.entry(module_id.to_string()).or_default();
420        if *in_flight >= limit {
421            return Err(*in_flight);
422        }
423        *in_flight += 1;
424        Ok(RouteBindConcurrencyGuard {
425            concurrency: self.clone(),
426            module_id: module_id.to_string(),
427        })
428    }
429}
430
431impl Drop for RouteBindConcurrencyGuard {
432    fn drop(&mut self) {
433        let mut modules = self
434            .concurrency
435            .modules
436            .lock()
437            .expect("route.bind concurrency mutex poisoned");
438        let remove = {
439            let in_flight = modules
440                .get_mut(&self.module_id)
441                .expect("admitted route.bind has a concurrency entry");
442            *in_flight -= 1;
443            *in_flight == 0
444        };
445        if remove {
446            modules.remove(&self.module_id);
447        }
448    }
449}
450
451#[derive(Debug, Default)]
452struct ModuleBreakerState {
453    /// Relay timeouts observed with no accepted relay in between.
454    consecutive_timeouts: u32,
455    /// `Some` while the breaker is open: the instant the cooldown expires and
456    /// the next arrival may probe. `None` means closed.
457    cooldown_until: Option<Instant>,
458    /// A half-open probe has been admitted and has not settled yet. This is
459    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
460    /// that read the cooldown, so concurrent opens arriving at the moment the
461    /// cooldown expires cannot all decide that they are the probe.
462    probe_in_flight: bool,
463}
464
465/// What the breaker decided for one `route.open`, before any relay work.
466enum RouteBindAdmission<'a> {
467    Admitted {
468        guard: RouteBindBreakerGuard<'a>,
469        /// This open is the single half-open probe, so the transition is worth
470        /// one log line.
471        probe: bool,
472    },
473    Refused {
474        consecutive_timeouts: u32,
475        /// What is left of the cooldown. Zero when the refusal is because the
476        /// one probe is already in flight rather than because the cooldown has
477        /// not elapsed.
478        retry_in: Duration,
479        probe_in_flight: bool,
480    },
481}
482
483/// An outstanding admission, which must be told how its relay settled.
484///
485/// `Drop` settles it as inconclusive, so an early return between admission and
486/// the relay -- or the whole handler being cancelled when the client
487/// disconnects -- releases a half-open probe slot instead of leaving the
488/// breaker wedged half-open with no further probes.
489struct RouteBindBreakerGuard<'a> {
490    breakers: RouteBindBreakers,
491    module_id: &'a str,
492    settled: bool,
493}
494
495impl RouteBindBreakerGuard<'_> {
496    /// The module answered within the budget and took the bind. THE ONLY
497    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
498    /// breaker, which is a transition worth logging.
499    fn record_accepted(&mut self) -> bool {
500        self.settled = true;
501        self.breakers.record_accepted(self.module_id)
502    }
503
504    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
505    /// COUNTS TOWARD OPENING.
506    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
507        self.settled = true;
508        self.breakers
509            .record_timeout(self.module_id, threshold, cooldown)
510    }
511
512    /// Everything else: the module REJECTED the bind, its connection went away
513    /// mid-relay, or the waiter was cancelled.
514    ///
515    /// None of these is evidence that a module is slow, and each already has
516    /// its own refusal with its own code. A module that rejects a bind in
517    /// microseconds is healthy and must never be convicted for it; a module
518    /// that died has said nothing about the module that replaces it. So these
519    /// neither increment nor reset the count -- they only release a probe slot.
520    fn record_inconclusive(&mut self) {
521        self.settled = true;
522        self.breakers.record_inconclusive(self.module_id);
523    }
524}
525
526impl Drop for RouteBindBreakerGuard<'_> {
527    fn drop(&mut self) {
528        if !self.settled {
529            self.breakers.record_inconclusive(self.module_id);
530        }
531    }
532}
533
534/// The breaker moved to open, reported so the caller can log it outside the
535/// lock. Opening is rare and load-bearing; the refusals that follow are
536/// frequent and are counted rather than logged.
537struct BreakerOpened {
538    consecutive_timeouts: u32,
539    /// True when a failed probe re-opened an already-open breaker, which reads
540    /// very differently in a log from a first opening.
541    reopened_after_probe: bool,
542}
543
544impl RouteBindBreakers {
545    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
546        self.modules
547            .lock()
548            .expect("route.bind breaker mutex poisoned")
549    }
550
551    /// Decide whether this `route.open` may attempt its relay. Takes the map
552    /// lock and nothing else, and never awaits.
553    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
554        let admitted = |probe| RouteBindAdmission::Admitted {
555            guard: RouteBindBreakerGuard {
556                breakers: self.clone(),
557                module_id,
558                settled: false,
559            },
560            probe,
561        };
562
563        let mut modules = self.lock();
564        let Some(state) = modules.get_mut(module_id) else {
565            return admitted(false);
566        };
567        let Some(cooldown_until) = state.cooldown_until else {
568            return admitted(false);
569        };
570        if state.probe_in_flight {
571            return RouteBindAdmission::Refused {
572                consecutive_timeouts: state.consecutive_timeouts,
573                retry_in: Duration::ZERO,
574                probe_in_flight: true,
575            };
576        }
577        let now = Instant::now();
578        if now < cooldown_until {
579            return RouteBindAdmission::Refused {
580                consecutive_timeouts: state.consecutive_timeouts,
581                retry_in: cooldown_until - now,
582                probe_in_flight: false,
583            };
584        }
585        state.probe_in_flight = true;
586        admitted(true)
587    }
588
589    fn record_accepted(&self, module_id: &str) -> bool {
590        self.lock()
591            .remove(module_id)
592            .is_some_and(|state| state.cooldown_until.is_some())
593    }
594
595    fn record_timeout(
596        &self,
597        module_id: &str,
598        threshold: u32,
599        cooldown: Duration,
600    ) -> Option<BreakerOpened> {
601        let mut modules = self.lock();
602        let state = modules.entry(module_id.to_string()).or_default();
603        let was_open = state.cooldown_until.is_some();
604        let was_probe = state.probe_in_flight;
605        state.probe_in_flight = false;
606        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
607        if state.consecutive_timeouts < threshold {
608            return None;
609        }
610        state.cooldown_until = Some(Instant::now() + cooldown);
611        Some(BreakerOpened {
612            consecutive_timeouts: state.consecutive_timeouts,
613            reopened_after_probe: was_open && was_probe,
614        })
615    }
616
617    fn record_inconclusive(&self, module_id: &str) {
618        if let Some(state) = self.lock().get_mut(module_id) {
619            state.probe_in_flight = false;
620        }
621    }
622
623    /// Discard what was learned about a module, because the process it was
624    /// learned about is gone. Returns the discarded count when it was non-zero.
625    ///
626    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
627    /// `module_id` is a configuration identity that outlives any particular
628    /// child; what the breaker observed was the process behind the module
629    /// connection of the moment. When a new connection registers under that id
630    /// the verdict's subject no longer exists, so the verdict is stale by
631    /// construction rather than merely likely to be wrong. Keeping it would
632    /// apply a dead process's record to a live one, which is the same defect
633    /// class this breaker exists to stop the daemon committing.
634    ///
635    /// A half-open probe in flight is discarded with the rest: it was a
636    /// question about the old process.
637    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
638        self.lock()
639            .remove(module_id)
640            .map(|state| state.consecutive_timeouts)
641            .filter(|discarded| *discarded > 0)
642    }
643
644    /// Open breakers, for the `server.describe` counters object. `None` when
645    /// none is open, so the key stays absent rather than present-and-empty.
646    ///
647    /// This is the operator's answer to "is this module refusing instantly or
648    /// is it fine?", which look identical from a client that retries and then
649    /// succeeds.
650    fn open_snapshot(&self) -> Option<serde_json::Value> {
651        let now = Instant::now();
652        let modules = self.lock();
653        let open = modules
654            .iter()
655            .filter_map(|(module_id, state)| {
656                let cooldown_until = state.cooldown_until?;
657                Some((
658                    module_id.clone(),
659                    serde_json::json!({
660                        "consecutive_timeouts": state.consecutive_timeouts,
661                        "cooldown_remaining_ms":
662                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
663                        "probe_in_flight": state.probe_in_flight,
664                    }),
665                ))
666            })
667            .collect::<serde_json::Map<String, serde_json::Value>>();
668        (!open.is_empty()).then_some(serde_json::Value::Object(open))
669    }
670}
671
672impl ControlHandler {
673    pub fn new(registry: Arc<Registry>) -> Self {
674        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
675    }
676
677    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
678        let counters = forwarding.counters();
679        // Taken from the forwarding table rather than created here, so that the
680        // breaker a `route.open` consults is the same one a module's
681        // registration resets, however many handlers are built over one table.
682        let route_bind_breakers = forwarding.route_bind_breakers();
683        let route_bind_concurrency = forwarding.route_bind_concurrency();
684        Self {
685            registry,
686            forwarding,
687            process_liveness: None,
688            supervisor: SupervisorHandle::new(),
689            subc_capabilities: Arc::from([
690                CAP_MANIFEST_REGISTRATION.to_string(),
691                CAP_CHANNEL_LIFECYCLE.to_string(),
692                CAP_PING_PONG.to_string(),
693                CAP_SESSION_ATTACH.to_string(),
694                CAP_ADMISSION_FACTS_RELAY.to_string(),
695            ]),
696            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
697            route_bind_relay_timeouts: BTreeMap::new(),
698            route_bind_breakers,
699            route_bind_concurrency,
700            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
701            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
702            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
703            storage_config: None,
704            machine_id: None,
705            admission_facts_carrier_module_id: None,
706            admission_facts_targets: None,
707            rescan: None,
708            connected_clients: ConnectedClients::new(),
709            counters,
710            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
711            daemon_provenance: DaemonProvenanceFacts::default(),
712            #[cfg(test)]
713            control_dispatch_delay: None,
714            #[cfg(test)]
715            provenance_probe_override: None,
716        }
717    }
718
719    /// Set the central storage policy: registering modules then receive their
720    /// resolved storage descriptor in HELLO_ACK.
721    pub fn with_storage_config(
722        mut self,
723        storage_config: Option<crate::daemon_config::StorageConfig>,
724    ) -> Self {
725        self.storage_config = storage_config;
726        self
727    }
728
729    /// Set the machine id served to every registering module (HELLO_ACK) and on
730    /// `server.describe`.
731    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
732        self.machine_id = machine_id;
733        self
734    }
735
736    /// Configure the exact reserved module and target ids permitted to relay
737    /// opaque admission facts. Config-file loading validates this authority;
738    /// this builder keeps the same policy available to embedded test daemons.
739    pub fn with_admission_facts_config(
740        mut self,
741        carrier_module_id: Option<String>,
742        targets: Option<Vec<String>>,
743    ) -> Self {
744        self.admission_facts_carrier_module_id = carrier_module_id;
745        self.admission_facts_targets = targets;
746        self
747    }
748
749    /// Override the route.bind relay timeout. Used by tests that assert the
750    /// timeout path so they don't block on the production-safe default.
751    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
752        self.route_bind_relay_timeout = timeout;
753        self
754    }
755
756    /// Install per-module route.bind relay budget overrides. A module id
757    /// listed here wins over the daemon-wide default set via
758    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
759    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
760    /// the same `Duration` the bind path will use.
761    pub fn with_route_bind_relay_timeouts(
762        mut self,
763        timeouts: impl IntoIterator<Item = (String, Duration)>,
764    ) -> Self {
765        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
766        self
767    }
768
769    /// Resolve the route.bind relay budget for a specific target module id.
770    /// Per-module overrides win; the daemon-wide value (set via
771    /// `with_route_bind_relay_timeout` or the built-in default) is the
772    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
773    /// the same resolution `handle_route_open` will use.
774    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
775        self.route_bind_relay_timeouts
776            .get(module_id)
777            .copied()
778            .unwrap_or(self.route_bind_relay_timeout)
779    }
780
781    /// Override the per-module bind-relay breaker policy.
782    ///
783    /// Used by tests, which cannot spend three production budgets opening a
784    /// breaker or twenty seconds waiting for its cooldown. The production
785    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
786    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
787    /// reasoning for the numbers.
788    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
789        self.route_bind_breaker_threshold = threshold.max(1);
790        self.route_bind_breaker_cooldown = cooldown;
791        self
792    }
793
794    #[cfg(test)]
795    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
796        self.health_probe_timeout = timeout;
797        self
798    }
799
800    #[cfg(test)]
801    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
802        self.control_dispatch_delay = Some(delay);
803        self
804    }
805
806    pub fn with_process_liveness(
807        mut self,
808        process_liveness: Arc<dyn ModuleProcessLiveness>,
809    ) -> Self {
810        self.process_liveness = Some(process_liveness);
811        self
812    }
813
814    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
815        self.supervisor = supervisor;
816        self
817    }
818
819    pub fn with_daemon_provenance(
820        mut self,
821        pid: u32,
822        started_at_ms: u64,
823        executable_path: Option<PathBuf>,
824        build_git_sha: Option<String>,
825        build_lock_digest: Option<String>,
826    ) -> Self {
827        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
828        let process_start_time = process_start_time(pid);
829        self.daemon_provenance = DaemonProvenanceFacts {
830            build: DaemonBuildProvenance {
831                build_git_sha,
832                build_lock_digest,
833            },
834            pid: Some(pid),
835            started_at_ms: Some(started_at_ms),
836            start_clock: None,
837            executable_path,
838            executable_identity,
839            process_start_time,
840            probe: ExecutableIdentityProbe::default(),
841        };
842        self
843    }
844
845    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
846        self.daemon_provenance.start_clock = Some(clock);
847        self
848    }
849
850    #[cfg(test)]
851    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
852        self.provenance_probe_override = Some(result);
853        self
854    }
855
856    /// Install the configured module set and its reserved capability bindings.
857    /// Bindings are configuration-scoped and may point at a provider that has not
858    /// been installed yet, so this does not require the bound module to exist.
859    pub fn with_capability_config(
860        self,
861        modules: impl IntoIterator<Item = (String, bool)>,
862        reserved_capabilities: BTreeMap<String, String>,
863    ) -> Self {
864        self.capability_evaluator
865            .configure(modules, reserved_capabilities);
866        self
867    }
868
869    pub fn with_supervisor_rescan(
870        mut self,
871        supervisor: Supervisor,
872        config_path: impl Into<PathBuf>,
873        configured_port: Option<u16>,
874    ) -> Self {
875        self.rescan = Some(SupervisorRescanContext {
876            supervisor,
877            config_path: config_path.into(),
878            configured_port,
879            storage_config: self.storage_config.clone(),
880            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
881            admission_facts_targets: self.admission_facts_targets.clone(),
882        });
883        self
884    }
885
886    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
887        self.connected_clients = connected_clients;
888        self
889    }
890
891    pub fn forwarding(&self) -> Arc<ForwardingTable> {
892        Arc::clone(&self.forwarding)
893    }
894
895    pub(crate) fn counters(&self) -> DaemonCounters {
896        self.counters.clone()
897    }
898
899    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
900    /// requirement event without depending on an operator polling a status command.
901    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
902        tokio::spawn(async move {
903            loop {
904                self.capability_evaluator
905                    .wait_for_change_or_deadline()
906                    .await;
907                self.refresh_capability_requirements();
908            }
909        });
910    }
911
912    fn runtime_capability_snapshot(
913        &self,
914    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
915        let runtime = self
916            .supervisor
917            .list()
918            .into_iter()
919            .map(|module| {
920                let status = module.status().map_err(|err| {
921                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
922                })?;
923                Ok(RuntimeModule {
924                    module_id: status.module_id,
925                    state: status.state,
926                    enabled: status.enabled,
927                })
928            })
929            .collect::<Result<Vec<_>, RouterError>>()?;
930        let (_, registrations) = self.registry.list_modules().map_err(|err| {
931            RouterError::backend(
932                0,
933                0,
934                format!("failed to list capability registrations: {err}"),
935            )
936        })?;
937        let registrations = registrations
938            .into_iter()
939            .map(|registration| RegisteredModule {
940                module_id: registration.manifest.module_id,
941                module_version: registration.manifest.module_version,
942                capabilities: registration.manifest.capabilities,
943            })
944            .collect();
945        Ok((runtime, registrations))
946    }
947
948    /// The capability side effects of a module becoming the active registration
949    /// for its id: cache its manifest (warning if its claims drifted), run the
950    /// deny census when its declarations call for one, and recompute the
951    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
952    /// candidate's does not, and the supervisor does it at promotion instead,
953    /// through [`crate::supervise::SwapPromotionObserver`].
954    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
955        let cached_registration = RegisteredModule {
956            module_id: registration.manifest.module_id.clone(),
957            module_version: registration.manifest.module_version.clone(),
958            capabilities: registration.manifest.capabilities.clone(),
959        };
960        if self.capability_evaluator.record_hello(&cached_registration) {
961            warn!(
962                module_id = %cached_registration.module_id,
963                "capability claims drifted from the cached manifest"
964            );
965        }
966        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
967            self.enforce_capability_denies();
968        }
969        self.refresh_capability_requirements();
970    }
971
972    /// Point the shared supervisor handle at this handler for swap promotions.
973    /// Called wherever a handler is put behind the `Arc` the router serves, so
974    /// it can be held weakly.
975    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
976        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
977            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
978        self.supervisor.set_swap_promotion_observer(observer);
979    }
980
981    pub fn refresh_capability_requirements(&self) {
982        match self.runtime_capability_snapshot() {
983            Ok((runtime, registrations)) => {
984                log_requirement_events(
985                    self.capability_evaluator
986                        .evaluate_now(&runtime, &registrations),
987                );
988            }
989            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
990        }
991    }
992
993    /// Reconcile only live, attested route bindings after a capability deny edge
994    /// or target claim was added. This is deliberately a control-plane census:
995    /// the opaque forwarding hot path must not grow a per-frame capability check.
996    fn enforce_capability_denies(&self) {
997        let (_, registrations) = match self.registry.list_modules() {
998            Ok(snapshot) => snapshot,
999            Err(err) => {
1000                warn!(error = %err, "failed to read registrations for capability deny census");
1001                return;
1002            }
1003        };
1004        let manifests = registrations
1005            .into_iter()
1006            .map(|registration| {
1007                (
1008                    registration.manifest.module_id.clone(),
1009                    registration.manifest,
1010                )
1011            })
1012            .collect::<BTreeMap<_, _>>();
1013        let census = match self.forwarding.route_census(None) {
1014            Ok(census) => census,
1015            Err(err) => {
1016                warn!(error = %err, "failed to read route census for capability deny enforcement");
1017                return;
1018            }
1019        };
1020
1021        for (target_module_id, routes) in census {
1022            let Some(target_manifest) = manifests.get(&target_module_id) else {
1023                continue;
1024            };
1025            let mut closed_routes = Vec::new();
1026            let mut module_goodbyes = Vec::new();
1027            for route in routes {
1028                let Principal::Reserved {
1029                    module_id: opening_module_id,
1030                } = &route.principal
1031                else {
1032                    continue;
1033                };
1034                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1035                    continue;
1036                };
1037                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1038                    continue;
1039                };
1040
1041                match self.forwarding.release_client_route(
1042                    route.goodbye_target.connection_id,
1043                    route.goodbye_target.channel,
1044                    route.goodbye_target.epoch,
1045                ) {
1046                    Ok(RouteRelease::Removed(module_goodbye)) => {
1047                        warn!(
1048                            opening_module_id,
1049                            target_module_id,
1050                            capability,
1051                            "force-closing route because an attested capability deny edge now matches"
1052                        );
1053                        closed_routes.push(route);
1054                        module_goodbyes.push(module_goodbye);
1055                    }
1056                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1057                    Err(err) => warn!(
1058                        opening_module_id,
1059                        target_module_id,
1060                        capability,
1061                        error = %err,
1062                        "failed to force-close capability-denied route"
1063                    ),
1064                }
1065            }
1066
1067            if closed_routes.is_empty() {
1068                continue;
1069            }
1070            send_route_control_pushes(
1071                &self.forwarding,
1072                closed_routes,
1073                ClientControlPush::RouteClosed {
1074                    module_id: target_module_id,
1075                    reason: RouteCloseReason::CapabilityDenied,
1076                    drained: false,
1077                    abandoned: 0,
1078                    excluded_subscriptions: 0,
1079                    terminal: Some(false),
1080                },
1081            );
1082            self.emit_route_goodbyes(module_goodbyes);
1083        }
1084    }
1085
1086    /// Why a registered module is not accepting new route binds, or `None` when
1087    /// it is. This is the module's effective readiness: its declared readiness
1088    /// first, then every `need: required` capability it declares evaluating to
1089    /// `provided`. `route.open` and `catalog.list` both read it here so the
1090    /// catalog never reports a module routable that `route.open` would refuse.
1091    fn not_ready_reason(
1092        &self,
1093        registration: &crate::registry::ModuleRegistration,
1094    ) -> Option<NotReadyReason> {
1095        if !registration.ready {
1096            return Some(NotReadyReason {
1097                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1098                capability: None,
1099            });
1100        }
1101        self.first_unprovided_required_capability(registration)
1102            .map(|capability| NotReadyReason {
1103                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1104                capability: Some(capability),
1105            })
1106    }
1107
1108    /// The lexicographically first capability this registration declares
1109    /// `need: required` whose evaluator verdict is not `provided`.
1110    ///
1111    /// The verdicts are the capability evaluator's own; nothing here decides
1112    /// what "provided" means. The evaluator counts a capability provided as
1113    /// soon as a module claiming it has REGISTERED, not once that module is
1114    /// ready. That distinction is what keeps two modules that require each
1115    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1116    /// is ready", each would wait for the other to become ready first and
1117    /// neither ever would. Do not tighten it to readiness.
1118    ///
1119    /// A required capability with no verdict at all means this registration's
1120    /// HELLO or catalog.update landed after the last recompute; recompute once
1121    /// rather than let a missing verdict read as either answer. If it is still
1122    /// missing (the recompute itself failed) the capability counts as
1123    /// unprovided: the refusal is retryable, and routing a module whose
1124    /// required provider is unknown is the outcome this check exists to stop.
1125    fn first_unprovided_required_capability(
1126        &self,
1127        registration: &crate::registry::ModuleRegistration,
1128    ) -> Option<String> {
1129        let required = registration
1130            .manifest
1131            .capabilities
1132            .iter()
1133            .flat_map(|declarations| declarations.requires.iter())
1134            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1135            .map(|requirement| requirement.capability.as_str())
1136            .collect::<BTreeSet<_>>();
1137        if required.is_empty() {
1138            return None;
1139        }
1140        let module_id = registration.manifest.module_id.as_str();
1141        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1142        if required
1143            .iter()
1144            .any(|capability| verdict(capability).is_none())
1145        {
1146            self.refresh_capability_requirements();
1147        }
1148        required
1149            .into_iter()
1150            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1151            .map(str::to_string)
1152    }
1153
1154    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1155        self.capability_evaluator
1156            .statuses()
1157            .into_iter()
1158            .map(capability_requirement_status)
1159            .collect()
1160    }
1161
1162    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1163    /// registration-release watch. The signal is what the supervisor waits on
1164    /// before spawning a replacement, so it must only fire once forwarding
1165    /// teardown is also done (see [`Self::cleanup_connection`] /
1166    /// [`Self::handle_goodbye`]). Used directly only where there is no forwarding
1167    /// state to tear down (a HELLO that failed before module registration).
1168    fn deregister_connection(
1169        &self,
1170        connection_id: ConnectionId,
1171    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1172        self.registry.deregister_connection(connection_id)
1173    }
1174
1175    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1176        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1177            return None;
1178        }
1179        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1180            parse_client_control_request(&frame.body)
1181        else {
1182            return None;
1183        };
1184        Some(target_module_id(&target).to_string())
1185    }
1186
1187    pub(crate) fn route_open_capacity_refusal(
1188        &self,
1189        ctx: &RouteCtx,
1190        frame: &Frame,
1191        target_module_id: &str,
1192        limit: usize,
1193    ) -> Result<Frame, RouterError> {
1194        self.route_open_admission_refusal_frame(
1195            ctx,
1196            frame,
1197            target_module_id,
1198            format!(
1199                "connection already has {limit} route.open binds in flight; retry after one settles"
1200            ),
1201        )
1202    }
1203
1204    /// Admission pressure clears as existing binds settle, so its refusal must
1205    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
1206    /// `module_reloading`, `module_warming`, `target_unavailable`, or
1207    /// `module_timeout`. `target_unavailable` is honest for an attempt that
1208    /// cannot currently reach its target; `module_timeout` would falsely claim
1209    /// that a wait expired. A new, cleaner code would be terminal to deployed
1210    /// clients, so it requires a client-tolerance rollout before daemon emission.
1211    fn route_open_admission_refusal_frame(
1212        &self,
1213        ctx: &RouteCtx,
1214        frame: &Frame,
1215        target_module_id: &str,
1216        message: impl Into<String>,
1217    ) -> Result<Frame, RouterError> {
1218        self.route_open_refusal_frame(
1219            ctx,
1220            frame,
1221            target_module_id,
1222            "open_admission_full",
1223            error_codes::TARGET_UNAVAILABLE,
1224            message,
1225        )
1226    }
1227
1228    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
1229    ///
1230    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
1231    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
1232    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
1233    #[cfg(test)]
1234    pub fn handle_control(
1235        &self,
1236        connection_id: ConnectionId,
1237        frame: Frame,
1238    ) -> Result<Vec<Frame>, RouterError> {
1239        match frame.header.ty {
1240            FrameType::Ping => Ok(vec![pong(&frame)?]),
1241            FrameType::Hello => self.handle_hello(connection_id, None, frame),
1242            FrameType::Goodbye => self.handle_goodbye(connection_id),
1243            ty => Ok(vec![control_error_frame(
1244                &frame,
1245                "unsupported_control_frame",
1246                format!("unsupported channel-0 frame {ty:?}"),
1247            )?]),
1248        }
1249    }
1250
1251    pub async fn handle_control_frame(
1252        &self,
1253        ctx: &RouteCtx,
1254        frame: Frame,
1255    ) -> Result<Vec<Frame>, RouterError> {
1256        self.handle_control_frame_timed(ctx, frame, None).await
1257    }
1258
1259    pub(crate) async fn handle_control_frame_timed(
1260        &self,
1261        ctx: &RouteCtx,
1262        frame: Frame,
1263        dispatch_started_at: Option<StdInstant>,
1264    ) -> Result<Vec<Frame>, RouterError> {
1265        match frame.header.ty {
1266            FrameType::Ping => Ok(vec![pong(&frame)?]),
1267            FrameType::Hello => {
1268                self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1269            }
1270            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1271            FrameType::Cancel => {
1272                if self
1273                    .supervisor
1274                    .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1275                {
1276                    Ok(Vec::new())
1277                } else {
1278                    Ok(vec![control_error_frame(
1279                        &frame,
1280                        "unknown_subscription",
1281                        "no supervisor spawn subscription has this correlation id",
1282                    )?])
1283                }
1284            }
1285            FrameType::Request => {
1286                if self
1287                    .forwarding
1288                    .module_endpoint_for_connection(ctx.connection_id)
1289                    .map_err(RouterError::Forwarding)?
1290                    .is_some()
1291                {
1292                    if !is_known_module_request_op(&frame.body) {
1293                        return Ok(vec![control_error_frame(
1294                            &frame,
1295                            "unsupported_control_frame",
1296                            "module-originated channel-0 REQUEST is not supported",
1297                        )?]);
1298                    }
1299                    let request = match parse_module_control_request_from_module(&frame.body) {
1300                        Ok(request) => request,
1301                        Err((err, ControlRequestBodyError::UnknownOp)) => {
1302                            return Ok(vec![control_error_frame(
1303                                &frame,
1304                                "unsupported_control_frame",
1305                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
1306                            )?])
1307                        }
1308                        Err((err, ControlRequestBodyError::InvalidBody)) => {
1309                            return Ok(vec![control_error_frame(
1310                                &frame,
1311                                "invalid_control_body",
1312                                format!("malformed module control body: {err}"),
1313                            )?])
1314                        }
1315                    };
1316                    let op = module_control_request_op(&request);
1317                    let corr = frame.header.corr;
1318                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
1319                    let result =
1320                        self.handle_module_control_request(ctx.connection_id, frame, request);
1321                    log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1322                    return result;
1323                }
1324
1325                if is_known_module_request_op(&frame.body) {
1326                    return Ok(vec![control_error_frame(
1327                        &frame,
1328                        "not_registered",
1329                        "catalog.update requires an active module registration owned by this connection",
1330                    )?]);
1331                }
1332
1333                let request = match parse_client_control_request(&frame.body) {
1334                    Ok(request) => request,
1335                    Err((err, ControlRequestBodyError::UnknownOp)) => {
1336                        return Ok(vec![control_error_frame(
1337                            &frame,
1338                            "unknown_control_op",
1339                            format!("unknown client control op: {err}"),
1340                        )?])
1341                    }
1342                    Err((err, ControlRequestBodyError::InvalidBody)) => {
1343                        return Ok(vec![control_error_frame(
1344                            &frame,
1345                            "invalid_control_body",
1346                            format!("malformed client control body: {err}"),
1347                        )?])
1348                    }
1349                };
1350                let op = client_control_request_op(&request);
1351                let corr = frame.header.corr;
1352                log_control_dispatch_arrival(op, ctx.connection_id, corr);
1353                #[cfg(test)]
1354                if let Some(delay) = self.control_dispatch_delay {
1355                    tokio::time::sleep(delay).await;
1356                }
1357                let result = self
1358                    .handle_client_control_request(ctx, frame, request)
1359                    .await;
1360                log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1361                result
1362            }
1363            FrameType::Push => {
1364                let Some(endpoint) = self
1365                    .forwarding
1366                    .module_endpoint_for_connection(ctx.connection_id)
1367                    .map_err(RouterError::Forwarding)?
1368                else {
1369                    return Ok(vec![control_error_frame(
1370                        &frame,
1371                        "unsupported_control_frame",
1372                        "client-originated channel-0 PUSH is not supported",
1373                    )?]);
1374                };
1375                self.handle_status_update(endpoint, frame)
1376            }
1377            FrameType::Response | FrameType::Error
1378                if self
1379                    .forwarding
1380                    .module_endpoint_for_connection(ctx.connection_id)
1381                    .map_err(RouterError::Forwarding)?
1382                    .is_some() =>
1383            {
1384                self.handle_module_relay_response(ctx.connection_id, frame)
1385            }
1386            ty => Ok(vec![control_error_frame(
1387                &frame,
1388                "unsupported_control_frame",
1389                format!("unsupported channel-0 frame {ty:?}"),
1390            )?]),
1391        }
1392    }
1393
1394    pub fn cleanup_connection(
1395        &self,
1396        connection_id: ConnectionId,
1397    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1398        let crash_closed = self
1399            .registry
1400            .get_module_by_connection(connection_id)?
1401            .and_then(|registration| {
1402                self.forwarding
1403                    .module_endpoint_for_connection(connection_id)
1404                    .ok()
1405                    .flatten()
1406                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1407                    .map(|routes| (registration.manifest.module_id, routes))
1408            });
1409        let crash_closed = crash_closed.map(|(module_id, routes)| {
1410            let terminal = match self.supervisor.get(&module_id) {
1411                None => false,
1412                Some(module) => match module.will_recover_after_connection_loss() {
1413                    Ok(will_recover) => !will_recover,
1414                    Err(err) => {
1415                        warn!(
1416                            %module_id,
1417                            error = %err,
1418                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
1419                        );
1420                        false
1421                    }
1422                },
1423            };
1424            // The forwarding table gates all providers at the start of daemon
1425            // shutdown, before their connections are closed. An ordinary
1426            // module disconnect still reports crash if that gate is not set.
1427            let reason = match self.forwarding.is_daemon_draining() {
1428                Ok(true) => RouteCloseReason::Restart,
1429                Ok(false) => RouteCloseReason::Crash,
1430                Err(err) => {
1431                    warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1432                    RouteCloseReason::Crash
1433                }
1434            };
1435            (module_id, routes, reason, terminal)
1436        });
1437        let registrations = self.deregister_connection(connection_id);
1438        let cleanup = self.forwarding.cleanup_connection_counted(connection_id);
1439        // The route.closed push waits for forwarding teardown because only
1440        // teardown knows how many pending route.bind relays it aborted. It still
1441        // goes out before the GOODBYEs for the released routes, and its targets
1442        // were captured above, before teardown removed those routes.
1443        if let Some((module_id, routes, reason, terminal)) = crash_closed {
1444            let abandoned = cleanup
1445                .as_ref()
1446                .map_or(0, |cleanup| cleanup.abandoned_relays);
1447            send_route_control_pushes(
1448                &self.forwarding,
1449                routes,
1450                ClientControlPush::RouteClosed {
1451                    module_id,
1452                    reason,
1453                    drained: false,
1454                    abandoned,
1455                    excluded_subscriptions: 0,
1456                    terminal: Some(terminal),
1457                },
1458            );
1459        }
1460        if let Ok(cleanup) = cleanup {
1461            self.emit_route_goodbyes(cleanup.released);
1462        }
1463        // Signal the registration-release watch only now that BOTH registry and
1464        // forwarding teardown are done, so a supervisor waiting to spawn a
1465        // replacement never observes release while old routes still exist.
1466        if matches!(&registrations, Ok(r) if !r.is_empty()) {
1467            crate::supervise::notify_registration_release();
1468            self.capability_evaluator.wake_deadline_loop();
1469            self.refresh_capability_requirements();
1470        }
1471        self.supervisor.remove_spawn_subscribers(connection_id);
1472        registrations
1473    }
1474
1475    pub(crate) fn handle_route_goodbye(
1476        &self,
1477        connection_id: ConnectionId,
1478        route_channel: u16,
1479        route_epoch: u32,
1480    ) -> Result<bool, RouterError> {
1481        debug!(
1482            connection_id = connection_id.get(),
1483            route_channel, route_epoch, "handling route GOODBYE"
1484        );
1485        let RouteRelease::Removed(released_route) = self
1486            .forwarding
1487            .release_client_route(connection_id, route_channel, route_epoch)
1488            .map_err(RouterError::Forwarding)?
1489        else {
1490            return Ok(false);
1491        };
1492        self.emit_route_goodbyes(vec![released_route]);
1493        Ok(true)
1494    }
1495
1496    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1497        for released in released_routes {
1498            let frame = match Frame::build_with_version(
1499                released.negotiated_ver,
1500                FrameType::Goodbye,
1501                control_flags(),
1502                released.channel,
1503                released.epoch,
1504                0,
1505                Vec::new(),
1506            ) {
1507                Ok(frame) => frame,
1508                Err(err) => {
1509                    warn!(
1510                        route_channel = released.channel,
1511                        error = %err,
1512                        "failed to build route GOODBYE frame"
1513                    );
1514                    continue;
1515                }
1516            };
1517            if !released.close_on_delivery_failure() {
1518                crate::forwarding::send_module_route_goodbye(
1519                    &self.counters,
1520                    &released.sink,
1521                    frame,
1522                    released.module_id.as_deref(),
1523                    "client route released",
1524                );
1525                continue;
1526            }
1527            if let Err(err) = released.sink.try_send(frame) {
1528                warn!(
1529                    target_connection_id = released.connection_id.get(),
1530                    route_channel = released.channel,
1531                    error = %err,
1532                    "route GOODBYE was not delivered to client; closing target connection"
1533                );
1534                if self
1535                    .forwarding
1536                    .escalate_client_delivery_failure(
1537                        released.connection_id,
1538                        released.channel,
1539                        released.epoch,
1540                        CloseReason::new(
1541                            "route_goodbye_delivery_failed",
1542                            format!(
1543                                "failed to enqueue route GOODBYE for channel {}: {err}",
1544                                released.channel
1545                            ),
1546                        ),
1547                        crate::forwarding::UndeliveredFrame {
1548                            module_id: released.module_id.as_deref(),
1549                            sink: &released.sink,
1550                        },
1551                    )
1552                    .unwrap_or(false)
1553                {
1554                    self.counters.increment_goodbye_relay_client_failed();
1555                }
1556            }
1557        }
1558    }
1559
1560    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
1561    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
1562    /// own commit failed after the module had already accepted). Without this, a
1563    /// module that accepts late keeps a binding subc has torn down, so a later
1564    /// frame on that module channel could misdeliver if the channel is reused.
1565    ///
1566    /// Never closes the shared module connection on failure: a dropped notification
1567    /// only wastes a bounded amount of warm module-side state, which the module's
1568    /// own idle reaper reclaims. Only call this once the route.bind relay was
1569    /// actually enqueued to the module — if the relay send itself failed, the
1570    /// module never created a binding and there is nothing to tear down.
1571    fn send_abandoned_route_bind_goodbye(
1572        &self,
1573        module_sink: &crate::FrameSink,
1574        negotiated_ver: u8,
1575        module_channel: u16,
1576        module_epoch: u32,
1577    ) {
1578        let frame = match Frame::build_with_version(
1579            negotiated_ver,
1580            FrameType::Goodbye,
1581            control_flags(),
1582            module_channel,
1583            module_epoch,
1584            0,
1585            Vec::new(),
1586        ) {
1587            Ok(frame) => frame,
1588            Err(err) => {
1589                warn!(
1590                    route_channel = module_channel,
1591                    error = %err,
1592                    "failed to build GOODBYE for abandoned route.bind"
1593                );
1594                return;
1595            }
1596        };
1597        crate::forwarding::send_module_route_goodbye(
1598            &self.counters,
1599            module_sink,
1600            frame,
1601            None,
1602            "abandoned route.bind",
1603        );
1604    }
1605
1606    fn handle_hello(
1607        &self,
1608        connection_id: ConnectionId,
1609        sink: Option<crate::FrameSink>,
1610        frame: Frame,
1611    ) -> Result<Vec<Frame>, RouterError> {
1612        debug!(
1613            connection_id = connection_id.get(),
1614            corr = frame.header.corr,
1615            "handling HELLO"
1616        );
1617        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1618            Ok(value) => value,
1619            Err(err) => {
1620                return Ok(vec![control_error_frame(
1621                    &frame,
1622                    "invalid_hello",
1623                    format!("malformed HELLO body: {err}"),
1624                )?])
1625            }
1626        };
1627        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1628            return Ok(vec![control_error_frame(
1629                &frame,
1630                "invalid_capability_grammar",
1631                err.to_string(),
1632            )?]);
1633        }
1634        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
1635            return Ok(vec![control_error_frame(
1636                &frame,
1637                "invalid_manifest",
1638                err.to_string(),
1639            )?]);
1640        }
1641        if let Some(provenance) = hello_value
1642            .get("manifest")
1643            .and_then(|manifest| manifest.get("provenance"))
1644        {
1645            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
1646                return Ok(vec![control_error_frame(
1647                    &frame,
1648                    "invalid_manifest",
1649                    format!("malformed manifest provenance: {err}"),
1650                )?]);
1651            }
1652        }
1653        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
1654            Ok(hello) => hello,
1655            Err(err) => {
1656                return Ok(vec![control_error_frame(
1657                    &frame,
1658                    "invalid_hello",
1659                    format!("malformed HELLO body: {err}"),
1660                )?])
1661            }
1662        };
1663
1664        if hello.protocol_ver != hello.manifest.protocol_ver {
1665            return Ok(vec![control_error_frame(
1666                &frame,
1667                "invalid_manifest",
1668                format!(
1669                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
1670                    hello.protocol_ver, hello.manifest.protocol_ver
1671                ),
1672            )?]);
1673        }
1674
1675        if hello.manifest.module_id.trim().is_empty() {
1676            return Ok(vec![control_error_frame(
1677                &frame,
1678                "invalid_manifest",
1679                "manifest module_id must not be empty",
1680            )?]);
1681        }
1682
1683        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
1684            Ok(negotiated_ver) => negotiated_ver,
1685            Err(message) => {
1686                return Ok(vec![control_error_frame(
1687                    &frame,
1688                    "version_unsupported",
1689                    message,
1690                )?])
1691            }
1692        };
1693
1694        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
1695        // swap is open for this id, the only HELLO admitted as a second process
1696        // is the one carrying the candidate's launch nonce (the swap token), and
1697        // it registers into the candidate slot rather than being refused as a
1698        // duplicate. Run after the reserved gate, a reserved module's candidate
1699        // would be refused `reserved_module` for presenting a nonce that gate
1700        // does not know. See `SupervisorHandle::swap_hello_admission`.
1701        let swap_admission = self
1702            .supervisor
1703            .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
1704        if swap_admission == SwapHelloAdmission::Refused {
1705            warn!(
1706                module_id = %hello.manifest.module_id,
1707                connection_id = connection_id.get(),
1708                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
1709            );
1710            return Ok(vec![control_error_frame(
1711                &frame,
1712                "swap_token_invalid",
1713                format!(
1714                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
1715                    hello.manifest.module_id
1716                ),
1717            )?]);
1718        }
1719        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
1720
1721        // Reserved-module identity gate: a module_id configured `reserved` may be
1722        // registered ONLY by the process subc spawned for it, proven by echoing the
1723        // one-time launch nonce subc injected. A non-reserved id has no recorded
1724        // nonce and always passes. This blocks a key-holder from impersonating a
1725        // security-boundary module (e.g. the credential vault) while the real one is
1726        // down/restarting and its registration slot is momentarily free. A swap
1727        // candidate has already proven the same thing with its own nonce above.
1728        if let Some(rejection) = (!swap_candidate)
1729            .then(|| {
1730                self.supervisor.reserved_hello_rejection(
1731                    &hello.manifest.module_id,
1732                    hello.launch_nonce.as_deref(),
1733                )
1734            })
1735            .flatten()
1736        {
1737            let message = match rejection {
1738                ReservedHelloRejection::Exact { module_id } => format!(
1739                    "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
1740                ),
1741                ReservedHelloRejection::Prefix {
1742                    prefix,
1743                    owner_module_id,
1744                } => format!(
1745                    "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
1746                    hello.manifest.module_id
1747                ),
1748            };
1749            return Ok(vec![control_error_frame(
1750                &frame,
1751                "reserved_module",
1752                message,
1753            )?]);
1754        }
1755
1756        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
1757            &hello.manifest.module_id,
1758            hello.manifest.capabilities.as_ref(),
1759        );
1760        if let Some(refusal) = reserved_capability_refusals.first() {
1761            let capability = refusal.capability.clone();
1762            let bound_module = refusal.claimants[0].clone();
1763            log_duplicate_claim_events(reserved_capability_refusals);
1764            return Ok(vec![control_error_frame(
1765                &frame,
1766                "reserved_capability",
1767                format!(
1768                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
1769                    capability, bound_module, hello.manifest.module_id
1770                ),
1771            )?]);
1772        }
1773
1774        // A connection that already opened client routes must not also register as
1775        // a module: cleanup would then release only one side and leak the other.
1776        if self
1777            .forwarding
1778            .connection_has_client_routes(connection_id)
1779            .map_err(RouterError::Forwarding)?
1780        {
1781            return Ok(vec![control_error_frame(
1782                &frame,
1783                "invalid_hello",
1784                "connection has open client routes and cannot also register as a module",
1785            )?]);
1786        }
1787
1788        let control_ops = effective_module_control_ops(hello.control_ops);
1789        // Built before anything is registered so an encoding failure leaves no
1790        // registry or forwarding state behind.
1791        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
1792        if swap_candidate {
1793            return self.register_swap_candidate(
1794                connection_id,
1795                sink,
1796                &frame,
1797                hello.manifest,
1798                negotiated_ver,
1799                control_ops,
1800                hello_ack,
1801            );
1802        }
1803        let registration = match self.registry.register_with_control_ops(
1804            hello.manifest,
1805            negotiated_ver,
1806            connection_id,
1807            control_ops,
1808        ) {
1809            Ok(registration) => registration,
1810            Err(RegistryError::DuplicateModuleId { module_id }) => {
1811                return Ok(vec![control_error_frame(
1812                    &frame,
1813                    "duplicate_module_id",
1814                    format!(
1815                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
1816                    ),
1817                )?])
1818            }
1819            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
1820                return Ok(vec![control_error_frame(
1821                    &frame,
1822                    "invalid_module_id",
1823                    err.to_string(),
1824                )?])
1825            }
1826            Err(err) => {
1827                return Ok(vec![control_error_frame(
1828                    &frame,
1829                    "registry_error",
1830                    err.to_string(),
1831                )?])
1832            }
1833        };
1834
1835        let reply = if let Some(sink) = sink {
1836            // The forwarding table's module store is also the daemon-to-module
1837            // control-RPC lane, so every HELLO gets a live endpoint even when the
1838            // manifest has no routable provider role. Non-routable modules still
1839            // cannot receive route.bind in production: `handle_route_open` checks
1840            // the registry manifest with `target_has_required_role` before the
1841            // only production call to `begin_route_bind_relay_for` below that
1842            // route.open path. The remaining direct relay callers are unit tests
1843            // and benchmark harnesses that construct forwarding state explicitly.
1844            //
1845            // The HELLO_ACK is queued by the forwarding table itself, before the
1846            // endpoint becomes visible, and is NOT returned as a reply. A module
1847            // reads HELLO_ACK first and exits on anything else; a reply is only
1848            // written after this handler returns, by which time a route.open on
1849            // another connection could already have queued a route.bind request
1850            // for this module ahead of it.
1851            let concurrency = manifest_concurrency(&registration.manifest);
1852            if let Err(err) = self.forwarding.register_module_connection_acked(
1853                connection_id,
1854                registration.manifest.module_id.clone(),
1855                negotiated_ver,
1856                concurrency,
1857                sink,
1858                hello_ack,
1859            ) {
1860                // Forwarding registration failed, so there is no forwarding
1861                // state to tear down. Remove the registry entry and signal the
1862                // release watch directly.
1863                if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
1864                    crate::supervise::notify_registration_release();
1865                }
1866                return Ok(vec![control_error_frame(
1867                    &frame,
1868                    forwarding_error_code(&err),
1869                    err.to_string(),
1870                )?]);
1871            }
1872            Vec::new()
1873        } else {
1874            // No sink means no forwarding endpoint, so nothing can be routed
1875            // ahead of the ack; it goes out as the reply.
1876            vec![hello_ack]
1877        };
1878
1879        // Exposure over assumption: Concurrency's serde default is pinned to the
1880        // pre-field behavior (ModuleManaged), so a management surface that is
1881        // genuinely Serial and just never declared it inherits concurrent
1882        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
1883        // turns "no module has been bitten yet" into the checkable claim "no
1884        // module is exposed" -- one read of the boot log instead of a fleet
1885        // audit. Detected from the raw HELLO bytes because the serde default
1886        // deliberately erases the absent/declared distinction from the type.
1887        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
1888            info!(
1889                module_id = %registration.manifest.module_id,
1890                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
1891            );
1892        }
1893
1894        self.apply_registration_capabilities(&registration);
1895
1896        info!(
1897            module_id = %registration.manifest.module_id,
1898            module_version = %registration.manifest.module_version,
1899            negotiated_ver,
1900            routable_provider = manifest_provides_routable_role(&registration.manifest),
1901            connection_id = connection_id.get(),
1902            "module registered"
1903        );
1904
1905        Ok(reply)
1906    }
1907
1908    /// Register a HELLO the swap gate admitted into the candidate slot of the
1909    /// registry and of forwarding, where it is reachable over its own
1910    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
1911    /// nothing routes to it until the supervisor cuts over.
1912    ///
1913    /// Registry first, then forwarding, the same order as an ordinary HELLO;
1914    /// a forwarding failure removes the registry entry again. The capability
1915    /// census is not run: it describes routable modules, and this one is not
1916    /// routable until promotion.
1917    #[allow(clippy::too_many_arguments)]
1918    fn register_swap_candidate(
1919        &self,
1920        connection_id: ConnectionId,
1921        sink: Option<crate::FrameSink>,
1922        frame: &Frame,
1923        manifest: ModuleManifest,
1924        negotiated_ver: u8,
1925        control_ops: Vec<String>,
1926        hello_ack: Frame,
1927    ) -> Result<Vec<Frame>, RouterError> {
1928        let module_id = manifest.module_id.clone();
1929        let registration = match self.registry.register_candidate_with_control_ops(
1930            manifest,
1931            negotiated_ver,
1932            connection_id,
1933            control_ops,
1934        ) {
1935            Ok(registration) => registration,
1936            Err(RegistryError::DuplicateModuleId { module_id }) => {
1937                return Ok(vec![control_error_frame(
1938                    frame,
1939                    "duplicate_module_id",
1940                    format!(
1941                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
1942                    ),
1943                )?])
1944            }
1945            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
1946                return Ok(vec![control_error_frame(
1947                    frame,
1948                    "invalid_module_id",
1949                    err.to_string(),
1950                )?])
1951            }
1952            Err(err) => {
1953                return Ok(vec![control_error_frame(
1954                    frame,
1955                    "registry_error",
1956                    err.to_string(),
1957                )?])
1958            }
1959        };
1960        let reply = if let Some(sink) = sink {
1961            // Same ordering as an ordinary HELLO: the forwarding table queues
1962            // the HELLO_ACK before the candidate endpoint is inserted, because
1963            // a module exits if its first frame after HELLO is anything else.
1964            let concurrency = manifest_concurrency(&registration.manifest);
1965            if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
1966                connection_id,
1967                module_id.clone(),
1968                negotiated_ver,
1969                concurrency,
1970                sink,
1971                hello_ack,
1972            ) {
1973                if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
1974                    crate::supervise::notify_registration_release();
1975                }
1976                return Ok(vec![control_error_frame(
1977                    frame,
1978                    forwarding_error_code(&err),
1979                    err.to_string(),
1980                )?]);
1981            }
1982            Vec::new()
1983        } else {
1984            vec![hello_ack]
1985        };
1986        self.supervisor.mark_swap_candidate_admitted(&module_id);
1987        info!(
1988            module_id = %module_id,
1989            module_version = %registration.manifest.module_version,
1990            negotiated_ver,
1991            ready = registration.ready,
1992            connection_id = connection_id.get(),
1993            "swap candidate registered; not routable until cutover"
1994        );
1995        Ok(reply)
1996    }
1997
1998    fn build_hello_ack(
1999        &self,
2000        frame: &Frame,
2001        negotiated_ver: u8,
2002        module_id: &str,
2003    ) -> Result<Frame, RouterError> {
2004        let ack = ModuleHelloAckBody {
2005            negotiated_ver,
2006            subc_ops: module_subc_ops(),
2007            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2008            storage: self
2009                .storage_config
2010                .as_ref()
2011                .map(|cfg| cfg.descriptor_for(module_id)),
2012            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2013        };
2014        let body = serde_json::to_vec(&ack).map_err(|err| {
2015            RouterError::backend(
2016                0,
2017                frame.header.corr,
2018                format!("failed to encode HELLO_ACK: {err}"),
2019            )
2020        })?;
2021
2022        Frame::build_with_version(
2023            negotiated_ver,
2024            FrameType::HelloAck,
2025            control_flags(),
2026            0,
2027            0,
2028            frame.header.corr,
2029            body,
2030        )
2031        .map_err(RouterError::FrameBuild)
2032    }
2033
2034    async fn handle_client_control_request(
2035        &self,
2036        ctx: &RouteCtx,
2037        frame: Frame,
2038        request: ClientControlRequest,
2039    ) -> Result<Vec<Frame>, RouterError> {
2040        match request {
2041            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2042            ClientControlRequest::CatalogList { module_id } => {
2043                self.handle_catalog_list(frame, module_id)
2044            }
2045            ClientControlRequest::RouteOpen {
2046                target,
2047                identity,
2048                consumer_identity,
2049                consumer_capabilities,
2050                admission_facts,
2051            } => {
2052                self.handle_route_open(
2053                    ctx,
2054                    frame,
2055                    RouteOpenRequest {
2056                        target,
2057                        identity,
2058                        consumer_identity,
2059                        consumer_capabilities,
2060                        admission_facts,
2061                    },
2062                )
2063                .await
2064            }
2065            ClientControlRequest::RoutePoll {
2066                route_channel,
2067                route_epoch,
2068                kind,
2069            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2070            ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2071            ClientControlRequest::SupervisorSpawnSnapshot {} => {
2072                self.handle_supervisor_spawn_snapshot(frame)
2073            }
2074            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2075                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2076            }
2077            ClientControlRequest::SupervisorRestart {
2078                module_id,
2079                drain_timeout_ms,
2080            } => {
2081                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2082                    .await
2083            }
2084            ClientControlRequest::SupervisorSwap {
2085                module_id,
2086                ready_timeout_ms,
2087            } => {
2088                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2089                    .await
2090            }
2091            ClientControlRequest::SupervisorReload { module_id } => {
2092                self.handle_supervisor_reload(frame, module_id).await
2093            }
2094            ClientControlRequest::SupervisorRescan { preview } => {
2095                self.handle_supervisor_rescan(frame, preview).await
2096            }
2097            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2098                self.handle_supervisor_release_reserved(frame, module_id)
2099                    .await
2100            }
2101            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2102                self.handle_supervisor_set_enabled(frame, module_id, enabled)
2103                    .await
2104            }
2105            ClientControlRequest::SupervisorHealthProbe { module_id } => {
2106                self.handle_supervisor_health_probe(frame, module_id).await
2107            }
2108            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2109            ClientControlRequest::SupervisorRoutes { module_id } => {
2110                self.handle_supervisor_routes(frame, module_id)
2111            }
2112            ClientControlRequest::SupervisorProvenance { module_id } => {
2113                self.handle_supervisor_provenance(frame, module_id).await
2114            }
2115            ClientControlRequest::SupervisorStderrTail {
2116                module_id,
2117                max_lines,
2118                max_bytes,
2119            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2120            ClientControlRequest::SupervisorTerminals { module_id } => {
2121                self.handle_supervisor_terminals(frame, module_id).await
2122            }
2123        }
2124    }
2125
2126    fn handle_module_control_request(
2127        &self,
2128        connection_id: ConnectionId,
2129        frame: Frame,
2130        request: ModuleControlRequestFromModule,
2131    ) -> Result<Vec<Frame>, RouterError> {
2132        match request {
2133            ModuleControlRequestFromModule::CatalogUpdate {
2134                provides,
2135                capabilities,
2136                ready,
2137            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2138            ModuleControlRequestFromModule::LiveRoots {} => {
2139                let registered = self
2140                    .registry
2141                    .get_module_by_connection(connection_id)
2142                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2143                let Some(registration) = registered else {
2144                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2145                };
2146                let response = self
2147                    .forwarding
2148                    .live_roots(&registration.manifest.module_id)
2149                    .map_err(RouterError::Forwarding)?;
2150                Ok(vec![control_response_body_frame(
2151                    &frame,
2152                    &response,
2153                    "ModuleControlResponseToModule::LiveRoots",
2154                )?])
2155            }
2156        }
2157    }
2158
2159    fn handle_catalog_update(
2160        &self,
2161        connection_id: ConnectionId,
2162        frame: Frame,
2163        provides: Vec<ProviderRole>,
2164        capabilities: Option<CapabilityDeclarations>,
2165        ready: Option<bool>,
2166    ) -> Result<Vec<Frame>, RouterError> {
2167        self.refresh_capability_requirements();
2168        let Some(registration) = self
2169            .registry
2170            .get_module_by_connection(connection_id)
2171            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2172        else {
2173            return Ok(vec![control_error_frame(
2174                &frame,
2175                "not_registered",
2176                "catalog.update requires an active module registration owned by this connection",
2177            )?]);
2178        };
2179
2180        if let Some(message) =
2181            catalog_update_frozen_field_message(&registration.manifest, &provides)
2182        {
2183            return Ok(vec![control_error_frame(
2184                &frame,
2185                "catalog_update_frozen_field",
2186                message,
2187            )?]);
2188        }
2189
2190        let mut candidate = registration.manifest.clone();
2191        candidate.provides = provides.clone();
2192        candidate.capabilities = capabilities
2193            .clone()
2194            .or_else(|| registration.manifest.capabilities.clone());
2195        if let Err(err) = candidate.validate_capability_grammar() {
2196            return Ok(vec![control_error_frame(
2197                &frame,
2198                "invalid_capability_grammar",
2199                err.to_string(),
2200            )?]);
2201        }
2202
2203        let updated = self
2204            .registry
2205            .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
2206            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2207        if updated.is_none() {
2208            return Ok(vec![control_error_frame(
2209                &frame,
2210                "not_registered",
2211                "catalog.update requires an active module registration owned by this connection",
2212            )?]);
2213        }
2214        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
2215            log_duplicate_claim_events(
2216                self.capability_evaluator
2217                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
2218            );
2219        }
2220        if capability_census_trigger(
2221            registration.manifest.capabilities.as_ref(),
2222            updated
2223                .as_ref()
2224                .and_then(|entry| entry.manifest.capabilities.as_ref()),
2225        ) {
2226            self.enforce_capability_denies();
2227        }
2228        self.refresh_capability_requirements();
2229
2230        let response = ModuleControlResponseToModule::CatalogUpdate {};
2231        control_response_body_frame(
2232            &frame,
2233            &response,
2234            "ModuleControlResponseToModule::CatalogUpdate",
2235        )
2236        .map(|frame| vec![frame])
2237    }
2238
2239    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2240        self.refresh_capability_requirements();
2241        // A bare connection count is ambiguous between many clients holding a
2242        // route each and one client accumulating hundreds, so publish the
2243        // concentration alongside it. Route state is best-effort here: a
2244        // diagnostic endpoint must still answer if the forwarding lock is
2245        // contended.
2246        let mut counters = self.counters.snapshot();
2247        if let (Ok((connections_with_routes, max)), Some(obj)) = (
2248            self.forwarding.client_route_concentration(),
2249            counters.as_object_mut(),
2250        ) {
2251            obj.insert(
2252                "client_connections_with_routes".into(),
2253                connections_with_routes.into(),
2254            );
2255            obj.insert("max_routes_on_one_connection".into(), max.into());
2256        }
2257        // A module that is being fast-refused and a module that is fine look
2258        // identical from a client that retries and succeeds, so name the open
2259        // breakers here. This rides the existing free-form counters object
2260        // rather than a new wire field, so no sibling that deserializes
2261        // `ServerDescribe` has to be rebuilt to keep reading it.
2262        if let (Some(open_breakers), Some(obj)) = (
2263            self.route_bind_breakers.open_snapshot(),
2264            counters.as_object_mut(),
2265        ) {
2266            obj.insert("route_bind_breakers_open".into(), open_breakers);
2267        }
2268        let response = ClientControlResponse::ServerDescribe {
2269            protocol_ver: PROTOCOL_VERSION,
2270            subc_ops: subc_ops(),
2271            capabilities: self.subc_capabilities.as_ref().to_vec(),
2272            connected_clients: self.connected_clients.count(),
2273            counters: Some(counters),
2274            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
2275            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
2276            capability_requirements: self.capability_requirement_statuses(),
2277            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2278        };
2279        Ok(vec![control_response_body_frame(
2280            &frame,
2281            &response,
2282            "ClientControlResponse::ServerDescribe",
2283        )?])
2284    }
2285
2286    fn handle_catalog_list(
2287        &self,
2288        frame: Frame,
2289        module_id: Option<String>,
2290    ) -> Result<Vec<Frame>, RouterError> {
2291        let (generation, modules) = self.registry.list_modules().map_err(|err| {
2292            RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
2293        })?;
2294        let entries = modules
2295            .into_iter()
2296            .filter(|registration| {
2297                module_id
2298                    .as_deref()
2299                    .map(|wanted| registration.manifest.module_id == wanted)
2300                    .unwrap_or(true)
2301            })
2302            .map(|registration| {
2303                let not_ready = self.not_ready_reason(&registration);
2304                let roles = registration.manifest.provides;
2305                CatalogEntry {
2306                    module_id: registration.manifest.module_id,
2307                    ready: not_ready.is_none(),
2308                    not_ready,
2309                    module_version: Some(registration.manifest.module_version),
2310                    roles,
2311                    control_ops: registration.control_ops,
2312                    capabilities: registration.manifest.capabilities,
2313                    self_signals: registration.manifest.self_signals,
2314                }
2315            })
2316            .collect();
2317        let response = ClientControlResponse::CatalogList {
2318            generation,
2319            modules: entries,
2320            subc_ops: subc_ops(),
2321        };
2322        Ok(vec![control_response_body_frame(
2323            &frame,
2324            &response,
2325            "ClientControlResponse::CatalogList",
2326        )?])
2327    }
2328
2329    fn route_open_principal(
2330        &self,
2331        frame: &Frame,
2332        consumer_identity: Option<ConsumerIdentity>,
2333    ) -> Result<Result<Principal, Frame>, RouterError> {
2334        let Some(consumer_identity) = consumer_identity else {
2335            return Ok(Ok(Principal::Direct));
2336        };
2337
2338        if self.supervisor.spawned_consumer_authorized(
2339            &consumer_identity.module_id,
2340            &consumer_identity.launch_nonce,
2341        ) {
2342            return Ok(Ok(Principal::Reserved {
2343                module_id: consumer_identity.module_id,
2344            }));
2345        }
2346
2347        Ok(Err(control_error_frame(
2348            frame,
2349            "bad_consumer_identity",
2350            format!(
2351                "consumer_identity for module_id '{}' did not match a supervised launch nonce",
2352                consumer_identity.module_id
2353            ),
2354        )?))
2355    }
2356
2357    /// Every refusal of a `route.open` goes through here so the daemon can
2358    /// attest which code it sent: without the event, a client's "the daemon
2359    /// refused me" and the daemon's own view could only be reconciled by
2360    /// argument. Malformed input (`invalid_project_root`) does not come here;
2361    /// rejecting a request that was never a valid open is not a refusal of one.
2362    fn route_open_refusal_frame(
2363        &self,
2364        ctx: &RouteCtx,
2365        frame: &Frame,
2366        module_id: &str,
2367        reason: &'static str,
2368        code: &'static str,
2369        message: impl Into<String>,
2370    ) -> Result<Frame, RouterError> {
2371        self.observe_route_open_refusal(ctx, module_id, reason, code);
2372        control_error_frame(frame, code, message.into())
2373    }
2374
2375    /// Refuse a `route.open` because the target module's bind-relay breaker is
2376    /// open, without attempting the relay.
2377    ///
2378    /// The wire code is `module_timeout`, which is the truth (the module has
2379    /// not been answering binds) and which both SDKs already classify as
2380    /// retryable with capped backoff. Reusing it is what keeps this change out
2381    /// of both SDKs; the daemon-side distinction lives in the counter key
2382    /// instead.
2383    ///
2384    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
2385    /// While a breaker is open this fires on every open to that module, and the
2386    /// stall written up in `docs/designs/route-open-head-of-line.md` already
2387    /// produced 261 lines about a single module inside 3000 lines of daemon
2388    /// log. The rare transitions are logged at warn/info instead and the volume
2389    /// is carried by the counter, so the evidence survives without the flood.
2390    /// The debug line keeps a per-refusal record reachable for whoever turns
2391    /// the level up.
2392    fn route_open_breaker_refusal_frame(
2393        &self,
2394        ctx: &RouteCtx,
2395        frame: &Frame,
2396        module_id: &str,
2397        consecutive_timeouts: u32,
2398        retry_in: Duration,
2399        probe_in_flight: bool,
2400    ) -> Result<Frame, RouterError> {
2401        self.counters
2402            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
2403        debug!(
2404            target: "control",
2405            code = "module_timeout",
2406            module_id = ?module_id,
2407            connection_id = ctx.connection_id.get(),
2408            consecutive_timeouts,
2409            retry_in_ms = retry_in.as_millis() as u64,
2410            probe_in_flight,
2411            "route.open refused by open bind-relay breaker"
2412        );
2413        let detail = if probe_in_flight {
2414            "one probe bind is already in flight; retry once it settles".to_string()
2415        } else {
2416            format!("not relaying for another {retry_in:?}")
2417        };
2418        control_error_frame(
2419            frame,
2420            "module_timeout",
2421            format!(
2422                "module_id '{module_id}' failed {consecutive_timeouts} consecutive route.bind \
2423                 relays; {detail}"
2424            ),
2425        )
2426    }
2427
2428    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
2429    /// requester's bytes (an unknown target is whatever the client sent) and
2430    /// is Debug-formatted so control characters land in the log escaped
2431    /// rather than as terminal sequences for whoever tails it.
2432    ///
2433    /// `reason` names the check that refused, because one wire code has
2434    /// several senders: after a module registers, `target_unavailable` can
2435    /// come from a missing role, an inactive registration, a supervisor that
2436    /// has not marked the process live, a missing forwarding connection, or a
2437    /// failed relay, and a log that records only the code cannot say which of
2438    /// them fired. It is a static, daemon-chosen label per branch, so it is
2439    /// safe to print plainly and stays a closed set.
2440    fn observe_route_open_refusal(
2441        &self,
2442        ctx: &RouteCtx,
2443        module_id: &str,
2444        reason: &'static str,
2445        code: &'static str,
2446    ) {
2447        self.counters.increment_route_open_refused(code);
2448        info!(
2449            target: "control",
2450            code,
2451            reason,
2452            module_id = ?module_id,
2453            connection_id = ctx.connection_id.get(),
2454            "route.open refused"
2455        );
2456    }
2457
2458    /// Record an ACCEPTED route.open.
2459    ///
2460    /// Refusals have been logged and counted since the attestation work; accepts
2461    /// were invisible, so the daemon knew every principal it stamped and wrote
2462    /// none of them down. The party that attests the identity was the only party
2463    /// not recording it, which left a credential vault unable to name the sender
2464    /// of a call that reached it (claustrum #43) and left the launch-nonce
2465    /// concurrency question unanswerable from the outside.
2466    ///
2467    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
2468    /// `code`/`module_id`/`connection_id` returns both directions of the same
2469    /// decision rather than two shapes a reader has to join by hand.
2470    ///
2471    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
2472    /// and the difference carries information rather than being an
2473    /// inconsistency. This line is only reachable after a successful bind to a
2474    /// REGISTERED module, so the value has already passed HELLO validation
2475    /// including the path-hazard refusal and cannot contain control bytes. A
2476    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
2477    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
2478    ///
2479    /// Bare is also what every other daemon line already emits (`module
2480    /// registered`, `configured module supervised`). Shipping `?module_id` here
2481    /// made this instrument the only one in the file whose ids did not answer
2482    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
2483    /// line whose whole purpose is being grepped beside its sibling.
2484    ///
2485    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
2486    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
2487    /// forwards every field type through it and a bare `&str` and a `?`-escaped
2488    /// one are recorded identically. A test written against that harness passes
2489    /// either way -- I wrote one, measured it, and deleted it rather than ship a
2490    /// green assertion that cannot fail. The same limit applies to the escaping
2491    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
2492    /// it reads as a guard on the Debug escaping and cannot detect its removal.
2493    /// Fencing either needs the real formatter, not the capture layer.
2494    ///
2495    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
2496    /// unix-socket options and subc is loopback TCP, so there is no peer identity
2497    /// to record. The ephemeral port would decay within minutes and answer only a
2498    /// live question. The identity question is instead answered by counting
2499    /// distinct live connections presenting one module's `consumer_identity` --
2500    /// "is anyone else holding this secret" rather than "is this the right
2501    /// process".
2502    fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
2503        self.counters.increment_route_open_accepted(principal);
2504        info!(
2505            target: "control",
2506            principal,
2507            module_id,
2508            connection_id = ctx.connection_id.get(),
2509            "route.open accepted"
2510        );
2511    }
2512
2513    fn supervised_absent_route_open_refusal_frame(
2514        &self,
2515        ctx: &RouteCtx,
2516        frame: &Frame,
2517        module_id: &str,
2518        code: &'static str,
2519        status: &crate::supervise::ModuleStatus,
2520    ) -> Result<Frame, RouterError> {
2521        self.counters.increment_route_open_refused(code);
2522        info!(
2523            target: "control",
2524            code,
2525            reason = "supervised_not_registered",
2526            module_id = ?module_id,
2527            connection_id = ctx.connection_id.get(),
2528            state = %status.state,
2529            enabled = status.enabled,
2530            live = status.live,
2531            "route.open refused"
2532        );
2533        control_error_frame(
2534            frame,
2535            code,
2536            format!(
2537                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
2538                status.state, status.enabled, status.live
2539            ),
2540        )
2541    }
2542
2543    async fn handle_route_open(
2544        &self,
2545        ctx: &RouteCtx,
2546        frame: Frame,
2547        request: RouteOpenRequest,
2548    ) -> Result<Vec<Frame>, RouterError> {
2549        let RouteOpenRequest {
2550            target,
2551            mut identity,
2552            consumer_identity,
2553            consumer_capabilities,
2554            admission_facts,
2555        } = request;
2556        let target_module_id = target_module_id(&target).to_string();
2557        debug!(
2558            connection_id = ctx.connection_id.get(),
2559            corr = frame.header.corr,
2560            module_id = %target_module_id,
2561            "handling route.open"
2562        );
2563
2564        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
2565        // opposite. Below, a caller learns whether a module is unregistered,
2566        // supervised-but-down (with state/enabled/live), or registered without the
2567        // requested role. Elsewhere that is an enumeration leak: a probe learning
2568        // the shape of a fleet it cannot otherwise see.
2569        //
2570        // It is not one here, and the reason is the ACCESS MODEL rather than
2571        // anything about these errors. Reaching route.open requires the
2572        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
2573        // connection file, so any caller who completes it already runs as this
2574        // user -- and can read subc.jsonc for the module list and `ck module
2575        // status` for live state. The reply discloses nothing the caller cannot
2576        // read more easily from disk, while the precision is load-bearing:
2577        // `unknown_module` is retryable and a missing role is not.
2578        //
2579        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
2580        // remote transport, a sandboxed caller, a shared-host mode -- THAT
2581        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
2582        let Some(registration) = self
2583            .registry
2584            .get_module(&target_module_id)
2585            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2586        else {
2587            if let Some((status, warming)) =
2588                self.supervisor_status(&target_module_id, frame.header.corr)?
2589            {
2590                // BEFORE the two availability codes below, because for a module
2591                // that speaks no subc wire both of them are false comfort: they
2592                // say "not right now" and are retried, and this module will
2593                // never register no matter how long the caller waits. The
2594                // absence here is the declaration being honoured, not a module
2595                // that is late.
2596                if status.protocol == ModuleProtocol::None {
2597                    return Ok(vec![self.route_open_refusal_frame(
2598                        ctx,
2599                        &frame,
2600                        &target_module_id,
2601                        "protocol_none",
2602                        error_codes::MODULE_NO_PROTOCOL,
2603                        format!(
2604                            "module_id '{target_module_id}' is declared protocol: none; \
2605                             it speaks no subc wire and serves no routes"
2606                        ),
2607                    )?]);
2608                }
2609                let code = if warming {
2610                    "module_warming"
2611                } else {
2612                    "target_unavailable"
2613                };
2614                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
2615                    ctx,
2616                    &frame,
2617                    &target_module_id,
2618                    code,
2619                    &status,
2620                )?]);
2621            }
2622            if let Some(removed_ago_ms) =
2623                self.supervisor.removal_tombstone_age_ms(&target_module_id)
2624            {
2625                return Ok(vec![self.route_open_refusal_frame(
2626                    ctx,
2627                    &frame,
2628                    &target_module_id,
2629                    "removed",
2630                    error_codes::MODULE_REMOVED,
2631                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
2632                )?]);
2633            }
2634            return Ok(vec![self.route_open_refusal_frame(
2635                ctx,
2636                &frame,
2637                &target_module_id,
2638                "not_registered",
2639                error_codes::UNKNOWN_MODULE,
2640                format!("module_id '{target_module_id}' is not registered"),
2641            )?]);
2642        };
2643
2644        // Best-effort only: registry readiness and forwarding reservation use
2645        // different locks, so a module can flip readiness between this read and
2646        // the relay. Modules must still tolerate an `on_bind` while not ready.
2647        if !registration.ready {
2648            self.counters
2649                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
2650            info!(
2651                target: "control",
2652                code = error_codes::MODULE_WARMING,
2653                module_id = ?target_module_id,
2654                connection_id = ctx.connection_id.get(),
2655                reason = "declared_not_ready",
2656                "route.open refused"
2657            );
2658            return Ok(vec![control_error_body_frame(
2659                &frame,
2660                ErrorBody {
2661                    code: error_codes::MODULE_WARMING.to_string(),
2662                    message: format!(
2663                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
2664                    ),
2665                    detail: Some(serde_json::json!({
2666                        "reason": "declared_not_ready"
2667                    })),
2668                },
2669            )?]);
2670        }
2671
2672        // Effective readiness, second half: a module that declares a capability
2673        // `need: required` is not routable while that capability has no
2674        // registered provider. It is enforced HERE, as a retryable routing
2675        // refusal, and deliberately not as spawn ordering or a boot block. The
2676        // module is still started and registered and can make its own calls;
2677        // spawn ordering is a promise that cannot be kept once a provider
2678        // crashes at runtime, and refusing to boot would stop the whole
2679        // machine, including the tools needed to fix its configuration.
2680        //
2681        // "Provided" is the evaluator's verdict, which counts a provider as
2682        // soon as it has REGISTERED, not once it is ready. Two modules that
2683        // require each other's capabilities are therefore both routable once
2684        // both register; counting readiness instead would deadlock them.
2685        //
2686        // Only new opens are refused. Routes already bound when a provider
2687        // goes away stay bound: nothing here tears them down, and the module
2688        // answers them as it can. Like the readiness read above this is
2689        // best-effort against a provider registering or leaving concurrently.
2690        if let Some(capability) = self.first_unprovided_required_capability(&registration) {
2691            self.counters
2692                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
2693            info!(
2694                target: "control",
2695                code = error_codes::MODULE_WARMING,
2696                module_id = ?target_module_id,
2697                connection_id = ctx.connection_id.get(),
2698                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
2699                capability = %capability,
2700                "route.open refused"
2701            );
2702            return Ok(vec![control_error_body_frame(
2703                &frame,
2704                ErrorBody {
2705                    code: error_codes::MODULE_WARMING.to_string(),
2706                    message: format!(
2707                        "module_id '{target_module_id}' requires capability '{capability}', \
2708                         which no registered module provides; retry"
2709                    ),
2710                    detail: Some(serde_json::json!({
2711                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
2712                        "capability": capability,
2713                    })),
2714                },
2715            )?]);
2716        }
2717
2718        if !target_has_required_role(&target, &registration.manifest.provides) {
2719            return Ok(vec![self.route_open_refusal_frame(
2720                ctx,
2721                &frame,
2722                &target_module_id,
2723                "role_not_provided",
2724                "target_unavailable",
2725                format!("module_id '{target_module_id}' does not provide the requested target"),
2726            )?]);
2727        }
2728
2729        if registration.state != ChannelState::Active {
2730            return Ok(vec![self.route_open_refusal_frame(
2731                ctx,
2732                &frame,
2733                &target_module_id,
2734                "registration_not_active",
2735                "target_unavailable",
2736                format!("module_id '{target_module_id}' is not active"),
2737            )?]);
2738        }
2739
2740        if self
2741            .forwarding
2742            .module_is_draining(&target_module_id)
2743            .map_err(RouterError::Forwarding)?
2744        {
2745            return Ok(vec![self.route_open_refusal_frame(
2746                ctx,
2747                &frame,
2748                &target_module_id,
2749                "reloading",
2750                "module_reloading",
2751                format!("module_id '{target_module_id}' is reloading"),
2752            )?]);
2753        }
2754
2755        if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
2756            process_liveness.process_live(&target_module_id) == Some(false)
2757        }) {
2758            // A module the supervisor is restarting or reloading can still hold
2759            // a registration: the old process before its connection closes, or
2760            // a new one that registered while the supervisor was draining. The
2761            // forwarding table does not see that as draining, but the consumer
2762            // should still be told to retry soon, exactly as for the drain
2763            // above, rather than that the target is unavailable.
2764            if process_liveness.process_replacing(&target_module_id) {
2765                return Ok(vec![self.route_open_refusal_frame(
2766                    ctx,
2767                    &frame,
2768                    &target_module_id,
2769                    "reloading",
2770                    "module_reloading",
2771                    format!("module_id '{target_module_id}' is reloading"),
2772                )?]);
2773            }
2774            return Ok(vec![self.route_open_refusal_frame(
2775                ctx,
2776                &frame,
2777                &target_module_id,
2778                "supervisor_not_live",
2779                "target_unavailable",
2780                format!("module_id '{target_module_id}' is not live"),
2781            )?]);
2782        }
2783
2784        if !self
2785            .forwarding
2786            .has_live_module_connection(&target_module_id)
2787            .map_err(RouterError::Forwarding)?
2788        {
2789            return Ok(vec![self.route_open_refusal_frame(
2790                ctx,
2791                &frame,
2792                &target_module_id,
2793                "no_forwarding_connection",
2794                "target_unavailable",
2795                format!("module_id '{target_module_id}' has no live forwarding connection"),
2796            )?]);
2797        }
2798
2799        if let Some(error) =
2800            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
2801        {
2802            self.observe_route_open_refusal(
2803                ctx,
2804                &target_module_id,
2805                "op_not_allowed",
2806                "op_not_allowed",
2807            );
2808            return Ok(vec![error]);
2809        }
2810
2811        let principal = match self.route_open_principal(&frame, consumer_identity)? {
2812            Ok(principal) => principal,
2813            Err(error) => {
2814                self.observe_route_open_refusal(
2815                    ctx,
2816                    &target_module_id,
2817                    "bad_consumer_identity",
2818                    "bad_consumer_identity",
2819                );
2820                return Ok(vec![error]);
2821            }
2822        };
2823
2824        // This is attested, control-plane policy for supervised module origins.
2825        // Keep it before route reservation and out of the opaque forwarding hot
2826        // path: data frames must never acquire a per-frame capability check.
2827        if let Principal::Reserved {
2828            module_id: opening_module_id,
2829        } = &principal
2830        {
2831            if let Some(opening_registration) = self
2832                .registry
2833                .get_module(opening_module_id)
2834                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2835            {
2836                if let Some(capability) =
2837                    denied_capability(&opening_registration.manifest, &registration.manifest)
2838                {
2839                    warn!(
2840                        opening_module_id,
2841                        target_module_id,
2842                        capability,
2843                        "refusing route.open because an attested capability deny edge matches"
2844                    );
2845                    return Ok(vec![self.route_open_refusal_frame(
2846                        ctx,
2847                        &frame,
2848                        &target_module_id,
2849                        "capability_deny_edge",
2850                        "capability_forbidden",
2851                        format!(
2852                            "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
2853                        ),
2854                    )?]);
2855                }
2856            }
2857        }
2858
2859        if admission_facts.is_some() {
2860            let carrier_matches = matches!(
2861                &principal,
2862                Principal::Reserved { module_id }
2863                    if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
2864            );
2865            if !carrier_matches {
2866                return Ok(vec![self.route_open_refusal_frame(
2867                    ctx,
2868                    &frame,
2869                    &target_module_id,
2870                    "admission_facts_carrier_not_permitted",
2871                    "admission_facts_not_permitted",
2872                    "admission facts may only be carried by the configured reserved module",
2873                )?]);
2874            }
2875
2876            let target_allowed = self
2877                .admission_facts_targets
2878                .as_ref()
2879                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
2880            if !target_allowed {
2881                return Ok(vec![self.route_open_refusal_frame(
2882                    ctx,
2883                    &frame,
2884                    &target_module_id,
2885                    "admission_facts_target_not_listed",
2886                    "admission_facts_target_not_allowed",
2887                    format!(
2888                        "admission facts are not permitted for target module_id '{target_module_id}'"
2889                    ),
2890                )?]);
2891            }
2892
2893            // Keep the value opaque to subc. The downstream admission validator owns
2894            // schema and semantic checks; this daemon only enforces carrier authority
2895            // and the configured destination allowlist.
2896        }
2897
2898        // Bind admits a root that no longer exists on disk, because refusing here
2899        // closes the only exit from a paused run: cancel needs a bound route, and a
2900        // renamed or reclaimed directory makes that route unopenable forever. The
2901        // run itself is intact and still addressable by its recorded identity.
2902        //
2903        // This does NOT relax the rule the strict constructor protects. That rule is
2904        // that no root is ever aliased into NEW durable state -- a missing component
2905        // can reappear as a symlink elsewhere, which would move the identity and
2906        // split a session's history across two of them. The engine now refuses the
2907        // two operations that create such state (send and import) at admission,
2908        // which is a narrower way to hold the same invariant: reads and terminations
2909        // are admitted, writes are not. That refusal had to ship before this line
2910        // changed, or there is an interval where a send commits under a provisional
2911        // identity -- the exact failure the original policy existed to prevent.
2912        //
2913        // Resolution follows realpath rather than lexical cleanup: the longest
2914        // existing ancestor is canonicalized and the missing tail re-appended, so a
2915        // live root is unchanged and a vanished leaf keeps the identity it was
2916        // admitted under. Lexical cleanup would mint a DIFFERENT identity for the
2917        // same caller the moment the directory vanished, which strands the run more
2918        // quietly than refusing it.
2919        let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
2920            Ok(project_root) => project_root,
2921            Err(err) => {
2922                return Ok(vec![control_error_frame(
2923                    &frame,
2924                    "invalid_project_root",
2925                    err.to_string(),
2926                )?])
2927            }
2928        };
2929        identity.project_root = project_root.as_path().to_path_buf();
2930
2931        // Last gate before any relay work, and deliberately after the cheap
2932        // registry and availability checks above: those name a more precise
2933        // condition (unknown, removed, reloading) and a caller is better served
2934        // by the precise code than by this one.
2935        //
2936        // Everything below this point costs an egress permit, a reserved handle
2937        // pair and, if the module does not answer, the whole relay budget. The
2938        // reader no longer waits for that budget, so cap each target explicitly;
2939        // serial dispatch used to provide the accidental cap of one relay per
2940        // connection. Admission is a mutex-protected count and never waits.
2941        let _concurrency_guard = match self
2942            .route_bind_concurrency
2943            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
2944        {
2945            Ok(guard) => guard,
2946            Err(in_flight) => {
2947                return Ok(vec![self.route_open_admission_refusal_frame(
2948                    ctx,
2949                    &frame,
2950                    &target_module_id,
2951                    format!(
2952                        "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
2953                    ),
2954                )?]);
2955            }
2956        };
2957
2958        // A module that has already burned the whole budget `threshold` times
2959        // in a row does not get to charge it again until a probe says it recovered.
2960        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
2961            RouteBindAdmission::Admitted { guard, probe } => {
2962                if probe {
2963                    info!(
2964                        module_id = %target_module_id,
2965                        connection_id = ctx.connection_id.get(),
2966                        "route.bind breaker half-open: admitting one probe"
2967                    );
2968                }
2969                guard
2970            }
2971            RouteBindAdmission::Refused {
2972                consecutive_timeouts,
2973                retry_in,
2974                probe_in_flight,
2975            } => {
2976                return Ok(vec![self.route_open_breaker_refusal_frame(
2977                    ctx,
2978                    &frame,
2979                    &target_module_id,
2980                    consecutive_timeouts,
2981                    retry_in,
2982                    probe_in_flight,
2983                )?]);
2984            }
2985        };
2986
2987        // Resolve the per-module budget here so the wait matches the operator's
2988        // intent for this specific target. A per-module override in
2989        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
2990        // daemons) wins over the daemon-wide default.
2991        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
2992        let relay_deadline = Instant::now() + route_bind_relay_timeout;
2993        let pending = match self
2994            .forwarding
2995            .begin_route_bind_relay_for(
2996                ctx.connection_id,
2997                ctx.egress.clone(),
2998                response_version(&frame),
2999                frame.header.corr,
3000                &target_module_id,
3001                principal.clone(),
3002                Some(project_root),
3003                relay_deadline,
3004            )
3005            .await
3006        {
3007            Ok(pending) => pending,
3008            Err(err) => {
3009                return Ok(vec![self.route_open_refusal_frame(
3010                    ctx,
3011                    &frame,
3012                    &target_module_id,
3013                    "relay_reservation_failed",
3014                    forwarding_error_code(&err),
3015                    err.to_string(),
3016                )?])
3017            }
3018        };
3019        let crate::forwarding::PendingRouteBindRelay {
3020            endpoint,
3021            module_sink,
3022            negotiated_ver,
3023            client_channel,
3024            client_epoch,
3025            module_channel,
3026            module_epoch,
3027            corr: relay_corr,
3028            receiver,
3029        } = pending;
3030        let mut reservation =
3031            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
3032
3033        debug!(
3034            connection_id = ctx.connection_id.get(),
3035            client_channel,
3036            client_epoch,
3037            module_channel,
3038            module_epoch,
3039            "reserved route handle pair"
3040        );
3041        // Rendered BEFORE the move into the relay, because the accept arm below
3042        // is where it is logged and the principal is gone by then.
3043        let principal_label = match &principal {
3044            Principal::Reserved { module_id } => format!("reserved:{module_id}"),
3045            Principal::Direct => "direct".to_string(),
3046            other => format!("{other:?}"),
3047        };
3048        let relay = ModuleControlRequest::RouteBind {
3049            route_channel: module_channel,
3050            epoch: module_epoch,
3051            target,
3052            identity,
3053            principal: Some(principal),
3054            consumer_capabilities,
3055            admission_facts,
3056        };
3057        let relay_body = serde_json::to_vec(&relay).map_err(|err| {
3058            RouterError::backend(
3059                0,
3060                frame.header.corr,
3061                format!("failed to encode route.bind request: {err}"),
3062            )
3063        })?;
3064        let relay_frame = Frame::build_with_version(
3065            negotiated_ver,
3066            FrameType::Request,
3067            control_flags(),
3068            0,
3069            0,
3070            relay_corr,
3071            relay_body,
3072        )
3073        .map_err(RouterError::FrameBuild)?;
3074
3075        if let Err(err) = module_sink.send(relay_frame).await {
3076            reservation.release_and_disarm();
3077            return Ok(vec![self.route_open_refusal_frame(
3078                ctx,
3079                &frame,
3080                &target_module_id,
3081                "relay_send_failed",
3082                "target_unavailable",
3083                err.to_string(),
3084            )?]);
3085        }
3086
3087        if !self
3088            .forwarding
3089            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
3090            .map_err(RouterError::Forwarding)?
3091        {
3092            self.send_abandoned_route_bind_goodbye(
3093                &module_sink,
3094                negotiated_ver,
3095                module_channel,
3096                module_epoch,
3097            );
3098        }
3099
3100        match timeout_at(relay_deadline, receiver).await {
3101            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
3102                reservation.disarm();
3103                if breaker.record_accepted() {
3104                    info!(
3105                        module_id = %target_module_id,
3106                        "route.bind breaker closed: the probe was accepted"
3107                    );
3108                }
3109                self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
3110                Ok(Vec::new())
3111            }
3112            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
3113                reservation.release_and_disarm();
3114                // A module that says no in microseconds is healthy. Rejection
3115                // is a different condition with its own refusal and must not
3116                // move the breaker.
3117                breaker.record_inconclusive();
3118                self.counters
3119                    .increment_route_open_refused("module_rejected");
3120                info!(
3121                    target: "control",
3122                    code = "module_rejected",
3123                    module_code = ?body.code,
3124                    module_id = ?target_module_id,
3125                    connection_id = ctx.connection_id.get(),
3126                    "route.open refused"
3127                );
3128                Ok(vec![control_error_body_frame(&frame, body)?])
3129            }
3130            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
3131                reservation.release_and_disarm();
3132                breaker.record_inconclusive();
3133                // Fires when the module's connection closes while a relayed
3134                // bind is pending -- typically a caller racing a module restart
3135                // whose bind was relayed BEFORE the drain mark went up. Logged
3136                // because the caller sees only its own error and the fleet has
3137                // already spent one diagnosis round unable to tell this arm
3138                // from a relay timeout without daemon-side evidence.
3139                tracing::warn!(
3140                    module_id = %target_module_id,
3141                    "route.bind relay abandoned: {message}"
3142                );
3143                Ok(vec![self.route_open_refusal_frame(
3144                    ctx,
3145                    &frame,
3146                    &target_module_id,
3147                    "relay_abandoned",
3148                    "target_unavailable",
3149                    message,
3150                )?])
3151            }
3152            Ok(Err(_)) => {
3153                reservation.release_and_disarm();
3154                breaker.record_inconclusive();
3155                Ok(vec![self.route_open_refusal_frame(
3156                    ctx,
3157                    &frame,
3158                    &target_module_id,
3159                    "relay_waiter_canceled",
3160                    "target_unavailable",
3161                    "route.bind relay waiter was canceled before the module responded",
3162                )?])
3163            }
3164            Err(_) => {
3165                reservation.release_and_disarm();
3166                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
3167                // answer at all is the one condition a fast refusal can
3168                // usefully stand in for; every other arm already answered.
3169                if let Some(opened) = breaker.record_timeout(
3170                    self.route_bind_breaker_threshold,
3171                    self.route_bind_breaker_cooldown,
3172                ) {
3173                    warn!(
3174                        module_id = %target_module_id,
3175                        consecutive_timeouts = opened.consecutive_timeouts,
3176                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
3177                        reopened_after_probe = opened.reopened_after_probe,
3178                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
3179                    );
3180                }
3181                // The generous budget just burned to no answer: the module is
3182                // registered and its connection is up, but its bind handler sat
3183                // on the ack for the full budget (warm-on-bind, cold configure,
3184                // or a wedged handler). Every earlier unavailability shape
3185                // fast-refuses BEFORE the relay, so this arm firing means the
3186                // slowness is module-side -- log it so the per-module timeline
3187                // is reconstructable without client audit rows.
3188                tracing::warn!(
3189                    module_id = %target_module_id,
3190                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
3191                    "route.bind relay timed out: module did not ack within budget"
3192                );
3193                Ok(vec![self.route_open_refusal_frame(
3194                    ctx,
3195                    &frame,
3196                    &target_module_id,
3197                    "relay_timed_out",
3198                    "module_timeout",
3199                    format!(
3200                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
3201                        route_bind_relay_timeout
3202                    ),
3203                )?])
3204            }
3205        }
3206    }
3207
3208    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3209        let response = ClientControlResponse::SupervisorSpawnSnapshot {
3210            snapshot: self.supervisor.spawn_snapshot(),
3211        };
3212        Ok(vec![control_response_body_frame(
3213            &frame,
3214            &response,
3215            "ClientControlResponse::SupervisorSpawnSnapshot",
3216        )?])
3217    }
3218
3219    fn handle_supervisor_spawn_subscribe(
3220        &self,
3221        ctx: &RouteCtx,
3222        frame: Frame,
3223        since: Option<SpawnCursor>,
3224    ) -> Result<Vec<Frame>, RouterError> {
3225        match self.supervisor.subscribe_spawns(
3226            ctx.connection_id,
3227            frame.header.corr,
3228            response_version(&frame),
3229            since,
3230            ctx.egress.clone(),
3231        ) {
3232            Ok(()) => Ok(Vec::new()),
3233            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
3234                Ok(vec![control_error_body_frame(
3235                    &frame,
3236                    ErrorBody {
3237                        code: "spawn_cursor_incarnation_mismatch".to_string(),
3238                        message: "spawn cursor belongs to a different daemon incarnation"
3239                            .to_string(),
3240                        detail: Some(serde_json::json!({
3241                            "current_daemon_incarnation": current
3242                        })),
3243                    },
3244                )?])
3245            }
3246            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
3247                &frame,
3248                ErrorBody {
3249                    code: "spawn_cursor_too_old".to_string(),
3250                    message: "spawn cursor predates the retained event ring".to_string(),
3251                    detail: Some(serde_json::json!({
3252                        "oldest_retained_cursor": oldest
3253                    })),
3254                },
3255            )?]),
3256            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
3257                0,
3258                frame.header.corr,
3259                format!("failed to open supervisor spawn subscription: {error}"),
3260            )),
3261        }
3262    }
3263
3264    async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3265        let generation = self
3266            .registry
3267            .generation()
3268            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3269        let mut modules = Vec::new();
3270        for module in self.supervisor.list() {
3271            let status = module.status_for_control("list").map_err(|err| {
3272                RouterError::backend(
3273                    0,
3274                    frame.header.corr,
3275                    format!("failed to read supervisor status: {err}"),
3276                )
3277            })?;
3278            let (configured, _) = module.configuration().map_err(|err| {
3279                RouterError::backend(
3280                    0,
3281                    frame.header.corr,
3282                    format!("failed to read module configuration: {err}"),
3283                )
3284            })?;
3285            // Status and configuration snapshots release their locks before the image probe awaits.
3286            let image = module.running_image_agreement().await;
3287            let pending_reload = Some(reload_verdict(
3288                &configured.program,
3289                status.spawned_from.as_deref(),
3290                image,
3291            ));
3292            modules.push(SupervisorEntry {
3293                module_id: status.module_id,
3294                state: status.state.to_string(),
3295                enabled: status.enabled,
3296                live: status.live,
3297                protocol: status.protocol,
3298                health: status.health.status,
3299                pending_reload,
3300                last_probe_ms: status.health.last_probe_ms,
3301                last_exit_code: status.last_exit.as_ref().and_then(|e| e.code),
3302                last_exit_signal: status.last_exit.as_ref().and_then(|e| e.signal),
3303                last_exit_ms: status.last_exit.as_ref().map(|e| e.at_ms),
3304                last_exit_kind: status.last_exit.as_ref().map(|e| e.kind.into()),
3305                restart_count: Some(status.restart_count),
3306                max_restarts: Some(status.max_restarts),
3307                lifetime_restarts: Some(status.lifetime_restarts),
3308                spawn_generation: Some(status.spawn_generation),
3309                restart_window_secs: Some(status.restart_window.as_secs()),
3310                drain_timeout_ms: Some(status.drain_timeout.as_millis() as u64),
3311                restart_backoff_ms: Some(status.restart_backoff.as_millis() as u64),
3312                restart_max_backoff_ms: Some(status.restart_max_backoff.as_millis() as u64),
3313            });
3314        }
3315        let response = ClientControlResponse::SupervisorList {
3316            generation,
3317            modules,
3318        };
3319        Ok(vec![control_response_body_frame(
3320            &frame,
3321            &response,
3322            "ClientControlResponse::SupervisorList",
3323        )?])
3324    }
3325
3326    fn handle_supervisor_stderr_tail(
3327        &self,
3328        frame: Frame,
3329        module_id: String,
3330        max_lines: Option<u32>,
3331        max_bytes: Option<u32>,
3332    ) -> Result<Vec<Frame>, RouterError> {
3333        let Some(module) = self.supervisor.get(&module_id) else {
3334            return Ok(vec![control_error_frame(
3335                &frame,
3336                "unknown_module",
3337                format!("module_id '{module_id}' is not supervised"),
3338            )?]);
3339        };
3340
3341        let snapshot = module.stderr_tail(
3342            max_lines.map(|value| value as usize),
3343            max_bytes.map(|value| value as usize),
3344        );
3345
3346        let response = ClientControlResponse::SupervisorStderrTail {
3347            module_id,
3348            tail: StderrTail {
3349                capture: match snapshot.capture {
3350                    CaptureState::Captured => StderrCaptureState::Captured,
3351                    CaptureState::Incomplete { reason } => {
3352                        StderrCaptureState::Incomplete { reason }
3353                    }
3354                    CaptureState::NotCaptured { reason } => {
3355                        StderrCaptureState::NotCaptured { reason }
3356                    }
3357                },
3358                entries: snapshot
3359                    .entries
3360                    .into_iter()
3361                    .map(|entry| match entry {
3362                        TailEntry::Line { text, truncated } => {
3363                            StderrTailEntry::Line { text, truncated }
3364                        }
3365                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
3366                    })
3367                    .collect(),
3368                dropped_lines: snapshot.dropped_lines,
3369            },
3370        };
3371        Ok(vec![control_response_body_frame(
3372            &frame,
3373            &response,
3374            "ClientControlResponse::SupervisorStderrTail",
3375        )?])
3376    }
3377
3378    async fn handle_supervisor_terminals(
3379        &self,
3380        frame: Frame,
3381        module_id: String,
3382    ) -> Result<Vec<Frame>, RouterError> {
3383        let Some(module) = self.supervisor.get(&module_id) else {
3384            return Ok(vec![control_error_frame(
3385                &frame,
3386                "unknown_module",
3387                format!("module_id '{module_id}' is not supervised"),
3388            )?]);
3389        };
3390
3391        // The journal read runs on a blocking thread: it can be megabytes of
3392        // file I/O and must not occupy a runtime worker.
3393        let terminals = module
3394            .read_durable_terminal_history()
3395            .await
3396            .map_err(|error| {
3397                RouterError::backend(
3398                    0,
3399                    frame.header.corr,
3400                    format!("failed to read terminal history: {error}"),
3401                )
3402            })?;
3403        let response = ClientControlResponse::SupervisorTerminals {
3404            module_id,
3405            terminals,
3406        };
3407        Ok(vec![control_response_body_frame(
3408            &frame,
3409            &response,
3410            "ClientControlResponse::SupervisorTerminals",
3411        )?])
3412    }
3413
3414    fn handle_supervisor_routes(
3415        &self,
3416        frame: Frame,
3417        module_id: Option<String>,
3418    ) -> Result<Vec<Frame>, RouterError> {
3419        let modules = self
3420            .forwarding
3421            .route_census(module_id.as_deref())
3422            .map_err(RouterError::Forwarding)?
3423            .into_iter()
3424            .map(|(module_id, routes)| SupervisorRouteModule {
3425                module_id,
3426                routes: routes
3427                    .into_iter()
3428                    .map(|route| SupervisorRoute {
3429                        consumer: match route.principal {
3430                            Principal::Reserved { module_id } => {
3431                                SupervisorRouteConsumer::Reserved { module_id }
3432                            }
3433                            Principal::Direct | Principal::Unverified => {
3434                                SupervisorRouteConsumer::Direct {
3435                                    connection_id: route.goodbye_target.connection_id.get(),
3436                                }
3437                            }
3438                        },
3439                        age_ms: Instant::now()
3440                            .saturating_duration_since(route.bound_at)
3441                            .as_millis()
3442                            .try_into()
3443                            .unwrap_or(u64::MAX),
3444                        draining: route.draining,
3445                        drain_reason: route.drain_reason,
3446                    })
3447                    .collect(),
3448            })
3449            .collect();
3450        let response = ClientControlResponse::SupervisorRoutes { modules };
3451        Ok(vec![control_response_body_frame(
3452            &frame,
3453            &response,
3454            "ClientControlResponse::SupervisorRoutes",
3455        )?])
3456    }
3457
3458    async fn handle_supervisor_provenance(
3459        &self,
3460        frame: Frame,
3461        module_id: Option<String>,
3462    ) -> Result<Vec<Frame>, RouterError> {
3463        let mut selected = if let Some(module_id) = module_id {
3464            let Some(module) = self.supervisor.get(&module_id) else {
3465                return Ok(vec![control_error_frame(
3466                    &frame,
3467                    "unknown_module",
3468                    format!("module_id '{module_id}' is not supervised"),
3469                )?]);
3470            };
3471            vec![module]
3472        } else {
3473            self.supervisor.list()
3474        };
3475
3476        let mut modules = Vec::with_capacity(selected.len());
3477        for module in selected.drain(..) {
3478            let status = module.status().map_err(|err| {
3479                RouterError::backend(
3480                    0,
3481                    frame.header.corr,
3482                    format!("failed to read supervisor status: {err}"),
3483                )
3484            })?;
3485            let module_declared = self
3486                .registry
3487                .get_module(&status.module_id)
3488                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3489                .and_then(|registration| registration.manifest.provenance)
3490                .map(|build| ModuleDeclaredProvenance::Reported { build })
3491                .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
3492            #[cfg(test)]
3493            let running_image = match &self.provenance_probe_override {
3494                Some(result) => result.clone(),
3495                None => module.running_image_agreement().await,
3496            };
3497            #[cfg(not(test))]
3498            let running_image = module.running_image_agreement().await;
3499            modules.push(SupervisorModuleProvenance {
3500                module_id: status.module_id,
3501                module_declared,
3502                daemon_observed: SupervisorObservedProcess {
3503                    pid: status.pid,
3504                    spawned_at_ms: status.spawned_at_ms,
3505                    spawned_from: status.spawned_from,
3506                    running_image,
3507                },
3508            });
3509        }
3510        let daemon = SupervisorDaemonProvenance {
3511            daemon_build: self.daemon_provenance.build.clone(),
3512            daemon_observed: DaemonObservedProcess {
3513                pid: self.daemon_provenance.pid,
3514                started_at_ms: self
3515                    .daemon_provenance
3516                    .start_clock
3517                    .map(|clock| clock.started_at_ms())
3518                    .or(self.daemon_provenance.started_at_ms),
3519                running_image: self
3520                    .daemon_provenance
3521                    .probe
3522                    .observe(
3523                        self.daemon_provenance.pid,
3524                        self.daemon_provenance.executable_path.as_deref(),
3525                        self.daemon_provenance.executable_identity,
3526                        self.daemon_provenance.process_start_time,
3527                    )
3528                    .await,
3529            },
3530        };
3531        let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
3532        Ok(vec![control_response_body_frame(
3533            &frame,
3534            &response,
3535            "ClientControlResponse::SupervisorProvenance",
3536        )?])
3537    }
3538
3539    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3540        self.refresh_capability_requirements();
3541        let generation = self
3542            .registry
3543            .generation()
3544            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3545        let modules = self
3546            .supervisor
3547            .list()
3548            .into_iter()
3549            .map(|module| {
3550                let status = module.status_for_control("health").map_err(|err| {
3551                    RouterError::backend(
3552                        0,
3553                        frame.header.corr,
3554                        format!("failed to read supervisor health: {err}"),
3555                    )
3556                })?;
3557                let module_id = status.module_id;
3558                let capability_detail = self
3559                    .capability_evaluator
3560                    .required_problem_detail(&module_id);
3561                Ok(SupervisorHealthEntry {
3562                    module_id,
3563                    status: status.health.status,
3564                    detail: append_capability_problem_detail(
3565                        status.health.detail,
3566                        capability_detail,
3567                    ),
3568                    metrics: status.health.metrics,
3569                    consecutive_failures: status.health.consecutive_failures,
3570                    late_answer_count: status.health.late_answer_count,
3571                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
3572                    last_action: status.health.last_action,
3573                    last_action_ms: status.health.last_action_ms,
3574                    last_probe_ms: status.health.last_probe_ms,
3575                })
3576            })
3577            .collect::<Result<Vec<_>, RouterError>>()?;
3578        let response = ClientControlResponse::SupervisorHealth {
3579            generation,
3580            modules,
3581        };
3582        Ok(vec![control_response_body_frame(
3583            &frame,
3584            &response,
3585            "ClientControlResponse::SupervisorHealth",
3586        )?])
3587    }
3588
3589    async fn handle_supervisor_restart(
3590        &self,
3591        frame: Frame,
3592        module_id: String,
3593        drain_timeout_ms: Option<u64>,
3594    ) -> Result<Vec<Frame>, RouterError> {
3595        let operation_lock = self.supervisor.operation_lock();
3596        let _operation_guard = operation_lock.lock().await;
3597        let Some(module) = self.supervisor.get(&module_id) else {
3598            return Ok(vec![control_error_frame(
3599                &frame,
3600                "unknown_module",
3601                format!("module_id '{module_id}' is not supervised"),
3602            )?]);
3603        };
3604
3605        if let Err(err) = module.restart(drain_timeout_ms).await {
3606            let (code, message) = match err {
3607                crate::supervise::SuperviseError::Disabled { .. } => {
3608                    ("module_disabled", err.to_string())
3609                }
3610                crate::supervise::SuperviseError::SwapInProgress { .. } => {
3611                    ("swap_in_progress", err.to_string())
3612                }
3613                _ => (
3614                    "target_unavailable",
3615                    format!("failed to restart module_id '{module_id}': {err}"),
3616                ),
3617            };
3618            return Ok(vec![control_error_frame(&frame, code, message)?]);
3619        }
3620
3621        let response = ClientControlResponse::SupervisorAck {
3622            module_id,
3623            applied: true,
3624        };
3625        Ok(vec![control_response_body_frame(
3626            &frame,
3627            &response,
3628            "ClientControlResponse::SupervisorAck",
3629        )?])
3630    }
3631
3632    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
3633    /// when the old process has finished draining: a caller whose own lane
3634    /// rides the old process must get its reply before that drain waits on it.
3635    async fn handle_supervisor_swap(
3636        &self,
3637        frame: Frame,
3638        module_id: String,
3639        ready_timeout_ms: Option<u64>,
3640    ) -> Result<Vec<Frame>, RouterError> {
3641        // The daemon-wide operation lock is held only to resolve the handle,
3642        // not across the swap. The swap can take its whole readiness budget,
3643        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
3644        // holding it here would park an operator's stop behind the swap it is
3645        // meant to abort. A rescan or stop that reaches the module during the
3646        // swap is served by the swap itself (see `supervise_swap`).
3647        let module = {
3648            let operation_lock = self.supervisor.operation_lock();
3649            let _operation_guard = operation_lock.lock().await;
3650            self.supervisor.get(&module_id)
3651        };
3652        let Some(module) = module else {
3653            return Ok(vec![control_error_frame(
3654                &frame,
3655                "unknown_module",
3656                format!("module_id '{module_id}' is not supervised"),
3657            )?]);
3658        };
3659
3660        if let Err(err) = module
3661            .swap(ready_timeout_ms.map(Duration::from_millis))
3662            .await
3663        {
3664            use crate::supervise::SuperviseError;
3665            let message = err.to_string();
3666            let error = match err {
3667                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
3668                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
3669                    code: "swap_refused".to_string(),
3670                    message,
3671                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
3672                },
3673                SuperviseError::SwapFailed {
3674                    arm,
3675                    candidate_exit,
3676                    ..
3677                } => ErrorBody {
3678                    code: "swap_failed".to_string(),
3679                    message,
3680                    detail: Some(serde_json::json!({
3681                        "arm": arm.as_str(),
3682                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
3683                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
3684                    })),
3685                },
3686                _ => ErrorBody::new(
3687                    "target_unavailable",
3688                    format!("failed to swap module_id '{module_id}': {message}"),
3689                ),
3690            };
3691            return Ok(vec![control_error_body_frame(&frame, error)?]);
3692        }
3693
3694        let response = ClientControlResponse::SupervisorAck {
3695            module_id,
3696            applied: true,
3697        };
3698        Ok(vec![control_response_body_frame(
3699            &frame,
3700            &response,
3701            "ClientControlResponse::SupervisorAck",
3702        )?])
3703    }
3704
3705    async fn handle_supervisor_reload(
3706        &self,
3707        frame: Frame,
3708        module_id: String,
3709    ) -> Result<Vec<Frame>, RouterError> {
3710        let operation_lock = self.supervisor.operation_lock();
3711        let _operation_guard = operation_lock.lock().await;
3712        let Some(module) = self.supervisor.get(&module_id) else {
3713            return Ok(vec![control_error_frame(
3714                &frame,
3715                "unknown_module",
3716                format!("module_id '{module_id}' is not supervised"),
3717            )?]);
3718        };
3719
3720        if let Err(err) = module.reload().await {
3721            let (code, message) = match err {
3722                crate::supervise::SuperviseError::Disabled { .. } => {
3723                    ("module_disabled", err.to_string())
3724                }
3725                crate::supervise::SuperviseError::SwapInProgress { .. } => {
3726                    ("swap_in_progress", err.to_string())
3727                }
3728                _ => (
3729                    "reload_failed",
3730                    format!("failed to reload module_id '{module_id}': {err}"),
3731                ),
3732            };
3733            return Ok(vec![control_error_frame(&frame, code, message)?]);
3734        }
3735
3736        let response = ClientControlResponse::SupervisorAck {
3737            module_id,
3738            applied: true,
3739        };
3740        Ok(vec![control_response_body_frame(
3741            &frame,
3742            &response,
3743            "ClientControlResponse::SupervisorAck",
3744        )?])
3745    }
3746
3747    async fn handle_supervisor_rescan(
3748        &self,
3749        frame: Frame,
3750        preview: bool,
3751    ) -> Result<Vec<Frame>, RouterError> {
3752        let Some(context) = self.rescan.clone() else {
3753            return Ok(vec![control_error_frame(
3754                &frame,
3755                "rescan_unavailable",
3756                "the daemon was not started with a reloadable config path".to_string(),
3757            )?]);
3758        };
3759
3760        let operation_lock = self.supervisor.operation_lock();
3761        let _operation_guard = operation_lock.lock().await;
3762        let loaded = match crate::daemon_config::load(&context.config_path) {
3763            Ok(config) => config,
3764            Err(err) => {
3765                return Ok(vec![control_error_frame(
3766                    &frame,
3767                    "invalid_daemon_config",
3768                    format!("supervisor rescan rejected daemon config: {err}"),
3769                )?])
3770            }
3771        };
3772        // `load` reports a missing file as Ok(None), which is correct at boot
3773        // (no config, nothing to supervise) and catastrophic here: rescan treats
3774        // "not in the config" as "remove it", so an absent file would read as an
3775        // empty module list and retire the entire running fleet. An editor
3776        // writing via write-new-then-rename, or a half-finished edit, is enough
3777        // to open that window. Refuse instead: a config that cannot be read
3778        // carries no instruction to remove anything.
3779        let Some(config) = loaded else {
3780            return Ok(vec![control_error_frame(
3781                &frame,
3782                "invalid_daemon_config",
3783                format!(
3784                    "daemon config not found at {}; refusing to rescan (an absent config would \
3785                     retire every supervised module)",
3786                    context.config_path.display()
3787                ),
3788            )?]);
3789        };
3790        let (
3791            configured_port,
3792            storage_config,
3793            admission_facts_carrier_module_id,
3794            admission_facts_targets,
3795            modules,
3796            reserved_capabilities,
3797        ) = (
3798            config.port,
3799            config.storage,
3800            config.admission_facts_carrier_module_id,
3801            config.admission_facts_targets,
3802            config.modules,
3803            config.reserved_capabilities,
3804        );
3805
3806        // Collect the sections rescan cannot apply, so the REPLY carries them.
3807        //
3808        // The warning below has always been correct and has always gone only to
3809        // the journal -- addressed to whoever reads logs, while the person who
3810        // just edited the config is looking at the CLI. Naming each section
3811        // individually rather than setting a flag: "something outside modules
3812        // changed" sends the operator back to diffing their own file, which is
3813        // the work this is meant to save.
3814        let mut restart_required = Vec::new();
3815        for section in RestartRequiredSection::ALL {
3816            let changed = match section {
3817                RestartRequiredSection::Port => configured_port != context.configured_port,
3818                RestartRequiredSection::Storage => storage_config != context.storage_config,
3819                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
3820                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
3821                }
3822                RestartRequiredSection::AdmissionFactsTargets => {
3823                    admission_facts_targets != context.admission_facts_targets
3824                }
3825            };
3826            if changed {
3827                restart_required.push(section.label().to_string());
3828            }
3829        }
3830        if !restart_required.is_empty() {
3831            warn!(
3832                config_path = %context.config_path.display(),
3833                sections = %restart_required.join(", "),
3834                "daemon config changed outside the modules section; restart the daemon to apply those changes"
3835            );
3836        }
3837
3838        for configured in &modules {
3839            if let Err(err) = validate_spec(&configured.module_spec()) {
3840                return Ok(vec![control_error_frame(
3841                    &frame,
3842                    "invalid_daemon_config",
3843                    format!("supervisor rescan rejected daemon config: {err}"),
3844                )?]);
3845            }
3846        }
3847
3848        let configured_capabilities = modules
3849            .iter()
3850            .map(|module| (module.module_id.clone(), module.enabled))
3851            .collect::<Vec<_>>();
3852        let preview_capability_warnings = if preview {
3853            let (_, registrations) = self.runtime_capability_snapshot()?;
3854            let current_modules = self
3855                .supervisor
3856                .list()
3857                .into_iter()
3858                .map(|module| module.module_id().to_string())
3859                .collect::<BTreeSet<_>>();
3860            let resulting_modules = configured_capabilities.clone();
3861            let removed = current_modules
3862                .into_iter()
3863                .filter(|module_id| {
3864                    !resulting_modules
3865                        .iter()
3866                        .any(|(configured_id, _)| configured_id == module_id)
3867                })
3868                .collect::<Vec<_>>();
3869            self.capability_evaluator.preview_removal_warnings(
3870                resulting_modules,
3871                &removed,
3872                &registrations,
3873            )
3874        } else {
3875            Vec::new()
3876        };
3877        let result = match self
3878            .reconcile_supervised_modules(&context.supervisor, modules, preview)
3879            .await
3880        {
3881            Ok(result) => result,
3882            Err(message) => {
3883                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
3884            }
3885        };
3886        if !preview {
3887            self.capability_evaluator
3888                .configure(configured_capabilities, reserved_capabilities);
3889            self.capability_evaluator.wake_deadline_loop();
3890            self.refresh_capability_requirements();
3891        }
3892        let mut result = result;
3893        result.restart_required = restart_required;
3894        result.capability_warnings = preview_capability_warnings;
3895        let response = ClientControlResponse::SupervisorRescan { result };
3896        Ok(vec![control_response_body_frame(
3897            &frame,
3898            &response,
3899            "ClientControlResponse::SupervisorRescan",
3900        )?])
3901    }
3902
3903    async fn handle_supervisor_release_reserved(
3904        &self,
3905        frame: Frame,
3906        module_id: String,
3907    ) -> Result<Vec<Frame>, RouterError> {
3908        let Some(context) = self.rescan.clone() else {
3909            return Ok(vec![control_error_frame(
3910                &frame,
3911                "release_unavailable",
3912                "reserved-id release requires a daemon started with a reloadable config path",
3913            )?]);
3914        };
3915        let operation_lock = self.supervisor.operation_lock();
3916        let _operation_guard = operation_lock.lock().await;
3917        let loaded = match crate::daemon_config::load(&context.config_path) {
3918            Ok(Some(config)) => config,
3919            Ok(None) => {
3920                return Ok(vec![control_error_frame(
3921                    &frame,
3922                    "invalid_daemon_config",
3923                    format!(
3924                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
3925                        context.config_path.display()
3926                    ),
3927                )?])
3928            }
3929            Err(err) => {
3930                return Ok(vec![control_error_frame(
3931                    &frame,
3932                    "invalid_daemon_config",
3933                    format!("unable to verify reserved-id release against daemon config: {err}"),
3934                )?])
3935            }
3936        };
3937        if loaded
3938            .modules
3939            .iter()
3940            .any(|configured| configured.module_id == module_id)
3941        {
3942            return Ok(vec![control_error_frame(
3943                &frame,
3944                "reserved_module_configured",
3945                format!(
3946                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
3947                ),
3948            )?]);
3949        }
3950        if !self.supervisor.release_retained_reserved_gate(&module_id) {
3951            return Ok(vec![control_error_frame(
3952                &frame,
3953                "reserved_gate_not_retained",
3954                format!(
3955                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
3956                ),
3957            )?]);
3958        }
3959
3960        let response = ClientControlResponse::SupervisorAck {
3961            module_id,
3962            applied: true,
3963        };
3964        Ok(vec![control_response_body_frame(
3965            &frame,
3966            &response,
3967            "ClientControlResponse::SupervisorAck",
3968        )?])
3969    }
3970
3971    /// Reconcile the running module set against the configured one.
3972    ///
3973    /// With `preview` set, the diff is computed and returned WITHOUT applying any
3974    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
3975    /// deliberately shares this function with the executing path rather than
3976    /// computing the same diff somewhere else -- two implementations of one
3977    /// decision agree until they do not, and the whole value of a preview is that
3978    /// it describes the operation that will actually run.
3979    async fn reconcile_supervised_modules(
3980        &self,
3981        supervisor: &Supervisor,
3982        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
3983        preview: bool,
3984    ) -> Result<SupervisorRescanResult, String> {
3985        let mut current = BTreeMap::new();
3986        for module in self.supervisor.list() {
3987            let (spec, health) = module.configuration().map_err(|err| {
3988                format!(
3989                    "failed to read configuration for module_id '{}': {err}",
3990                    module.module_id()
3991                )
3992            })?;
3993            let enabled = module
3994                .status()
3995                .map_err(|err| {
3996                    format!(
3997                        "failed to read status for module_id '{}': {err}",
3998                        module.module_id()
3999                    )
4000                })?
4001                .enabled;
4002            current.insert(
4003                module.module_id().to_string(),
4004                (module, spec, health, enabled),
4005            );
4006        }
4007        let configured = configured_modules
4008            .into_iter()
4009            .map(|module| (module.module_id.clone(), module))
4010            .collect::<BTreeMap<_, _>>();
4011
4012        let added = configured
4013            .keys()
4014            .filter(|module_id| !current.contains_key(*module_id))
4015            .cloned()
4016            .collect::<Vec<_>>();
4017        let removed = current
4018            .keys()
4019            .filter(|module_id| !configured.contains_key(*module_id))
4020            .cloned()
4021            .collect::<Vec<_>>();
4022        let mut changed_pending_reload = Vec::new();
4023        let mut configuration_changes = BTreeSet::new();
4024        let mut enabled_changes = BTreeSet::new();
4025        let mut unchanged = 0_u32;
4026
4027        for (module_id, configured_module) in &configured {
4028            let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
4029            else {
4030                continue;
4031            };
4032            let configuration_changed = *current_spec != configured_module.module_spec()
4033                || *current_health != configured_module.health;
4034            let enabled_changed = *current_enabled != configured_module.enabled;
4035            if configuration_changed {
4036                configuration_changes.insert(module_id.clone());
4037                changed_pending_reload.push(module_id.clone());
4038            }
4039            if enabled_changed {
4040                enabled_changes.insert(module_id.clone());
4041            }
4042            if !configuration_changed && !enabled_changed {
4043                unchanged = unchanged.saturating_add(1);
4044            }
4045        }
4046
4047        // Everything above this point is pure computation over two snapshots.
4048        // Everything below MUTATES. The preview returns here so the boundary is a
4049        // single early return rather than a condition repeated at each mutation
4050        // site, where one missed guard would apply part of a change the caller was
4051        // told would not happen.
4052        if preview {
4053            return Ok(SupervisorRescanResult {
4054                added,
4055                removed,
4056                changed_pending_reload,
4057                enabled_changes: enabled_changes.iter().cloned().collect(),
4058                unchanged,
4059                preview: true,
4060                // Filled by the caller on both paths, so the preview reports
4061                // restart-required sections identically to an executed rescan --
4062                // the preview is where an operator is most likely to be looking.
4063                restart_required: Vec::new(),
4064                capability_warnings: Vec::new(),
4065            });
4066        }
4067
4068        for module_id in &removed {
4069            let module = &current
4070                .get(module_id)
4071                .expect("removed module came from current supervisor state")
4072                .0;
4073            module.retire().await.map_err(|err| {
4074                format!("failed to retire module_id '{module_id}' during rescan: {err}")
4075            })?;
4076            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
4077            //
4078            // `handle_route_open` resolves an absent module in three steps:
4079            // registry, then supervisor status, then tombstone. Retiring first
4080            // opens a window where ALL THREE ARE ABSENT -- the registry entry
4081            // went with the teardown above, the supervisor entry went with
4082            // `retire`, and the tombstone does not exist yet -- so a route.open
4083            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
4084            // it") for a module that was deliberately removed and whose caller
4085            // should get `module_removed` (TERMINAL, carrying a removal age).
4086            //
4087            // Writing the tombstone first closes it: during the window the
4088            // supervisor entry still answers, so the caller gets
4089            // `target_unavailable` -- retryable, and TRUE, because the module
4090            // is mid-teardown. After both statements it is `module_removed`.
4091            // No instant remains where a removed module reads as one that
4092            // never existed.
4093            //
4094            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
4095            // the absence of a test beside a fix invites deletion: these are
4096            // two sync statements with no await between them, so reaching the
4097            // window needs a second worker thread to land exactly between them
4098            // and there is no hook to force it. MEASURED: the 25 daemon_config
4099            // tests pass identically with the old order and the new one, so
4100            // the existing suite cannot see this and a green run is not
4101            // evidence either way. What the suite does hold is the
4102            // post-condition -- a removed module answers `module_removed` --
4103            // which this preserves.
4104            //
4105            // Found by an Athena panel reading the shipped tree against a
4106            // design note (2026-09-19), as the one concrete instance of that
4107            // note's class that survived contact with source. Direction is
4108            // benign: retryable where terminal was intended, never the reverse.
4109            self.supervisor.record_rescan_removal(module_id);
4110            self.supervisor.retire(module_id);
4111        }
4112
4113        for module_id in configured.keys() {
4114            let Some((module, _, _, _)) = current.get(module_id) else {
4115                continue;
4116            };
4117            let configured_module = configured
4118                .get(module_id)
4119                .expect("configured module id came from configured map");
4120            if configuration_changes.contains(module_id) {
4121                module
4122                    .update_configuration(
4123                        configured_module.module_spec(),
4124                        configured_module.health,
4125                        configured_module.drain_timeout_ms,
4126                    )
4127                    .await
4128                    .map_err(|err| {
4129                        format!(
4130                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
4131                        )
4132                    })?;
4133            }
4134            if enabled_changes.contains(module_id) {
4135                module
4136                    .set_enabled(configured_module.enabled)
4137                    .await
4138                    .map_err(|err| {
4139                        format!(
4140                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
4141                            configured_module.enabled
4142                        )
4143                    })?;
4144            }
4145        }
4146
4147        for module_id in &added {
4148            let configured_module = configured
4149                .get(module_id)
4150                .expect("added module id came from configured map");
4151            supervisor
4152                .supervise_configured_with_health(
4153                    configured_module.module_spec(),
4154                    configured_module.enabled,
4155                    configured_module.health,
4156                    configured_module.drain_timeout_ms,
4157                    configured_module.restart,
4158                )
4159                .map_err(|err| {
4160                    format!("failed to add module_id '{module_id}' during rescan: {err}")
4161                })?;
4162        }
4163
4164        Ok(SupervisorRescanResult {
4165            added,
4166            removed,
4167            changed_pending_reload,
4168            enabled_changes: enabled_changes.iter().cloned().collect(),
4169            unchanged,
4170            preview: false,
4171            // Filled by the caller, which is the only layer that can see the
4172            // previous config to diff against.
4173            restart_required: Vec::new(),
4174            capability_warnings: Vec::new(),
4175        })
4176    }
4177
4178    async fn handle_supervisor_set_enabled(
4179        &self,
4180        frame: Frame,
4181        module_id: String,
4182        enabled: bool,
4183    ) -> Result<Vec<Frame>, RouterError> {
4184        let operation_lock = self.supervisor.operation_lock();
4185        let _operation_guard = operation_lock.lock().await;
4186        let Some(module) = self.supervisor.get(&module_id) else {
4187            return Ok(vec![control_error_frame(
4188                &frame,
4189                "unknown_module",
4190                format!("module_id '{module_id}' is not supervised"),
4191            )?]);
4192        };
4193
4194        let applied = match module.set_enabled(enabled).await {
4195            Ok(applied) => applied,
4196            Err(err) => {
4197                return Ok(vec![control_error_frame(
4198                    &frame,
4199                    "target_unavailable",
4200                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
4201                )?])
4202            }
4203        };
4204
4205        self.capability_evaluator.wake_deadline_loop();
4206        self.refresh_capability_requirements();
4207        let response = ClientControlResponse::SupervisorAck { module_id, applied };
4208        Ok(vec![control_response_body_frame(
4209            &frame,
4210            &response,
4211            "ClientControlResponse::SupervisorAck",
4212        )?])
4213    }
4214
4215    async fn handle_supervisor_health_probe(
4216        &self,
4217        frame: Frame,
4218        module_id: String,
4219    ) -> Result<Vec<Frame>, RouterError> {
4220        self.refresh_capability_requirements();
4221        let Some(registration) = self
4222            .registry
4223            .get_module(&module_id)
4224            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4225        else {
4226            return Ok(vec![control_error_frame(
4227                &frame,
4228                "unknown_module",
4229                format!("module_id '{module_id}' is not registered"),
4230            )?]);
4231        };
4232
4233        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
4234        // named for it. Making `module_registration_grants_op` return false
4235        // unconditionally reddens five tests, and every one is named for something
4236        // else -- capability relay, probe/bind demultiplexing, supervision-only
4237        // probing. They exercise a successful advertisement check on the way to their
4238        // own subject.
4239        //
4240        // Real protection, fragile in a specific way: narrowing any of those tests to
4241        // focus on its stated subject would silently remove coverage nobody knows
4242        // they are carrying. Recorded here rather than as a sixth test, because the
4243        // useful fact is WHICH tests hold the guard up -- a new test would add
4244        // coverage without telling the next person what the existing ones quietly do.
4245        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
4246        {
4247            return Ok(vec![control_error_frame(
4248                &frame,
4249                "health_not_advertised",
4250                format!("module_id '{module_id}' did not advertise health.check"),
4251            )?]);
4252        }
4253
4254        let deadline = Instant::now() + self.health_probe_timeout;
4255        let pending = match self.forwarding.begin_module_control_rpc_for(
4256            &module_id,
4257            MODULE_CONTROL_OP_HEALTH_CHECK,
4258            deadline,
4259        ) {
4260            Ok(pending) => pending,
4261            Err(err) => {
4262                return Ok(vec![control_error_frame(
4263                    &frame,
4264                    forwarding_error_code(&err),
4265                    err.to_string(),
4266                )?])
4267            }
4268        };
4269
4270        let PendingModuleControlRpc {
4271            endpoint,
4272            module_sink,
4273            negotiated_ver,
4274            corr: probe_corr,
4275            receiver,
4276        } = pending;
4277        let mut guard =
4278            ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
4279        let probe_body =
4280            serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
4281                RouterError::backend(
4282                    0,
4283                    frame.header.corr,
4284                    format!("failed to encode health.check request: {err}"),
4285                )
4286            })?;
4287        let probe_frame = Frame::build_with_version(
4288            negotiated_ver,
4289            FrameType::Request,
4290            control_flags(),
4291            0,
4292            0,
4293            probe_corr,
4294            probe_body,
4295        )
4296        .map_err(RouterError::FrameBuild)?;
4297
4298        if let Err(err) = module_sink.send(probe_frame).await {
4299            return Ok(vec![control_error_frame(
4300                &frame,
4301                "target_unavailable",
4302                err.to_string(),
4303            )?]);
4304        }
4305
4306        match timeout_at(deadline, receiver).await {
4307            Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
4308                guard.disarm();
4309                let Some(report) = response.health_report() else {
4310                    return Ok(vec![control_error_frame(
4311                        &frame,
4312                        "invalid_control_body",
4313                        "health.check RPC returned a non-health response",
4314                    )?]);
4315                };
4316                // Metrics go out whole here. The supervisor's cached snapshot
4317                // caps this blob (see truncate_health_metrics), and this path
4318                // exists precisely to answer without that cap -- so applying it
4319                // here would leave no way to see what the cached view drops.
4320                let HealthReport {
4321                    status,
4322                    detail,
4323                    metrics,
4324                } = report;
4325                let capability_detail = self
4326                    .capability_evaluator
4327                    .required_problem_detail(&module_id);
4328                let response = ClientControlResponse::SupervisorHealthProbe {
4329                    module_id,
4330                    status,
4331                    detail: append_capability_problem_detail(detail, capability_detail),
4332                    metrics,
4333                };
4334                Ok(vec![control_response_body_frame(
4335                    &frame,
4336                    &response,
4337                    "ClientControlResponse::SupervisorHealthProbe",
4338                )?])
4339            }
4340            Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
4341                guard.disarm();
4342                Ok(vec![control_error_body_frame(&frame, body)?])
4343            }
4344            Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
4345                guard.disarm();
4346                Ok(vec![control_error_frame(
4347                    &frame,
4348                    "target_unavailable",
4349                    message,
4350                )?])
4351            }
4352            Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
4353                guard.disarm();
4354                Ok(vec![control_error_frame(
4355                    &frame,
4356                    "invalid_control_body",
4357                    message,
4358                )?])
4359            }
4360            Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
4361                guard.disarm();
4362                Ok(vec![control_error_frame(
4363                    &frame,
4364                    "invalid_control_body",
4365                    format!("expected module-control op '{expected}', got '{actual}'"),
4366                )?])
4367            }
4368            Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
4369                guard.disarm();
4370                Ok(vec![control_error_frame(
4371                    &frame,
4372                    "module_timeout",
4373                    format!(
4374                        "module_id '{module_id}' answered health.check after {:?}",
4375                        self.health_probe_timeout
4376                    ),
4377                )?])
4378            }
4379            Ok(Err(_)) => Ok(vec![control_error_frame(
4380                &frame,
4381                "target_unavailable",
4382                "health.check waiter was canceled before the module responded",
4383            )?]),
4384            Err(_) => Ok(vec![control_error_frame(
4385                &frame,
4386                "module_timeout",
4387                format!(
4388                    "module_id '{module_id}' did not answer health.check within {:?}",
4389                    self.health_probe_timeout
4390                ),
4391            )?]),
4392        }
4393    }
4394
4395    fn supervisor_status(
4396        &self,
4397        module_id: &str,
4398        corr: u64,
4399    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
4400        self.supervisor
4401            .get(module_id)
4402            .map(|module| {
4403                let warming = module.is_warming_for_control("status").map_err(|err| {
4404                    RouterError::backend(
4405                        0,
4406                        corr,
4407                        format!(
4408                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
4409                        ),
4410                    )
4411                })?;
4412                module.status_for_control("status").map_err(|err| {
4413                    RouterError::backend(
4414                        0,
4415                        corr,
4416                        format!(
4417                            "failed to read supervisor status for module_id '{module_id}': {err}"
4418                        ),
4419                    )
4420                }).map(|status| (status, warming))
4421            })
4422            .transpose()
4423    }
4424
4425    fn guard_module_control_op(
4426        &self,
4427        frame: &Frame,
4428        module_id: &str,
4429        op: &str,
4430    ) -> Result<Option<Frame>, RouterError> {
4431        if self.module_grants_op(module_id, op, frame.header.corr)? {
4432            return Ok(None);
4433        }
4434
4435        Ok(Some(control_error_frame(
4436            frame,
4437            "op_not_allowed",
4438            format!("module_id '{module_id}' did not grant control op '{op}'"),
4439        )?))
4440    }
4441
4442    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
4443        let Some(registration) = self
4444            .registry
4445            .get_module(module_id)
4446            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
4447        else {
4448            return Ok(false);
4449        };
4450        Ok(module_registration_grants_op(&registration.control_ops, op))
4451    }
4452
4453    fn handle_status_update(
4454        &self,
4455        endpoint: ModuleEndpointId,
4456        frame: Frame,
4457    ) -> Result<Vec<Frame>, RouterError> {
4458        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
4459            Ok(update) => update,
4460            Err(err) => {
4461                // Forward-compat: a newer module may push a channel-0 op this subc
4462                // version doesn't know. The control contract says unknown push ops
4463                // are IGNORED, never answered with an error. Only a malformed body
4464                // for an op we DO know is a real error worth surfacing.
4465                if is_known_module_push_op(&frame.body) {
4466                    return Ok(vec![control_error_frame(
4467                        &frame,
4468                        "invalid_control_body",
4469                        format!("malformed module control push body: {err}"),
4470                    )?]);
4471                }
4472                return Ok(Vec::new());
4473            }
4474        };
4475
4476        match update {
4477            ModuleControlPush::RouteStatus {
4478                route_channel,
4479                route_epoch,
4480                status,
4481            } => {
4482                self.forwarding
4483                    .cache_status(endpoint, route_channel, route_epoch, status)
4484                    .map_err(RouterError::Forwarding)?;
4485            }
4486        }
4487        Ok(Vec::new())
4488    }
4489
4490    fn handle_route_poll(
4491        &self,
4492        ctx: &RouteCtx,
4493        frame: Frame,
4494        route_channel: u16,
4495        route_epoch: u32,
4496        kind: PollKind,
4497    ) -> Result<Vec<Frame>, RouterError> {
4498        let snapshot = self
4499            .forwarding
4500            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
4501            .map_err(RouterError::Forwarding)?;
4502        let response = match (kind, snapshot) {
4503            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
4504                ClientControlResponse::RoutePoll {
4505                    route_channel,
4506                    route_epoch,
4507                    status,
4508                    live: None,
4509                }
4510            }
4511            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
4512                route_channel,
4513                route_epoch,
4514                status: None,
4515                live: None,
4516            },
4517            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
4518                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
4519                // is what makes reporting `true` correct rather than a
4520                // confident guess. `process_live` returns None only when the
4521                // module id has no supervisor snapshot at all -- an
4522                // externally-started module the daemon did not spawn -- and
4523                // for those the supervisor has no opinion to offer, ever. It
4524                // is never None for a supervised module in an unknown state:
4525                // a supervised module always has a snapshot, and the answer
4526                // comes from `state == Running && process_alive`.
4527                //
4528                // The route is Bound, so the module completed a HELLO on a
4529                // live connection; "the process this route points at is
4530                // running" is therefore attested by the binding rather than
4531                // assumed. Reporting `false` for an unsupervised module would
4532                // be the actual lie -- it would tell a client its healthy
4533                // route is dead because the daemon does not manage the
4534                // process.
4535                //
4536                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
4537                // module whose liveness is genuinely unknown, e.g. a snapshot
4538                // that has not been populated yet -- THIS DEFAULT BECOMES
4539                // WRONG and must split: unsupervised stays true, unknown
4540                // becomes null so the client can tell the two apart. The
4541                // response field is already `Option<bool>`, so the wire can
4542                // carry that distinction today.
4543                let live = self
4544                    .process_liveness
4545                    .as_ref()
4546                    .and_then(|source| source.process_live(&module_id))
4547                    .unwrap_or(true);
4548                ClientControlResponse::RoutePoll {
4549                    route_channel,
4550                    route_epoch,
4551                    status: None,
4552                    live: Some(live),
4553                }
4554            }
4555            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
4556                route_channel,
4557                route_epoch,
4558                status: None,
4559                live: Some(false),
4560            },
4561        };
4562
4563        Ok(vec![control_response_body_frame(
4564            &frame,
4565            &response,
4566            "ClientControlResponse::RoutePoll",
4567        )?])
4568    }
4569
4570    pub(crate) fn observe_module_control_completion(
4571        &self,
4572        completion: ModuleControlRpcCompletion,
4573    ) -> bool {
4574        match completion {
4575            ModuleControlRpcCompletion::Unknown => false,
4576            ModuleControlRpcCompletion::Settled => true,
4577            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
4578                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
4579                info!(
4580                    module_id = %module_id,
4581                    latency_ms,
4582                    "late health.check answer proves the module is alive"
4583                );
4584                match self
4585                    .supervisor
4586                    .record_late_health_answer(&module_id, latency_ms)
4587                {
4588                    Ok(true) => {}
4589                    Ok(false) => debug!(
4590                        module_id = %module_id,
4591                        latency_ms,
4592                        "late health.check answer has no active supervisor snapshot"
4593                    ),
4594                    Err(err) => warn!(
4595                        module_id = %module_id,
4596                        latency_ms,
4597                        error = %err,
4598                        "failed to record late health.check answer"
4599                    ),
4600                }
4601                true
4602            }
4603        }
4604    }
4605
4606    /// Decide whether a failure while settling a relayed `route.bind` belongs to
4607    /// the module connection whose frame is being handled, or to the client that
4608    /// relay was opened for.
4609    ///
4610    /// This runs on the MODULE connection's frame handler, where returning `Err`
4611    /// ends that connection -- and a module connection carries every client's
4612    /// routes to that module, so ending it costs the whole fleet its tools.
4613    /// `ConnectionClosing` carries the id of the connection that is closing, and
4614    /// when that id is a CLIENT's, the condition is entirely about that one
4615    /// client's route.open. A client-scoped condition has no authority over a
4616    /// shared module connection, so it is logged and the single relay is dropped:
4617    /// the client is going away, and `complete_pending_relay` already removed the
4618    /// relay before failing, so there is nothing left to settle. Anything that
4619    /// relay still reserved is released by that client's own connection teardown,
4620    /// which is already under way -- that is what "closing" means.
4621    ///
4622    /// Every other failure is a statement about THIS connection and stays fatal:
4623    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
4624    /// id in `ConnectionClosing` all mean this connection cannot keep serving
4625    /// frames correctly.
4626    fn refuse_to_end_module_connection_for_a_client(
4627        &self,
4628        module_connection_id: ConnectionId,
4629        corr: u64,
4630        err: ForwardingError,
4631    ) -> Result<(), RouterError> {
4632        if let ForwardingError::ConnectionClosing { connection_id } = err {
4633            if connection_id != module_connection_id {
4634                warn!(
4635                    module_connection_id = module_connection_id.get(),
4636                    client_connection_id = connection_id.get(),
4637                    corr,
4638                    "dropping a route.bind response for a closing client; the module connection keeps serving"
4639                );
4640                return Ok(());
4641            }
4642        }
4643        Err(RouterError::Forwarding(err))
4644    }
4645
4646    fn handle_module_relay_response(
4647        &self,
4648        connection_id: ConnectionId,
4649        frame: Frame,
4650    ) -> Result<Vec<Frame>, RouterError> {
4651        let mut secondary_error = None;
4652        let outcome = match frame.header.ty {
4653            FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
4654                Ok(probe) if probe.op == "route.bind" => {
4655                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
4656                        Ok(ModuleControlResponse::RouteBindAck {}) => {
4657                            RouteBindRelayOutcome::Accepted
4658                        }
4659                        Ok(other) => {
4660                            let message =
4661                                format!("route.bind response carried unexpected body: {other:?}");
4662                            secondary_error = Some(control_error_frame(
4663                                &frame,
4664                                "invalid_control_body",
4665                                message.clone(),
4666                            )?);
4667                            RouteBindRelayOutcome::ModuleGone(message)
4668                        }
4669                        Err(err) => {
4670                            let message = format!("malformed route.bind response body: {err}");
4671                            secondary_error = Some(control_error_frame(
4672                                &frame,
4673                                "invalid_control_body",
4674                                message.clone(),
4675                            )?);
4676                            RouteBindRelayOutcome::ModuleGone(message)
4677                        }
4678                    }
4679                }
4680                Ok(probe) => {
4681                    let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
4682                    {
4683                        Ok(response) => ModuleControlRpcOutcome::Response(response),
4684                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
4685                            "malformed {} response body: {err}",
4686                            probe.op
4687                        )),
4688                    };
4689                    let completion = self
4690                        .forwarding
4691                        .complete_module_control_rpc(
4692                            connection_id,
4693                            frame.header.corr,
4694                            Some(&probe.op),
4695                            outcome,
4696                        )
4697                        .map_err(RouterError::Forwarding)?;
4698                    if !self.observe_module_control_completion(completion) {
4699                        debug!(
4700                            connection_id = connection_id.get(),
4701                            corr = frame.header.corr,
4702                            op = %probe.op,
4703                            "dropping late or unknown module-control RPC response"
4704                        );
4705                    }
4706                    return Ok(Vec::new());
4707                }
4708                Err(err) => {
4709                    if let Some(expected_op) = self
4710                        .forwarding
4711                        .pending_module_control_op(connection_id, frame.header.corr)
4712                        .map_err(RouterError::Forwarding)?
4713                    {
4714                        let completion = self
4715                            .forwarding
4716                            .complete_module_control_rpc(
4717                                connection_id,
4718                                frame.header.corr,
4719                                None,
4720                                ModuleControlRpcOutcome::MalformedResponse(format!(
4721                                    "malformed {expected_op} response body: {err}"
4722                                )),
4723                            )
4724                            .map_err(RouterError::Forwarding)?;
4725                        if !self.observe_module_control_completion(completion) {
4726                            debug!(
4727                                connection_id = connection_id.get(),
4728                                corr = frame.header.corr,
4729                                "dropping late malformed module-control RPC response"
4730                            );
4731                        }
4732                        return Ok(Vec::new());
4733                    }
4734                    let message = format!("malformed route.bind response body: {err}");
4735                    secondary_error = Some(control_error_frame(
4736                        &frame,
4737                        "invalid_control_body",
4738                        message.clone(),
4739                    )?);
4740                    RouteBindRelayOutcome::ModuleGone(message)
4741                }
4742            },
4743            FrameType::Error => {
4744                if self
4745                    .forwarding
4746                    .pending_module_control_op(connection_id, frame.header.corr)
4747                    .map_err(RouterError::Forwarding)?
4748                    .is_some()
4749                {
4750                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
4751                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
4752                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
4753                            "malformed module-control ERROR body: {err}"
4754                        )),
4755                    };
4756                    let completion = self
4757                        .forwarding
4758                        .complete_module_control_rpc(
4759                            connection_id,
4760                            frame.header.corr,
4761                            None,
4762                            outcome,
4763                        )
4764                        .map_err(RouterError::Forwarding)?;
4765                    if !self.observe_module_control_completion(completion) {
4766                        debug!(
4767                            connection_id = connection_id.get(),
4768                            corr = frame.header.corr,
4769                            "dropping late or unknown module-control RPC error"
4770                        );
4771                    }
4772                    return Ok(Vec::new());
4773                }
4774                match serde_json::from_slice::<ErrorBody>(&frame.body) {
4775                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
4776                    Err(err) => {
4777                        let message = format!("malformed route.bind ERROR body: {err}");
4778                        secondary_error = Some(control_error_frame(
4779                            &frame,
4780                            "invalid_control_body",
4781                            message.clone(),
4782                        )?);
4783                        RouteBindRelayOutcome::ModuleGone(message)
4784                    }
4785                }
4786            }
4787            ty => {
4788                return Ok(vec![control_error_frame(
4789                    &frame,
4790                    "unsupported_control_frame",
4791                    format!("unsupported module channel-0 frame {ty:?}"),
4792                )?])
4793            }
4794        };
4795
4796        let settled =
4797            self.forwarding
4798                .complete_pending_relay(connection_id, frame.header.corr, outcome);
4799        let completion = match settled {
4800            Ok(completion) => completion,
4801            Err(err) => {
4802                self.refuse_to_end_module_connection_for_a_client(
4803                    connection_id,
4804                    frame.header.corr,
4805                    err,
4806                )?;
4807                return Ok(secondary_error.into_iter().collect());
4808            }
4809        };
4810        if let Some(target) = completion.abandoned.as_ref() {
4811            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
4812        }
4813        if !completion.settled {
4814            debug!(
4815                connection_id = connection_id.get(),
4816                corr = frame.header.corr,
4817                frame_type = ?frame.header.ty,
4818                "dropping late or unknown route.bind relay response"
4819            );
4820        }
4821        Ok(secondary_error.into_iter().collect())
4822    }
4823
4824    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
4825        debug!(connection_id = connection_id.get(), "handling GOODBYE");
4826        let registrations = self
4827            .deregister_connection(connection_id)
4828            .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
4829        let released_routes = self
4830            .forwarding
4831            .cleanup_connection(connection_id)
4832            .map_err(RouterError::Forwarding)?;
4833        self.emit_route_goodbyes(released_routes);
4834        // Notify only after forwarding teardown completes (see cleanup_connection).
4835        if !registrations.is_empty() {
4836            crate::supervise::notify_registration_release();
4837        }
4838        Ok(Vec::new())
4839    }
4840}
4841
4842impl Default for ControlHandler {
4843    fn default() -> Self {
4844        Self::new(Arc::new(Registry::default()))
4845    }
4846}
4847
4848impl crate::supervise::SwapPromotionObserver for ControlHandler {
4849    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
4850        self.apply_registration_capabilities(registration);
4851    }
4852}
4853
4854fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
4855    CapabilityRequirementStatus {
4856        consumer: status.consumer,
4857        capability: status.capability,
4858        need: match status.need {
4859            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
4860            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
4861        },
4862        verdict: status.verdict.as_str().to_string(),
4863        episode_seq: status.episode_seq,
4864        config_satisfiable: status.config_satisfiable,
4865        runtime_available: status.runtime_available,
4866        detail: status.detail,
4867    }
4868}
4869
4870fn append_capability_problem_detail(
4871    detail: Option<String>,
4872    capability_detail: Option<String>,
4873) -> Option<String> {
4874    match (detail, capability_detail) {
4875        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
4876        (Some(detail), None) => Some(detail),
4877        (None, Some(capability_detail)) => Some(capability_detail),
4878        (None, None) => None,
4879    }
4880}
4881
4882fn subc_ops() -> Vec<String> {
4883    SUBC_CONTROL_OPS
4884        .iter()
4885        .map(|op| (*op).to_string())
4886        .collect()
4887}
4888
4889fn module_subc_ops() -> Vec<String> {
4890    SUBC_CONTROL_OPS
4891        .iter()
4892        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
4893        .map(|op| (*op).to_string())
4894        .collect()
4895}
4896
4897#[cfg(test)]
4898fn module_baseline_control_ops() -> Vec<String> {
4899    MODULE_BASELINE_CONTROL_OPS
4900        .iter()
4901        .map(|op| (*op).to_string())
4902        .collect()
4903}
4904
4905fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
4906    let mut seen = HashSet::new();
4907    let mut effective = Vec::new();
4908    for op in MODULE_BASELINE_CONTROL_OPS {
4909        if seen.insert((*op).to_string()) {
4910            effective.push((*op).to_string());
4911        }
4912    }
4913    for op in declared.unwrap_or_default() {
4914        if seen.insert(op.clone()) {
4915            effective.push(op);
4916        }
4917    }
4918    effective
4919}
4920
4921fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
4922    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
4923}
4924
4925fn target_module_id(target: &RouteTarget) -> &str {
4926    match target {
4927        RouteTarget::ToolProvider { module_id }
4928        | RouteTarget::ManagementSurface { module_id }
4929        | RouteTarget::InternalService { module_id, .. } => module_id,
4930    }
4931}
4932
4933fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
4934    roles.iter().any(|role| match (target, role) {
4935        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
4936        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
4937        (
4938            RouteTarget::InternalService { service_id, .. },
4939            ProviderRole::InternalService {
4940                service_id: provided,
4941                ..
4942            },
4943        ) => service_id == provided,
4944        _ => false,
4945    })
4946}
4947
4948fn is_routable_role(role: &ProviderRole) -> bool {
4949    matches!(
4950        role,
4951        ProviderRole::ToolProvider { .. }
4952            | ProviderRole::ManagementSurface { .. }
4953            | ProviderRole::InternalService { .. }
4954    )
4955}
4956
4957#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4958enum ControlRequestBodyError {
4959    UnknownOp,
4960    InvalidBody,
4961}
4962
4963#[derive(Debug, Deserialize)]
4964struct ControlOpProbe {
4965    op: String,
4966}
4967
4968/// Channel-0 push ops this subc version understands. A push whose `op` is not in
4969/// this set is treated as a forward-compat unknown and ignored rather than errored.
4970const MODULE_PUSH_OPS: &[&str] = &["route.status"];
4971
4972fn is_known_module_push_op(body: &[u8]) -> bool {
4973    serde_json::from_slice::<ControlOpProbe>(body)
4974        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
4975        .unwrap_or(false)
4976}
4977
4978fn is_known_module_request_op(body: &[u8]) -> bool {
4979    serde_json::from_slice::<ControlOpProbe>(body)
4980        .map(|probe| MODULE_TO_SUBC_CONTROL_OPS.contains(&probe.op.as_str()))
4981        .unwrap_or(false)
4982}
4983
4984fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
4985    debug!(
4986        op = %op,
4987        connection_id = connection_id.get(),
4988        corr,
4989        "control dispatch"
4990    );
4991}
4992
4993fn log_slow_control_dispatch(
4994    dispatch_started_at: Option<StdInstant>,
4995    op: &'static str,
4996    connection_id: ConnectionId,
4997    corr: u64,
4998) {
4999    let Some(dispatch_started_at) = dispatch_started_at else {
5000        return;
5001    };
5002    let elapsed = dispatch_started_at.elapsed();
5003    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
5004        warn!(
5005            op = %op,
5006            connection_id = connection_id.get(),
5007            corr,
5008            elapsed_ms = elapsed.as_millis() as u64,
5009            "slow control dispatch"
5010        );
5011    }
5012}
5013
5014fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
5015    match request {
5016        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
5017        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
5018        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
5019        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
5020        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
5021        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
5022        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
5023        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
5024        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
5025        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
5026        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
5027        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
5028        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
5029        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
5030        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
5031        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
5032        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
5033        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
5034        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
5035    }
5036}
5037
5038fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
5039    match request {
5040        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
5041        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
5042    }
5043}
5044
5045fn parse_client_control_request(
5046    body: &[u8],
5047) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
5048    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
5049        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5050            Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5051                ControlRequestBodyError::InvalidBody
5052            }
5053            Ok(_) => ControlRequestBodyError::UnknownOp,
5054            Err(_) => ControlRequestBodyError::InvalidBody,
5055        };
5056        (err, classification)
5057    })
5058}
5059
5060fn parse_module_control_request_from_module(
5061    body: &[u8],
5062) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
5063    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
5064        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5065            Ok(probe) if MODULE_TO_SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5066                ControlRequestBodyError::InvalidBody
5067            }
5068            Ok(_) => ControlRequestBodyError::UnknownOp,
5069            Err(_) => ControlRequestBodyError::InvalidBody,
5070        };
5071        (err, classification)
5072    })
5073}
5074
5075#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
5076enum ProviderRoleKind {
5077    ToolProvider,
5078    PipelineStage,
5079    ManagementSurface,
5080    InternalService,
5081}
5082
5083fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
5084    match role {
5085        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
5086        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
5087        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
5088        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
5089    }
5090}
5091
5092fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
5093    roles.iter().map(provider_role_kind).collect()
5094}
5095
5096/// Return whether a catalog change can create a newly violating live route.
5097/// Removing an attested claim is intentionally excluded: it makes fewer routes
5098/// forbidden and therefore must leave the existing route census untouched.
5099fn capability_census_trigger(
5100    old: Option<&CapabilityDeclarations>,
5101    new: Option<&CapabilityDeclarations>,
5102) -> bool {
5103    let old_provides = old
5104        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
5105        .unwrap_or_default();
5106    let old_denies = old
5107        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
5108        .unwrap_or_default();
5109    let new = new.cloned().unwrap_or(CapabilityDeclarations {
5110        provides: Vec::new(),
5111        requires: Vec::new(),
5112        must_never_reach: Vec::new(),
5113    });
5114
5115    new.provides
5116        .iter()
5117        .any(|capability| !old_provides.contains(capability))
5118        || new
5119            .must_never_reach
5120            .iter()
5121            .any(|capability| !old_denies.contains(capability))
5122}
5123
5124/// Find the first capability an attested opener denies that an attested target
5125/// claims. Both manifests are live registry records, never cached or client data.
5126fn denied_capability<'a>(
5127    opening_manifest: &'a ModuleManifest,
5128    target_manifest: &ModuleManifest,
5129) -> Option<&'a str> {
5130    let opening_capabilities = opening_manifest.capabilities.as_ref()?;
5131    let target_capabilities = target_manifest.capabilities.as_ref()?;
5132    opening_capabilities
5133        .must_never_reach
5134        .iter()
5135        .find(|denied| {
5136            target_capabilities
5137                .provides
5138                .iter()
5139                .any(|provided| provided == *denied)
5140        })
5141        .map(String::as_str)
5142}
5143
5144fn catalog_update_frozen_field_message(
5145    registered: &ModuleManifest,
5146    provides: &[ProviderRole],
5147) -> Option<String> {
5148    let old_has_provides = !registered.provides.is_empty();
5149    let new_has_provides = !provides.is_empty();
5150    if old_has_provides != new_has_provides {
5151        return Some(format!(
5152            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
5153            registered.module_id
5154        ));
5155    }
5156
5157    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
5158        return Some(format!(
5159            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
5160            registered.module_id
5161        ));
5162    }
5163
5164    let registered_concurrency = manifest_concurrency(registered);
5165    let mut candidate = registered.clone();
5166    candidate.provides = provides.to_vec();
5167    let candidate_concurrency = manifest_concurrency(&candidate);
5168    if candidate_concurrency != registered_concurrency {
5169        return Some(format!(
5170            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
5171            registered.module_id, registered_concurrency, candidate_concurrency
5172        ));
5173    }
5174
5175    // control_ops live beside the manifest in the HELLO body, not inside
5176    // ModuleManifest, so a provides-only catalog.update cannot change them.
5177    None
5178}
5179
5180fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
5181    manifest.provides.iter().any(is_routable_role)
5182}
5183
5184/// Returns the routable-provider concurrency subc should enforce for this manifest.
5185///
5186/// ToolProvider and ManagementSurface store their delivery concurrency directly.
5187/// InternalService has no role-specific concurrency field, so it retains the
5188/// existing ModuleManaged default for backward compatibility.
5189fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
5190    manifest
5191        .provides
5192        .iter()
5193        .find_map(|provider| match provider {
5194            ProviderRole::ToolProvider { concurrency, .. }
5195            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
5196            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
5197        })
5198        .unwrap_or(Concurrency::ModuleManaged)
5199}
5200
5201/// True when the manifest carries a ManagementSurface role whose concurrency
5202/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
5203/// bytes because the typed manifest deliberately erases that distinction: the
5204/// default exists for wire compatibility, and this probe exists so the default
5205/// stays observable. Any parse irregularity returns false -- the caller only
5206/// logs, and a malformed body already failed registration upstream.
5207fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
5208    let has_management_surface = manifest
5209        .provides
5210        .iter()
5211        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
5212    if !has_management_surface {
5213        return false;
5214    }
5215    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
5216        return false;
5217    };
5218    let Some(provides) = raw
5219        .get("manifest")
5220        .and_then(|manifest| manifest.get("provides"))
5221        .and_then(serde_json::Value::as_array)
5222    else {
5223        return false;
5224    };
5225    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
5226    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
5227    // against the management_surface_manifest_without_concurrency golden, not
5228    // recalled (the externally-tagged guess was this function's first bug).
5229    provides.iter().any(|role| {
5230        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
5231            && role.get("concurrency").is_none()
5232    })
5233}
5234
5235fn negotiate_version(peer_version: u8) -> Result<u8, String> {
5236    if peer_version != PROTOCOL_VERSION {
5237        return Err(format!(
5238            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
5239        ));
5240    }
5241    Ok(PROTOCOL_VERSION)
5242}
5243
5244fn pong(frame: &Frame) -> Result<Frame, RouterError> {
5245    Frame::build_with_version(
5246        response_version(frame),
5247        FrameType::Pong,
5248        frame.header.flags,
5249        0,
5250        0,
5251        frame.header.corr,
5252        Vec::new(),
5253    )
5254    .map_err(RouterError::FrameBuild)
5255}
5256
5257fn control_error_frame(
5258    frame: &Frame,
5259    code: &'static str,
5260    message: impl Into<String>,
5261) -> Result<Frame, RouterError> {
5262    control_error_body_frame(
5263        frame,
5264        ErrorBody {
5265            code: code.to_string(),
5266            message: message.into(),
5267            detail: None,
5268        },
5269    )
5270}
5271
5272fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
5273    let body = serde_json::to_vec(&error).map_err(|err| {
5274        RouterError::backend(
5275            0,
5276            frame.header.corr,
5277            format!("failed to encode control ERROR: {err}"),
5278        )
5279    })?;
5280
5281    Frame::build_with_version(
5282        response_version(frame),
5283        FrameType::Error,
5284        control_flags(),
5285        0,
5286        0,
5287        frame.header.corr,
5288        body,
5289    )
5290    .map_err(RouterError::FrameBuild)
5291}
5292
5293fn control_response_body_frame<T: Serialize>(
5294    frame: &Frame,
5295    reply: &T,
5296    label: &'static str,
5297) -> Result<Frame, RouterError> {
5298    let body = serde_json::to_vec(reply).map_err(|err| {
5299        RouterError::backend(
5300            0,
5301            frame.header.corr,
5302            format!("failed to encode {label}: {err}"),
5303        )
5304    })?;
5305
5306    Frame::build_with_version(
5307        response_version(frame),
5308        FrameType::Response,
5309        control_flags(),
5310        0,
5311        0,
5312        frame.header.corr,
5313        body,
5314    )
5315    .map_err(RouterError::FrameBuild)
5316}
5317
5318/// Map a forwarding failure to the wire code a client sees.
5319///
5320/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
5321/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
5322/// chosen here decides whether a caller retries or gives up.
5323///
5324/// That makes attribution the load-bearing property, not merely having a code. A
5325/// permanent fault published as a retryable one produces a fleet-wide retry storm
5326/// against something that can never recover; a transient fault published as
5327/// permanent gives up on work that would have succeeded. Both look correct in a
5328/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
5329/// the mapping per variant rather than merely asserting that some code exists.
5330///
5331/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
5332/// two codes on the same side of the boundary passes it. Measured rather than
5333/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
5334/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
5335/// a test named for something else that happens to assert the string.
5336///
5337/// That accidental coverage is deliberately left alone rather than promoted to a
5338/// named test, because it guards a property this function does not promise.
5339/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
5340/// specific code within a class, so identity is free to change and only the
5341/// partition is a contract. Splitting it out would assert a guarantee nothing
5342/// depends on — and a suite that promises more than the code does is the harder
5343/// thing to correct later, because the next reader cannot tell which assertions
5344/// are load-bearing.
5345///
5346/// Pin identity here the moment a consumer branches on a specific code.
5347fn forwarding_error_code(err: &ForwardingError) -> &'static str {
5348    match err {
5349        ForwardingError::NoModuleConnection => "target_unavailable",
5350        ForwardingError::ModuleReloading { .. } => "module_reloading",
5351        ForwardingError::ClientRouteChannelExhausted { .. }
5352        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
5353        ForwardingError::StaleModuleEndpoint
5354        | ForwardingError::UnknownReservation { .. }
5355        | ForwardingError::ConnectionClosing { .. }
5356        | ForwardingError::ClientEgressClosed { .. }
5357        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
5358        // Only a swap candidate's registration can produce this, and it means
5359        // exactly what a second active HELLO for a live id means.
5360        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
5361        ForwardingError::RelayCorrelationExhausted
5362        | ForwardingError::RouteOpenBuild(_)
5363        | ForwardingError::Poisoned => "forwarding_error",
5364    }
5365}
5366
5367fn response_version(frame: &Frame) -> u8 {
5368    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
5369        frame.header.ver
5370    } else {
5371        PROTOCOL_VERSION
5372    }
5373}
5374
5375fn control_flags() -> Flags {
5376    Flags::new(false, Priority::Passive, false)
5377}
5378
5379/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
5380/// channel. The target is the module (a client never saw the route), so this
5381/// takes the module path: delivered late rather than dropped when the module's
5382/// queue is momentarily full, and never closing its connection.
5383fn send_goodbye_target_best_effort(
5384    counters: &DaemonCounters,
5385    target: &GoodbyeTarget,
5386    context: &'static str,
5387) {
5388    let Ok(frame) = Frame::build_with_version(
5389        target.negotiated_ver,
5390        FrameType::Goodbye,
5391        control_flags(),
5392        target.channel,
5393        target.epoch,
5394        0,
5395        Vec::new(),
5396    ) else {
5397        return;
5398    };
5399    crate::forwarding::send_module_route_goodbye(
5400        counters,
5401        &target.sink,
5402        frame,
5403        target.module_id.as_deref(),
5404        context,
5405    );
5406}
5407
5408pub(crate) fn send_route_control_pushes(
5409    forwarding: &ForwardingTable,
5410    routes: Vec<EndpointRoute>,
5411    push: ClientControlPush,
5412) {
5413    let body = match serde_json::to_vec(&push) {
5414        Ok(body) => body,
5415        Err(err) => {
5416            warn!(error = %err, "failed to serialize route lifecycle control PUSH");
5417            return;
5418        }
5419    };
5420    let mut targets = Vec::new();
5421    for route in routes {
5422        let target = route.goodbye_target;
5423        if let Some(existing) = targets
5424            .iter()
5425            .find(|existing: &&GoodbyeTarget| existing.connection_id == target.connection_id)
5426        {
5427            debug_assert_eq!(
5428                existing.negotiated_ver, target.negotiated_ver,
5429                "one connection cannot negotiate multiple frame versions"
5430            );
5431            continue;
5432        }
5433        targets.push(target);
5434    }
5435    for target in targets {
5436        let frame = match Frame::build_with_version(
5437            target.negotiated_ver,
5438            FrameType::Push,
5439            control_flags(),
5440            0,
5441            0,
5442            0,
5443            body.clone(),
5444        ) {
5445            Ok(frame) => frame,
5446            Err(err) => {
5447                warn!(
5448                    route_channel = target.channel,
5449                    error = %err,
5450                    "failed to build route lifecycle control PUSH frame"
5451                );
5452                continue;
5453            }
5454        };
5455        if let Err(err) = target.sink.try_send(frame) {
5456            if target.close_on_delivery_failure() {
5457                warn!(
5458                    target_connection_id = target.connection_id.get(),
5459                    route_channel = target.channel,
5460                    error = %err,
5461                    "route lifecycle control PUSH was not delivered to client; closing target connection"
5462                );
5463                let _ = forwarding.escalate_client_delivery_failure(
5464                    target.connection_id,
5465                    target.channel,
5466                    target.epoch,
5467                    CloseReason::new(
5468                        "route_lifecycle_push_delivery_failed",
5469                        format!(
5470                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
5471                            target.channel
5472                        ),
5473                    ),
5474                    crate::forwarding::UndeliveredFrame {
5475                        module_id: target.module_id.as_deref(),
5476                        sink: &target.sink,
5477                    },
5478                );
5479            }
5480        }
5481    }
5482}
5483
5484#[cfg(test)]
5485mod tests {
5486    use std::{
5487        collections::BTreeMap,
5488        fmt,
5489        path::PathBuf,
5490        sync::{Arc, Mutex},
5491        time::Duration,
5492    };
5493
5494    use serde_json::{json, Value};
5495    use subc_protocol::{
5496        manifest::{
5497            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
5498            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
5499        },
5500        session::HealthStatus,
5501        FrameType,
5502    };
5503
5504    use super::*;
5505    use crate::{
5506        forwarding::{DataRoute, DataRouteState},
5507        registry::ChannelState,
5508        router::FrameSink,
5509        stderr_tail::DEFAULT_MAX_LINE_BYTES,
5510        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
5511        test_support::TestTempDir,
5512        RouteCtx, Router,
5513    };
5514    use tokio::{
5515        sync::mpsc,
5516        time::{sleep, Instant},
5517    };
5518    use tracing::{
5519        field::{Field, Visit},
5520        Event, Subscriber,
5521    };
5522    use tracing_subscriber::{layer::Context, prelude::*, Layer};
5523
5524    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
5525    ///
5526    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
5527    /// is only populated for `tests/*.rs` integration test binaries -- this file
5528    /// compiles as part of the library target, which gets neither. This test's
5529    /// own executable path is `<target-dir>/<profile>/deps/subc_core-<hash>`,
5530    /// and the sibling binary lives two directories up at
5531    /// `<target-dir>/<profile>/fake-aft-stub`.
5532    ///
5533    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
5534    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
5535    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
5536    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
5537    /// `NotFound`, which reads as a broken test rather than an unbuilt
5538    /// dependency -- so state the cause and the remedy instead. Deliberately a
5539    /// panic and not a silent skip: a test that quietly passes when it could not
5540    /// run is worse than one that fails, because it reports health it never
5541    /// verified.
5542    fn fake_aft_stub_path() -> PathBuf {
5543        let mut path = std::env::current_exe().expect("current_exe available in tests");
5544        path.pop(); // .../deps/
5545        path.pop(); // .../<profile>/
5546        path.push(if cfg!(windows) {
5547            "fake-aft-stub.exe"
5548        } else {
5549            "fake-aft-stub"
5550        });
5551        assert!(
5552            path.exists(),
5553            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
5554             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
5555            path.display()
5556        );
5557        path
5558    }
5559
5560    /// Whether clients retry `code` in place: the predicate itself, never a copy
5561    /// of its set. A copied list breaks silently when a code is added to or
5562    /// removed from the real one, and a stale copy here would let exactly the
5563    /// failure this test exists to catch pass.
5564    fn client_retries(code: &str) -> bool {
5565        subc_protocol::error_codes::is_retryable_route_open(code)
5566    }
5567
5568    /// A code is not a label — clients branch on it, so publishing the wrong KIND
5569    /// of failure is worse than publishing none. A permanent fault dressed as
5570    /// retryable makes every client in the fleet retry forever against something
5571    /// that cannot recover; a transient fault dressed as permanent abandons work
5572    /// that would have succeeded.
5573    ///
5574    /// Asserting "a code exists" cannot catch either, because the string is free
5575    /// to say anything. This enumerates every variant and pins which side of the
5576    /// retry boundary it lands on, so a new variant must be classified here
5577    /// deliberately rather than inheriting whichever arm it was appended to.
5578    #[test]
5579    fn retryability_of_forwarding_codes_matches_the_failure() {
5580        // Transient by nature: the target is booting, reloading, or its endpoint
5581        // was swapped mid-flight. Retrying is how these resolve.
5582        let transient = [
5583            ForwardingError::NoModuleConnection,
5584            ForwardingError::ModuleReloading {
5585                module_id: "m".into(),
5586            },
5587            ForwardingError::StaleModuleEndpoint,
5588            ForwardingError::UnknownReservation {
5589                client_channel: 1,
5590                module_channel: 1,
5591            },
5592            ForwardingError::ConnectionClosing {
5593                connection_id: ConnectionId::new(1),
5594            },
5595            ForwardingError::ClientEgressClosed {
5596                connection_id: ConnectionId::new(1),
5597            },
5598            ForwardingError::ModuleEgressUnavailable {
5599                connection_id: ConnectionId::new(1),
5600            },
5601        ];
5602        for err in transient {
5603            let code = forwarding_error_code(&err);
5604            assert!(
5605                client_retries(code),
5606                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
5607            );
5608        }
5609
5610        // Not fixed by retrying. Channel and correlation exhaustion need the
5611        // caller to close routes, and a poisoned lock is a daemon that cannot
5612        // recover at all — the worst thing to advertise as retryable, since every
5613        // client would storm a daemon that will never answer.
5614        let permanent = [
5615            ForwardingError::ClientRouteChannelExhausted {
5616                connection_id: ConnectionId::new(1),
5617            },
5618            ForwardingError::ModuleRouteChannelExhausted {
5619                endpoint: ModuleEndpointId {
5620                    connection_id: ConnectionId::new(1),
5621                    generation: 1,
5622                },
5623            },
5624            ForwardingError::RelayCorrelationExhausted,
5625            ForwardingError::RouteOpenBuild("x".into()),
5626            ForwardingError::Poisoned,
5627        ];
5628        for err in permanent {
5629            let code = forwarding_error_code(&err);
5630            assert!(
5631                !client_retries(code),
5632                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
5633            );
5634        }
5635    }
5636
5637    /// The principal is the daemon's answer to "who is calling", and modules
5638    /// branch on it: aft gates bash on it, cerebellum gates browser control,
5639    /// plexus gates connector invocation. So a stamp is an authorization input in
5640    /// another process, not a label — and both possible answers SUCCEED, which is
5641    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
5642    /// first-party capability to something that never proved it; a supervised one
5643    /// stamped `Direct` silently strips a module of capability it is entitled to.
5644    ///
5645    /// Neither shows up in a test that only checks the bind succeeded. Before this
5646    /// test the only coverage was accidental —
5647    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
5648    /// stamped principal on its way past, so narrowing that wire-shape test to its
5649    /// stated subject would have deleted the last assertion on this value. It
5650    /// still asserts the stamp, which is now redundancy rather than the only
5651    /// guard: both fail under the same mutation, and this one names the reason.
5652    /// SCOPE: this handler's supervisor has spawned nothing, so
5653    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
5654    /// is unreachable here. Both assertions below are refusals, and a mutant that
5655    /// refuses everything would satisfy them.
5656    ///
5657    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
5658    /// spawns a supervised consumer, reads its live nonce, and asserts the module
5659    /// observed `principal.kind == "reserved"` carrying that module_id — verified
5660    /// at source rather than assumed, since a citation is a claim about another
5661    /// file and ages like one. Recorded because a harness that structurally
5662    /// cannot reach an arm reports "none" for that arm identically to one that
5663    /// covers it and found nothing.
5664    #[tokio::test]
5665    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
5666        let handler = ControlHandler::default();
5667        let frame =
5668            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
5669
5670        // Absent consumer_identity is the ordinary case: a human at a terminal, or
5671        // any process holding the connection file. Nothing was proved, so nothing
5672        // may be granted beyond the unattested floor.
5673        let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
5674        assert_eq!(
5675            stamped,
5676            Principal::Direct,
5677            "a caller that proved nothing must not be stamped as a supervised module"
5678        );
5679
5680        // A claimed module_id with a nonce no supervised child was given is a
5681        // forgery attempt, not a weaker caller: it must be REFUSED rather than
5682        // quietly demoted to Direct, or an impersonation attempt looks identical
5683        // to an ordinary unattested connection.
5684        let forged = handler
5685            .route_open_principal(
5686                &frame,
5687                Some(ConsumerIdentity {
5688                    module_id: "aft".to_string(),
5689                    launch_nonce: "not-a-real-nonce".to_string(),
5690                }),
5691            )
5692            .unwrap();
5693        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
5694        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
5695    }
5696
5697    /// The test above hands `route_open_principal` an identity it built itself,
5698    /// which proves the stamping rule and nothing about where the identity comes
5699    /// from. The real producer is a wire body, and the two are joined by a serde
5700    /// field name that nothing else asserts.
5701    ///
5702    /// That join fails quietly in one specific way: an unrecognised key is simply
5703    /// absent after parsing, so a renamed or misspelled `consumer_identity`
5704    /// yields `None` and every supervised module silently drops to `Direct`.
5705    /// Capability-wise that is the safe direction, but it surfaces far from its
5706    /// cause — as a module mysteriously refused bash — and it would pass every
5707    /// test that builds its own input.
5708    ///
5709    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
5710    /// would break every client the moment the daemon gains a field, trading a
5711    /// quiet demotion for a hard refusal on additive change. Asserting the join
5712    /// instead means a rename breaks a test here rather than the fleet.
5713    #[test]
5714    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
5715        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
5716        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
5717        let ClientControlRequest::RouteOpen {
5718            consumer_identity, ..
5719        } = parsed
5720        else {
5721            panic!("route.open body must parse as RouteOpen");
5722        };
5723        assert_eq!(
5724            consumer_identity,
5725            Some(ConsumerIdentity {
5726                module_id: "aft".to_string(),
5727                launch_nonce: "n".to_string(),
5728            }),
5729            "the wire field name must reach the value route_open_principal reads"
5730        );
5731    }
5732
5733    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
5734        ModuleManifest::builder(module_id, "0.1.0")
5735            .protocol_ver(protocol_ver)
5736            .provides(vec![ProviderRole::ToolProvider {
5737                tools: vec![Tool {
5738                    name: "read".to_string(),
5739                    description: None,
5740                    execution_mode: ExecutionMode::Pure,
5741                    schema: json!({"type": "object"}),
5742                }],
5743                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
5744                concurrency: Concurrency::ModuleManaged,
5745                emits_push: true,
5746                sub_supervises: true,
5747            }])
5748            .build()
5749    }
5750
5751    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
5752        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
5753    }
5754
5755    fn hello_frame_with_control_ops(
5756        module_id: &str,
5757        protocol_ver: u8,
5758        corr: u64,
5759        control_ops: Option<Vec<String>>,
5760    ) -> Frame {
5761        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
5762    }
5763
5764    fn hello_frame_with_nonce(
5765        module_id: &str,
5766        protocol_ver: u8,
5767        corr: u64,
5768        launch_nonce: Option<&str>,
5769    ) -> Frame {
5770        hello_frame_full(
5771            module_id,
5772            protocol_ver,
5773            corr,
5774            None,
5775            launch_nonce.map(ToOwned::to_owned),
5776        )
5777    }
5778
5779    fn hello_frame_full(
5780        module_id: &str,
5781        protocol_ver: u8,
5782        corr: u64,
5783        control_ops: Option<Vec<String>>,
5784        launch_nonce: Option<String>,
5785    ) -> Frame {
5786        let body = serde_json::to_vec(&ModuleHelloBody {
5787            manifest: manifest(module_id, protocol_ver),
5788            protocol_ver,
5789            control_ops,
5790            launch_nonce,
5791        })
5792        .unwrap();
5793        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
5794    }
5795
5796    fn non_routable_hello_frame_with_control_ops(
5797        module_id: &str,
5798        corr: u64,
5799        control_ops: Option<Vec<String>>,
5800    ) -> Frame {
5801        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
5802        manifest.provides.clear();
5803        let body = serde_json::to_vec(&ModuleHelloBody {
5804            manifest,
5805            protocol_ver: PROTOCOL_VERSION,
5806            control_ops,
5807            launch_nonce: None,
5808        })
5809        .unwrap();
5810        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
5811    }
5812
5813    fn capability_grammar_hello_frame(
5814        capabilities: Value,
5815        runtime_computed: Option<Value>,
5816        corr: u64,
5817    ) -> Frame {
5818        let mut body = serde_json::to_value(ModuleHelloBody {
5819            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
5820            protocol_ver: PROTOCOL_VERSION,
5821            control_ops: None,
5822            launch_nonce: None,
5823        })
5824        .expect("HELLO body serializes");
5825        body["manifest"]["capabilities"] = capabilities;
5826        if let Some(runtime_computed) = runtime_computed {
5827            body["runtime_computed"] = runtime_computed;
5828        }
5829        Frame::build(
5830            FrameType::Hello,
5831            control_flags(),
5832            0,
5833            0,
5834            corr,
5835            serde_json::to_vec(&body).expect("HELLO body reserializes"),
5836        )
5837        .expect("HELLO frame builds")
5838    }
5839
5840    fn channel_request(channel: u16, corr: u64) -> Frame {
5841        Frame::build(
5842            FrameType::Request,
5843            Flags::new(true, Priority::Interactive, false),
5844            channel,
5845            0,
5846            corr,
5847            b"opaque".to_vec(),
5848        )
5849        .unwrap()
5850    }
5851
5852    fn route_ctx(
5853        connection_id: ConnectionId,
5854    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
5855        let (tx, rx) = mpsc::channel(8);
5856        (
5857            RouteCtx {
5858                connection_id,
5859                egress: FrameSink::new(tx),
5860            },
5861            rx,
5862        )
5863    }
5864
5865    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
5866        serde_json::from_slice(&frame.body).unwrap()
5867    }
5868
5869    /// Register a module over a connection that has a sink and return the
5870    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
5871    /// module's own sink rather than returning it as a reply, so the ack is
5872    /// taken off `rx` here and whatever the test reads next is what followed it.
5873    async fn hello_via_sink(
5874        handler: &ControlHandler,
5875        ctx: &RouteCtx,
5876        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
5877        hello: Frame,
5878    ) -> Frame {
5879        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
5880        assert!(
5881            replies.is_empty(),
5882            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
5883        );
5884        let ack = rx
5885            .try_recv()
5886            .expect("HELLO_ACK is queued on the module sink")
5887            .frame;
5888        assert_eq!(ack.header.ty, FrameType::HelloAck);
5889        ack
5890    }
5891
5892    fn parse_error(frame: &Frame) -> Value {
5893        serde_json::from_slice(&frame.body).unwrap()
5894    }
5895
5896    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
5897        serde_json::from_slice(&frame.body).unwrap()
5898    }
5899
5900    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
5901        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
5902            route_channel,
5903            route_epoch: 0,
5904            kind,
5905        })
5906        .unwrap();
5907        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5908    }
5909
5910    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
5911        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
5912            module_id: module_id.to_string(),
5913        })
5914        .unwrap();
5915        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5916    }
5917
5918    fn route_open_frame(corr: u64, module_id: &str, project_root: TestTempDir) -> Frame {
5919        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
5920    }
5921
5922    fn route_open_frame_with_consumer_capabilities(
5923        corr: u64,
5924        module_id: &str,
5925        project_root: TestTempDir,
5926        consumer_capabilities: Option<Vec<String>>,
5927    ) -> Frame {
5928        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
5929            target: RouteTarget::ToolProvider {
5930                module_id: module_id.to_string(),
5931            },
5932            identity: BindIdentity::new(
5933                project_root.path().to_path_buf(),
5934                "unit".to_string(),
5935                "session".to_string(),
5936            ),
5937            consumer_identity: None,
5938            consumer_capabilities,
5939            admission_facts: None,
5940        })
5941        .unwrap();
5942        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5943    }
5944
5945    fn route_open_frame_with_admission_facts(
5946        corr: u64,
5947        module_id: &str,
5948        project_root: TestTempDir,
5949        consumer_identity: Option<subc_control::ConsumerIdentity>,
5950        facts: Option<Value>,
5951    ) -> Frame {
5952        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
5953            target: RouteTarget::ToolProvider {
5954                module_id: module_id.to_string(),
5955            },
5956            identity: BindIdentity::new(
5957                project_root.path().to_path_buf(),
5958                "unit".to_string(),
5959                format!("session-{corr}"),
5960            ),
5961            consumer_identity,
5962            consumer_capabilities: None,
5963            admission_facts: facts,
5964        })
5965        .unwrap();
5966        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5967    }
5968
5969    #[derive(Clone, Default)]
5970    struct EventCapture {
5971        events: Arc<Mutex<Vec<CapturedEvent>>>,
5972    }
5973
5974    #[derive(Clone, Debug)]
5975    struct CapturedEvent {
5976        target: String,
5977        fields: BTreeMap<String, String>,
5978    }
5979
5980    impl EventCapture {
5981        fn events(&self) -> Vec<CapturedEvent> {
5982            self.events.lock().unwrap().clone()
5983        }
5984    }
5985
5986    impl<S> Layer<S> for EventCapture
5987    where
5988        S: Subscriber,
5989    {
5990        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
5991            let mut visitor = EventFieldVisitor::default();
5992            event.record(&mut visitor);
5993            self.events.lock().unwrap().push(CapturedEvent {
5994                target: event.metadata().target().to_string(),
5995                fields: visitor.fields,
5996            });
5997        }
5998    }
5999
6000    #[derive(Default)]
6001    struct EventFieldVisitor {
6002        fields: BTreeMap<String, String>,
6003    }
6004
6005    impl Visit for EventFieldVisitor {
6006        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
6007            self.fields
6008                .insert(field.name().to_string(), format!("{value:?}"));
6009        }
6010    }
6011
6012    fn health_response(corr: u64, status: HealthStatus) -> Frame {
6013        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
6014            status,
6015            detail: Some("warming".to_string()),
6016            metrics: Some(json!({"queue_depth": 3})),
6017        })
6018        .unwrap();
6019        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6020    }
6021
6022    fn route_bind_ack(corr: u64) -> Frame {
6023        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
6024        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6025    }
6026
6027    fn unique_project_root(label: &str) -> TestTempDir {
6028        TestTempDir::new(label)
6029    }
6030
6031    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
6032        match parse_route_poll(frame) {
6033            ClientControlResponse::RoutePoll {
6034                status: None,
6035                live: Some(live),
6036                ..
6037            } => assert_eq!(live, expected_live),
6038            other => panic!("unexpected route.poll response: {other:?}"),
6039        }
6040    }
6041
6042    fn bind_liveness_route(
6043        registry: &Registry,
6044        forwarding: &ForwardingTable,
6045        module_id: &str,
6046    ) -> (RouteCtx, u16, u32) {
6047        let module_connection = ConnectionId::new(101);
6048        let client_connection = ConnectionId::new(202);
6049        let registration = registry
6050            .register_with_control_ops(
6051                manifest(module_id, PROTOCOL_VERSION),
6052                PROTOCOL_VERSION,
6053                module_connection,
6054                module_baseline_control_ops(),
6055            )
6056            .unwrap();
6057        let (module_tx, _module_rx) = mpsc::channel(8);
6058        let endpoint = forwarding
6059            .register_module_connection(
6060                module_connection,
6061                module_id.to_string(),
6062                PROTOCOL_VERSION,
6063                manifest_concurrency(&registration.manifest),
6064                FrameSink::new(module_tx),
6065            )
6066            .unwrap();
6067        let (client_ctx, _client_rx) = route_ctx(client_connection);
6068        let pending = forwarding
6069            .begin_route_bind_relay_for_test(
6070                client_connection,
6071                client_ctx.egress.clone(),
6072                1,
6073                module_id,
6074            )
6075            .unwrap();
6076        assert_eq!(pending.endpoint, endpoint);
6077        let route_channel = pending.client_channel;
6078        let route_epoch = pending.client_epoch;
6079        forwarding
6080            .complete_pending_relay(
6081                module_connection,
6082                pending.corr,
6083                RouteBindRelayOutcome::Accepted,
6084            )
6085            .unwrap();
6086        (client_ctx, route_channel, route_epoch)
6087    }
6088
6089    struct FakeProcessLiveness {
6090        live: Option<bool>,
6091    }
6092
6093    impl ModuleProcessLiveness for FakeProcessLiveness {
6094        fn process_live(&self, _module_id: &str) -> Option<bool> {
6095            self.live
6096        }
6097    }
6098
6099    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6100    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
6101    {
6102        let registry = Arc::new(Registry::default());
6103        let supervisor_handle = SupervisorHandle::new();
6104        let supervisor = Supervisor::new(
6105            Arc::clone(&registry),
6106            RestartPolicy::new(1, Duration::from_millis(10)),
6107        )
6108        .with_handle(supervisor_handle.clone());
6109        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
6110        let module = supervisor
6111            .spawn(ModuleSpec {
6112                module_id: "stderr-tail-wire".to_string(),
6113                program: fake_aft_stub_path(),
6114                args: Vec::new(),
6115                env: vec![
6116                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
6117                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
6118                ],
6119                reserved: false,
6120                reserved_prefixes: Vec::new(),
6121                protocol: ModuleProtocol::Subc,
6122                overlap: Default::default(),
6123            })
6124            .unwrap();
6125
6126        let deadline = Instant::now() + Duration::from_secs(5);
6127        loop {
6128            let tail = module.stderr_tail(None, None);
6129            if tail
6130                .entries
6131                .iter()
6132                .any(|entry| matches!(entry, TailEntry::ProcessStart))
6133                && tail.entries.iter().any(|entry| {
6134                    matches!(
6135                        entry,
6136                        TailEntry::Line {
6137                            truncated: true,
6138                            ..
6139                        }
6140                    )
6141                })
6142            {
6143                break;
6144            }
6145            assert!(
6146                Instant::now() < deadline,
6147                "module did not produce a truncated line and restart boundary: {tail:?}"
6148            );
6149            sleep(Duration::from_millis(10)).await;
6150        }
6151
6152        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
6153        let request = ClientControlRequest::SupervisorStderrTail {
6154            module_id: "stderr-tail-wire".to_string(),
6155            max_lines: None,
6156            max_bytes: None,
6157        };
6158        let frame = Frame::build(
6159            FrameType::Request,
6160            control_flags(),
6161            0,
6162            0,
6163            1,
6164            serde_json::to_vec(&request).unwrap(),
6165        )
6166        .unwrap();
6167        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6168        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6169        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
6170            serde_json::from_slice(&responses[0].body).unwrap()
6171        else {
6172            panic!("expected supervisor.stderr_tail response");
6173        };
6174
6175        assert!(
6176            tail.entries
6177                .iter()
6178                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
6179            "the control response lost the restart boundary"
6180        );
6181        let Some(StderrTailEntry::Line { text, truncated }) = tail.entries.iter().find(|entry| {
6182            matches!(
6183                entry,
6184                StderrTailEntry::Line {
6185                    truncated: true,
6186                    ..
6187                }
6188            )
6189        }) else {
6190            panic!("the control response lost the truncated line");
6191        };
6192        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
6193        assert!(*truncated);
6194    }
6195
6196    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
6197    /// read done on the worker thread would stall every other task until it
6198    /// finished; the read must run off the worker so this test's own task keeps
6199    /// running while the read is paused.
6200    #[tokio::test(flavor = "current_thread")]
6201    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
6202        let dir = TestTempDir::new("terminals-off-worker");
6203        let journal_path = dir.join("terminals.jsonl");
6204        let registry = Arc::new(Registry::default());
6205        let supervisor_handle = SupervisorHandle::new();
6206        let supervisor =
6207            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
6208                .with_handle(supervisor_handle.clone())
6209                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
6210        let module = supervisor
6211            .spawn(ModuleSpec {
6212                module_id: "terminal-off-worker".to_string(),
6213                program: fake_aft_stub_path(),
6214                args: Vec::new(),
6215                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6216                reserved: false,
6217                reserved_prefixes: Vec::new(),
6218                protocol: ModuleProtocol::Subc,
6219                overlap: Default::default(),
6220            })
6221            .unwrap();
6222        let deadline = Instant::now() + Duration::from_secs(5);
6223        while module.terminal_history().entries.len() != 2 {
6224            assert!(Instant::now() < deadline, "module did not record two exits");
6225            sleep(Duration::from_millis(10)).await;
6226        }
6227
6228        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
6229        let handler =
6230            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
6231        let frame = Frame::build(
6232            FrameType::Request,
6233            control_flags(),
6234            0,
6235            0,
6236            1,
6237            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
6238                module_id: "terminal-off-worker".to_string(),
6239            })
6240            .unwrap(),
6241        )
6242        .unwrap();
6243        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6244        let spawned_at = std::time::Instant::now();
6245        let read = tokio::spawn({
6246            let handler = Arc::clone(&handler);
6247            async move { handler.handle_control_frame(&ctx, frame).await }
6248        });
6249        // Waiting for the pause from a blocking thread keeps this task pending,
6250        // so the runtime's single worker is free to run the read task.
6251        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
6252            .await
6253            .unwrap()
6254            .expect("the history read reached its pause");
6255        let elapsed = spawned_at.elapsed();
6256        assert!(
6257            elapsed < Duration::from_secs(2) && !read.is_finished(),
6258            "this task could not run while the history read was paused \
6259             (resumed after {elapsed:?}, read finished: {})",
6260            read.is_finished()
6261        );
6262
6263        drop(release);
6264        let responses = read.await.unwrap().unwrap();
6265        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6266        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
6267            panic!("expected supervisor.terminals response");
6268        };
6269        assert_eq!(terminals.entries.len(), 2);
6270        assert_eq!(terminals.journal_skipped_lines, 0);
6271        assert_eq!(terminals.journal_read_errors, 0);
6272    }
6273
6274    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6275    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
6276        let registry = Arc::new(Registry::default());
6277        let supervisor_handle = SupervisorHandle::new();
6278        let supervisor =
6279            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
6280                .with_handle(supervisor_handle.clone());
6281        let module = supervisor
6282            .spawn(ModuleSpec {
6283                module_id: "terminal-golden".to_string(),
6284                program: fake_aft_stub_path(),
6285                args: Vec::new(),
6286                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6287                reserved: false,
6288                reserved_prefixes: Vec::new(),
6289                protocol: ModuleProtocol::Subc,
6290                overlap: Default::default(),
6291            })
6292            .unwrap();
6293
6294        let deadline = Instant::now() + Duration::from_secs(5);
6295        while module.terminal_history().entries.len() != 2 {
6296            assert!(
6297                Instant::now() < deadline,
6298                "module did not retain two terminal exits: {:?}",
6299                module.terminal_history()
6300            );
6301            sleep(Duration::from_millis(10)).await;
6302        }
6303
6304        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
6305        let request = ClientControlRequest::SupervisorTerminals {
6306            module_id: "terminal-golden".to_string(),
6307        };
6308        let frame = Frame::build(
6309            FrameType::Request,
6310            control_flags(),
6311            0,
6312            0,
6313            1,
6314            serde_json::to_vec(&request).unwrap(),
6315        )
6316        .unwrap();
6317        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6318        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6319        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6320        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
6321            panic!("expected supervisor.terminals response");
6322        };
6323        assert_eq!(terminals.entries.len(), 2);
6324        assert_eq!(terminals.dropped, 0);
6325
6326        let mut rendered = serde_json::to_value(response).unwrap();
6327        // Wall-clock fields are the observation contract, but not stable fixture
6328        // bytes; normalize only them after the real handler has shaped the response.
6329        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
6330        for (index, entry) in rendered["entries"]
6331            .as_array_mut()
6332            .expect("terminal response entries array")
6333            .iter_mut()
6334            .enumerate()
6335        {
6336            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
6337        }
6338
6339        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
6340            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
6341        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
6342        if std::env::var_os("UPDATE_GOLDEN").is_some() {
6343            std::fs::write(&golden_path, &serialized).unwrap();
6344        }
6345        let expected: Value =
6346            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
6347        assert_eq!(rendered, expected);
6348    }
6349
6350    #[test]
6351    fn hello_registers_manifest_and_returns_ack() {
6352        let registry = Arc::new(Registry::default());
6353        let handler = ControlHandler::new(Arc::clone(&registry));
6354        let conn = ConnectionId::new(1);
6355
6356        let responses = handler
6357            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
6358            .unwrap();
6359
6360        assert_eq!(responses.len(), 1);
6361        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6362        assert_eq!(responses[0].header.channel, 0);
6363        assert_eq!(responses[0].header.corr, 7);
6364        let ack = parse_ack(&responses[0]);
6365        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
6366        assert!(ack
6367            .subc_capabilities
6368            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
6369        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
6370        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
6371        assert!(ack
6372            .subc_ops
6373            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
6374        assert!(ack
6375            .subc_ops
6376            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
6377
6378        let registration = registry.get_module("aft").unwrap().unwrap();
6379        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
6380        assert_eq!(registration.state, ChannelState::Active);
6381        assert_eq!(registration.connection_id, conn);
6382        assert_eq!(registration.control_ops, module_baseline_control_ops());
6383    }
6384
6385    #[test]
6386    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
6387        let invalid_identifiers = [
6388            ("case_change", "credentials-Provider/v1"),
6389            ("leading_zero", "credentials-provider/v01"),
6390            ("trailing_hyphen", "credentials-provider-/v1"),
6391            ("consecutive_hyphens", "credentials--provider/v1"),
6392            ("uppercase", "Credentials-provider/v1"),
6393            ("missing_v", "credentials-provider/1"),
6394            ("whitespace", "credentials provider/v1"),
6395            ("zero_version", "credentials-provider/v0"),
6396            ("out_of_range_version", "credentials-provider/v4294967296"),
6397            (
6398                "overlength_name",
6399                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
6400            ),
6401        ];
6402        let mut cases = invalid_identifiers
6403            .into_iter()
6404            .map(|(name, identifier)| {
6405                (
6406                    format!("identifier_{name}"),
6407                    "capabilities.provides[0]".to_string(),
6408                    identifier.to_string(),
6409                    json!({ "provides": [identifier] }),
6410                    None,
6411                )
6412            })
6413            .collect::<Vec<_>>();
6414        cases.extend([
6415            (
6416                "unknown_need".to_string(),
6417                "capabilities.requires[0].need".to_string(),
6418                "deferred".to_string(),
6419                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
6420                None,
6421            ),
6422            (
6423                "duplicate_provides".to_string(),
6424                "capabilities.provides[1]".to_string(),
6425                "credentials-provider/v1".to_string(),
6426                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
6427                None,
6428            ),
6429            (
6430                "duplicate_must_never_reach".to_string(),
6431                "capabilities.must_never_reach[1]".to_string(),
6432                "credentials-provider/v1".to_string(),
6433                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
6434                None,
6435            ),
6436            (
6437                "duplicate_requires_same_need".to_string(),
6438                "capabilities.requires[1]".to_string(),
6439                "credentials-provider/v1".to_string(),
6440                json!({ "requires": [
6441                    { "capability": "credentials-provider/v1", "need": "required" },
6442                    { "capability": "credentials-provider/v1", "need": "required" }
6443                ] }),
6444                None,
6445            ),
6446            (
6447                "duplicate_requires_conflicting_need".to_string(),
6448                "capabilities.requires[1]".to_string(),
6449                "credentials-provider/v1".to_string(),
6450                json!({ "requires": [
6451                    { "capability": "credentials-provider/v1", "need": "required" },
6452                    { "capability": "credentials-provider/v1", "need": "optional" }
6453                ] }),
6454                None,
6455            ),
6456            (
6457                "capabilities_root_pointer".to_string(),
6458                "runtime_computed[0]".to_string(),
6459                "/capabilities".to_string(),
6460                json!({}),
6461                Some(json!(["/capabilities"])),
6462            ),
6463            (
6464                "capabilities_descendant_pointer".to_string(),
6465                "runtime_computed[0]".to_string(),
6466                "/capabilities/provides".to_string(),
6467                json!({}),
6468                Some(json!(["/capabilities/provides"])),
6469            ),
6470            (
6471                "malformed_pointer_without_leading_slash".to_string(),
6472                "runtime_computed[0]".to_string(),
6473                "capabilities".to_string(),
6474                json!({}),
6475                Some(json!(["capabilities"])),
6476            ),
6477            (
6478                "malformed_pointer_escape".to_string(),
6479                "runtime_computed[0]".to_string(),
6480                "/roles/~2/tools".to_string(),
6481                json!({}),
6482                Some(json!(["/roles/~2/tools"])),
6483            ),
6484            (
6485                "unknown_capabilities_field".to_string(),
6486                "capabilities.future".to_string(),
6487                "<array>".to_string(),
6488                json!({ "future": [] }),
6489                None,
6490            ),
6491        ]);
6492
6493        for (index, (name, field, value, capabilities, runtime_computed)) in
6494            cases.into_iter().enumerate()
6495        {
6496            let registry = Arc::new(Registry::default());
6497            let handler = ControlHandler::new(Arc::clone(&registry));
6498            let response = handler
6499                .handle_control(
6500                    ConnectionId::new((index + 1) as u64),
6501                    capability_grammar_hello_frame(
6502                        capabilities,
6503                        runtime_computed,
6504                        index as u64 + 1,
6505                    ),
6506                )
6507                .expect("invalid HELLO returns a refusal");
6508
6509            assert_eq!(response.len(), 1, "{name} must emit one refusal");
6510            let error = parse_error(&response[0]);
6511            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
6512            let message = error["message"]
6513                .as_str()
6514                .expect("error message is a string");
6515            assert!(
6516                message.contains(&field),
6517                "{name}: field missing from {message}"
6518            );
6519            assert!(
6520                message.contains(&value),
6521                "{name}: value missing from {message}"
6522            );
6523            assert_eq!(
6524                registry
6525                    .active_registration_count()
6526                    .expect("registry reads"),
6527                0,
6528                "{name}: refused HELLO must not create a catalog entry"
6529            );
6530        }
6531    }
6532
6533    #[test]
6534    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
6535        let registry = Arc::new(Registry::default());
6536        let handler = ControlHandler::new(Arc::clone(&registry));
6537        let capabilities = json!({
6538            "provides": ["credentials-provider/v1"],
6539            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
6540            "must_never_reach": ["federation-transport/v1"]
6541        });
6542        let response = handler
6543            .handle_control(
6544                ConnectionId::new(99),
6545                capability_grammar_hello_frame(
6546                    capabilities.clone(),
6547                    Some(json!(["/roles/0/tools"])),
6548                    99,
6549                ),
6550            )
6551            .expect("valid HELLO registers");
6552        assert_eq!(response[0].header.ty, FrameType::HelloAck);
6553
6554        let request = Frame::build(
6555            FrameType::Request,
6556            control_flags(),
6557            0,
6558            0,
6559            100,
6560            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6561                .expect("catalog request serializes"),
6562        )
6563        .expect("catalog request frame builds");
6564        let response = handler
6565            .handle_catalog_list(request, None)
6566            .expect("catalog list succeeds");
6567        let ClientControlResponse::CatalogList { modules, .. } =
6568            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
6569        else {
6570            panic!("catalog request must return catalog.list");
6571        };
6572        assert_eq!(modules.len(), 1);
6573        assert_eq!(
6574            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
6575            capabilities
6576        );
6577    }
6578
6579    #[test]
6580    fn catalog_list_mirrors_management_operation_description() {
6581        let registry = Arc::new(Registry::default());
6582        let handler = ControlHandler::new(Arc::clone(&registry));
6583        let description = "List managed records and return their identifiers and metadata.";
6584        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
6585        manifest.provides = vec![ProviderRole::ManagementSurface {
6586            operations: vec![ManagementOperation {
6587                name: "records.list".to_string(),
6588                kind: ManagementOperationKind::Query,
6589                description: Some(description.to_string()),
6590            }],
6591            config_schema: json!({"type": "object"}),
6592            observability: vec![ObservabilitySurface {
6593                name: "records.stats".to_string(),
6594                kind: ObservabilityKind::Snapshot,
6595            }],
6596            identity_scope: vec![IdentityScope::Project],
6597            concurrency: Concurrency::ModuleManaged,
6598        }];
6599        registry
6600            .register_with_control_ops(
6601                manifest,
6602                PROTOCOL_VERSION,
6603                ConnectionId::new(99),
6604                Vec::new(),
6605            )
6606            .expect("described management manifest registers");
6607
6608        let request = Frame::build(
6609            FrameType::Request,
6610            control_flags(),
6611            0,
6612            0,
6613            100,
6614            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6615                .expect("catalog request serializes"),
6616        )
6617        .expect("catalog request frame builds");
6618        let response = handler
6619            .handle_catalog_list(request, None)
6620            .expect("catalog list succeeds");
6621        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
6622        assert_eq!(
6623            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
6624            "catalog.list must preserve the declared operation description verbatim"
6625        );
6626    }
6627
6628    #[test]
6629    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
6630        let registry = Arc::new(Registry::default());
6631        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
6632            [("vault".to_string(), true), ("squatter".to_string(), true)],
6633            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
6634        );
6635        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
6636        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
6637            provides: vec!["credentials-provider/v1".to_string()],
6638            requires: Vec::new(),
6639            must_never_reach: Vec::new(),
6640        });
6641        let frame = Frame::build(
6642            FrameType::Hello,
6643            control_flags(),
6644            0,
6645            0,
6646            77,
6647            serde_json::to_vec(&ModuleHelloBody {
6648                manifest: squatter,
6649                protocol_ver: PROTOCOL_VERSION,
6650                control_ops: None,
6651                launch_nonce: None,
6652            })
6653            .expect("HELLO serializes"),
6654        )
6655        .expect("HELLO frame builds");
6656        let response = handler
6657            .handle_control(ConnectionId::new(77), frame)
6658            .expect("reserved claim receives a typed refusal");
6659        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
6660        assert_eq!(
6661            registry
6662                .active_registration_count()
6663                .expect("registry reads"),
6664            0,
6665            "a reserved capability refusal must not leave a catalog entry"
6666        );
6667    }
6668
6669    #[test]
6670    fn server_describe_surfaces_required_capability_verdict_fields() {
6671        let registry = Arc::new(Registry::default());
6672        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
6673            [
6674                ("consumer".to_string(), true),
6675                ("provider".to_string(), false),
6676            ],
6677            BTreeMap::new(),
6678        );
6679        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
6680        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
6681            provides: Vec::new(),
6682            requires: vec![subc_protocol::manifest::CapabilityRequirement {
6683                capability: "credentials-provider/v1".to_string(),
6684                need: subc_protocol::manifest::CapabilityNeed::Required,
6685            }],
6686            must_never_reach: Vec::new(),
6687        });
6688        let hello = Frame::build(
6689            FrameType::Hello,
6690            control_flags(),
6691            0,
6692            0,
6693            78,
6694            serde_json::to_vec(&ModuleHelloBody {
6695                manifest: consumer,
6696                protocol_ver: PROTOCOL_VERSION,
6697                control_ops: None,
6698                launch_nonce: None,
6699            })
6700            .expect("HELLO serializes"),
6701        )
6702        .expect("HELLO frame builds");
6703        handler
6704            .handle_control(ConnectionId::new(78), hello)
6705            .expect("consumer registers");
6706        let describe = Frame::build(
6707            FrameType::Request,
6708            control_flags(),
6709            0,
6710            0,
6711            79,
6712            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
6713                .expect("request serializes"),
6714        )
6715        .expect("describe frame builds");
6716        let response = handler
6717            .handle_server_describe(describe)
6718            .expect("server.describe succeeds");
6719        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
6720        let requirement = &rendered["capability_requirements"][0];
6721        assert_eq!(requirement["consumer"], "consumer");
6722        assert_eq!(requirement["verdict"], "never_provided");
6723        assert_eq!(requirement["episode_seq"], 1);
6724        assert_eq!(requirement["config_satisfiable"], false);
6725        assert_eq!(requirement["runtime_available"], false);
6726        assert!(requirement["detail"]
6727            .as_str()
6728            .expect("detail string")
6729            .contains("credentials-provider/v1"));
6730    }
6731
6732    #[test]
6733    fn catalog_list_omits_capabilities_for_legacy_manifest() {
6734        let registry = Arc::new(Registry::default());
6735        let handler = ControlHandler::new(Arc::clone(&registry));
6736        let hello = handler
6737            .handle_control(
6738                ConnectionId::new(101),
6739                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
6740            )
6741            .expect("legacy HELLO registers");
6742        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
6743
6744        let request = Frame::build(
6745            FrameType::Request,
6746            control_flags(),
6747            0,
6748            0,
6749            102,
6750            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6751                .expect("catalog request serializes"),
6752        )
6753        .expect("catalog request frame builds");
6754        let response = handler
6755            .handle_catalog_list(request, None)
6756            .expect("catalog list succeeds");
6757        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
6758        assert!(
6759            body["modules"][0].get("capabilities").is_none(),
6760            "legacy manifest must retain an absent capabilities field on catalog.list"
6761        );
6762    }
6763
6764    #[test]
6765    fn hello_ack_omits_storage_when_no_storage_config() {
6766        let registry = Arc::new(Registry::default());
6767        let handler = ControlHandler::new(Arc::clone(&registry));
6768        let responses = handler
6769            .handle_control(
6770                ConnectionId::new(1),
6771                hello_frame("aft", PROTOCOL_VERSION, 7),
6772            )
6773            .unwrap();
6774        let ack = parse_ack(&responses[0]);
6775        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
6776        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
6777    }
6778
6779    #[tokio::test]
6780    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
6781        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
6782        let registry = Arc::new(Registry::default());
6783        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
6784        let responses = handler
6785            .handle_control(
6786                ConnectionId::new(1),
6787                hello_frame("aft", PROTOCOL_VERSION, 7),
6788            )
6789            .unwrap();
6790        let ack = parse_ack(&responses[0]);
6791        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
6792
6793        let described = handler
6794            .handle_control_frame(
6795                &route_ctx(ConnectionId::new(2)).0,
6796                Frame::build(
6797                    FrameType::Request,
6798                    control_flags(),
6799                    0,
6800                    0,
6801                    9,
6802                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
6803                )
6804                .unwrap(),
6805            )
6806            .await
6807            .unwrap();
6808        let ClientControlResponse::ServerDescribe { machine_id, .. } =
6809            serde_json::from_slice(&described[0].body).unwrap()
6810        else {
6811            panic!("server.describe answered with another shape");
6812        };
6813        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
6814    }
6815
6816    #[test]
6817    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
6818        // With a central sqlite storage policy, each registering module gets its
6819        // own resolved descriptor in HELLO_ACK, keyed by its module id.
6820        let registry = Arc::new(Registry::default());
6821        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
6822            crate::daemon_config::StorageConfig::Sqlite {
6823                data_home: std::path::PathBuf::from("/data"),
6824            },
6825        ));
6826
6827        let responses = handler
6828            .handle_control(
6829                ConnectionId::new(1),
6830                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
6831            )
6832            .unwrap();
6833        let ack = parse_ack(&responses[0]);
6834        assert_eq!(
6835            ack.storage,
6836            Some(serde_json::json!({
6837                "module_id": "alfonso-routing",
6838                "storage_namespace": "default",
6839                "isolation": { "kind": "module" },
6840                "backend": {
6841                    "backend": "sqlite",
6842                    "path": "/data/cortexkit/alfonso-routing/store.db"
6843                }
6844            })),
6845            "the delivered descriptor is the module's own sqlite store path"
6846        );
6847    }
6848
6849    #[test]
6850    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
6851        let registry = Arc::new(Registry::default());
6852        let handler = ControlHandler::new(Arc::clone(&registry));
6853        let conn = ConnectionId::new(1);
6854        let responses = handler
6855            .handle_control(
6856                conn,
6857                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
6858            )
6859            .unwrap();
6860        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6861        let registration = registry.get_module("aft").unwrap().unwrap();
6862        assert_eq!(registration.control_ops, module_baseline_control_ops());
6863
6864        let frame =
6865            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
6866        assert!(handler
6867            .guard_module_control_op(&frame, "aft", "route.bind")
6868            .unwrap()
6869            .is_none());
6870        let error = handler
6871            .guard_module_control_op(&frame, "aft", "test.synthetic")
6872            .unwrap()
6873            .expect("synthetic ungranted op should be rejected");
6874        assert_eq!(error.header.ty, FrameType::Error);
6875        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
6876    }
6877
6878    #[test]
6879    fn hello_control_ops_some_adds_optional_grants() {
6880        let registry = Arc::new(Registry::default());
6881        let handler = ControlHandler::new(Arc::clone(&registry));
6882        handler
6883            .handle_control(
6884                ConnectionId::new(1),
6885                hello_frame_with_control_ops(
6886                    "aft",
6887                    PROTOCOL_VERSION,
6888                    7,
6889                    Some(vec![
6890                        "future.synthetic".to_string(),
6891                        "route.bind".to_string(),
6892                    ]),
6893                ),
6894            )
6895            .unwrap();
6896        let registration = registry.get_module("aft").unwrap().unwrap();
6897        assert_eq!(
6898            registration.control_ops,
6899            vec![
6900                "route.bind".to_string(),
6901                "route.status".to_string(),
6902                "future.synthetic".to_string(),
6903            ]
6904        );
6905        let frame =
6906            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
6907        assert!(handler
6908            .guard_module_control_op(&frame, "aft", "future.synthetic")
6909            .unwrap()
6910            .is_none());
6911    }
6912
6913    #[tokio::test]
6914    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
6915        let registry = Arc::new(Registry::default());
6916        let forwarding = Arc::new(ForwardingTable::default());
6917        let handler =
6918            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
6919        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
6920        hello_via_sink(
6921            &handler,
6922            &module_ctx,
6923            &mut module_rx,
6924            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
6925        )
6926        .await;
6927
6928        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
6929        let responses = handler
6930            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
6931            .await
6932            .unwrap();
6933        assert_eq!(responses.len(), 1);
6934        assert_eq!(responses[0].header.ty, FrameType::Error);
6935        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
6936        assert!(module_rx.try_recv().is_err());
6937    }
6938
6939    #[tokio::test]
6940    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
6941        let registry = Arc::new(Registry::default());
6942        let forwarding = Arc::new(ForwardingTable::default());
6943        let handler =
6944            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
6945        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
6946        hello_via_sink(
6947            &handler,
6948            &module_ctx,
6949            &mut module_rx,
6950            hello_frame_with_control_ops(
6951                "aft",
6952                PROTOCOL_VERSION,
6953                7,
6954                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
6955            ),
6956        )
6957        .await;
6958
6959        let project_root = unique_project_root("demux");
6960        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
6961        let route_handler = handler.clone();
6962        let route_task = tokio::spawn(async move {
6963            route_handler
6964                .handle_control_frame(
6965                    &route_client_ctx,
6966                    route_open_frame(100, "aft", project_root),
6967                )
6968                .await
6969                .unwrap()
6970        });
6971        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
6972            .await
6973            .unwrap()
6974            .unwrap();
6975        assert!(matches!(
6976            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
6977            ModuleControlRequest::RouteBind { .. }
6978        ));
6979
6980        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
6981        let health_handler = handler.clone();
6982        let health_task = tokio::spawn(async move {
6983            health_handler
6984                .handle_control_frame(
6985                    &health_client_ctx,
6986                    supervisor_health_probe_frame(101, "aft"),
6987                )
6988                .await
6989                .unwrap()
6990        });
6991        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
6992            .await
6993            .unwrap()
6994            .unwrap();
6995        assert_eq!(
6996            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
6997            ModuleControlRequest::HealthCheck {}
6998        );
6999
7000        handler
7001            .handle_control_frame(
7002                &module_ctx,
7003                health_response(health_frame.header.corr, HealthStatus::Degraded),
7004            )
7005            .await
7006            .unwrap();
7007        let health_response = health_task.await.unwrap();
7008        assert_eq!(health_response.len(), 1);
7009        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
7010            ClientControlResponse::SupervisorHealthProbe {
7011                module_id,
7012                status,
7013                detail,
7014                metrics,
7015            } => {
7016                assert_eq!(module_id, "aft");
7017                assert_eq!(status, HealthStatus::Degraded);
7018                assert_eq!(detail.as_deref(), Some("warming"));
7019                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
7020            }
7021            other => panic!("unexpected health response: {other:?}"),
7022        }
7023
7024        handler
7025            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
7026            .await
7027            .unwrap();
7028        let route_response = route_task.await.unwrap();
7029        assert!(route_response.is_empty());
7030        let published = route_client_rx.recv().await.unwrap();
7031        assert!(matches!(
7032            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
7033            ClientControlResponse::RouteOpen { .. }
7034        ));
7035    }
7036
7037    /// Start one `route.open` on `client_connection` and return its still-running
7038    /// handler task together with the `route.bind` the module received for it.
7039    /// The handler blocks until the module answers, so it has to run as a task
7040    /// while the test drives the module side.
7041    async fn relay_route_open(
7042        handler: &ControlHandler,
7043        client_connection: ConnectionId,
7044        client_egress: &FrameSink,
7045        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7046        corr: u64,
7047        module_id: &str,
7048        project_root_label: &str,
7049    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
7050        let ctx = RouteCtx {
7051            connection_id: client_connection,
7052            egress: client_egress.clone(),
7053        };
7054        let handler = handler.clone();
7055        let project_root = unique_project_root(project_root_label);
7056        let module_id = module_id.to_string();
7057        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
7058        let task = tokio::spawn(async move {
7059            let _guard = tracing::dispatcher::set_default(&dispatch);
7060            handler
7061                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
7062                .await
7063                .unwrap()
7064        });
7065        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
7066            .await
7067            .expect("module receives the relayed route.bind")
7068            .expect("module egress is open");
7069        (task, bind.frame)
7070    }
7071
7072    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
7073        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
7074            ModuleControlRequest::RouteBind {
7075                route_channel,
7076                epoch,
7077                ..
7078            } => (route_channel, epoch),
7079            other => panic!("expected a route.bind request, got {other:?}"),
7080        }
7081    }
7082
7083    fn published_route(frame: &Frame) -> (u16, u32) {
7084        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
7085            ClientControlResponse::RouteOpen {
7086                route_channel,
7087                route_epoch,
7088            } => (route_channel, route_epoch),
7089            other => panic!("expected a route.open response, got {other:?}"),
7090        }
7091    }
7092
7093    /// Reproduction of a production outage. A client had `route.open`s in
7094    /// flight to a module and was already marked closing -- its egress had refused a
7095    /// module frame, so the daemon asked its connection to end -- while its sink
7096    /// was still open. When the module acked those binds, the daemon refused to
7097    /// commit a route for a closing client, and that refusal was returned from
7098    /// the MODULE connection's frame handler, where a router error that has no
7099    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
7100    /// the supervisor correctly did not respawn a clean exit, and every seat lost
7101    /// its tools for hours -- one client's teardown took down a connection
7102    /// carrying ~170 other routes.
7103    ///
7104    /// The window is opened here by calling the production path that opens it
7105    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
7106    /// state that matters is "in `closing_connections`, sink still open, relay
7107    /// still pending", and it lasts only from the close request until the
7108    /// connection loop reacts to it; a socket-level test can flood a client into
7109    /// that escalation but cannot pin the module's ack inside the window. Closing
7110    /// the socket instead takes the other path entirely -- connection teardown
7111    /// removes the pending relay under the same lock, so the ack finds nothing.
7112    #[tokio::test]
7113    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
7114        let registry = Arc::new(Registry::default());
7115        let forwarding = Arc::new(ForwardingTable::default());
7116        let handler =
7117            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
7118
7119        let module_connection = ConnectionId::new(30);
7120        let (module_ctx, mut module_rx) = route_ctx(module_connection);
7121        hello_via_sink(
7122            &handler,
7123            &module_ctx,
7124            &mut module_rx,
7125            hello_frame("aft", PROTOCOL_VERSION, 7),
7126        )
7127        .await;
7128
7129        let dying_client = ConnectionId::new(31);
7130        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
7131
7132        // A published route on the dying client. The escalation below only marks
7133        // a connection closing for a route it has already published.
7134        let (first_task, first_bind) = relay_route_open(
7135            &handler,
7136            dying_client,
7137            &dying_ctx.egress,
7138            &mut module_rx,
7139            100,
7140            "aft",
7141            "closing-first",
7142        )
7143        .await;
7144        handler
7145            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
7146            .await
7147            .unwrap();
7148        assert!(first_task.await.unwrap().is_empty());
7149        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
7150
7151        // A second route.open from the same client, relayed and awaiting its ack.
7152        let (second_task, second_bind) = relay_route_open(
7153            &handler,
7154            dying_client,
7155            &dying_ctx.egress,
7156            &mut module_rx,
7157            101,
7158            "aft",
7159            "closing-second",
7160        )
7161        .await;
7162        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
7163
7164        // The window: the client is closing, its sink is still open, and its
7165        // second bind is still pending.
7166        assert!(forwarding
7167            .escalate_client_delivery_failure(
7168                dying_client,
7169                first_channel,
7170                first_epoch,
7171                CloseReason::new(
7172                    "module_to_client_delivery_failed",
7173                    "client egress refused a module frame",
7174                ),
7175                crate::forwarding::UndeliveredFrame {
7176                    module_id: None,
7177                    sink: &dying_ctx.egress,
7178                },
7179            )
7180            .unwrap());
7181        assert!(!dying_ctx.egress.is_closed());
7182
7183        // The frame that used to end the module connection.
7184        let ack = handler
7185            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
7186            .await;
7187        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
7188        if module_loop_error.is_some() {
7189            // What the server's connection loop does with a router error that has
7190            // no ERROR-frame translation: end the connection, which releases the
7191            // module's registration and every route on it.
7192            handler.cleanup_connection(module_connection).unwrap();
7193        }
7194        // Read the module's next frame before opening the co-tenant's route, so
7195        // the GOODBYE assertion below is about THIS ack and not about later
7196        // traffic. `None` means the module was told nothing.
7197        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7198            .await
7199            .ok()
7200            .flatten();
7201
7202        // 1. The module connection is still registered.
7203        assert!(
7204            registry
7205                .get_module_by_connection(module_connection)
7206                .unwrap()
7207                .is_some(),
7208            "one client's closing connection ended the shared module connection: \
7209             {module_loop_error:?}"
7210        );
7211        // ...and still serving: another client can open and use a route on it.
7212        let cotenant = ConnectionId::new(32);
7213        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
7214        let (cotenant_task, cotenant_bind) = relay_route_open(
7215            &handler,
7216            cotenant,
7217            &cotenant_ctx.egress,
7218            &mut module_rx,
7219            102,
7220            "aft",
7221            "closing-cotenant",
7222        )
7223        .await;
7224        handler
7225            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
7226            .await
7227            .unwrap();
7228        assert!(cotenant_task.await.unwrap().is_empty());
7229        let (cotenant_channel, cotenant_epoch) =
7230            published_route(&cotenant_rx.recv().await.unwrap());
7231        assert!(matches!(
7232            forwarding
7233                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
7234                .unwrap(),
7235            DataRoute::Client(DataRouteState::Bound(_))
7236        ));
7237
7238        // 2. The module was told to drop the binding it created for the route
7239        //    that will never be published.
7240        let goodbye = post_ack_module_frame
7241            .expect("module receives a GOODBYE for the abandoned route channel");
7242        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
7243        assert_eq!(goodbye.header.channel, abandoned_channel);
7244        assert_eq!(goodbye.header.epoch, abandoned_epoch);
7245
7246        // 3. The dying client received nothing: no route was ever published to
7247        //    it. Its route.open is answered as unavailable, which the connection
7248        //    loop would write to a socket that is already going away.
7249        assert!(dying_rx.try_recv().is_err());
7250        let second_response = second_task.await.unwrap();
7251        assert_eq!(second_response.len(), 1);
7252        assert_eq!(
7253            parse_error(&second_response[0])["code"],
7254            "target_unavailable"
7255        );
7256    }
7257
7258    /// The fence at the module-loop boundary, stated as its own contract: which
7259    /// forwarding failures are allowed to end the module connection that is being
7260    /// served. A `ConnectionClosing` naming some client is about that client, and
7261    /// a module connection is shared; the same error naming the module's own
7262    /// connection is about this connection and must stay fatal, as must failures
7263    /// that are about the forwarding table itself.
7264    #[test]
7265    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
7266        let handler = ControlHandler::default();
7267        let module_connection = ConnectionId::new(30);
7268        let client_connection = ConnectionId::new(31);
7269
7270        handler
7271            .refuse_to_end_module_connection_for_a_client(
7272                module_connection,
7273                77,
7274                ForwardingError::ConnectionClosing {
7275                    connection_id: client_connection,
7276                },
7277            )
7278            .expect("a closing client must never end the module connection");
7279
7280        assert!(matches!(
7281            handler.refuse_to_end_module_connection_for_a_client(
7282                module_connection,
7283                78,
7284                ForwardingError::ConnectionClosing {
7285                    connection_id: module_connection,
7286                },
7287            ),
7288            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
7289                connection_id
7290            })) if connection_id == module_connection
7291        ));
7292        assert!(matches!(
7293            handler.refuse_to_end_module_connection_for_a_client(
7294                module_connection,
7295                79,
7296                ForwardingError::Poisoned,
7297            ),
7298            Err(RouterError::Forwarding(ForwardingError::Poisoned))
7299        ));
7300        assert!(matches!(
7301            handler.refuse_to_end_module_connection_for_a_client(
7302                module_connection,
7303                80,
7304                ForwardingError::StaleModuleEndpoint,
7305            ),
7306            Err(RouterError::Forwarding(
7307                ForwardingError::StaleModuleEndpoint
7308            ))
7309        ));
7310    }
7311
7312    /// The spawn-attestation guard is what stops a connected module from claiming
7313    /// another module's identity and being stamped `Reserved` for it. Every other
7314    /// test that supplies a consumer_identity supplies a CORRECT one, because a
7315    /// correct one is what the rest of the flow needs -- so the guard's rejection
7316    /// branch was never the subject of an assertion, only its acceptance branch.
7317    ///
7318    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
7319    /// whole subc-core library suite green; only the forwarding integration tests
7320    /// notice, and they notice for unrelated reasons. This test exists so the
7321    /// refusal itself is asserted where the guard lives: it fails if the identity
7322    /// check stops refusing, which is the direction that matters, since a guard
7323    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
7324    #[tokio::test]
7325    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
7326        let registry = Arc::new(Registry::default());
7327        let forwarding = Arc::new(ForwardingTable::default());
7328        let supervisor = SupervisorHandle::new();
7329        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7330        let handler =
7331            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
7332                .with_supervisor(supervisor);
7333
7334        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
7335        hello_via_sink(
7336            &handler,
7337            &target_ctx,
7338            &mut target_rx,
7339            hello_frame("target", PROTOCOL_VERSION, 1),
7340        )
7341        .await;
7342
7343        // A real supervised module id presenting the wrong nonce. This is the
7344        // impersonation case: the attacker knows a privileged module_id, which is
7345        // public, and guesses at the nonce, which is not.
7346        let wrong_nonce = handler
7347            .handle_control_frame(
7348                &route_ctx(ConnectionId::new(91)).0,
7349                route_open_frame_with_admission_facts(
7350                    20,
7351                    "target",
7352                    unique_project_root("admission-facts"),
7353                    Some(subc_control::ConsumerIdentity {
7354                        module_id: "fed".to_string(),
7355                        launch_nonce: "not-the-real-nonce".to_string(),
7356                    }),
7357                    None,
7358                ),
7359            )
7360            .await
7361            .unwrap();
7362        assert_eq!(
7363            parse_error(&wrong_nonce[0])["code"],
7364            "bad_consumer_identity",
7365            "a mismatched launch nonce must be refused, not stamped Reserved"
7366        );
7367
7368        // A module id the supervisor never spawned at all, so no nonce exists to
7369        // compare against. An implementation that treats "no record" as "nothing
7370        // to check" fails open here while passing the case above.
7371        let never_spawned = handler
7372            .handle_control_frame(
7373                &route_ctx(ConnectionId::new(92)).0,
7374                route_open_frame_with_admission_facts(
7375                    21,
7376                    "target",
7377                    unique_project_root("admission-facts"),
7378                    Some(subc_control::ConsumerIdentity {
7379                        module_id: "never-spawned".to_string(),
7380                        launch_nonce: "any-nonce".to_string(),
7381                    }),
7382                    None,
7383                ),
7384            )
7385            .await
7386            .unwrap();
7387        assert_eq!(
7388            parse_error(&never_spawned[0])["code"],
7389            "bad_consumer_identity",
7390            "an unspawned module_id must be refused rather than accepted for lack of a record"
7391        );
7392    }
7393
7394    /// The refusal test above proves the guard says NO. Nothing proved it can say
7395    /// YES, and the difference is not academic: replacing the whole authorization
7396    /// with `false` -- admitting no consumer identity at all, revoking Reserved
7397    /// standing for every supervised module in the fleet -- leaves 110 of the 111
7398    /// library tests GREEN. The one that notices does so by HANGING, because it
7399    /// waits for a bind that can no longer happen.
7400    ///
7401    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
7402    /// or flaky test, invites a RETRY rather than an investigation, and the retry
7403    /// hangs too and gets blamed on the runner. So a total revocation of the
7404    /// daemon's trust grant would have shipped behind a symptom nobody attributes
7405    /// to code.
7406    ///
7407    /// The bias is structural rather than accidental. A REFUSAL looks like a
7408    /// failure someone writes a test for; a GRANT looks like the happy path. Every
7409    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
7410    /// suite for that reason, and this one is the purest case in the daemon.
7411    ///
7412    /// This test asserts the EFFECT rather than the absence of an error: the module
7413    /// receives a RouteBind and it carries `Reserved` naming the attested module.
7414    /// A guard that admitted nobody would produce no bind at all; one that admitted
7415    /// everybody would stamp the wrong principal, which the refusal test catches.
7416    #[tokio::test]
7417    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
7418        let registry = Arc::new(Registry::default());
7419        let forwarding = Arc::new(ForwardingTable::default());
7420        let supervisor = SupervisorHandle::new();
7421        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7422        let handler =
7423            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
7424                .with_supervisor(supervisor);
7425
7426        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
7427        hello_via_sink(
7428            &handler,
7429            &target_ctx,
7430            &mut target_rx,
7431            hello_frame("target", PROTOCOL_VERSION, 1),
7432        )
7433        .await;
7434
7435        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
7436        let route_handler = handler.clone();
7437        let route_task = tokio::spawn(async move {
7438            route_handler
7439                .handle_control_frame(
7440                    &client_ctx,
7441                    route_open_frame_with_admission_facts(
7442                        30,
7443                        "target",
7444                        unique_project_root("admission-facts"),
7445                        Some(subc_control::ConsumerIdentity {
7446                            module_id: "fed".to_string(),
7447                            launch_nonce: "fed-nonce".to_string(),
7448                        }),
7449                        None,
7450                    ),
7451                )
7452                .await
7453                .unwrap()
7454        });
7455
7456        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
7457        // the very mutation it exists to catch -- a guard that admits nobody -- no
7458        // bind is ever sent, so it HUNG rather than failing. That reproduces the
7459        // exact defect being fixed: a total revocation detected only as a stalled
7460        // suite, which reads as flakiness and invites a retry. An acceptance test
7461        // that waits for an effect must bound the wait, or a red becomes a hang.
7462        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7463            .await
7464            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
7465            .expect("module control channel closed before route.bind");
7466        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
7467        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
7468            panic!("expected route.bind")
7469        };
7470        assert_eq!(
7471            principal,
7472            Some(Principal::Reserved {
7473                module_id: "fed".to_string()
7474            }),
7475            "a correctly attested consumer must be stamped Reserved for its own id"
7476        );
7477
7478        handler
7479            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
7480            .await
7481            .unwrap();
7482        assert!(route_task.await.unwrap().is_empty());
7483        assert!(
7484            matches!(
7485                serde_json::from_slice::<ClientControlResponse>(
7486                    &client_rx.recv().await.unwrap().body
7487                )
7488                .unwrap(),
7489                ClientControlResponse::RouteOpen { .. }
7490            ),
7491            "the route must actually open, not merely avoid an error"
7492        );
7493    }
7494
7495    #[tokio::test(start_paused = true)]
7496    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
7497        let registry = Arc::new(Registry::default());
7498        let forwarding = Arc::new(ForwardingTable::default());
7499        let supervisor = SupervisorHandle::new();
7500        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7501        let handler =
7502            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
7503                .with_supervisor(supervisor);
7504
7505        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
7506        hello_via_sink(
7507            &handler,
7508            &target_ctx,
7509            &mut target_rx,
7510            hello_frame("target", PROTOCOL_VERSION, 1),
7511        )
7512        .await;
7513
7514        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
7515        let direct_handler = handler.clone();
7516        let direct_open = tokio::spawn(async move {
7517            direct_handler
7518                .handle_control_frame(
7519                    &direct_ctx,
7520                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
7521                )
7522                .await
7523                .unwrap()
7524        });
7525        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7526            .await
7527            .expect("no direct route.bind within 5s")
7528            .expect("target control channel closed before direct route.bind");
7529        handler
7530            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
7531            .await
7532            .unwrap();
7533        assert!(direct_open.await.unwrap().is_empty());
7534        let _ = direct_rx.recv().await.unwrap();
7535
7536        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
7537        let reserved_handler = handler.clone();
7538        let reserved_open = tokio::spawn(async move {
7539            reserved_handler
7540                .handle_control_frame(
7541                    &reserved_ctx,
7542                    route_open_frame_with_admission_facts(
7543                        3,
7544                        "target",
7545                        unique_project_root("admission-facts"),
7546                        Some(ConsumerIdentity {
7547                            module_id: "fed".to_string(),
7548                            launch_nonce: "fed-nonce".to_string(),
7549                        }),
7550                        None,
7551                    ),
7552                )
7553                .await
7554                .unwrap()
7555        });
7556        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7557            .await
7558            .expect("no reserved route.bind within 5s")
7559            .expect("target control channel closed before reserved route.bind");
7560        handler
7561            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
7562            .await
7563            .unwrap();
7564        assert!(reserved_open.await.unwrap().is_empty());
7565        let _ = reserved_rx.recv().await.unwrap();
7566
7567        forwarding
7568            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
7569            .unwrap();
7570        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
7571        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
7572            module_id: Some("target".to_string()),
7573        })
7574        .unwrap();
7575        let census_frame =
7576            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
7577        let response = handler
7578            .handle_control_frame(&census_ctx, census_frame)
7579            .await
7580            .unwrap()
7581            .pop()
7582            .unwrap();
7583        let actual: Value = serde_json::from_slice(&response.body).unwrap();
7584        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
7585        assert!(matches!(
7586            decoded,
7587            ClientControlResponse::SupervisorRoutes { .. }
7588        ));
7589        let routes = actual["modules"][0]["routes"].as_array().unwrap();
7590        assert_eq!(routes.len(), 2);
7591        assert!(routes.iter().all(|route| route["draining"] == true));
7592        // The census carries WHY: the reason the drain was begun with, in the
7593        // route.closing vocabulary, on every draining route this drain marked.
7594        assert!(
7595            routes.iter().all(|route| route["drain_reason"] == "reload"),
7596            "draining routes must name the drain's reason: {routes:?}"
7597        );
7598        assert!(routes.iter().any(|route| {
7599            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
7600        }));
7601        assert!(routes.iter().any(|route| {
7602            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
7603        }));
7604
7605        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7606            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
7607        if std::env::var_os("UPDATE_GOLDEN").is_some() {
7608            std::fs::write(
7609                &golden_path,
7610                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
7611            )
7612            .unwrap();
7613        }
7614        let expected: Value =
7615            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
7616        assert_eq!(actual, expected);
7617    }
7618
7619    async fn query_live_roots(
7620        handler: &ControlHandler,
7621        module_ctx: &RouteCtx,
7622    ) -> ModuleControlResponseToModule {
7623        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
7624        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
7625        let response = handler
7626            .handle_control_frame(module_ctx, frame)
7627            .await
7628            .unwrap()
7629            .pop()
7630            .unwrap();
7631        serde_json::from_slice(&response.body).unwrap()
7632    }
7633
7634    #[tokio::test(start_paused = true)]
7635    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
7636        let registry = Arc::new(Registry::default());
7637        let forwarding = Arc::new(ForwardingTable::default());
7638        let handler = ControlHandler::with_forwarding(registry, forwarding);
7639        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
7640        hello_via_sink(
7641            &handler,
7642            &target_ctx,
7643            &mut target_rx,
7644            hello_frame("target", PROTOCOL_VERSION, 1),
7645        )
7646        .await;
7647        let root = unique_project_root("live-roots-known");
7648        let path = ProjectRootId::from_path_allowing_missing(root.path())
7649            .unwrap()
7650            .as_path()
7651            .to_path_buf();
7652        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
7653        let open_handler = handler.clone();
7654        let opened = tokio::spawn(async move {
7655            open_handler
7656                .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
7657                .await
7658                .unwrap()
7659        });
7660        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7661            .await
7662            .unwrap()
7663            .unwrap();
7664        handler
7665            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
7666            .await
7667            .unwrap();
7668        assert!(opened.await.unwrap().is_empty());
7669        let _ = client_rx.recv().await.unwrap();
7670
7671        let root = unique_project_root("live-roots-pending");
7672        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
7673            .unwrap()
7674            .as_path()
7675            .to_path_buf();
7676        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
7677        let open_handler = handler.clone();
7678        let pending = tokio::spawn(async move {
7679            open_handler
7680                .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
7681                .await
7682                .unwrap()
7683        });
7684        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7685            .await
7686            .unwrap()
7687            .unwrap();
7688        let actual = query_live_roots(&handler, &target_ctx).await;
7689        let ModuleControlResponseToModule::LiveRoots {
7690            roots,
7691            unknown_root_bindings,
7692            total_bindings,
7693        } = actual
7694        else {
7695            panic!("expected live roots")
7696        };
7697        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
7698        assert_eq!(unknown_root_bindings, 0);
7699        assert_eq!(
7700            roots.len(),
7701            2,
7702            "root-known arm must retain each canonical root"
7703        );
7704        assert_eq!(
7705            total_bindings,
7706            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7707        );
7708        let counts = roots
7709            .iter()
7710            .map(|root| (root.project_root.clone(), root.bound, root.pending))
7711            .collect::<Vec<_>>();
7712        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
7713        expected.sort_by(|a, b| a.0.cmp(&b.0));
7714        assert_eq!(
7715            counts, expected,
7716            "roots must sort by path and count pending separately"
7717        );
7718        handler
7719            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
7720            .await
7721            .unwrap();
7722        assert!(pending.await.unwrap().is_empty());
7723    }
7724
7725    #[tokio::test(start_paused = true)]
7726    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
7727        let forwarding = Arc::new(ForwardingTable::default());
7728        let handler =
7729            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
7730        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
7731        hello_via_sink(
7732            &handler,
7733            &target_ctx,
7734            &mut target_rx,
7735            hello_frame("target", PROTOCOL_VERSION, 1),
7736        )
7737        .await;
7738        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
7739        let pending = forwarding
7740            .begin_route_bind_relay_for_test(
7741                client_ctx.connection_id,
7742                client_ctx.egress.clone(),
7743                2,
7744                "target",
7745            )
7746            .unwrap();
7747        forwarding
7748            .complete_pending_relay(
7749                target_ctx.connection_id,
7750                pending.corr,
7751                RouteBindRelayOutcome::Accepted,
7752            )
7753            .unwrap();
7754        let actual = query_live_roots(&handler, &target_ctx).await;
7755        let ModuleControlResponseToModule::LiveRoots {
7756            roots,
7757            unknown_root_bindings,
7758            total_bindings,
7759        } = actual
7760        else {
7761            panic!("expected live roots")
7762        };
7763        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
7764        assert_eq!(
7765            unknown_root_bindings, 1,
7766            "unknown-root arm must not read as no bindings"
7767        );
7768        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
7769        assert_eq!(
7770            total_bindings,
7771            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7772        );
7773    }
7774
7775    /// A module reads HELLO_ACK as its first frame and exits on anything else,
7776    /// so the ack has to be on its outbound queue before the module is
7777    /// routable. The connection loop writes a handler's replies only after the
7778    /// handler returns; this test stops in exactly that gap, runs a real
7779    /// route.open from another connection, and only then writes whatever the
7780    /// HELLO handler returned, the way the loop would. If the ack were still a
7781    /// reply, the route.bind request would reach the module first.
7782    #[tokio::test(start_paused = true)]
7783    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
7784        let forwarding = Arc::new(ForwardingTable::default());
7785        let handler =
7786            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
7787        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
7788        let replies = handler
7789            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
7790            .await
7791            .unwrap();
7792        let queued_by_hello = module_rx.len();
7793
7794        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
7795        let open_handler = handler.clone();
7796        let open = tokio::spawn(async move {
7797            open_handler
7798                .handle_control_frame(
7799                    &client_ctx,
7800                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
7801                )
7802                .await
7803                .unwrap()
7804        });
7805        // Let the route.open run until its route.bind is on the module's queue.
7806        let mut spins = 0;
7807        while module_rx.len() == queued_by_hello {
7808            spins += 1;
7809            assert!(spins < 10_000, "route.open never queued a route.bind");
7810            tokio::task::yield_now().await;
7811        }
7812
7813        // Now the connection loop's half: write the HELLO handler's replies.
7814        for reply in replies {
7815            module_ctx.egress.send(reply).await.unwrap();
7816        }
7817
7818        let first = module_rx.recv().await.unwrap().frame;
7819        assert_eq!(
7820            first.header.ty,
7821            FrameType::HelloAck,
7822            "the first frame a registering module reads must be its HELLO_ACK"
7823        );
7824        assert_eq!(first.header.corr, 7);
7825        let second = module_rx.recv().await.unwrap().frame;
7826        assert_eq!(second.header.ty, FrameType::Request);
7827        assert!(
7828            matches!(
7829                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
7830                ModuleControlRequest::RouteBind { .. }
7831            ),
7832            "the route.bind follows the ack"
7833        );
7834        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
7835
7836        handler
7837            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
7838            .await
7839            .unwrap();
7840        assert!(open.await.unwrap().is_empty());
7841        let _ = client_rx.recv().await.unwrap();
7842    }
7843
7844    #[tokio::test(start_paused = true)]
7845    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
7846        let handler = ControlHandler::with_forwarding(
7847            Arc::new(Registry::default()),
7848            Arc::new(ForwardingTable::default()),
7849        );
7850        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
7851        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
7852        hello_via_sink(
7853            &handler,
7854            &first_ctx,
7855            &mut first_rx,
7856            hello_frame("first", PROTOCOL_VERSION, 1),
7857        )
7858        .await;
7859        hello_via_sink(
7860            &handler,
7861            &second_ctx,
7862            &mut second_rx,
7863            hello_frame("second", PROTOCOL_VERSION, 2),
7864        )
7865        .await;
7866        let root = unique_project_root("second-only");
7867        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
7868        let cloned = handler.clone();
7869        let open = tokio::spawn(async move {
7870            cloned
7871                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
7872                .await
7873                .unwrap()
7874        });
7875        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
7876            .await
7877            .unwrap()
7878            .unwrap();
7879        let first = query_live_roots(&handler, &first_ctx).await;
7880        let second = query_live_roots(&handler, &second_ctx).await;
7881        assert!(
7882            matches!(
7883                first,
7884                ModuleControlResponseToModule::LiveRoots {
7885                    total_bindings: 0,
7886                    ..
7887                }
7888            ),
7889            "cross-module scope must not expose another module's roots"
7890        );
7891        assert!(
7892            matches!(
7893                second,
7894                ModuleControlResponseToModule::LiveRoots {
7895                    total_bindings: 1,
7896                    ..
7897                }
7898            ),
7899            "second module must see its pending route"
7900        );
7901        handler
7902            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
7903            .await
7904            .unwrap();
7905        assert!(open.await.unwrap().is_empty());
7906    }
7907
7908    #[tokio::test(start_paused = true)]
7909    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
7910        let handler = ControlHandler::with_forwarding(
7911            Arc::new(Registry::default()),
7912            Arc::new(ForwardingTable::default()),
7913        );
7914        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
7915        hello_via_sink(
7916            &handler,
7917            &target_ctx,
7918            &mut target_rx,
7919            hello_frame("target", PROTOCOL_VERSION, 1),
7920        )
7921        .await;
7922        let actual = query_live_roots(&handler, &target_ctx).await;
7923        let ModuleControlResponseToModule::LiveRoots {
7924            roots,
7925            unknown_root_bindings,
7926            total_bindings,
7927        } = actual
7928        else {
7929            panic!("expected live roots")
7930        };
7931        assert!(roots.is_empty());
7932        assert_eq!(unknown_root_bindings, 0);
7933        assert_eq!(total_bindings, 0);
7934        assert_eq!(
7935            total_bindings,
7936            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7937        );
7938    }
7939
7940    /// Read the vendored fed corpus rather than hand-building a package.
7941    ///
7942    /// A hand-built object encodes what the test author believed the carrier
7943    /// emits. These vectors are what it actually emits, and one of them exists
7944    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
7945    /// ignores additive unknown fields at the traversal emit terminus."
7946    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
7947        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7948            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
7949        let text = std::fs::read_to_string(&path)
7950            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
7951        let vectors: Vec<(String, Value)> = text
7952            .lines()
7953            .filter(|line| !line.trim().is_empty())
7954            .map(|line| {
7955                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
7956                let id = entry["corpus_id"]
7957                    .as_str()
7958                    .expect("every vector carries a corpus_id")
7959                    .to_string();
7960                (id, entry["package"].clone())
7961            })
7962            .collect();
7963        // Pin the count: a corpus that silently shrinks would take its coverage
7964        // with it, and a suite reading N-1 vectors reports the same clean pass
7965        // as one reading N.
7966        assert_eq!(
7967            vectors.len(),
7968            3,
7969            "vendored fed corpus changed size; re-sync from subc-federation"
7970        );
7971
7972        // Pin what makes the corpus DISCRIMINATING, not just present.
7973        //
7974        // The relay test below takes its expected value from the corpus, so the
7975        // corpus supplies the test's power to detect a lossy relay rather than
7976        // its correctness. A relay that dropped unrecognised fields would still
7977        // be caught -- but only by a package carrying fields it does not know.
7978        // Shrink every package to the handful of keys any implementation would
7979        // recognise and the test keeps passing over an input that can no longer
7980        // fail, which is the same clean green as a corpus that shrank away.
7981        //
7982        // So assert the precondition rather than duplicating the packages here:
7983        // at least one vector must carry a field beyond the small common set.
7984        // That is one claim to maintain instead of nine, and it fails loudly if
7985        // a re-sync ever flattens the corpus.
7986        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
7987        let richest = vectors
7988            .iter()
7989            .filter_map(|(_, package)| package.as_object())
7990            .map(|object| {
7991                object
7992                    .keys()
7993                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
7994                    .count()
7995            })
7996            .max()
7997            .unwrap_or(0);
7998        assert!(
7999            richest >= 2,
8000            "vendored corpus no longer carries a package with unmodelled fields, \
8001             so the relay test can no longer distinguish a verbatim relay from a lossy one"
8002        );
8003
8004        vectors
8005    }
8006
8007    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
8008    ///
8009    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
8010    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
8011    /// three-key object, so a relay that quietly dropped fields it did not
8012    /// recognise would satisfy it. These vectors carry nine keys including ones
8013    /// this crate has no type for, so a typed relay fails here and only here.
8014    #[tokio::test]
8015    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
8016        for (corpus_id, package) in fed_admission_facts_vectors() {
8017            let registry = Arc::new(Registry::default());
8018            let forwarding = Arc::new(ForwardingTable::default());
8019            let supervisor = SupervisorHandle::new();
8020            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8021            let handler =
8022                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8023                    .with_supervisor(supervisor)
8024                    .with_admission_facts_config(
8025                        Some("fed".to_string()),
8026                        Some(vec!["target".to_string()]),
8027                    );
8028
8029            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
8030            hello_via_sink(
8031                &handler,
8032                &target_ctx,
8033                &mut target_rx,
8034                hello_frame("target", PROTOCOL_VERSION, 1),
8035            )
8036            .await;
8037
8038            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
8039            let route_handler = handler.clone();
8040            let expected = package.clone();
8041            let route_task = tokio::spawn(async move {
8042                route_handler
8043                    .handle_control_frame(
8044                        &client_ctx,
8045                        route_open_frame_with_admission_facts(
8046                            20,
8047                            "target",
8048                            unique_project_root("admission-facts"),
8049                            Some(subc_control::ConsumerIdentity {
8050                                module_id: "fed".to_string(),
8051                                launch_nonce: "fed-nonce".to_string(),
8052                            }),
8053                            Some(package),
8054                        ),
8055                    )
8056                    .await
8057                    .unwrap()
8058            });
8059
8060            let bind_frame = target_rx.recv().await.unwrap();
8061            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8062            let ModuleControlRequest::RouteBind {
8063                admission_facts, ..
8064            } = bind
8065            else {
8066                panic!("{corpus_id}: expected route.bind")
8067            };
8068            assert_eq!(
8069                admission_facts,
8070                Some(expected),
8071                "{corpus_id}: relay must not add, drop or reshape any field"
8072            );
8073
8074            handler
8075                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8076                .await
8077                .unwrap();
8078            route_task.await.unwrap();
8079        }
8080    }
8081
8082    #[tokio::test]
8083    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
8084        let registry = Arc::new(Registry::default());
8085        let forwarding = Arc::new(ForwardingTable::default());
8086        let supervisor = SupervisorHandle::new();
8087        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8088        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
8089        let handler =
8090            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8091                .with_supervisor(supervisor)
8092                .with_admission_facts_config(
8093                    Some("fed".to_string()),
8094                    Some(vec!["target".to_string()]),
8095                );
8096
8097        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
8098        hello_via_sink(
8099            &handler,
8100            &target_ctx,
8101            &mut target_rx,
8102            hello_frame("target", PROTOCOL_VERSION, 1),
8103        )
8104        .await;
8105        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
8106        hello_via_sink(
8107            &handler,
8108            &other_ctx,
8109            &mut other_rx,
8110            hello_frame("other", PROTOCOL_VERSION, 2),
8111        )
8112        .await;
8113
8114        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
8115        let expected_facts = facts.clone();
8116        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
8117        let route_handler = handler.clone();
8118        let route_task = tokio::spawn(async move {
8119            route_handler
8120                .handle_control_frame(
8121                    &client_ctx,
8122                    route_open_frame_with_admission_facts(
8123                        10,
8124                        "target",
8125                        unique_project_root("admission-facts"),
8126                        Some(subc_control::ConsumerIdentity {
8127                            module_id: "fed".to_string(),
8128                            launch_nonce: "fed-nonce".to_string(),
8129                        }),
8130                        Some(facts.clone()),
8131                    ),
8132                )
8133                .await
8134                .unwrap()
8135        });
8136        let bind_frame = target_rx.recv().await.unwrap();
8137        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8138        let ModuleControlRequest::RouteBind {
8139            admission_facts, ..
8140        } = bind
8141        else {
8142            panic!("expected route.bind")
8143        };
8144        assert_eq!(admission_facts, Some(expected_facts));
8145        handler
8146            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8147            .await
8148            .unwrap();
8149        assert!(route_task.await.unwrap().is_empty());
8150        assert!(matches!(
8151            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
8152                .unwrap(),
8153            ClientControlResponse::RouteOpen { .. }
8154        ));
8155
8156        let direct = handler
8157            .handle_control_frame(
8158                &route_ctx(ConnectionId::new(73)).0,
8159                route_open_frame_with_admission_facts(
8160                    11,
8161                    "target",
8162                    unique_project_root("admission-facts"),
8163                    None,
8164                    Some(json!({"x": 1})),
8165                ),
8166            )
8167            .await
8168            .unwrap();
8169        assert_eq!(
8170            parse_error(&direct[0])["code"],
8171            "admission_facts_not_permitted"
8172        );
8173
8174        let different_reserved = handler
8175            .handle_control_frame(
8176                &route_ctx(ConnectionId::new(77)).0,
8177                route_open_frame_with_admission_facts(
8178                    15,
8179                    "target",
8180                    unique_project_root("admission-facts"),
8181                    Some(subc_control::ConsumerIdentity {
8182                        module_id: "other".to_string(),
8183                        launch_nonce: "other-nonce".to_string(),
8184                    }),
8185                    Some(json!({"x": 1})),
8186                ),
8187            )
8188            .await
8189            .unwrap();
8190        assert_eq!(
8191            parse_error(&different_reserved[0])["code"],
8192            "admission_facts_not_permitted"
8193        );
8194
8195        let other_target = handler
8196            .handle_control_frame(
8197                &route_ctx(ConnectionId::new(74)).0,
8198                route_open_frame_with_admission_facts(
8199                    12,
8200                    "other",
8201                    unique_project_root("admission-facts"),
8202                    Some(subc_control::ConsumerIdentity {
8203                        module_id: "fed".to_string(),
8204                        launch_nonce: "fed-nonce".to_string(),
8205                    }),
8206                    Some(json!({"x": 1})),
8207                ),
8208            )
8209            .await
8210            .unwrap();
8211        assert_eq!(
8212            parse_error(&other_target[0])["code"],
8213            "admission_facts_target_not_allowed"
8214        );
8215
8216        let nonexistent = handler
8217            .handle_control_frame(
8218                &route_ctx(ConnectionId::new(75)).0,
8219                route_open_frame_with_admission_facts(
8220                    13,
8221                    "missing",
8222                    unique_project_root("admission-facts"),
8223                    None,
8224                    Some(json!({"x": 1})),
8225                ),
8226            )
8227            .await
8228            .unwrap();
8229        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
8230
8231        let described = handler
8232            .handle_control_frame(
8233                &route_ctx(ConnectionId::new(76)).0,
8234                Frame::build(
8235                    FrameType::Request,
8236                    control_flags(),
8237                    0,
8238                    0,
8239                    14,
8240                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8241                )
8242                .unwrap(),
8243            )
8244            .await
8245            .unwrap();
8246        let ClientControlResponse::ServerDescribe { capabilities, .. } =
8247            serde_json::from_slice(&described[0].body).unwrap()
8248        else {
8249            panic!("expected server.describe response")
8250        };
8251        assert!(capabilities
8252            .iter()
8253            .any(|cap| cap == "admission_facts_relay_v1"));
8254    }
8255
8256    #[tokio::test]
8257    async fn admission_facts_without_configured_carrier_are_rejected() {
8258        let registry = Arc::new(Registry::default());
8259        let forwarding = Arc::new(ForwardingTable::default());
8260        let handler = ControlHandler::with_forwarding(registry, forwarding);
8261        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
8262        hello_via_sink(
8263            &handler,
8264            &target_ctx,
8265            &mut target_rx,
8266            hello_frame("target", PROTOCOL_VERSION, 1),
8267        )
8268        .await;
8269
8270        let responses = handler
8271            .handle_control_frame(
8272                &route_ctx(ConnectionId::new(79)).0,
8273                route_open_frame_with_admission_facts(
8274                    16,
8275                    "target",
8276                    unique_project_root("admission-facts"),
8277                    None,
8278                    Some(json!({"x": 1})),
8279                ),
8280            )
8281            .await
8282            .unwrap();
8283        assert_eq!(
8284            parse_error(&responses[0])["code"],
8285            "admission_facts_not_permitted"
8286        );
8287    }
8288
8289    #[tokio::test]
8290    async fn route_open_relays_consumer_capabilities_verbatim() {
8291        let registry = Arc::new(Registry::default());
8292        let forwarding = Arc::new(ForwardingTable::default());
8293        let handler =
8294            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
8295        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
8296        hello_via_sink(
8297            &handler,
8298            &module_ctx,
8299            &mut module_rx,
8300            hello_frame("aft", PROTOCOL_VERSION, 7),
8301        )
8302        .await;
8303
8304        let expected = vec!["elicitation".to_string(), "roots".to_string()];
8305        let expected_for_request = expected.clone();
8306        let project_root = unique_project_root("consumer-capabilities-present");
8307        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
8308        let route_handler = handler.clone();
8309        let route_task = tokio::spawn(async move {
8310            route_handler
8311                .handle_control_frame(
8312                    &client_ctx,
8313                    route_open_frame_with_consumer_capabilities(
8314                        401,
8315                        "aft",
8316                        project_root,
8317                        Some(expected_for_request),
8318                    ),
8319                )
8320                .await
8321                .unwrap()
8322        });
8323        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8324            .await
8325            .unwrap()
8326            .unwrap();
8327        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8328        let ModuleControlRequest::RouteBind {
8329            consumer_capabilities,
8330            ..
8331        } = bind
8332        else {
8333            panic!("expected route.bind request, got {bind:?}");
8334        };
8335        assert_eq!(consumer_capabilities, Some(expected.clone()));
8336
8337        handler
8338            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8339            .await
8340            .unwrap();
8341        let route_response = route_task.await.unwrap();
8342        assert!(route_response.is_empty());
8343        let published = client_rx.recv().await.unwrap();
8344        assert!(matches!(
8345            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8346            ClientControlResponse::RouteOpen { .. }
8347        ));
8348    }
8349
8350    #[tokio::test]
8351    async fn route_open_without_consumer_capabilities_relays_none() {
8352        let registry = Arc::new(Registry::default());
8353        let forwarding = Arc::new(ForwardingTable::default());
8354        let handler =
8355            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
8356        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
8357        hello_via_sink(
8358            &handler,
8359            &module_ctx,
8360            &mut module_rx,
8361            hello_frame("aft", PROTOCOL_VERSION, 7),
8362        )
8363        .await;
8364
8365        let project_root = unique_project_root("consumer-capabilities-absent");
8366        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
8367        let route_handler = handler.clone();
8368        let route_task = tokio::spawn(async move {
8369            route_handler
8370                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
8371                .await
8372                .unwrap()
8373        });
8374        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8375            .await
8376            .unwrap()
8377            .unwrap();
8378        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8379        let ModuleControlRequest::RouteBind {
8380            consumer_capabilities,
8381            ..
8382        } = bind
8383        else {
8384            panic!("expected route.bind request, got {bind:?}");
8385        };
8386        assert_eq!(consumer_capabilities, None);
8387
8388        handler
8389            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8390            .await
8391            .unwrap();
8392        let route_response = route_task.await.unwrap();
8393        assert!(route_response.is_empty());
8394        let published = client_rx.recv().await.unwrap();
8395        assert!(matches!(
8396            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8397            ClientControlResponse::RouteOpen { .. }
8398        ));
8399    }
8400
8401    #[tokio::test]
8402    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
8403        let registry = Arc::new(Registry::default());
8404        let forwarding = Arc::new(ForwardingTable::default());
8405        let handler =
8406            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8407                .with_health_probe_timeout(Duration::from_secs(5));
8408        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
8409        hello_via_sink(
8410            &handler,
8411            &module_ctx,
8412            &mut module_rx,
8413            non_routable_hello_frame_with_control_ops(
8414                "mcp",
8415                300,
8416                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8417            ),
8418        )
8419        .await;
8420        assert!(registry
8421            .get_module("mcp")
8422            .unwrap()
8423            .unwrap()
8424            .manifest
8425            .provides
8426            .is_empty());
8427
8428        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
8429        let route_response = handler
8430            .handle_control_frame(
8431                &route_client_ctx,
8432                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
8433            )
8434            .await
8435            .unwrap();
8436        assert_eq!(route_response[0].header.ty, FrameType::Error);
8437        assert_eq!(
8438            parse_error(&route_response[0])["code"],
8439            "target_unavailable"
8440        );
8441        assert!(parse_error(&route_response[0])["message"]
8442            .as_str()
8443            .unwrap()
8444            .contains("does not provide the requested target"));
8445        assert!(module_rx.try_recv().is_err());
8446
8447        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
8448        let health_handler = handler.clone();
8449        let health_task = tokio::spawn(async move {
8450            health_handler
8451                .handle_control_frame(
8452                    &health_client_ctx,
8453                    supervisor_health_probe_frame(302, "mcp"),
8454                )
8455                .await
8456                .unwrap()
8457        });
8458        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8459            .await
8460            .unwrap()
8461            .unwrap();
8462        assert_eq!(
8463            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
8464            ModuleControlRequest::HealthCheck {}
8465        );
8466        handler
8467            .handle_control_frame(
8468                &module_ctx,
8469                health_response(health_frame.header.corr, HealthStatus::Ok),
8470            )
8471            .await
8472            .unwrap();
8473        let health_response = health_task.await.unwrap();
8474        assert_eq!(health_response[0].header.ty, FrameType::Response);
8475        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
8476            ClientControlResponse::SupervisorHealthProbe {
8477                module_id, status, ..
8478            } => {
8479                assert_eq!(module_id, "mcp");
8480                assert_eq!(status, HealthStatus::Ok);
8481            }
8482            other => panic!("unexpected health response: {other:?}"),
8483        }
8484
8485        // Exercise the forwarding cleanup path directly while leaving the registry
8486        // advertisement in place. If cleanup leaves a stale control sink behind,
8487        // the next probe will enqueue onto it and wait for the long probe timeout
8488        // instead of returning an immediate no-connection error.
8489        forwarding
8490            .cleanup_connection(module_ctx.connection_id)
8491            .unwrap();
8492        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
8493        let cleanup_response = tokio::time::timeout(
8494            Duration::from_millis(200),
8495            handler.handle_control_frame(
8496                &cleanup_probe_ctx,
8497                supervisor_health_probe_frame(303, "mcp"),
8498            ),
8499        )
8500        .await
8501        .expect("probe should fail immediately when the control lane is gone")
8502        .unwrap();
8503        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
8504        assert_eq!(
8505            parse_error(&cleanup_response[0])["code"],
8506            "target_unavailable"
8507        );
8508        assert!(parse_error(&cleanup_response[0])["message"]
8509            .as_str()
8510            .unwrap()
8511            .contains("no module connection"));
8512
8513        handler
8514            .cleanup_connection(module_ctx.connection_id)
8515            .unwrap();
8516    }
8517
8518    #[tokio::test]
8519    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
8520        let registry = Arc::new(Registry::default());
8521        let supervisor_handle = SupervisorHandle::new();
8522        let supervisor =
8523            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
8524                .with_handle(supervisor_handle.clone())
8525                .with_connection_file_path(
8526                    std::env::temp_dir()
8527                        .join(format!("subc-route-open-warming-{}", std::process::id())),
8528                );
8529        let module = supervisor
8530            .supervise_configured(
8531                ModuleSpec {
8532                    module_id: "warming".to_string(),
8533                    program: fake_aft_stub_path(),
8534                    args: Vec::new(),
8535                    env: Vec::new(),
8536                    reserved: false,
8537                    reserved_prefixes: Vec::new(),
8538                    protocol: ModuleProtocol::Subc,
8539                    overlap: Default::default(),
8540                },
8541                true,
8542            )
8543            .unwrap();
8544        assert_eq!(module.state().unwrap(), ModuleState::Running);
8545
8546        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8547        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
8548        let response = handler
8549            .handle_control_frame(
8550                &ctx,
8551                route_open_frame(304, "warming", unique_project_root("warming")),
8552            )
8553            .await
8554            .unwrap();
8555        module.stop().await.unwrap();
8556
8557        assert_eq!(response[0].header.ty, FrameType::Error);
8558        let error = parse_error(&response[0]);
8559        assert_eq!(error["code"], "module_warming");
8560        assert!(error["message"]
8561            .as_str()
8562            .unwrap()
8563            .contains("state=running, enabled=true, live=false"));
8564    }
8565
8566    /// One wire code has several senders, so the refusal line names the check
8567    /// that refused. This drives the shared refusal path (every non-supervised
8568    /// refusal goes through `route_open_refusal_frame`) with an unregistered
8569    /// target and requires the branch label on the event.
8570    #[tokio::test]
8571    async fn route_open_refusal_names_the_check_that_refused() {
8572        let handler = ControlHandler::new(Arc::new(Registry::default()));
8573        let capture = EventCapture::default();
8574        let _subscriber =
8575            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8576        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
8577        let response = handler
8578            .handle_control_frame(
8579                &ctx,
8580                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
8581            )
8582            .await
8583            .unwrap();
8584
8585        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
8586        let event = capture
8587            .events()
8588            .into_iter()
8589            .find(|event| {
8590                event.target == "control"
8591                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
8592            })
8593            .expect("route.open refusal event");
8594        assert_eq!(
8595            event.fields.get("reason"),
8596            Some(&"\"not_registered\"".to_string())
8597        );
8598    }
8599
8600    #[tokio::test]
8601    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
8602        let registry = Arc::new(Registry::default());
8603        let supervisor_handle = SupervisorHandle::new();
8604        let supervisor =
8605            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
8606                .with_handle(supervisor_handle.clone())
8607                .with_connection_file_path(std::env::temp_dir().join(format!(
8608                    "subc-route-open-refusal-info-{}",
8609                    std::process::id()
8610                )));
8611        let module = supervisor
8612            .supervise_configured(
8613                ModuleSpec {
8614                    module_id: "warming".to_string(),
8615                    program: fake_aft_stub_path(),
8616                    args: Vec::new(),
8617                    env: Vec::new(),
8618                    reserved: false,
8619                    reserved_prefixes: Vec::new(),
8620                    protocol: ModuleProtocol::Subc,
8621                    overlap: Default::default(),
8622                },
8623                true,
8624            )
8625            .unwrap();
8626        assert_eq!(module.state().unwrap(), ModuleState::Running);
8627
8628        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8629        assert!(handler
8630            .counters()
8631            .snapshot()
8632            .get("route_open_refused_by_code")
8633            .is_none());
8634        let capture = EventCapture::default();
8635        let _subscriber =
8636            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8637        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
8638        let response = handler
8639            .handle_control_frame(
8640                &ctx,
8641                route_open_frame(394, "warming", unique_project_root("refusal-info")),
8642            )
8643            .await
8644            .unwrap();
8645        module.stop().await.unwrap();
8646
8647        assert_eq!(parse_error(&response[0])["code"], "module_warming");
8648        let event = capture
8649            .events()
8650            .into_iter()
8651            .find(|event| {
8652                event.target == "control"
8653                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
8654            })
8655            .expect("route.open refusal event");
8656        assert_eq!(
8657            event.fields.get("module_id"),
8658            Some(&"\"warming\"".to_string())
8659        );
8660        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
8661        assert_eq!(
8662            event.fields.get("reason"),
8663            Some(&"\"supervised_not_registered\"".to_string())
8664        );
8665        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
8666        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
8667        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
8668        assert_eq!(
8669            handler.counters().snapshot()["route_open_refused_by_code"],
8670            json!({ "module_warming": 1 })
8671        );
8672    }
8673
8674    #[tokio::test(flavor = "current_thread")]
8675    async fn route_open_unknown_module_escapes_target_module_id() {
8676        let handler = ControlHandler::new(Arc::new(Registry::default()));
8677        let capture = EventCapture::default();
8678        let _subscriber =
8679            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8680        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
8681        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
8682        let response = handler
8683            .handle_control_frame(
8684                &ctx,
8685                route_open_frame(
8686                    395,
8687                    hostile_module_id,
8688                    unique_project_root("hostile-target-module-id"),
8689                ),
8690            )
8691            .await
8692            .unwrap();
8693
8694        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
8695        let event = capture
8696            .events()
8697            .into_iter()
8698            .find(|event| {
8699                event.target == "control"
8700                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
8701            })
8702            .expect("route.open unknown-module refusal event");
8703        let logged = event.fields.get("module_id").expect("module_id field");
8704        assert!(!logged.bytes().any(|byte| byte < 0x20));
8705        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
8706    }
8707
8708    #[tokio::test(flavor = "current_thread")]
8709    async fn route_open_module_rejection_uses_daemon_counter_key() {
8710        let registry = Arc::new(Registry::default());
8711        let forwarding = Arc::new(ForwardingTable::default());
8712        let handler =
8713            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
8714        let module_connection = ConnectionId::new(95);
8715        let (module_ctx, mut module_rx) = route_ctx(module_connection);
8716        hello_via_sink(
8717            &handler,
8718            &module_ctx,
8719            &mut module_rx,
8720            hello_frame("aft", PROTOCOL_VERSION, 395),
8721        )
8722        .await;
8723
8724        let client_connection = ConnectionId::new(96);
8725        let (client_ctx, _client_rx) = route_ctx(client_connection);
8726        let capture = EventCapture::default();
8727        let _subscriber =
8728            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8729        let (route_task, bind) = relay_route_open(
8730            &handler,
8731            client_connection,
8732            &client_ctx.egress,
8733            &mut module_rx,
8734            396,
8735            "aft",
8736            "hostile-module-code",
8737        )
8738        .await;
8739        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
8740        let rejection = Frame::build(
8741            FrameType::Error,
8742            control_flags(),
8743            0,
8744            0,
8745            bind.header.corr,
8746            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
8747        )
8748        .unwrap();
8749        handler
8750            .handle_control_frame(&module_ctx, rejection)
8751            .await
8752            .unwrap();
8753
8754        let response = route_task.await.unwrap();
8755        assert_eq!(parse_error(&response[0])["code"], hostile_code);
8756        let counters = handler.counters().snapshot();
8757        assert_eq!(
8758            counters["route_open_refused_by_code"],
8759            json!({ "module_rejected": 1 })
8760        );
8761        assert!(counters["route_open_refused_by_code"]
8762            .get(hostile_code)
8763            .is_none());
8764
8765        let event = capture
8766            .events()
8767            .into_iter()
8768            .find(|event| {
8769                event.target == "control"
8770                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
8771            })
8772            .expect("route.open module-rejection refusal event");
8773        let logged = event.fields.get("module_code").expect("module_code field");
8774        assert!(!logged.bytes().any(|byte| byte < 0x20));
8775        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
8776    }
8777
8778    #[tokio::test]
8779    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
8780        let registry = Arc::new(Registry::default());
8781        let supervisor_handle = SupervisorHandle::new();
8782        let missing_program = std::env::temp_dir().join(format!(
8783            "subc-route-open-missing-program-{}",
8784            std::process::id()
8785        ));
8786        let supervisor =
8787            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
8788                .with_handle(supervisor_handle.clone());
8789        let module = supervisor
8790            .supervise_configured(
8791                ModuleSpec {
8792                    module_id: "failed".to_string(),
8793                    program: missing_program,
8794                    args: Vec::new(),
8795                    env: Vec::new(),
8796                    reserved: false,
8797                    reserved_prefixes: Vec::new(),
8798                    protocol: ModuleProtocol::Subc,
8799                    overlap: Default::default(),
8800                },
8801                true,
8802            )
8803            .unwrap();
8804        assert_eq!(module.state().unwrap(), ModuleState::Failed);
8805
8806        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8807        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
8808        let response = handler
8809            .handle_control_frame(
8810                &ctx,
8811                route_open_frame(305, "failed", unique_project_root("failed")),
8812            )
8813            .await
8814            .unwrap();
8815
8816        assert_eq!(response[0].header.ty, FrameType::Error);
8817        let error = parse_error(&response[0]);
8818        assert_eq!(error["code"], "target_unavailable");
8819        assert!(error["message"]
8820            .as_str()
8821            .unwrap()
8822            .contains("state=failed, enabled=true, live=false"));
8823    }
8824
8825    #[tokio::test]
8826    async fn route_open_role_mismatch_remains_target_unavailable() {
8827        let registry = Arc::new(Registry::default());
8828        let handler = ControlHandler::new(Arc::clone(&registry));
8829        handler
8830            .handle_control(
8831                ConnectionId::new(41),
8832                non_routable_hello_frame_with_control_ops("health-only", 306, None),
8833            )
8834            .unwrap();
8835
8836        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
8837        let response = handler
8838            .handle_control_frame(
8839                &ctx,
8840                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
8841            )
8842            .await
8843            .unwrap();
8844
8845        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8846        assert!(parse_error(&response[0])["message"]
8847            .as_str()
8848            .unwrap()
8849            .contains("does not provide the requested target"));
8850    }
8851
8852    #[tokio::test]
8853    async fn route_open_inactive_registration_remains_target_unavailable() {
8854        let registry = Arc::new(Registry::default());
8855        let handler = ControlHandler::new(Arc::clone(&registry));
8856        handler
8857            .handle_control(
8858                ConnectionId::new(43),
8859                hello_frame("inactive", PROTOCOL_VERSION, 308),
8860            )
8861            .unwrap();
8862        assert!(registry
8863            .set_module_state_for_test("inactive", ChannelState::Closed)
8864            .unwrap());
8865
8866        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
8867        let response = handler
8868            .handle_control_frame(
8869                &ctx,
8870                route_open_frame(309, "inactive", unique_project_root("inactive")),
8871            )
8872            .await
8873            .unwrap();
8874
8875        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8876        assert!(parse_error(&response[0])["message"]
8877            .as_str()
8878            .unwrap()
8879            .contains("is not active"));
8880    }
8881
8882    #[tokio::test]
8883    async fn late_health_reply_is_recorded_through_the_module_response_path() {
8884        let registry = Arc::new(Registry::default());
8885        let forwarding = Arc::new(ForwardingTable::default());
8886        let supervisor_handle = SupervisorHandle::new();
8887        let supervisor = Supervisor::new(Arc::clone(&registry), crate::RestartPolicy::default())
8888            .with_forwarding(Arc::clone(&forwarding))
8889            .with_handle(supervisor_handle.clone());
8890        let module = supervisor
8891            .supervise_configured(
8892                crate::ModuleSpec {
8893                    module_id: "late-health-response".to_string(),
8894                    program: PathBuf::from("disabled-module"),
8895                    args: Vec::new(),
8896                    env: Vec::new(),
8897                    reserved: false,
8898                    reserved_prefixes: Vec::new(),
8899                    protocol: ModuleProtocol::Subc,
8900                    overlap: Default::default(),
8901                },
8902                false,
8903            )
8904            .unwrap();
8905        let handler =
8906            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8907                .with_supervisor(supervisor_handle);
8908        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
8909        handler
8910            .handle_control_frame(
8911                &module_ctx,
8912                hello_frame_with_control_ops(
8913                    "late-health-response",
8914                    PROTOCOL_VERSION,
8915                    7,
8916                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8917                ),
8918            )
8919            .await
8920            .unwrap();
8921        let probe_started_at = Instant::now() - Duration::from_millis(80);
8922        let pending = forwarding
8923            .begin_health_probe_rpc_for(
8924                "late-health-response",
8925                MODULE_CONTROL_OP_HEALTH_CHECK,
8926                probe_started_at,
8927                Instant::now() - Duration::from_millis(1),
8928            )
8929            .unwrap();
8930        assert!(forwarding
8931            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
8932            .unwrap());
8933
8934        let responses = handler
8935            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
8936            .await
8937            .unwrap();
8938
8939        assert!(responses.is_empty());
8940        let health = module.status().unwrap().health;
8941        assert_eq!(health.late_answer_count, 1);
8942        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
8943    }
8944
8945    #[tokio::test]
8946    async fn health_probe_timeout_and_module_death_are_typed() {
8947        let registry = Arc::new(Registry::default());
8948        let forwarding = Arc::new(ForwardingTable::default());
8949        let handler =
8950            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8951                .with_health_probe_timeout(Duration::from_millis(50));
8952        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
8953        hello_via_sink(
8954            &handler,
8955            &module_ctx,
8956            &mut module_rx,
8957            hello_frame_with_control_ops(
8958                "aft",
8959                PROTOCOL_VERSION,
8960                7,
8961                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8962            ),
8963        )
8964        .await;
8965
8966        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
8967        let responses = handler
8968            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
8969            .await
8970            .unwrap();
8971        assert_eq!(responses[0].header.ty, FrameType::Error);
8972        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
8973        let _ = module_rx.try_recv();
8974
8975        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
8976        let health_handler = handler.clone();
8977        let death_task = tokio::spawn(async move {
8978            health_handler
8979                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
8980                .await
8981                .unwrap()
8982        });
8983        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8984            .await
8985            .unwrap()
8986            .unwrap();
8987        handler
8988            .cleanup_connection(module_ctx.connection_id)
8989            .unwrap();
8990        let responses = death_task.await.unwrap();
8991        assert_eq!(responses[0].header.ty, FrameType::Error);
8992        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
8993    }
8994
8995    #[test]
8996    fn hello_requires_exact_protocol_version() {
8997        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
8998            let registry = Arc::new(Registry::default());
8999            let handler = ControlHandler::new(Arc::clone(&registry));
9000            let responses = handler
9001                .handle_control(
9002                    ConnectionId::new(connection),
9003                    hello_frame("aft", offered, 9),
9004                )
9005                .unwrap();
9006
9007            assert_eq!(responses.len(), 1);
9008            assert_eq!(responses[0].header.ty, FrameType::Error);
9009            let error = parse_error(&responses[0]);
9010            assert_eq!(error["code"], "version_unsupported");
9011            assert!(registry.get_module("aft").unwrap().is_none());
9012            assert_eq!(registry.active_registration_count().unwrap(), 0);
9013        }
9014    }
9015
9016    #[test]
9017    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
9018        let registry = Arc::new(Registry::default());
9019        let forwarding = Arc::new(ForwardingTable::default());
9020        let handler =
9021            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9022        let module_connection = ConnectionId::new(301);
9023        let registration = registry
9024            .register_with_control_ops(
9025                manifest("aft-push", PROTOCOL_VERSION),
9026                PROTOCOL_VERSION,
9027                module_connection,
9028                module_baseline_control_ops(),
9029            )
9030            .unwrap();
9031        let (module_tx, _module_rx) = mpsc::channel(8);
9032        let endpoint = forwarding
9033            .register_module_connection(
9034                module_connection,
9035                "aft-push".to_string(),
9036                PROTOCOL_VERSION,
9037                manifest_concurrency(&registration.manifest),
9038                FrameSink::new(module_tx),
9039            )
9040            .unwrap();
9041
9042        // A push op this version does not know is ignored (forward-compat), not errored.
9043        let unknown = Frame::build(
9044            FrameType::Push,
9045            control_flags(),
9046            0,
9047            0,
9048            5,
9049            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
9050        )
9051        .unwrap();
9052        let out = handler.handle_status_update(endpoint, unknown).unwrap();
9053        assert!(
9054            out.is_empty(),
9055            "unknown push op must be ignored, got {out:?}"
9056        );
9057
9058        // A malformed body for a KNOWN op is a real error worth surfacing.
9059        let malformed = Frame::build(
9060            FrameType::Push,
9061            control_flags(),
9062            0,
9063            0,
9064            6,
9065            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
9066        )
9067        .unwrap();
9068        let out = handler.handle_status_update(endpoint, malformed).unwrap();
9069        assert_eq!(out.len(), 1);
9070        assert_eq!(out[0].header.ty, FrameType::Error);
9071        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
9072    }
9073
9074    #[test]
9075    fn hello_rejected_when_connection_already_owns_client_routes() {
9076        let registry = Arc::new(Registry::default());
9077        let forwarding = Arc::new(ForwardingTable::default());
9078        let handler =
9079            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9080        // Commits a client route on connection 202 (bound to a module on conn 101).
9081        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
9082        let client_connection = ConnectionId::new(202);
9083
9084        // That same connection now tries to register as a module: rejected, so one
9085        // connection never holds both client-route and module-endpoint state.
9086        let responses = handler
9087            .handle_control(
9088                client_connection,
9089                hello_frame("aft-second", PROTOCOL_VERSION, 9),
9090            )
9091            .unwrap();
9092        assert_eq!(responses[0].header.ty, FrameType::Error);
9093        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
9094        assert!(registry.get_module("aft-second").unwrap().is_none());
9095    }
9096
9097    #[test]
9098    fn reserved_module_hello_requires_matching_launch_nonce() {
9099        let registry = Arc::new(Registry::default());
9100        let supervisor = SupervisorHandle::new();
9101        // The supervisor recorded the nonce it injected when it spawned the reserved
9102        // module; the HELLO verifier checks against the same shared handle.
9103        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9104        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
9105
9106        // A HELLO with NO nonce is rejected.
9107        let no_nonce = handler
9108            .handle_control(
9109                ConnectionId::new(1),
9110                hello_frame("vault", PROTOCOL_VERSION, 1),
9111            )
9112            .unwrap();
9113        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
9114        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
9115        assert!(registry.get_module("vault").unwrap().is_none());
9116
9117        // A HELLO with the WRONG nonce is rejected.
9118        let wrong = handler
9119            .handle_control(
9120                ConnectionId::new(2),
9121                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
9122            )
9123            .unwrap();
9124        assert_eq!(wrong[0].header.ty, FrameType::Error);
9125        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
9126        assert!(registry.get_module("vault").unwrap().is_none());
9127
9128        // A HELLO with the CORRECT nonce registers.
9129        let ok = handler
9130            .handle_control(
9131                ConnectionId::new(3),
9132                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
9133            )
9134            .unwrap();
9135        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
9136        assert!(registry.get_module("vault").unwrap().is_some());
9137    }
9138
9139    #[test]
9140    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
9141        let registry = Arc::new(Registry::default());
9142        let supervisor = SupervisorHandle::new();
9143        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
9144        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
9145        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
9146
9147        let squat = handler
9148            .handle_control(
9149                ConnectionId::new(1),
9150                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
9151            )
9152            .unwrap();
9153        assert_eq!(squat[0].header.ty, FrameType::Error);
9154        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
9155        assert!(parse_error(&squat[0])["message"]
9156            .as_str()
9157            .unwrap()
9158            .contains("fed:"));
9159
9160        let accepted_peer = handler
9161            .handle_control(
9162                ConnectionId::new(2),
9163                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
9164            )
9165            .unwrap();
9166        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
9167
9168        let accepted_short = handler
9169            .handle_control(
9170                ConnectionId::new(3),
9171                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
9172            )
9173            .unwrap();
9174        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
9175
9176        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
9177            let response = handler
9178                .handle_control(
9179                    ConnectionId::new(conn),
9180                    hello_frame(module_id, PROTOCOL_VERSION, conn),
9181                )
9182                .unwrap();
9183            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
9184        }
9185    }
9186
9187    #[test]
9188    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
9189        let registry = Arc::new(Registry::default());
9190        let supervisor = SupervisorHandle::new();
9191        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
9192        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
9193        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
9194        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
9195
9196        let owner_nonce = handler
9197            .handle_control(
9198                ConnectionId::new(1),
9199                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
9200            )
9201            .unwrap();
9202        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
9203        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
9204        assert!(registry.get_module("fed:special").unwrap().is_none());
9205
9206        let exact_nonce = handler
9207            .handle_control(
9208                ConnectionId::new(2),
9209                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
9210            )
9211            .unwrap();
9212        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
9213        assert!(registry.get_module("fed:special").unwrap().is_some());
9214    }
9215
9216    #[test]
9217    fn non_reserved_module_ignores_launch_nonce() {
9218        let registry = Arc::new(Registry::default());
9219        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
9220        // registration succeeds whether a spawned process echoes a nonce or not.
9221        let handler = ControlHandler::new(Arc::clone(&registry));
9222        let no_nonce = handler
9223            .handle_control(
9224                ConnectionId::new(1),
9225                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
9226            )
9227            .unwrap();
9228        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
9229        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
9230
9231        let echoed_nonce = handler
9232            .handle_control(
9233                ConnectionId::new(2),
9234                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
9235            )
9236            .unwrap();
9237        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
9238        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
9239    }
9240
9241    #[test]
9242    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
9243        let handler = ControlHandler::default();
9244        let conn = ConnectionId::new(1);
9245        let malformed = Frame::build(
9246            FrameType::Hello,
9247            control_flags(),
9248            0,
9249            0,
9250            3,
9251            b"{not json".to_vec(),
9252        )
9253        .unwrap();
9254
9255        let error = handler.handle_control(conn, malformed).unwrap();
9256        assert_eq!(error[0].header.ty, FrameType::Error);
9257        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
9258
9259        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
9260        let pong = handler.handle_control(conn, ping).unwrap();
9261        assert_eq!(pong[0].header.ty, FrameType::Pong);
9262        assert_eq!(pong[0].header.corr, 4);
9263    }
9264
9265    #[test]
9266    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
9267        let registry = Arc::new(Registry::default());
9268        let handler = ControlHandler::new(Arc::clone(&registry));
9269
9270        handler
9271            .handle_control(
9272                ConnectionId::new(1),
9273                hello_frame("aft", PROTOCOL_VERSION, 1),
9274            )
9275            .unwrap();
9276        let duplicate = handler
9277            .handle_control(
9278                ConnectionId::new(2),
9279                hello_frame("aft", PROTOCOL_VERSION, 2),
9280            )
9281            .unwrap();
9282
9283        assert_eq!(duplicate[0].header.ty, FrameType::Error);
9284        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
9285        let registration = registry.get_module("aft").unwrap().unwrap();
9286        assert_eq!(registration.connection_id, ConnectionId::new(1));
9287    }
9288
9289    #[test]
9290    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
9291        let registry = Arc::new(Registry::default());
9292        let forwarding = Arc::new(ForwardingTable::default());
9293        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
9294        let handler =
9295            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9296                .with_process_liveness(process_liveness);
9297        let (ctx, route_channel, route_epoch) =
9298            bind_liveness_route(&registry, &forwarding, "aft-dead");
9299        let responses = handler
9300            .handle_route_poll(
9301                &ctx,
9302                route_poll_frame(41, PollKind::Liveness, route_channel),
9303                route_channel,
9304                route_epoch,
9305                PollKind::Liveness,
9306            )
9307            .unwrap();
9308
9309        assert_eq!(responses.len(), 1);
9310        assert_eq!(responses[0].header.ty, FrameType::Response);
9311        assert_route_poll_liveness(&responses[0], false);
9312    }
9313
9314    #[test]
9315    fn liveness_poll_without_process_source_uses_bound_route() {
9316        let registry = Arc::new(Registry::default());
9317        let forwarding = Arc::new(ForwardingTable::default());
9318        let handler =
9319            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9320        let (ctx, route_channel, route_epoch) =
9321            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
9322        let responses = handler
9323            .handle_route_poll(
9324                &ctx,
9325                route_poll_frame(42, PollKind::Liveness, route_channel),
9326                route_channel,
9327                route_epoch,
9328                PollKind::Liveness,
9329            )
9330            .unwrap();
9331
9332        assert_route_poll_liveness(&responses[0], true);
9333    }
9334
9335    #[test]
9336    fn liveness_poll_untracked_process_source_uses_bound_route() {
9337        let registry = Arc::new(Registry::default());
9338        let forwarding = Arc::new(ForwardingTable::default());
9339        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
9340        let handler =
9341            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9342                .with_process_liveness(process_liveness);
9343        let (ctx, route_channel, route_epoch) =
9344            bind_liveness_route(&registry, &forwarding, "aft-untracked");
9345        let responses = handler
9346            .handle_route_poll(
9347                &ctx,
9348                route_poll_frame(43, PollKind::Liveness, route_channel),
9349                route_channel,
9350                route_epoch,
9351                PollKind::Liveness,
9352            )
9353            .unwrap();
9354
9355        assert_route_poll_liveness(&responses[0], true);
9356    }
9357
9358    #[tokio::test]
9359    async fn unknown_op_returns_unknown_control_op() {
9360        let handler = ControlHandler::default();
9361        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
9362        let request = Frame::build(
9363            FrameType::Request,
9364            control_flags(),
9365            0,
9366            0,
9367            55,
9368            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
9369        )
9370        .unwrap();
9371
9372        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9373
9374        assert_eq!(response.len(), 1);
9375        assert_eq!(response[0].header.ty, FrameType::Error);
9376        assert_eq!(response[0].header.corr, 55);
9377        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
9378    }
9379
9380    #[tokio::test]
9381    async fn supervisor_provenance_rejects_unknown_exact_module() {
9382        let handler = ControlHandler::default();
9383        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
9384        let request = Frame::build(
9385            FrameType::Request,
9386            control_flags(),
9387            0,
9388            0,
9389            57,
9390            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
9391        )
9392        .unwrap();
9393
9394        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9395
9396        assert_eq!(response.len(), 1);
9397        assert_eq!(response[0].header.ty, FrameType::Error);
9398        assert_eq!(response[0].header.corr, 57);
9399        let error = parse_error(&response[0]);
9400        assert_eq!(error["code"], "unknown_module");
9401        assert_eq!(error["message"], "module_id 'missing' is not supervised");
9402    }
9403
9404    #[test]
9405    fn provenance_probe_override_keeps_handler_tests_deterministic() {
9406        let expected = subc_control::RunningImageAgreement::Unavailable {
9407            reason: subc_control::RunningImageUnavailableReason::HashFailed,
9408        };
9409        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
9410        assert_eq!(handler.provenance_probe_override, Some(expected));
9411    }
9412
9413    #[test]
9414    fn reload_verdict_detects_configured_program_different_from_spawned_path() {
9415        let verdict = reload_verdict(
9416            std::path::Path::new("/bin/new"),
9417            Some(std::path::Path::new("/bin/old")),
9418            subc_control::RunningImageAgreement::Unavailable {
9419                reason: subc_control::RunningImageUnavailableReason::HashFailed,
9420            },
9421        );
9422        assert!(matches!(
9423            verdict.path,
9424            subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
9425                if configured == std::path::Path::new("/bin/new")
9426                    && spawned_from == std::path::Path::new("/bin/old")
9427        ));
9428    }
9429
9430    #[test]
9431    fn reload_verdict_detects_replaced_image_at_same_path() {
9432        let image = subc_control::RunningImageAgreement::Mismatch {
9433            running: subc_control::RunningImageEvidence::LinuxProcSha256 {
9434                digest: "old".into(),
9435            },
9436            disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
9437                digest: "new".into(),
9438            },
9439        };
9440        let verdict = reload_verdict(
9441            std::path::Path::new("/bin/same"),
9442            Some(std::path::Path::new("/bin/same")),
9443            image.clone(),
9444        );
9445        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
9446        assert_eq!(verdict.image, image);
9447    }
9448
9449    #[test]
9450    fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
9451        let image = subc_control::RunningImageAgreement::Unavailable {
9452            reason: subc_control::RunningImageUnavailableReason::NotRunning,
9453        };
9454        let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
9455        assert_eq!(
9456            verdict.path,
9457            subc_control::ReloadPathAgreement::Unavailable {
9458                reason: subc_control::ReloadPathUnavailableReason::NotRunning,
9459            }
9460        );
9461        assert_eq!(verdict.image, image);
9462
9463        let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
9464            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
9465        };
9466        let verdict = reload_verdict(
9467            std::path::Path::new("/bin/same"),
9468            Some(std::path::Path::new("/bin/same")),
9469            unconfirmed.clone(),
9470        );
9471        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
9472        assert_eq!(verdict.image, unconfirmed);
9473    }
9474
9475    #[test]
9476    fn reload_verdict_preserves_each_image_unavailability_reason() {
9477        use subc_control::RunningImageUnavailableReason as Reason;
9478
9479        for reason in [
9480            Reason::NotRunning,
9481            Reason::UnsupportedPlatform,
9482            Reason::RunningExecutableUnreadable,
9483            Reason::SpawnedPathUnreadable,
9484            Reason::HashFailed,
9485            Reason::ProcessIdentityUnconfirmed,
9486            Reason::Unknown("future_probe_reason".to_string()),
9487        ] {
9488            let image = subc_control::RunningImageAgreement::Unavailable {
9489                reason: reason.clone(),
9490            };
9491            let verdict = reload_verdict(
9492                std::path::Path::new("/bin/same"),
9493                Some(std::path::Path::new("/bin/same")),
9494                image.clone(),
9495            );
9496            assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
9497            assert_eq!(verdict.image, image, "{reason:?}");
9498        }
9499    }
9500
9501    #[tokio::test]
9502    async fn malformed_control_bodies_return_invalid_control_body() {
9503        let handler = ControlHandler::default();
9504        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
9505
9506        for (corr, body) in [
9507            (56, br#"{"route_channel":1}"#.as_slice()),
9508            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
9509            (
9510                58,
9511                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
9512            ),
9513        ] {
9514            let request = Frame::build(
9515                FrameType::Request,
9516                control_flags(),
9517                0,
9518                0,
9519                corr,
9520                body.to_vec(),
9521            )
9522            .unwrap();
9523            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9524
9525            assert_eq!(response.len(), 1);
9526            assert_eq!(response[0].header.ty, FrameType::Error);
9527            assert_eq!(response[0].header.corr, corr);
9528            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
9529        }
9530    }
9531
9532    #[tokio::test]
9533    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
9534        let registry = Arc::new(Registry::default());
9535        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
9536        let router = Router::with_control_handler(Arc::clone(&control));
9537        let connection = router.begin_connection();
9538        let (ctx, mut rx) = route_ctx(connection.id());
9539
9540        router
9541            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
9542            .await
9543            .unwrap();
9544        let response = rx.recv().await.unwrap();
9545        let ack = parse_ack(&response);
9546        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9547        let channel = 1;
9548
9549        let goodbye =
9550            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
9551        router.route_for_connection(&ctx, goodbye).await.unwrap();
9552        assert!(rx.try_recv().is_err());
9553        assert!(registry.get_module("aft").unwrap().is_none());
9554
9555        router
9556            .route_for_connection(&ctx, channel_request(channel, 13))
9557            .await
9558            .unwrap();
9559        let error_frame = rx.recv().await.unwrap();
9560        assert_eq!(error_frame.header.ty, FrameType::Error);
9561        assert_eq!(error_frame.header.channel, channel);
9562    }
9563
9564    #[tokio::test]
9565    async fn dropping_router_connection_releases_registration() {
9566        let registry = Arc::new(Registry::default());
9567        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
9568        let router = Router::with_control_handler(control);
9569        let connection = router.begin_connection();
9570        let (ctx, mut rx) = route_ctx(connection.id());
9571
9572        router
9573            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
9574            .await
9575            .unwrap();
9576        let response = rx.recv().await.unwrap();
9577        let ack = parse_ack(&response);
9578        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9579        assert!(registry.get_module("aft").unwrap().is_some());
9580
9581        drop(connection);
9582
9583        assert!(registry.get_module("aft").unwrap().is_none());
9584        assert_eq!(registry.active_registration_count().unwrap(), 0);
9585    }
9586
9587    fn capability_manifest(
9588        module_id: &str,
9589        provides: &[&str],
9590        must_never_reach: &[&str],
9591    ) -> ModuleManifest {
9592        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
9593        manifest.capabilities = Some(CapabilityDeclarations {
9594            provides: provides
9595                .iter()
9596                .map(|capability| (*capability).to_string())
9597                .collect(),
9598            requires: Vec::new(),
9599            must_never_reach: must_never_reach
9600                .iter()
9601                .map(|capability| (*capability).to_string())
9602                .collect(),
9603        });
9604        manifest
9605    }
9606
9607    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
9608        Frame::build(
9609            FrameType::Hello,
9610            control_flags(),
9611            0,
9612            0,
9613            corr,
9614            serde_json::to_vec(&ModuleHelloBody {
9615                protocol_ver: manifest.protocol_ver,
9616                manifest,
9617                control_ops: None,
9618                launch_nonce: None,
9619            })
9620            .expect("capability test HELLO serializes"),
9621        )
9622        .expect("capability test HELLO frame builds")
9623    }
9624
9625    fn catalog_update_with_capabilities_frame(
9626        corr: u64,
9627        capabilities: CapabilityDeclarations,
9628    ) -> Frame {
9629        Frame::build(
9630            FrameType::Request,
9631            control_flags(),
9632            0,
9633            0,
9634            corr,
9635            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
9636                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
9637                capabilities: Some(capabilities),
9638                ready: None,
9639            })
9640            .expect("capability catalog.update serializes"),
9641        )
9642        .expect("capability catalog.update frame builds")
9643    }
9644
9645    async fn register_capability_manifest(
9646        handler: &ControlHandler,
9647        ctx: &RouteCtx,
9648        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9649        manifest: ModuleManifest,
9650        corr: u64,
9651    ) {
9652        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
9653    }
9654
9655    async fn open_route_for_capability_test(
9656        handler: &ControlHandler,
9657        target_ctx: &RouteCtx,
9658        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9659        client_connection_id: u64,
9660        corr: u64,
9661        target_module_id: &str,
9662        consumer_identity: Option<ConsumerIdentity>,
9663    ) -> (
9664        mpsc::Receiver<crate::router::OutboundFrame>,
9665        ModuleControlRequest,
9666    ) {
9667        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
9668        let route_handler = handler.clone();
9669        let target_module_id = target_module_id.to_string();
9670        let route_task = tokio::spawn(async move {
9671            route_handler
9672                .handle_control_frame(
9673                    &client_ctx,
9674                    route_open_frame_with_admission_facts(
9675                        corr,
9676                        &target_module_id,
9677                        unique_project_root("admission-facts"),
9678                        consumer_identity,
9679                        None,
9680                    ),
9681                )
9682                .await
9683                .expect("capability test route.open succeeds")
9684        });
9685        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
9686            .await
9687            .expect("capability test route.open must reach route.bind")
9688            .expect("target control receiver stays open");
9689        let bind_request: ModuleControlRequest =
9690            serde_json::from_slice(&bind.body).expect("route.bind decodes");
9691        handler
9692            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
9693            .await
9694            .expect("capability test route.bind ACK succeeds");
9695        assert!(route_task.await.expect("route.open task joins").is_empty());
9696        let opened = client_rx
9697            .recv()
9698            .await
9699            .expect("successful route.open publishes a response");
9700        assert!(matches!(
9701            serde_json::from_slice::<ClientControlResponse>(&opened.body),
9702            Ok(ClientControlResponse::RouteOpen { .. })
9703        ));
9704        (client_rx, bind_request)
9705    }
9706
9707    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
9708        assert_eq!(frame.header.ty, FrameType::Push);
9709        assert_eq!(frame.header.channel, 0);
9710        assert_eq!(
9711            serde_json::from_slice::<ClientControlPush>(&frame.body)
9712                .expect("route.closed control push decodes"),
9713            ClientControlPush::RouteClosed {
9714                module_id: target_module_id.to_string(),
9715                reason: RouteCloseReason::CapabilityDenied,
9716                drained: false,
9717                abandoned: 0,
9718                excluded_subscriptions: 0,
9719                terminal: Some(false),
9720            }
9721        );
9722    }
9723
9724    #[tokio::test]
9725    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
9726        let registry = Arc::new(Registry::default());
9727        let forwarding = Arc::new(ForwardingTable::default());
9728        let supervisor = SupervisorHandle::new();
9729        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9730        let handler =
9731            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9732                .with_supervisor(supervisor);
9733        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
9734        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
9735        register_capability_manifest(
9736            &handler,
9737            &target_ctx,
9738            &mut target_rx,
9739            capability_manifest("target", &["credentials-provider/v1"], &[]),
9740            1,
9741        )
9742        .await;
9743        register_capability_manifest(
9744            &handler,
9745            &opener_ctx,
9746            &mut opener_rx,
9747            capability_manifest("opener", &[], &["credentials-provider/v1"]),
9748            2,
9749        )
9750        .await;
9751
9752        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
9753        let replies = handler
9754            .handle_control_frame(
9755                &client_ctx,
9756                route_open_frame_with_admission_facts(
9757                    3,
9758                    "target",
9759                    unique_project_root("admission-facts"),
9760                    Some(ConsumerIdentity {
9761                        module_id: "opener".to_string(),
9762                        launch_nonce: "opener-nonce".to_string(),
9763                    }),
9764                    None,
9765                ),
9766            )
9767            .await
9768            .expect("denied route.open returns a typed frame");
9769        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
9770        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9771        assert!(
9772            target_rx.try_recv().is_err(),
9773            "forbidden route.open must not relay route.bind"
9774        );
9775    }
9776
9777    #[tokio::test]
9778    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
9779        let registry = Arc::new(Registry::default());
9780        let forwarding = Arc::new(ForwardingTable::default());
9781        let supervisor = SupervisorHandle::new();
9782        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9783        let handler =
9784            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9785                .with_supervisor(supervisor);
9786        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
9787        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
9788        register_capability_manifest(
9789            &handler,
9790            &target_ctx,
9791            &mut target_rx,
9792            capability_manifest("target", &["credentials-provider/v1"], &[]),
9793            1,
9794        )
9795        .await;
9796        register_capability_manifest(
9797            &handler,
9798            &old_opener_ctx,
9799            &mut old_opener_rx,
9800            capability_manifest("opener", &[], &[]),
9801            2,
9802        )
9803        .await;
9804        let (mut client_rx, _) = open_route_for_capability_test(
9805            &handler,
9806            &target_ctx,
9807            &mut target_rx,
9808            712,
9809            3,
9810            "target",
9811            Some(ConsumerIdentity {
9812                module_id: "opener".to_string(),
9813                launch_nonce: "opener-nonce".to_string(),
9814            }),
9815        )
9816        .await;
9817        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9818
9819        handler
9820            .cleanup_connection(old_opener_ctx.connection_id)
9821            .expect("old opener registration cleans up");
9822        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
9823        register_capability_manifest(
9824            &handler,
9825            &new_opener_ctx,
9826            &mut new_opener_rx,
9827            capability_manifest("opener", &[], &["credentials-provider/v1"]),
9828            4,
9829        )
9830        .await;
9831
9832        assert_capability_denied_push(
9833            client_rx
9834                .try_recv()
9835                .expect("HELLO deny addition must emit route.closed")
9836                .frame,
9837            "target",
9838        );
9839        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9840        assert!(matches!(
9841            target_rx.try_recv(),
9842            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
9843        ));
9844    }
9845
9846    #[tokio::test]
9847    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
9848        let registry = Arc::new(Registry::default());
9849        let forwarding = Arc::new(ForwardingTable::default());
9850        let supervisor = SupervisorHandle::new();
9851        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9852        let handler =
9853            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9854                .with_supervisor(supervisor);
9855        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
9856        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
9857        register_capability_manifest(
9858            &handler,
9859            &target_ctx,
9860            &mut target_rx,
9861            capability_manifest("target", &[], &[]),
9862            1,
9863        )
9864        .await;
9865        register_capability_manifest(
9866            &handler,
9867            &opener_ctx,
9868            &mut opener_rx,
9869            capability_manifest("opener", &[], &["credentials-provider/v1"]),
9870            2,
9871        )
9872        .await;
9873        let (mut client_rx, _) = open_route_for_capability_test(
9874            &handler,
9875            &target_ctx,
9876            &mut target_rx,
9877            722,
9878            3,
9879            "target",
9880            Some(ConsumerIdentity {
9881                module_id: "opener".to_string(),
9882                launch_nonce: "opener-nonce".to_string(),
9883            }),
9884        )
9885        .await;
9886        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9887
9888        let replies = handler
9889            .handle_control_frame(
9890                &target_ctx,
9891                catalog_update_with_capabilities_frame(
9892                    4,
9893                    CapabilityDeclarations {
9894                        provides: vec!["credentials-provider/v1".to_string()],
9895                        requires: Vec::new(),
9896                        must_never_reach: Vec::new(),
9897                    },
9898                ),
9899            )
9900            .await
9901            .expect("claim catalog.update succeeds");
9902        assert!(matches!(
9903            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
9904            Ok(ModuleControlResponseToModule::CatalogUpdate {})
9905        ));
9906        assert_capability_denied_push(
9907            client_rx
9908                .try_recv()
9909                .expect("claim addition must emit route.closed")
9910                .frame,
9911            "target",
9912        );
9913        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9914        assert!(matches!(
9915            target_rx.try_recv(),
9916            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
9917        ));
9918    }
9919
9920    #[tokio::test]
9921    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
9922        let registry = Arc::new(Registry::default());
9923        let forwarding = Arc::new(ForwardingTable::default());
9924        let supervisor = SupervisorHandle::new();
9925        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9926        let handler =
9927            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9928                .with_supervisor(supervisor);
9929        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
9930        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
9931        register_capability_manifest(
9932            &handler,
9933            &target_ctx,
9934            &mut target_rx,
9935            capability_manifest("target", &["credentials-provider/v1"], &[]),
9936            1,
9937        )
9938        .await;
9939        register_capability_manifest(
9940            &handler,
9941            &opener_ctx,
9942            &mut opener_rx,
9943            capability_manifest("opener", &[], &[]),
9944            2,
9945        )
9946        .await;
9947        let (mut client_rx, _) = open_route_for_capability_test(
9948            &handler,
9949            &target_ctx,
9950            &mut target_rx,
9951            732,
9952            3,
9953            "target",
9954            Some(ConsumerIdentity {
9955                module_id: "opener".to_string(),
9956                launch_nonce: "opener-nonce".to_string(),
9957            }),
9958        )
9959        .await;
9960        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9961
9962        handler
9963            .handle_control_frame(
9964                &target_ctx,
9965                catalog_update_with_capabilities_frame(
9966                    4,
9967                    CapabilityDeclarations {
9968                        provides: Vec::new(),
9969                        requires: Vec::new(),
9970                        must_never_reach: Vec::new(),
9971                    },
9972                ),
9973            )
9974            .await
9975            .expect("claim removal catalog.update succeeds");
9976        assert_eq!(
9977            forwarding.active_binding_count().unwrap(),
9978            1,
9979            "removing an attested target claim must leave the route census unchanged"
9980        );
9981        assert!(
9982            client_rx.try_recv().is_err(),
9983            "claim removal must not emit route.closed capability_denied"
9984        );
9985        assert!(
9986            target_rx.try_recv().is_err(),
9987            "claim removal must not send the target a route GOODBYE"
9988        );
9989    }
9990
9991    /// A direct client may open a route to a denied capability provider; this
9992    /// policy applies only to attested supervised module origins, not to direct clients.
9993    #[tokio::test]
9994    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
9995        let registry = Arc::new(Registry::default());
9996        let forwarding = Arc::new(ForwardingTable::default());
9997        let supervisor = SupervisorHandle::new();
9998        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9999        let handler =
10000            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10001                .with_supervisor(supervisor);
10002        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
10003        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
10004        register_capability_manifest(
10005            &handler,
10006            &target_ctx,
10007            &mut target_rx,
10008            capability_manifest("target", &["credentials-provider/v1"], &[]),
10009            1,
10010        )
10011        .await;
10012        register_capability_manifest(
10013            &handler,
10014            &opener_ctx,
10015            &mut opener_rx,
10016            capability_manifest("opener", &[], &["credentials-provider/v1"]),
10017            2,
10018        )
10019        .await;
10020
10021        let (_client_rx, bind) = open_route_for_capability_test(
10022            &handler,
10023            &target_ctx,
10024            &mut target_rx,
10025            742,
10026            3,
10027            "target",
10028            None,
10029        )
10030        .await;
10031        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
10032            panic!("direct scope-honesty route must bind");
10033        };
10034        assert_eq!(principal, Some(Principal::Direct));
10035        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10036    }
10037
10038    /// A module that denies a capability receives no self-route exemption when it
10039    /// also attestedly provides that capability.
10040    #[tokio::test]
10041    async fn must_never_reach_self_route_is_capability_forbidden() {
10042        let registry = Arc::new(Registry::default());
10043        let forwarding = Arc::new(ForwardingTable::default());
10044        let supervisor = SupervisorHandle::new();
10045        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
10046        let handler =
10047            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10048                .with_supervisor(supervisor);
10049        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
10050        register_capability_manifest(
10051            &handler,
10052            &self_ctx,
10053            &mut self_rx,
10054            capability_manifest(
10055                "self-provider",
10056                &["credentials-provider/v1"],
10057                &["credentials-provider/v1"],
10058            ),
10059            1,
10060        )
10061        .await;
10062
10063        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
10064        let replies = handler
10065            .handle_control_frame(
10066                &client_ctx,
10067                route_open_frame_with_admission_facts(
10068                    2,
10069                    "self-provider",
10070                    unique_project_root("admission-facts"),
10071                    Some(ConsumerIdentity {
10072                        module_id: "self-provider".to_string(),
10073                        launch_nonce: "self-nonce".to_string(),
10074                    }),
10075                    None,
10076                ),
10077            )
10078            .await
10079            .expect("self-route refusal returns a typed frame");
10080        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
10081        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10082        assert!(
10083            self_rx.try_recv().is_err(),
10084            "self denial must not relay route.bind"
10085        );
10086    }
10087
10088    #[test]
10089    fn unsupported_channel_zero_frame_returns_error() {
10090        let handler = ControlHandler::default();
10091        let request = Frame::build(
10092            FrameType::Request,
10093            control_flags(),
10094            0,
10095            0,
10096            21,
10097            b"opaque".to_vec(),
10098        )
10099        .unwrap();
10100
10101        let response = handler
10102            .handle_control(ConnectionId::new(1), request)
10103            .unwrap();
10104
10105        assert_eq!(response[0].header.ty, FrameType::Error);
10106        assert_eq!(
10107            parse_error(&response[0])["code"],
10108            "unsupported_control_frame"
10109        );
10110    }
10111
10112    /// Blue/green swap at the control-plane boundary. The supervisor that opens
10113    /// a swap is not wired yet, so the candidate is registered here directly
10114    /// into the registry and forwarding candidate slots, the way the swap's
10115    /// HELLO admission will.
10116    mod swap {
10117        use super::*;
10118
10119        const INCUMBENT: ConnectionId = ConnectionId::new(30);
10120        const CANDIDATE: ConnectionId = ConnectionId::new(40);
10121
10122        struct Swap {
10123            registry: Arc<Registry>,
10124            forwarding: Arc<ForwardingTable>,
10125            handler: ControlHandler,
10126            incumbent_ctx: RouteCtx,
10127            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
10128            candidate_ctx: RouteCtx,
10129            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
10130        }
10131
10132        async fn swap_with_incumbent() -> Swap {
10133            let registry = Arc::new(Registry::default());
10134            let forwarding = Arc::new(ForwardingTable::default());
10135            let handler =
10136                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10137            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
10138            hello_via_sink(
10139                &handler,
10140                &incumbent_ctx,
10141                &mut incumbent_rx,
10142                hello_frame("aft", PROTOCOL_VERSION, 7),
10143            )
10144            .await;
10145            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
10146            Swap {
10147                registry,
10148                forwarding,
10149                handler,
10150                incumbent_ctx,
10151                incumbent_rx,
10152                candidate_ctx,
10153                candidate_rx,
10154            }
10155        }
10156
10157        fn register_candidate(swap: &Swap, ready: Option<bool>) {
10158            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
10159            candidate_manifest.ready = ready;
10160            let registration = swap
10161                .registry
10162                .register_candidate_with_control_ops(
10163                    candidate_manifest,
10164                    PROTOCOL_VERSION,
10165                    CANDIDATE,
10166                    module_baseline_control_ops(),
10167                )
10168                .unwrap();
10169            swap.forwarding
10170                .register_candidate_module_connection(
10171                    CANDIDATE,
10172                    "aft".to_string(),
10173                    PROTOCOL_VERSION,
10174                    manifest_concurrency(&registration.manifest),
10175                    swap.candidate_ctx.egress.clone(),
10176                )
10177                .unwrap();
10178        }
10179
10180        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
10181            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
10182            swap.registry.promote_candidate("aft").unwrap().unwrap();
10183            cutover.incumbent.unwrap()
10184        }
10185
10186        fn keyed_total(counters: &Value, key: &str) -> u64 {
10187            counters[key]
10188                .as_object()
10189                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
10190                .unwrap_or(0)
10191        }
10192
10193        /// An ack from the incumbent for a bind it was sent before cutover,
10194        /// arriving before the incumbent is drained. The incumbent is the live
10195        /// connection carrying every other client's routes, so the ack must
10196        /// not end it: the waiting client is told to retry, the reservation is
10197        /// given back, and the incumbent is told to drop just that binding.
10198        #[tokio::test]
10199        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
10200            let mut swap = swap_with_incumbent().await;
10201            let handler = swap.handler.clone();
10202
10203            // A co-tenant route, bound on the incumbent before the swap.
10204            let cotenant = ConnectionId::new(31);
10205            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
10206            let (cotenant_task, cotenant_bind) = relay_route_open(
10207                &handler,
10208                cotenant,
10209                &cotenant_ctx.egress,
10210                &mut swap.incumbent_rx,
10211                100,
10212                "aft",
10213                "swap-cotenant",
10214            )
10215            .await;
10216            handler
10217                .handle_control_frame(
10218                    &swap.incumbent_ctx,
10219                    route_bind_ack(cotenant_bind.header.corr),
10220                )
10221                .await
10222                .unwrap();
10223            assert!(cotenant_task.await.unwrap().is_empty());
10224            let (cotenant_channel, cotenant_epoch) =
10225                published_route(&cotenant_rx.recv().await.unwrap());
10226
10227            // A second route.open, relayed to the incumbent and not yet acked.
10228            let caller = ConnectionId::new(32);
10229            let (caller_ctx, mut caller_rx) = route_ctx(caller);
10230            let (caller_task, caller_bind) = relay_route_open(
10231                &handler,
10232                caller,
10233                &caller_ctx.egress,
10234                &mut swap.incumbent_rx,
10235                101,
10236                "aft",
10237                "swap-caller",
10238            )
10239            .await;
10240            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
10241
10242            register_candidate(&swap, None);
10243            cutover(&swap);
10244
10245            // The incumbent acks after promotion and before any drain.
10246            let ack = handler
10247                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
10248                .await;
10249            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
10250            if module_loop_error.is_some() {
10251                // What the connection loop does with an untranslated router
10252                // error: end the connection, releasing every route on it.
10253                handler.cleanup_connection(INCUMBENT).unwrap();
10254            }
10255
10256            // 1. The incumbent's other routes survive.
10257            assert!(
10258                cotenant_rx.try_recv().is_err(),
10259                "the co-tenant route on the incumbent was torn down by one late ack: \
10260                 {module_loop_error:?}"
10261            );
10262            assert!(matches!(
10263                swap.forwarding
10264                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
10265                    .unwrap(),
10266                DataRoute::Client(DataRouteState::Bound(_))
10267            ));
10268            assert_eq!(module_loop_error, None);
10269            assert!(swap
10270                .registry
10271                .get_module_by_connection(INCUMBENT)
10272                .unwrap()
10273                .is_some());
10274
10275            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
10276            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
10277                .await
10278                .expect("the incumbent is told to drop the abandoned binding")
10279                .unwrap()
10280                .frame;
10281            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10282            assert_eq!(goodbye.header.channel, abandoned_channel);
10283            assert_eq!(goodbye.header.epoch, abandoned_epoch);
10284            assert!(swap.incumbent_rx.try_recv().is_err());
10285
10286            // 3. The waiting client gets a retryable refusal and no route.
10287            let response = caller_task.await.unwrap();
10288            assert_eq!(response.len(), 1);
10289            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
10290            assert!(caller_rx.try_recv().is_err());
10291
10292            // 4. The reservation pair is given back, and the pending bind
10293            //    settled exactly once: one accepted open (the co-tenant) and one
10294            //    refused open (the caller), nothing counted twice.
10295            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
10296            let counters = handler.counters().snapshot();
10297            assert_eq!(
10298                keyed_total(&counters, "route_open_accepted_by_principal"),
10299                1
10300            );
10301            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
10302            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
10303        }
10304
10305        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
10306        /// id would resolve to the promoted candidate and every new route.open
10307        /// would be refused as reloading, leaving neither process routable.
10308        #[tokio::test]
10309        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
10310            let mut swap = swap_with_incumbent().await;
10311            register_candidate(&swap, None);
10312            let incumbent = cutover(&swap);
10313            swap.forwarding
10314                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
10315                .unwrap()
10316                .expect("the incumbent is still registered");
10317
10318            let client = ConnectionId::new(33);
10319            let (client_ctx, mut client_rx) = route_ctx(client);
10320            let route_handler = swap.handler.clone();
10321            let open_ctx = RouteCtx {
10322                connection_id: client,
10323                egress: client_ctx.egress.clone(),
10324            };
10325            let mut route_task = tokio::spawn(async move {
10326                route_handler
10327                    .handle_control_frame(
10328                        &open_ctx,
10329                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
10330                    )
10331                    .await
10332                    .unwrap()
10333            });
10334            let bind = tokio::select! {
10335                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
10336                response = &mut route_task => {
10337                    let response = response.unwrap();
10338                    panic!(
10339                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
10340                        parse_error(&response[0])["code"]
10341                    );
10342                }
10343            };
10344            swap.handler
10345                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
10346                .await
10347                .unwrap();
10348            assert!(route_task.await.unwrap().is_empty());
10349            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
10350            match swap
10351                .forwarding
10352                .lookup_data_route(client, channel, epoch)
10353                .unwrap()
10354            {
10355                DataRoute::Client(DataRouteState::Bound(route)) => {
10356                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
10357                }
10358                other => panic!("expected a bound route on the candidate, got {other:?}"),
10359            }
10360            assert!(swap.incumbent_rx.try_recv().is_err());
10361        }
10362
10363        /// A candidate declares itself ready with `catalog.update` on its own
10364        /// connection. If the connection-keyed registry lookups searched only the
10365        /// active slot, this would answer `not_registered` and the candidate
10366        /// would never become ready.
10367        #[tokio::test]
10368        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
10369            let swap = swap_with_incumbent().await;
10370            register_candidate(&swap, Some(false));
10371            let update = Frame::build(
10372                FrameType::Request,
10373                control_flags(),
10374                0,
10375                0,
10376                55,
10377                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
10378                    provides: manifest("aft", PROTOCOL_VERSION).provides,
10379                    capabilities: None,
10380                    ready: Some(true),
10381                })
10382                .unwrap(),
10383            )
10384            .unwrap();
10385
10386            let replies = swap
10387                .handler
10388                .handle_control_frame(&swap.candidate_ctx, update)
10389                .await
10390                .unwrap();
10391
10392            assert_eq!(replies.len(), 1);
10393            assert_eq!(
10394                replies[0].header.ty,
10395                FrameType::Response,
10396                "candidate catalog.update was refused: {:?}",
10397                serde_json::from_slice::<Value>(&replies[0].body).ok()
10398            );
10399            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
10400            assert_eq!(
10401                swap.registry
10402                    .get_module("aft")
10403                    .unwrap()
10404                    .unwrap()
10405                    .connection_id,
10406                INCUMBENT
10407            );
10408        }
10409    }
10410
10411    /// The HELLO gate while the supervisor has a swap open: only the nonce it
10412    /// minted for the candidate admits a second process, into the candidate
10413    /// slot, and that check runs ahead of the reserved-module gate.
10414    mod swap_admission {
10415        use super::*;
10416
10417        const INCUMBENT_NONCE: &str = "incumbent-nonce";
10418        const CANDIDATE_NONCE: &str = "candidate-nonce";
10419
10420        fn handler_with_incumbent(
10421            module_id: &str,
10422            reserved: bool,
10423        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
10424            let registry = Arc::new(Registry::default());
10425            let supervisor = SupervisorHandle::new();
10426            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
10427            if reserved {
10428                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
10429            }
10430            let handler =
10431                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
10432            let incumbent = handler
10433                .handle_control(
10434                    ConnectionId::new(1),
10435                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
10436                )
10437                .unwrap();
10438            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
10439            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
10440            (registry, supervisor, handler)
10441        }
10442
10443        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
10444        /// admits every nonce, so while a swap is open the swap gate is the only
10445        /// thing between a key-holder and the candidate slot. A nonce the
10446        /// supervisor did not mint, or none at all, is refused, and neither the
10447        /// incumbent's registration nor the candidate slot moves.
10448        #[test]
10449        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
10450            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
10451
10452            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
10453                let replies = handler
10454                    .handle_control(
10455                        ConnectionId::new(connection),
10456                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
10457                    )
10458                    .unwrap();
10459                assert_eq!(replies[0].header.ty, FrameType::Error);
10460                assert_eq!(
10461                    parse_error(&replies[0])["code"],
10462                    "swap_token_invalid",
10463                    "nonce {nonce:?}"
10464                );
10465            }
10466            assert!(registry.get_candidate("aft").unwrap().is_none());
10467            assert_eq!(
10468                registry.get_module("aft").unwrap().unwrap().connection_id,
10469                ConnectionId::new(1)
10470            );
10471
10472            // Control: the minted token is admitted, into the candidate slot,
10473            // and only once.
10474            let admitted = handler
10475                .handle_control(
10476                    ConnectionId::new(4),
10477                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
10478                )
10479                .unwrap();
10480            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
10481            assert_eq!(
10482                registry
10483                    .get_candidate("aft")
10484                    .unwrap()
10485                    .unwrap()
10486                    .connection_id,
10487                ConnectionId::new(4)
10488            );
10489            assert_eq!(
10490                registry.get_module("aft").unwrap().unwrap().connection_id,
10491                ConnectionId::new(1),
10492                "the candidate must not take the active slot"
10493            );
10494            let replayed = handler
10495                .handle_control(
10496                    ConnectionId::new(5),
10497                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
10498                )
10499                .unwrap();
10500            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
10501
10502            // The case only this gate covers: the incumbent has died mid-swap,
10503            // so its duplicate refusal is gone too, and without the gate a
10504            // key-holder would take the id's ACTIVE slot.
10505            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
10506            let squatter = handler
10507                .handle_control(
10508                    ConnectionId::new(6),
10509                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
10510                )
10511                .unwrap();
10512            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
10513            assert!(
10514                registry.get_module("aft").unwrap().is_none(),
10515                "a squatter took the active slot of an id being swapped"
10516            );
10517        }
10518
10519        /// Design mutation arm (iii). A reserved module's candidate presents a
10520        /// nonce the reserved gate has never seen (that gate holds the
10521        /// incumbent's), so the swap gate must run first or the candidate is
10522        /// refused `reserved_module` and a reserved module can never be swapped.
10523        #[test]
10524        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
10525            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
10526
10527            let replies = handler
10528                .handle_control(
10529                    ConnectionId::new(2),
10530                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
10531                )
10532                .unwrap();
10533
10534            assert_eq!(
10535                replies[0].header.ty,
10536                FrameType::HelloAck,
10537                "reserved candidate refused: {:?}",
10538                serde_json::from_slice::<Value>(&replies[0].body).ok()
10539            );
10540            assert_eq!(
10541                registry
10542                    .get_candidate("vault")
10543                    .unwrap()
10544                    .unwrap()
10545                    .connection_id,
10546                ConnectionId::new(2)
10547            );
10548        }
10549
10550        /// With no swap open the gate is inert: the incumbent's reserved gate
10551        /// and duplicate refusal behave exactly as before.
10552        #[test]
10553        fn without_an_open_swap_the_ordinary_gates_decide() {
10554            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
10555            supervisor.close_swap("vault");
10556
10557            let candidate = handler
10558                .handle_control(
10559                    ConnectionId::new(2),
10560                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
10561                )
10562                .unwrap();
10563            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
10564            let duplicate = handler
10565                .handle_control(
10566                    ConnectionId::new(3),
10567                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
10568                )
10569                .unwrap();
10570            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10571            assert!(registry.get_candidate("vault").unwrap().is_none());
10572        }
10573    }
10574}
10575
10576#[cfg(test)]
10577mod concurrency_default_exposure_tests {
10578    use super::*;
10579
10580    fn hello_body(role_json: &str) -> Vec<u8> {
10581        format!(
10582            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
10583        )
10584        .into_bytes()
10585    }
10586
10587    fn manifest_from(body: &[u8]) -> ModuleManifest {
10588        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
10589        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
10590            .expect("manifest parses")
10591    }
10592
10593    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
10594
10595    #[test]
10596    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
10597        let body = hello_body(&format!(
10598            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
10599        ));
10600        let manifest = manifest_from(&body);
10601        // Precondition: serde really resolved it to the default, so the typed
10602        // manifest alone cannot answer the question this probe exists for.
10603        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
10604        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
10605    }
10606
10607    #[test]
10608    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
10609        let body = hello_body(&format!(
10610            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
10611        ));
10612        let manifest = manifest_from(&body);
10613        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
10614        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
10615    }
10616
10617    #[test]
10618    fn non_management_roles_are_never_reported() {
10619        let body = hello_body(
10620            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
10621        );
10622        let manifest = manifest_from(&body);
10623        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
10624    }
10625}