Skip to main content

subc_daemon/
control.rs

1use std::{
2    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3    fmt,
4    path::{Path, PathBuf},
5    sync::{Arc, Mutex},
6    time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14    ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20    error_codes,
21    manifest::{
22        validate_hello_capability_grammar, validate_hello_self_signal_declarations,
23        CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
24        ProviderRole,
25    },
26    session::{
27        HealthReport, ModuleControlPush, ModuleControlRequest, ModuleControlRequestFromModule,
28        ModuleControlResponse, ModuleControlResponseToModule, MODULE_CONTROL_OP_HEALTH_CHECK,
29        MODULE_TO_SUBC_OP_CATALOG_UPDATE,
30    },
31    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
32    Priority, RouteTarget, PROTOCOL_VERSION,
33};
34use tokio::time::{timeout_at, Instant};
35use tracing::{debug, info, warn};
36
37use crate::{
38    capability_requirements::{
39        log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
40        CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
41        RuntimeModule,
42    },
43    daemon_config::RestartRequiredSection,
44    forwarding::{
45        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
46        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
47        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
48    },
49    observability::{
50        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
51    },
52    provenance::{
53        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
54    },
55    registry::{ChannelState, ConnectionId, Registry, RegistryError},
56    router::{RouteCtx, RouterError},
57    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
58    stderr_tail::{CaptureState, TailEntry},
59    supervise::{
60        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
61        SupervisorHandle, SwapHelloAdmission,
62    },
63    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
64};
65
66/// Lowest envelope version this subc build will negotiate.
67///
68/// Module HELLO negotiation is exact: peers must use the daemon's locked
69/// protocol version. Older and newer peers receive `version_unsupported` and
70/// are not registered.
71pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
72
73const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
74const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
75const CAP_PING_PONG: &str = "ping_pong_v1";
76const CAP_SESSION_ATTACH: &str = "session_attach_v1";
77const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
78
79const SUBC_CONTROL_OPS: &[&str] = &[
80    ops::SERVER_DESCRIBE,
81    ops::CATALOG_LIST,
82    ops::ROUTE_OPEN,
83    ops::ROUTE_POLL,
84    ops::ROUTE_CLOSING,
85    ops::ROUTE_CLOSED,
86    ops::SUPERVISOR_LIST,
87    ops::SUPERVISOR_RESTART,
88    ops::SUPERVISOR_SWAP,
89    ops::SUPERVISOR_RELOAD,
90    ops::SUPERVISOR_RESCAN,
91    ops::SUPERVISOR_RELEASE_RESERVED,
92    ops::SUPERVISOR_SET_ENABLED,
93    ops::SUPERVISOR_HEALTH_PROBE,
94    ops::SUPERVISOR_HEALTH,
95    ops::SUPERVISOR_STDERR_TAIL,
96    ops::SUPERVISOR_TERMINALS,
97    ops::SUPERVISOR_ROUTES,
98    ops::SUPERVISOR_PROVENANCE,
99    ops::SUPERVISOR_SPAWN_SNAPSHOT,
100    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
101];
102
103const MODULE_TO_SUBC_CONTROL_OPS: &[&str] =
104    &[MODULE_TO_SUBC_OP_CATALOG_UPDATE, "supervisor.live_roots"];
105
106const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
107
108/// How long subc waits for a module to ack a relayed route.bind before returning
109/// `module_timeout`. The ack waits on the module's own configure, which for AFT
110/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
111/// and DB-open work — on a cold page cache or a large repo that legitimately
112/// exceeds a couple of seconds. The default is generous because rejecting a VALID
113/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
114/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
115pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
116
117/// How many CONSECUTIVE full-budget relay timeouts against one target module
118/// open that module's bind-relay breaker.
119///
120/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
121/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
122/// so three consecutive full-budget timeouts take ~36s to observe: every client
123/// whose open contributed to opening the breaker had already given up on its
124/// own. That is what makes opening the breaker unable to turn a call that would
125/// have succeeded into a refusal — it can only make an already-failing module
126/// fail faster.
127///
128/// Two would be reachable inside one default deadline. One would convict a
129/// module on a single cold-cache bind, which is exactly the valid-but-slow case
130/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
131pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
132
133/// How long a module's bind-relay breaker stays open before exactly one
134/// `route.open` is let through as a probe.
135///
136/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
137/// re-pays a full-budget stall almost continuously, and the breaker stops being
138/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
139/// deadline: a client that starts retrying after the module recovers has to get
140/// a probe opportunity inside its own deadline, or the breaker converts a
141/// recovered module into a failed call — the failure it exists to prevent,
142/// pointed the other way.
143///
144/// 20s sits between those with room on both sides, and it caps what a wedged
145/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
146/// rather than one per `route.open` per connection. The stall that motivated
147/// this, with its measurements, is written up in
148/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
149/// waited the whole budget out.
150pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
151
152const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
153const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
154
155fn reload_verdict(
156    configured: &Path,
157    spawned_from: Option<&Path>,
158    image: subc_control::RunningImageAgreement,
159) -> PendingReloadVerdict {
160    let path = match spawned_from {
161        Some(spawned_from) if configured == spawned_from => ReloadPathAgreement::Match,
162        Some(spawned_from) => ReloadPathAgreement::Mismatch {
163            configured: configured.to_path_buf(),
164            spawned_from: spawned_from.to_path_buf(),
165        },
166        None => ReloadPathAgreement::Unavailable {
167            reason: if matches!(
168                image,
169                subc_control::RunningImageAgreement::Unavailable {
170                    reason: subc_control::RunningImageUnavailableReason::NotRunning
171                }
172            ) {
173                ReloadPathUnavailableReason::NotRunning
174            } else {
175                ReloadPathUnavailableReason::SpawnedPathUnavailable
176            },
177        },
178    };
179    PendingReloadVerdict { path, image }
180}
181
182#[derive(Clone)]
183struct DaemonProvenanceFacts {
184    build: DaemonBuildProvenance,
185    pid: Option<u32>,
186    started_at_ms: Option<u64>,
187    start_clock: Option<crate::clock::StartClock>,
188    executable_path: Option<PathBuf>,
189    executable_identity: Option<SpawnedFileIdentity>,
190    process_start_time: Option<u64>,
191    probe: ExecutableIdentityProbe,
192}
193
194impl Default for DaemonProvenanceFacts {
195    fn default() -> Self {
196        Self {
197            build: DaemonBuildProvenance {
198                build_git_sha: None,
199                build_lock_digest: None,
200            },
201            pid: None,
202            started_at_ms: None,
203            start_clock: None,
204            executable_path: None,
205            executable_identity: None,
206            process_start_time: None,
207            probe: ExecutableIdentityProbe::default(),
208        }
209    }
210}
211
212#[derive(Debug, Clone)]
213struct SupervisorRescanContext {
214    supervisor: Supervisor,
215    config_path: PathBuf,
216    configured_port: Option<u16>,
217    storage_config: Option<crate::daemon_config::StorageConfig>,
218    admission_facts_carrier_module_id: Option<String>,
219    admission_facts_targets: Option<Vec<String>>,
220}
221
222/// Real channel-0 control handler for subc itself.
223#[derive(Clone)]
224pub struct ControlHandler {
225    registry: Arc<Registry>,
226    forwarding: Arc<ForwardingTable>,
227    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
228    supervisor: SupervisorHandle,
229    subc_capabilities: Arc<[String]>,
230    /// Daemon-wide route.bind relay budget. Used as the fallback when the
231    /// target module has no per-module override in
232    /// `route_bind_relay_timeouts`.
233    route_bind_relay_timeout: Duration,
234    /// Per-module route.bind relay budget overrides, keyed by module id. When
235    /// `handle_route_open` resolves the deadline for a target module, a
236    /// per-module entry wins over the daemon-wide value above.
237    route_bind_relay_timeouts: BTreeMap<String, Duration>,
238    /// Per-target-module bind-relay breaker state. Shared with the forwarding
239    /// table, which is where a new module connection resets it.
240    route_bind_breakers: RouteBindBreakers,
241    /// Live relay admissions keyed by target module. Shared through the
242    /// forwarding table so cloned or separately built handlers enforce one cap.
243    route_bind_concurrency: RouteBindConcurrency,
244    /// Consecutive relay timeouts that open a module's breaker.
245    route_bind_breaker_threshold: u32,
246    /// How long a breaker stays open before one probe is admitted.
247    route_bind_breaker_cooldown: Duration,
248    health_probe_timeout: Duration,
249    /// Central storage policy. When set, each registering module receives its
250    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
251    storage_config: Option<crate::daemon_config::StorageConfig>,
252    /// The machine id established at boot, served on every HELLO_ACK and on
253    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
254    /// changes the file, never this value. `None` serves no id.
255    machine_id: Option<crate::machine_id::MachineId>,
256    admission_facts_carrier_module_id: Option<String>,
257    admission_facts_targets: Option<Vec<String>>,
258    rescan: Option<SupervisorRescanContext>,
259    connected_clients: ConnectedClients,
260    counters: DaemonCounters,
261    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
262    daemon_provenance: DaemonProvenanceFacts,
263    #[cfg(test)]
264    control_dispatch_delay: Option<Duration>,
265    #[cfg(test)]
266    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
267}
268
269impl fmt::Debug for ControlHandler {
270    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
271        f.debug_struct("ControlHandler")
272            .field("registry", &self.registry)
273            .field("forwarding", &self.forwarding)
274            .field("process_liveness", &self.process_liveness.is_some())
275            .field("supervisor", &self.supervisor)
276            .field("subc_capabilities", &self.subc_capabilities)
277            .finish()
278    }
279}
280
281struct RouteOpenRequest {
282    target: RouteTarget,
283    identity: BindIdentity,
284    consumer_identity: Option<ConsumerIdentity>,
285    consumer_capabilities: Option<Vec<String>>,
286    admission_facts: Option<serde_json::Value>,
287}
288
289struct RouteBindReservationGuard {
290    forwarding: Arc<ForwardingTable>,
291    endpoint: ModuleEndpointId,
292    relay_corr: u64,
293    armed: bool,
294}
295
296struct ModuleControlRpcGuard {
297    forwarding: Arc<ForwardingTable>,
298    endpoint: ModuleEndpointId,
299    corr: u64,
300    armed: bool,
301}
302
303impl ModuleControlRpcGuard {
304    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
305        Self {
306            forwarding,
307            endpoint,
308            corr,
309            armed: true,
310        }
311    }
312
313    fn disarm(&mut self) {
314        self.armed = false;
315    }
316}
317
318impl Drop for ModuleControlRpcGuard {
319    fn drop(&mut self) {
320        if self.armed {
321            let _ = self
322                .forwarding
323                .cancel_module_control_rpc(self.endpoint, self.corr);
324        }
325    }
326}
327
328impl RouteBindReservationGuard {
329    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
330        Self {
331            forwarding,
332            endpoint,
333            relay_corr,
334            armed: true,
335        }
336    }
337
338    fn release_and_disarm(&mut self) {
339        if !self.armed {
340            return;
341        }
342        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
343            self.endpoint,
344            self.relay_corr,
345            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
346        ) {
347            send_goodbye_target_best_effort(
348                &self.forwarding.counters(),
349                &target,
350                "canceled route.bind",
351            );
352        }
353        self.armed = false;
354    }
355
356    fn disarm(&mut self) {
357        self.armed = false;
358    }
359}
360
361impl Drop for RouteBindReservationGuard {
362    fn drop(&mut self) {
363        self.release_and_disarm();
364    }
365}
366
367/// Per-target-module circuit breaker around the `route.bind` relay.
368///
369/// The connection reader is serial per connection, so a module whose `on_bind`
370/// sits on the ack blocks every LATER frame on the connections that call it,
371/// including calls to unrelated modules. This does not make any module's bind
372/// fast; it stops the daemon paying the full budget again and again for a
373/// condition it has already observed.
374///
375/// State is keyed by TARGET MODULE and shared by every connection: a wedged
376/// module wedges everyone, so what one connection learned should protect the
377/// rest.
378///
379/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
380/// relay to that module has actually timed out, and is removed again when a
381/// relay is accepted or the module reconnects, so it cannot grow with traffic
382/// or with modules that behave.
383///
384/// # Why a `std` mutex here is not the head-of-line defect again
385///
386/// Acquisition never awaits. The critical section is a hash lookup plus a few
387/// integer updates, with no I/O and no `.await` inside it, so a reader task
388/// cannot be descheduled behind it the way it can behind
389/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
390/// primitive, held for the same kind of work, as the refusal counter this very
391/// path already increments.
392///
393/// It is also NOT on the data-plane splice path: only `route.open` and module
394/// registration touch it, so bound-route frames gain no state check and no
395/// contention.
396#[derive(Debug, Clone, Default)]
397pub(crate) struct RouteBindBreakers {
398    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
399}
400
401#[derive(Debug, Clone, Default)]
402pub(crate) struct RouteBindConcurrency {
403    modules: Arc<Mutex<HashMap<String, usize>>>,
404}
405
406struct RouteBindConcurrencyGuard {
407    concurrency: RouteBindConcurrency,
408    module_id: String,
409}
410
411impl RouteBindConcurrency {
412    /// Admit without waiting. Waiting here would move the bind stall from the
413    /// module reply to a semaphore and restore reader head-of-line blocking.
414    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
415        let mut modules = self
416            .modules
417            .lock()
418            .expect("route.bind concurrency mutex poisoned");
419        let in_flight = modules.entry(module_id.to_string()).or_default();
420        if *in_flight >= limit {
421            return Err(*in_flight);
422        }
423        *in_flight += 1;
424        Ok(RouteBindConcurrencyGuard {
425            concurrency: self.clone(),
426            module_id: module_id.to_string(),
427        })
428    }
429}
430
431impl Drop for RouteBindConcurrencyGuard {
432    fn drop(&mut self) {
433        let mut modules = self
434            .concurrency
435            .modules
436            .lock()
437            .expect("route.bind concurrency mutex poisoned");
438        let remove = {
439            let in_flight = modules
440                .get_mut(&self.module_id)
441                .expect("admitted route.bind has a concurrency entry");
442            *in_flight -= 1;
443            *in_flight == 0
444        };
445        if remove {
446            modules.remove(&self.module_id);
447        }
448    }
449}
450
451#[derive(Debug, Default)]
452struct ModuleBreakerState {
453    /// Relay timeouts observed with no accepted relay in between.
454    consecutive_timeouts: u32,
455    /// `Some` while the breaker is open: the instant the cooldown expires and
456    /// the next arrival may probe. `None` means closed.
457    cooldown_until: Option<Instant>,
458    /// A half-open probe has been admitted and has not settled yet. This is
459    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
460    /// that read the cooldown, so concurrent opens arriving at the moment the
461    /// cooldown expires cannot all decide that they are the probe.
462    probe_in_flight: bool,
463}
464
465/// What the breaker decided for one `route.open`, before any relay work.
466enum RouteBindAdmission<'a> {
467    Admitted {
468        guard: RouteBindBreakerGuard<'a>,
469        /// This open is the single half-open probe, so the transition is worth
470        /// one log line.
471        probe: bool,
472    },
473    Refused {
474        consecutive_timeouts: u32,
475        /// What is left of the cooldown. Zero when the refusal is because the
476        /// one probe is already in flight rather than because the cooldown has
477        /// not elapsed.
478        retry_in: Duration,
479        probe_in_flight: bool,
480    },
481}
482
483/// An outstanding admission, which must be told how its relay settled.
484///
485/// `Drop` settles it as inconclusive, so an early return between admission and
486/// the relay -- or the whole handler being cancelled when the client
487/// disconnects -- releases a half-open probe slot instead of leaving the
488/// breaker wedged half-open with no further probes.
489struct RouteBindBreakerGuard<'a> {
490    breakers: RouteBindBreakers,
491    module_id: &'a str,
492    settled: bool,
493}
494
495impl RouteBindBreakerGuard<'_> {
496    /// The module answered within the budget and took the bind. THE ONLY
497    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
498    /// breaker, which is a transition worth logging.
499    fn record_accepted(&mut self) -> bool {
500        self.settled = true;
501        self.breakers.record_accepted(self.module_id)
502    }
503
504    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
505    /// COUNTS TOWARD OPENING.
506    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
507        self.settled = true;
508        self.breakers
509            .record_timeout(self.module_id, threshold, cooldown)
510    }
511
512    /// Everything else: the module REJECTED the bind, its connection went away
513    /// mid-relay, or the waiter was cancelled.
514    ///
515    /// None of these is evidence that a module is slow, and each already has
516    /// its own refusal with its own code. A module that rejects a bind in
517    /// microseconds is healthy and must never be convicted for it; a module
518    /// that died has said nothing about the module that replaces it. So these
519    /// neither increment nor reset the count -- they only release a probe slot.
520    fn record_inconclusive(&mut self) {
521        self.settled = true;
522        self.breakers.record_inconclusive(self.module_id);
523    }
524}
525
526impl Drop for RouteBindBreakerGuard<'_> {
527    fn drop(&mut self) {
528        if !self.settled {
529            self.breakers.record_inconclusive(self.module_id);
530        }
531    }
532}
533
534/// The breaker moved to open, reported so the caller can log it outside the
535/// lock. Opening is rare and load-bearing; the refusals that follow are
536/// frequent and are counted rather than logged.
537struct BreakerOpened {
538    consecutive_timeouts: u32,
539    /// True when a failed probe re-opened an already-open breaker, which reads
540    /// very differently in a log from a first opening.
541    reopened_after_probe: bool,
542}
543
544impl RouteBindBreakers {
545    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
546        self.modules
547            .lock()
548            .expect("route.bind breaker mutex poisoned")
549    }
550
551    /// Decide whether this `route.open` may attempt its relay. Takes the map
552    /// lock and nothing else, and never awaits.
553    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
554        let admitted = |probe| RouteBindAdmission::Admitted {
555            guard: RouteBindBreakerGuard {
556                breakers: self.clone(),
557                module_id,
558                settled: false,
559            },
560            probe,
561        };
562
563        let mut modules = self.lock();
564        let Some(state) = modules.get_mut(module_id) else {
565            return admitted(false);
566        };
567        let Some(cooldown_until) = state.cooldown_until else {
568            return admitted(false);
569        };
570        if state.probe_in_flight {
571            return RouteBindAdmission::Refused {
572                consecutive_timeouts: state.consecutive_timeouts,
573                retry_in: Duration::ZERO,
574                probe_in_flight: true,
575            };
576        }
577        let now = Instant::now();
578        if now < cooldown_until {
579            return RouteBindAdmission::Refused {
580                consecutive_timeouts: state.consecutive_timeouts,
581                retry_in: cooldown_until - now,
582                probe_in_flight: false,
583            };
584        }
585        state.probe_in_flight = true;
586        admitted(true)
587    }
588
589    fn record_accepted(&self, module_id: &str) -> bool {
590        self.lock()
591            .remove(module_id)
592            .is_some_and(|state| state.cooldown_until.is_some())
593    }
594
595    fn record_timeout(
596        &self,
597        module_id: &str,
598        threshold: u32,
599        cooldown: Duration,
600    ) -> Option<BreakerOpened> {
601        let mut modules = self.lock();
602        let state = modules.entry(module_id.to_string()).or_default();
603        let was_open = state.cooldown_until.is_some();
604        let was_probe = state.probe_in_flight;
605        state.probe_in_flight = false;
606        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
607        if state.consecutive_timeouts < threshold {
608            return None;
609        }
610        state.cooldown_until = Some(Instant::now() + cooldown);
611        Some(BreakerOpened {
612            consecutive_timeouts: state.consecutive_timeouts,
613            reopened_after_probe: was_open && was_probe,
614        })
615    }
616
617    fn record_inconclusive(&self, module_id: &str) {
618        if let Some(state) = self.lock().get_mut(module_id) {
619            state.probe_in_flight = false;
620        }
621    }
622
623    /// Discard what was learned about a module, because the process it was
624    /// learned about is gone. Returns the discarded count when it was non-zero.
625    ///
626    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
627    /// `module_id` is a configuration identity that outlives any particular
628    /// child; what the breaker observed was the process behind the module
629    /// connection of the moment. When a new connection registers under that id
630    /// the verdict's subject no longer exists, so the verdict is stale by
631    /// construction rather than merely likely to be wrong. Keeping it would
632    /// apply a dead process's record to a live one, which is the same defect
633    /// class this breaker exists to stop the daemon committing.
634    ///
635    /// A half-open probe in flight is discarded with the rest: it was a
636    /// question about the old process.
637    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
638        self.lock()
639            .remove(module_id)
640            .map(|state| state.consecutive_timeouts)
641            .filter(|discarded| *discarded > 0)
642    }
643
644    /// Open breakers, for the `server.describe` counters object. `None` when
645    /// none is open, so the key stays absent rather than present-and-empty.
646    ///
647    /// This is the operator's answer to "is this module refusing instantly or
648    /// is it fine?", which look identical from a client that retries and then
649    /// succeeds.
650    fn open_snapshot(&self) -> Option<serde_json::Value> {
651        let now = Instant::now();
652        let modules = self.lock();
653        let open = modules
654            .iter()
655            .filter_map(|(module_id, state)| {
656                let cooldown_until = state.cooldown_until?;
657                Some((
658                    module_id.clone(),
659                    serde_json::json!({
660                        "consecutive_timeouts": state.consecutive_timeouts,
661                        "cooldown_remaining_ms":
662                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
663                        "probe_in_flight": state.probe_in_flight,
664                    }),
665                ))
666            })
667            .collect::<serde_json::Map<String, serde_json::Value>>();
668        (!open.is_empty()).then_some(serde_json::Value::Object(open))
669    }
670}
671
672impl ControlHandler {
673    pub fn new(registry: Arc<Registry>) -> Self {
674        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
675    }
676
677    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
678        let counters = forwarding.counters();
679        // Taken from the forwarding table rather than created here, so that the
680        // breaker a `route.open` consults is the same one a module's
681        // registration resets, however many handlers are built over one table.
682        let route_bind_breakers = forwarding.route_bind_breakers();
683        let route_bind_concurrency = forwarding.route_bind_concurrency();
684        Self {
685            registry,
686            forwarding,
687            process_liveness: None,
688            supervisor: SupervisorHandle::new(),
689            subc_capabilities: Arc::from([
690                CAP_MANIFEST_REGISTRATION.to_string(),
691                CAP_CHANNEL_LIFECYCLE.to_string(),
692                CAP_PING_PONG.to_string(),
693                CAP_SESSION_ATTACH.to_string(),
694                CAP_ADMISSION_FACTS_RELAY.to_string(),
695            ]),
696            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
697            route_bind_relay_timeouts: BTreeMap::new(),
698            route_bind_breakers,
699            route_bind_concurrency,
700            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
701            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
702            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
703            storage_config: None,
704            machine_id: None,
705            admission_facts_carrier_module_id: None,
706            admission_facts_targets: None,
707            rescan: None,
708            connected_clients: ConnectedClients::new(),
709            counters,
710            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
711            daemon_provenance: DaemonProvenanceFacts::default(),
712            #[cfg(test)]
713            control_dispatch_delay: None,
714            #[cfg(test)]
715            provenance_probe_override: None,
716        }
717    }
718
719    /// Set the central storage policy: registering modules then receive their
720    /// resolved storage descriptor in HELLO_ACK.
721    pub fn with_storage_config(
722        mut self,
723        storage_config: Option<crate::daemon_config::StorageConfig>,
724    ) -> Self {
725        self.storage_config = storage_config;
726        self
727    }
728
729    /// Set the machine id served to every registering module (HELLO_ACK) and on
730    /// `server.describe`.
731    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
732        self.machine_id = machine_id;
733        self
734    }
735
736    /// Configure the exact reserved module and target ids permitted to relay
737    /// opaque admission facts. Config-file loading validates this authority;
738    /// this builder keeps the same policy available to embedded test daemons.
739    pub fn with_admission_facts_config(
740        mut self,
741        carrier_module_id: Option<String>,
742        targets: Option<Vec<String>>,
743    ) -> Self {
744        self.admission_facts_carrier_module_id = carrier_module_id;
745        self.admission_facts_targets = targets;
746        self
747    }
748
749    /// Override the route.bind relay timeout. Used by tests that assert the
750    /// timeout path so they don't block on the production-safe default.
751    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
752        self.route_bind_relay_timeout = timeout;
753        self
754    }
755
756    /// Install per-module route.bind relay budget overrides. A module id
757    /// listed here wins over the daemon-wide default set via
758    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
759    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
760    /// the same `Duration` the bind path will use.
761    pub fn with_route_bind_relay_timeouts(
762        mut self,
763        timeouts: impl IntoIterator<Item = (String, Duration)>,
764    ) -> Self {
765        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
766        self
767    }
768
769    /// Resolve the route.bind relay budget for a specific target module id.
770    /// Per-module overrides win; the daemon-wide value (set via
771    /// `with_route_bind_relay_timeout` or the built-in default) is the
772    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
773    /// the same resolution `handle_route_open` will use.
774    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
775        self.route_bind_relay_timeouts
776            .get(module_id)
777            .copied()
778            .unwrap_or(self.route_bind_relay_timeout)
779    }
780
781    /// Override the per-module bind-relay breaker policy.
782    ///
783    /// Used by tests, which cannot spend three production budgets opening a
784    /// breaker or twenty seconds waiting for its cooldown. The production
785    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
786    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
787    /// reasoning for the numbers.
788    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
789        self.route_bind_breaker_threshold = threshold.max(1);
790        self.route_bind_breaker_cooldown = cooldown;
791        self
792    }
793
794    #[cfg(test)]
795    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
796        self.health_probe_timeout = timeout;
797        self
798    }
799
800    #[cfg(test)]
801    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
802        self.control_dispatch_delay = Some(delay);
803        self
804    }
805
806    pub fn with_process_liveness(
807        mut self,
808        process_liveness: Arc<dyn ModuleProcessLiveness>,
809    ) -> Self {
810        self.process_liveness = Some(process_liveness);
811        self
812    }
813
814    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
815        self.supervisor = supervisor;
816        self
817    }
818
819    pub fn with_daemon_provenance(
820        mut self,
821        pid: u32,
822        started_at_ms: u64,
823        executable_path: Option<PathBuf>,
824        build_git_sha: Option<String>,
825        build_lock_digest: Option<String>,
826    ) -> Self {
827        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
828        let process_start_time = process_start_time(pid);
829        self.daemon_provenance = DaemonProvenanceFacts {
830            build: DaemonBuildProvenance {
831                build_git_sha,
832                build_lock_digest,
833            },
834            pid: Some(pid),
835            started_at_ms: Some(started_at_ms),
836            start_clock: None,
837            executable_path,
838            executable_identity,
839            process_start_time,
840            probe: ExecutableIdentityProbe::default(),
841        };
842        self
843    }
844
845    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
846        self.daemon_provenance.start_clock = Some(clock);
847        self
848    }
849
850    #[cfg(test)]
851    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
852        self.provenance_probe_override = Some(result);
853        self
854    }
855
856    /// Install the configured module set and its reserved capability bindings.
857    /// Bindings are configuration-scoped and may point at a provider that has not
858    /// been installed yet, so this does not require the bound module to exist.
859    pub fn with_capability_config(
860        self,
861        modules: impl IntoIterator<Item = (String, bool)>,
862        reserved_capabilities: BTreeMap<String, String>,
863    ) -> Self {
864        self.capability_evaluator
865            .configure(modules, reserved_capabilities);
866        self
867    }
868
869    pub fn with_supervisor_rescan(
870        mut self,
871        supervisor: Supervisor,
872        config_path: impl Into<PathBuf>,
873        configured_port: Option<u16>,
874    ) -> Self {
875        self.rescan = Some(SupervisorRescanContext {
876            supervisor,
877            config_path: config_path.into(),
878            configured_port,
879            storage_config: self.storage_config.clone(),
880            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
881            admission_facts_targets: self.admission_facts_targets.clone(),
882        });
883        self
884    }
885
886    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
887        self.connected_clients = connected_clients;
888        self
889    }
890
891    pub fn forwarding(&self) -> Arc<ForwardingTable> {
892        Arc::clone(&self.forwarding)
893    }
894
895    pub(crate) fn counters(&self) -> DaemonCounters {
896        self.counters.clone()
897    }
898
899    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
900    /// requirement event without depending on an operator polling a status command.
901    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
902        tokio::spawn(async move {
903            loop {
904                self.capability_evaluator
905                    .wait_for_change_or_deadline()
906                    .await;
907                self.refresh_capability_requirements();
908            }
909        });
910    }
911
912    fn runtime_capability_snapshot(
913        &self,
914    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
915        let runtime = self
916            .supervisor
917            .list()
918            .into_iter()
919            .map(|module| {
920                let status = module.status().map_err(|err| {
921                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
922                })?;
923                Ok(RuntimeModule {
924                    module_id: status.module_id,
925                    state: status.state,
926                    enabled: status.enabled,
927                })
928            })
929            .collect::<Result<Vec<_>, RouterError>>()?;
930        let (_, registrations) = self.registry.list_modules().map_err(|err| {
931            RouterError::backend(
932                0,
933                0,
934                format!("failed to list capability registrations: {err}"),
935            )
936        })?;
937        let registrations = registrations
938            .into_iter()
939            .map(|registration| RegisteredModule {
940                module_id: registration.manifest.module_id,
941                module_version: registration.manifest.module_version,
942                capabilities: registration.manifest.capabilities,
943            })
944            .collect();
945        Ok((runtime, registrations))
946    }
947
948    /// The capability side effects of a module becoming the active registration
949    /// for its id: cache its manifest (warning if its claims drifted), run the
950    /// deny census when its declarations call for one, and recompute the
951    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
952    /// candidate's does not, and the supervisor does it at promotion instead,
953    /// through [`crate::supervise::SwapPromotionObserver`].
954    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
955        let cached_registration = RegisteredModule {
956            module_id: registration.manifest.module_id.clone(),
957            module_version: registration.manifest.module_version.clone(),
958            capabilities: registration.manifest.capabilities.clone(),
959        };
960        if self.capability_evaluator.record_hello(&cached_registration) {
961            warn!(
962                module_id = %cached_registration.module_id,
963                "capability claims drifted from the cached manifest"
964            );
965        }
966        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
967            self.enforce_capability_denies();
968        }
969        self.refresh_capability_requirements();
970    }
971
972    /// Point the shared supervisor handle at this handler for swap promotions.
973    /// Called wherever a handler is put behind the `Arc` the router serves, so
974    /// it can be held weakly.
975    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
976        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
977            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
978        self.supervisor.set_swap_promotion_observer(observer);
979    }
980
981    pub fn refresh_capability_requirements(&self) {
982        match self.runtime_capability_snapshot() {
983            Ok((runtime, registrations)) => {
984                log_requirement_events(
985                    self.capability_evaluator
986                        .evaluate_now(&runtime, &registrations),
987                );
988            }
989            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
990        }
991    }
992
993    /// Reconcile only live, attested route bindings after a capability deny edge
994    /// or target claim was added. This is deliberately a control-plane census:
995    /// the opaque forwarding hot path must not grow a per-frame capability check.
996    fn enforce_capability_denies(&self) {
997        let (_, registrations) = match self.registry.list_modules() {
998            Ok(snapshot) => snapshot,
999            Err(err) => {
1000                warn!(error = %err, "failed to read registrations for capability deny census");
1001                return;
1002            }
1003        };
1004        let manifests = registrations
1005            .into_iter()
1006            .map(|registration| {
1007                (
1008                    registration.manifest.module_id.clone(),
1009                    registration.manifest,
1010                )
1011            })
1012            .collect::<BTreeMap<_, _>>();
1013        let census = match self.forwarding.route_census(None) {
1014            Ok(census) => census,
1015            Err(err) => {
1016                warn!(error = %err, "failed to read route census for capability deny enforcement");
1017                return;
1018            }
1019        };
1020
1021        for (target_module_id, routes) in census {
1022            let Some(target_manifest) = manifests.get(&target_module_id) else {
1023                continue;
1024            };
1025            let mut closed_routes = Vec::new();
1026            let mut module_goodbyes = Vec::new();
1027            for route in routes {
1028                let Principal::Reserved {
1029                    module_id: opening_module_id,
1030                } = &route.principal
1031                else {
1032                    continue;
1033                };
1034                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1035                    continue;
1036                };
1037                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1038                    continue;
1039                };
1040
1041                match self.forwarding.release_client_route(
1042                    route.goodbye_target.connection_id,
1043                    route.goodbye_target.channel,
1044                    route.goodbye_target.epoch,
1045                ) {
1046                    Ok(RouteRelease::Removed(module_goodbye)) => {
1047                        warn!(
1048                            opening_module_id,
1049                            target_module_id,
1050                            capability,
1051                            "force-closing route because an attested capability deny edge now matches"
1052                        );
1053                        closed_routes.push(route);
1054                        module_goodbyes.push(module_goodbye);
1055                    }
1056                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1057                    Err(err) => warn!(
1058                        opening_module_id,
1059                        target_module_id,
1060                        capability,
1061                        error = %err,
1062                        "failed to force-close capability-denied route"
1063                    ),
1064                }
1065            }
1066
1067            if closed_routes.is_empty() {
1068                continue;
1069            }
1070            send_route_control_pushes(
1071                &self.forwarding,
1072                closed_routes,
1073                ClientControlPush::RouteClosed {
1074                    module_id: target_module_id,
1075                    reason: RouteCloseReason::CapabilityDenied,
1076                    drained: false,
1077                    abandoned: 0,
1078                    excluded_subscriptions: 0,
1079                    terminal: Some(false),
1080                },
1081            );
1082            self.emit_route_goodbyes(module_goodbyes);
1083        }
1084    }
1085
1086    /// Why a registered module is not accepting new route binds, or `None` when
1087    /// it is. This is the module's effective readiness: its declared readiness
1088    /// first, then every `need: required` capability it declares evaluating to
1089    /// `provided`. `route.open` and `catalog.list` both read it here so the
1090    /// catalog never reports a module routable that `route.open` would refuse.
1091    fn not_ready_reason(
1092        &self,
1093        registration: &crate::registry::ModuleRegistration,
1094    ) -> Option<NotReadyReason> {
1095        if !registration.ready {
1096            return Some(NotReadyReason {
1097                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1098                capability: None,
1099            });
1100        }
1101        self.first_unprovided_required_capability(registration)
1102            .map(|capability| NotReadyReason {
1103                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1104                capability: Some(capability),
1105            })
1106    }
1107
1108    /// The lexicographically first capability this registration declares
1109    /// `need: required` whose evaluator verdict is not `provided`.
1110    ///
1111    /// The verdicts are the capability evaluator's own; nothing here decides
1112    /// what "provided" means. The evaluator counts a capability provided as
1113    /// soon as a module claiming it has REGISTERED, not once that module is
1114    /// ready. That distinction is what keeps two modules that require each
1115    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1116    /// is ready", each would wait for the other to become ready first and
1117    /// neither ever would. Do not tighten it to readiness.
1118    ///
1119    /// A required capability with no verdict at all means this registration's
1120    /// HELLO or catalog.update landed after the last recompute; recompute once
1121    /// rather than let a missing verdict read as either answer. If it is still
1122    /// missing (the recompute itself failed) the capability counts as
1123    /// unprovided: the refusal is retryable, and routing a module whose
1124    /// required provider is unknown is the outcome this check exists to stop.
1125    fn first_unprovided_required_capability(
1126        &self,
1127        registration: &crate::registry::ModuleRegistration,
1128    ) -> Option<String> {
1129        let required = registration
1130            .manifest
1131            .capabilities
1132            .iter()
1133            .flat_map(|declarations| declarations.requires.iter())
1134            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1135            .map(|requirement| requirement.capability.as_str())
1136            .collect::<BTreeSet<_>>();
1137        if required.is_empty() {
1138            return None;
1139        }
1140        let module_id = registration.manifest.module_id.as_str();
1141        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1142        if required
1143            .iter()
1144            .any(|capability| verdict(capability).is_none())
1145        {
1146            self.refresh_capability_requirements();
1147        }
1148        required
1149            .into_iter()
1150            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1151            .map(str::to_string)
1152    }
1153
1154    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1155        self.capability_evaluator
1156            .statuses()
1157            .into_iter()
1158            .map(capability_requirement_status)
1159            .collect()
1160    }
1161
1162    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1163    /// registration-release watch. The signal is what the supervisor waits on
1164    /// before spawning a replacement, so it must only fire once forwarding
1165    /// teardown is also done (see [`Self::cleanup_connection`] /
1166    /// [`Self::handle_goodbye`]). Used directly only where there is no forwarding
1167    /// state to tear down (a HELLO that failed before module registration).
1168    fn deregister_connection(
1169        &self,
1170        connection_id: ConnectionId,
1171    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1172        self.registry.deregister_connection(connection_id)
1173    }
1174
1175    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1176        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1177            return None;
1178        }
1179        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1180            parse_client_control_request(&frame.body)
1181        else {
1182            return None;
1183        };
1184        Some(target_module_id(&target).to_string())
1185    }
1186
1187    pub(crate) fn route_open_capacity_refusal(
1188        &self,
1189        ctx: &RouteCtx,
1190        frame: &Frame,
1191        target_module_id: &str,
1192        limit: usize,
1193    ) -> Result<Frame, RouterError> {
1194        self.route_open_admission_refusal_frame(
1195            ctx,
1196            frame,
1197            target_module_id,
1198            format!(
1199                "connection already has {limit} route.open binds in flight; retry after one settles"
1200            ),
1201        )
1202    }
1203
1204    /// Admission pressure clears as existing binds settle, so its refusal must
1205    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
1206    /// `module_reloading`, `module_warming`, `target_unavailable`, or
1207    /// `module_timeout`. `target_unavailable` is honest for an attempt that
1208    /// cannot currently reach its target; `module_timeout` would falsely claim
1209    /// that a wait expired. A new, cleaner code would be terminal to deployed
1210    /// clients, so it requires a client-tolerance rollout before daemon emission.
1211    fn route_open_admission_refusal_frame(
1212        &self,
1213        ctx: &RouteCtx,
1214        frame: &Frame,
1215        target_module_id: &str,
1216        message: impl Into<String>,
1217    ) -> Result<Frame, RouterError> {
1218        self.route_open_refusal_frame(
1219            ctx,
1220            frame,
1221            target_module_id,
1222            "open_admission_full",
1223            error_codes::TARGET_UNAVAILABLE,
1224            message,
1225        )
1226    }
1227
1228    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
1229    ///
1230    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
1231    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
1232    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
1233    #[cfg(test)]
1234    pub fn handle_control(
1235        &self,
1236        connection_id: ConnectionId,
1237        frame: Frame,
1238    ) -> Result<Vec<Frame>, RouterError> {
1239        match frame.header.ty {
1240            FrameType::Ping => Ok(vec![pong(&frame)?]),
1241            FrameType::Hello => self.handle_hello(connection_id, None, frame),
1242            FrameType::Goodbye => self.handle_goodbye(connection_id),
1243            ty => Ok(vec![control_error_frame(
1244                &frame,
1245                "unsupported_control_frame",
1246                format!("unsupported channel-0 frame {ty:?}"),
1247            )?]),
1248        }
1249    }
1250
1251    pub async fn handle_control_frame(
1252        &self,
1253        ctx: &RouteCtx,
1254        frame: Frame,
1255    ) -> Result<Vec<Frame>, RouterError> {
1256        self.handle_control_frame_timed(ctx, frame, None).await
1257    }
1258
1259    pub(crate) async fn handle_control_frame_timed(
1260        &self,
1261        ctx: &RouteCtx,
1262        frame: Frame,
1263        dispatch_started_at: Option<StdInstant>,
1264    ) -> Result<Vec<Frame>, RouterError> {
1265        match frame.header.ty {
1266            FrameType::Ping => Ok(vec![pong(&frame)?]),
1267            FrameType::Hello => {
1268                self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1269            }
1270            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1271            FrameType::Cancel => {
1272                if self
1273                    .supervisor
1274                    .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1275                {
1276                    Ok(Vec::new())
1277                } else {
1278                    Ok(vec![control_error_frame(
1279                        &frame,
1280                        "unknown_subscription",
1281                        "no supervisor spawn subscription has this correlation id",
1282                    )?])
1283                }
1284            }
1285            FrameType::Request => {
1286                if self
1287                    .forwarding
1288                    .module_endpoint_for_connection(ctx.connection_id)
1289                    .map_err(RouterError::Forwarding)?
1290                    .is_some()
1291                {
1292                    if !is_known_module_request_op(&frame.body) {
1293                        return Ok(vec![control_error_frame(
1294                            &frame,
1295                            "unsupported_control_frame",
1296                            "module-originated channel-0 REQUEST is not supported",
1297                        )?]);
1298                    }
1299                    let request = match parse_module_control_request_from_module(&frame.body) {
1300                        Ok(request) => request,
1301                        Err((err, ControlRequestBodyError::UnknownOp)) => {
1302                            return Ok(vec![control_error_frame(
1303                                &frame,
1304                                "unsupported_control_frame",
1305                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
1306                            )?])
1307                        }
1308                        Err((err, ControlRequestBodyError::InvalidBody)) => {
1309                            return Ok(vec![control_error_frame(
1310                                &frame,
1311                                "invalid_control_body",
1312                                format!("malformed module control body: {err}"),
1313                            )?])
1314                        }
1315                    };
1316                    let op = module_control_request_op(&request);
1317                    let corr = frame.header.corr;
1318                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
1319                    let result =
1320                        self.handle_module_control_request(ctx.connection_id, frame, request);
1321                    log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1322                    return result;
1323                }
1324
1325                if is_known_module_request_op(&frame.body) {
1326                    return Ok(vec![control_error_frame(
1327                        &frame,
1328                        "not_registered",
1329                        "catalog.update requires an active module registration owned by this connection",
1330                    )?]);
1331                }
1332
1333                let request = match parse_client_control_request(&frame.body) {
1334                    Ok(request) => request,
1335                    Err((err, ControlRequestBodyError::UnknownOp)) => {
1336                        return Ok(vec![control_error_frame(
1337                            &frame,
1338                            "unknown_control_op",
1339                            format!("unknown client control op: {err}"),
1340                        )?])
1341                    }
1342                    Err((err, ControlRequestBodyError::InvalidBody)) => {
1343                        return Ok(vec![control_error_frame(
1344                            &frame,
1345                            "invalid_control_body",
1346                            format!("malformed client control body: {err}"),
1347                        )?])
1348                    }
1349                };
1350                let op = client_control_request_op(&request);
1351                let corr = frame.header.corr;
1352                log_control_dispatch_arrival(op, ctx.connection_id, corr);
1353                #[cfg(test)]
1354                if let Some(delay) = self.control_dispatch_delay {
1355                    tokio::time::sleep(delay).await;
1356                }
1357                let result = self
1358                    .handle_client_control_request(ctx, frame, request)
1359                    .await;
1360                log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1361                result
1362            }
1363            FrameType::Push => {
1364                let Some(endpoint) = self
1365                    .forwarding
1366                    .module_endpoint_for_connection(ctx.connection_id)
1367                    .map_err(RouterError::Forwarding)?
1368                else {
1369                    return Ok(vec![control_error_frame(
1370                        &frame,
1371                        "unsupported_control_frame",
1372                        "client-originated channel-0 PUSH is not supported",
1373                    )?]);
1374                };
1375                self.handle_status_update(endpoint, frame)
1376            }
1377            FrameType::Response | FrameType::Error
1378                if self
1379                    .forwarding
1380                    .module_endpoint_for_connection(ctx.connection_id)
1381                    .map_err(RouterError::Forwarding)?
1382                    .is_some() =>
1383            {
1384                self.handle_module_relay_response(ctx.connection_id, frame)
1385            }
1386            ty => Ok(vec![control_error_frame(
1387                &frame,
1388                "unsupported_control_frame",
1389                format!("unsupported channel-0 frame {ty:?}"),
1390            )?]),
1391        }
1392    }
1393
1394    pub fn cleanup_connection(
1395        &self,
1396        connection_id: ConnectionId,
1397    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1398        let crash_closed = self
1399            .registry
1400            .get_module_by_connection(connection_id)?
1401            .and_then(|registration| {
1402                self.forwarding
1403                    .module_endpoint_for_connection(connection_id)
1404                    .ok()
1405                    .flatten()
1406                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1407                    .map(|routes| (registration.manifest.module_id, routes))
1408            });
1409        if let Some((module_id, routes)) = crash_closed {
1410            let terminal = match self.supervisor.get(&module_id) {
1411                None => false,
1412                Some(module) => match module.will_recover_after_connection_loss() {
1413                    Ok(will_recover) => !will_recover,
1414                    Err(err) => {
1415                        warn!(
1416                            %module_id,
1417                            error = %err,
1418                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
1419                        );
1420                        false
1421                    }
1422                },
1423            };
1424            // The forwarding table gates all providers at the start of daemon
1425            // shutdown, before their connections are closed. An ordinary
1426            // module disconnect still reports crash if that gate is not set.
1427            let reason = match self.forwarding.is_daemon_draining() {
1428                Ok(true) => RouteCloseReason::Restart,
1429                Ok(false) => RouteCloseReason::Crash,
1430                Err(err) => {
1431                    warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1432                    RouteCloseReason::Crash
1433                }
1434            };
1435            send_route_control_pushes(
1436                &self.forwarding,
1437                routes,
1438                ClientControlPush::RouteClosed {
1439                    module_id,
1440                    reason,
1441                    drained: false,
1442                    abandoned: 0,
1443                    excluded_subscriptions: 0,
1444                    terminal: Some(terminal),
1445                },
1446            );
1447        }
1448        let registrations = self.deregister_connection(connection_id);
1449        if let Ok(released_routes) = self.forwarding.cleanup_connection(connection_id) {
1450            self.emit_route_goodbyes(released_routes);
1451        }
1452        // Signal the registration-release watch only now that BOTH registry and
1453        // forwarding teardown are done, so a supervisor waiting to spawn a
1454        // replacement never observes release while old routes still exist.
1455        if matches!(&registrations, Ok(r) if !r.is_empty()) {
1456            crate::supervise::notify_registration_release();
1457            self.capability_evaluator.wake_deadline_loop();
1458            self.refresh_capability_requirements();
1459        }
1460        self.supervisor.remove_spawn_subscribers(connection_id);
1461        registrations
1462    }
1463
1464    pub(crate) fn handle_route_goodbye(
1465        &self,
1466        connection_id: ConnectionId,
1467        route_channel: u16,
1468        route_epoch: u32,
1469    ) -> Result<bool, RouterError> {
1470        debug!(
1471            connection_id = connection_id.get(),
1472            route_channel, route_epoch, "handling route GOODBYE"
1473        );
1474        let RouteRelease::Removed(released_route) = self
1475            .forwarding
1476            .release_client_route(connection_id, route_channel, route_epoch)
1477            .map_err(RouterError::Forwarding)?
1478        else {
1479            return Ok(false);
1480        };
1481        self.emit_route_goodbyes(vec![released_route]);
1482        Ok(true)
1483    }
1484
1485    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1486        for released in released_routes {
1487            let frame = match Frame::build_with_version(
1488                released.negotiated_ver,
1489                FrameType::Goodbye,
1490                control_flags(),
1491                released.channel,
1492                released.epoch,
1493                0,
1494                Vec::new(),
1495            ) {
1496                Ok(frame) => frame,
1497                Err(err) => {
1498                    warn!(
1499                        route_channel = released.channel,
1500                        error = %err,
1501                        "failed to build route GOODBYE frame"
1502                    );
1503                    continue;
1504                }
1505            };
1506            if !released.close_on_delivery_failure() {
1507                crate::forwarding::send_module_route_goodbye(
1508                    &self.counters,
1509                    &released.sink,
1510                    frame,
1511                    released.module_id.as_deref(),
1512                    "client route released",
1513                );
1514                continue;
1515            }
1516            if let Err(err) = released.sink.try_send(frame) {
1517                warn!(
1518                    target_connection_id = released.connection_id.get(),
1519                    route_channel = released.channel,
1520                    error = %err,
1521                    "route GOODBYE was not delivered to client; closing target connection"
1522                );
1523                if self
1524                    .forwarding
1525                    .escalate_client_delivery_failure(
1526                        released.connection_id,
1527                        released.channel,
1528                        released.epoch,
1529                        CloseReason::new(
1530                            "route_goodbye_delivery_failed",
1531                            format!(
1532                                "failed to enqueue route GOODBYE for channel {}: {err}",
1533                                released.channel
1534                            ),
1535                        ),
1536                        crate::forwarding::UndeliveredFrame {
1537                            module_id: released.module_id.as_deref(),
1538                            sink: &released.sink,
1539                        },
1540                    )
1541                    .unwrap_or(false)
1542                {
1543                    self.counters.increment_goodbye_relay_client_failed();
1544                }
1545            }
1546        }
1547    }
1548
1549    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
1550    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
1551    /// own commit failed after the module had already accepted). Without this, a
1552    /// module that accepts late keeps a binding subc has torn down, so a later
1553    /// frame on that module channel could misdeliver if the channel is reused.
1554    ///
1555    /// Never closes the shared module connection on failure: a dropped notification
1556    /// only wastes a bounded amount of warm module-side state, which the module's
1557    /// own idle reaper reclaims. Only call this once the route.bind relay was
1558    /// actually enqueued to the module — if the relay send itself failed, the
1559    /// module never created a binding and there is nothing to tear down.
1560    fn send_abandoned_route_bind_goodbye(
1561        &self,
1562        module_sink: &crate::FrameSink,
1563        negotiated_ver: u8,
1564        module_channel: u16,
1565        module_epoch: u32,
1566    ) {
1567        let frame = match Frame::build_with_version(
1568            negotiated_ver,
1569            FrameType::Goodbye,
1570            control_flags(),
1571            module_channel,
1572            module_epoch,
1573            0,
1574            Vec::new(),
1575        ) {
1576            Ok(frame) => frame,
1577            Err(err) => {
1578                warn!(
1579                    route_channel = module_channel,
1580                    error = %err,
1581                    "failed to build GOODBYE for abandoned route.bind"
1582                );
1583                return;
1584            }
1585        };
1586        crate::forwarding::send_module_route_goodbye(
1587            &self.counters,
1588            module_sink,
1589            frame,
1590            None,
1591            "abandoned route.bind",
1592        );
1593    }
1594
1595    fn handle_hello(
1596        &self,
1597        connection_id: ConnectionId,
1598        sink: Option<crate::FrameSink>,
1599        frame: Frame,
1600    ) -> Result<Vec<Frame>, RouterError> {
1601        debug!(
1602            connection_id = connection_id.get(),
1603            corr = frame.header.corr,
1604            "handling HELLO"
1605        );
1606        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1607            Ok(value) => value,
1608            Err(err) => {
1609                return Ok(vec![control_error_frame(
1610                    &frame,
1611                    "invalid_hello",
1612                    format!("malformed HELLO body: {err}"),
1613                )?])
1614            }
1615        };
1616        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1617            return Ok(vec![control_error_frame(
1618                &frame,
1619                "invalid_capability_grammar",
1620                err.to_string(),
1621            )?]);
1622        }
1623        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
1624            return Ok(vec![control_error_frame(
1625                &frame,
1626                "invalid_manifest",
1627                err.to_string(),
1628            )?]);
1629        }
1630        if let Some(provenance) = hello_value
1631            .get("manifest")
1632            .and_then(|manifest| manifest.get("provenance"))
1633        {
1634            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
1635                return Ok(vec![control_error_frame(
1636                    &frame,
1637                    "invalid_manifest",
1638                    format!("malformed manifest provenance: {err}"),
1639                )?]);
1640            }
1641        }
1642        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
1643            Ok(hello) => hello,
1644            Err(err) => {
1645                return Ok(vec![control_error_frame(
1646                    &frame,
1647                    "invalid_hello",
1648                    format!("malformed HELLO body: {err}"),
1649                )?])
1650            }
1651        };
1652
1653        if hello.protocol_ver != hello.manifest.protocol_ver {
1654            return Ok(vec![control_error_frame(
1655                &frame,
1656                "invalid_manifest",
1657                format!(
1658                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
1659                    hello.protocol_ver, hello.manifest.protocol_ver
1660                ),
1661            )?]);
1662        }
1663
1664        if hello.manifest.module_id.trim().is_empty() {
1665            return Ok(vec![control_error_frame(
1666                &frame,
1667                "invalid_manifest",
1668                "manifest module_id must not be empty",
1669            )?]);
1670        }
1671
1672        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
1673            Ok(negotiated_ver) => negotiated_ver,
1674            Err(message) => {
1675                return Ok(vec![control_error_frame(
1676                    &frame,
1677                    "version_unsupported",
1678                    message,
1679                )?])
1680            }
1681        };
1682
1683        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
1684        // swap is open for this id, the only HELLO admitted as a second process
1685        // is the one carrying the candidate's launch nonce (the swap token), and
1686        // it registers into the candidate slot rather than being refused as a
1687        // duplicate. Run after the reserved gate, a reserved module's candidate
1688        // would be refused `reserved_module` for presenting a nonce that gate
1689        // does not know. See `SupervisorHandle::swap_hello_admission`.
1690        let swap_admission = self
1691            .supervisor
1692            .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
1693        if swap_admission == SwapHelloAdmission::Refused {
1694            warn!(
1695                module_id = %hello.manifest.module_id,
1696                connection_id = connection_id.get(),
1697                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
1698            );
1699            return Ok(vec![control_error_frame(
1700                &frame,
1701                "swap_token_invalid",
1702                format!(
1703                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
1704                    hello.manifest.module_id
1705                ),
1706            )?]);
1707        }
1708        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
1709
1710        // Reserved-module identity gate: a module_id configured `reserved` may be
1711        // registered ONLY by the process subc spawned for it, proven by echoing the
1712        // one-time launch nonce subc injected. A non-reserved id has no recorded
1713        // nonce and always passes. This blocks a key-holder from impersonating a
1714        // security-boundary module (e.g. the credential vault) while the real one is
1715        // down/restarting and its registration slot is momentarily free. A swap
1716        // candidate has already proven the same thing with its own nonce above.
1717        if let Some(rejection) = (!swap_candidate)
1718            .then(|| {
1719                self.supervisor.reserved_hello_rejection(
1720                    &hello.manifest.module_id,
1721                    hello.launch_nonce.as_deref(),
1722                )
1723            })
1724            .flatten()
1725        {
1726            let message = match rejection {
1727                ReservedHelloRejection::Exact { module_id } => format!(
1728                    "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
1729                ),
1730                ReservedHelloRejection::Prefix {
1731                    prefix,
1732                    owner_module_id,
1733                } => format!(
1734                    "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
1735                    hello.manifest.module_id
1736                ),
1737            };
1738            return Ok(vec![control_error_frame(
1739                &frame,
1740                "reserved_module",
1741                message,
1742            )?]);
1743        }
1744
1745        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
1746            &hello.manifest.module_id,
1747            hello.manifest.capabilities.as_ref(),
1748        );
1749        if let Some(refusal) = reserved_capability_refusals.first() {
1750            let capability = refusal.capability.clone();
1751            let bound_module = refusal.claimants[0].clone();
1752            log_duplicate_claim_events(reserved_capability_refusals);
1753            return Ok(vec![control_error_frame(
1754                &frame,
1755                "reserved_capability",
1756                format!(
1757                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
1758                    capability, bound_module, hello.manifest.module_id
1759                ),
1760            )?]);
1761        }
1762
1763        // A connection that already opened client routes must not also register as
1764        // a module: cleanup would then release only one side and leak the other.
1765        if self
1766            .forwarding
1767            .connection_has_client_routes(connection_id)
1768            .map_err(RouterError::Forwarding)?
1769        {
1770            return Ok(vec![control_error_frame(
1771                &frame,
1772                "invalid_hello",
1773                "connection has open client routes and cannot also register as a module",
1774            )?]);
1775        }
1776
1777        let control_ops = effective_module_control_ops(hello.control_ops);
1778        // Built before anything is registered so an encoding failure leaves no
1779        // registry or forwarding state behind.
1780        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
1781        if swap_candidate {
1782            return self.register_swap_candidate(
1783                connection_id,
1784                sink,
1785                &frame,
1786                hello.manifest,
1787                negotiated_ver,
1788                control_ops,
1789                hello_ack,
1790            );
1791        }
1792        let registration = match self.registry.register_with_control_ops(
1793            hello.manifest,
1794            negotiated_ver,
1795            connection_id,
1796            control_ops,
1797        ) {
1798            Ok(registration) => registration,
1799            Err(RegistryError::DuplicateModuleId { module_id }) => {
1800                return Ok(vec![control_error_frame(
1801                    &frame,
1802                    "duplicate_module_id",
1803                    format!(
1804                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
1805                    ),
1806                )?])
1807            }
1808            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
1809                return Ok(vec![control_error_frame(
1810                    &frame,
1811                    "invalid_module_id",
1812                    err.to_string(),
1813                )?])
1814            }
1815            Err(err) => {
1816                return Ok(vec![control_error_frame(
1817                    &frame,
1818                    "registry_error",
1819                    err.to_string(),
1820                )?])
1821            }
1822        };
1823
1824        let reply = if let Some(sink) = sink {
1825            // The forwarding table's module store is also the daemon-to-module
1826            // control-RPC lane, so every HELLO gets a live endpoint even when the
1827            // manifest has no routable provider role. Non-routable modules still
1828            // cannot receive route.bind in production: `handle_route_open` checks
1829            // the registry manifest with `target_has_required_role` before the
1830            // only production call to `begin_route_bind_relay_for` below that
1831            // route.open path. The remaining direct relay callers are unit tests
1832            // and benchmark harnesses that construct forwarding state explicitly.
1833            //
1834            // The HELLO_ACK is queued by the forwarding table itself, before the
1835            // endpoint becomes visible, and is NOT returned as a reply. A module
1836            // reads HELLO_ACK first and exits on anything else; a reply is only
1837            // written after this handler returns, by which time a route.open on
1838            // another connection could already have queued a route.bind request
1839            // for this module ahead of it.
1840            let concurrency = manifest_concurrency(&registration.manifest);
1841            if let Err(err) = self.forwarding.register_module_connection_acked(
1842                connection_id,
1843                registration.manifest.module_id.clone(),
1844                negotiated_ver,
1845                concurrency,
1846                sink,
1847                hello_ack,
1848            ) {
1849                // Forwarding registration failed, so there is no forwarding
1850                // state to tear down. Remove the registry entry and signal the
1851                // release watch directly.
1852                if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
1853                    crate::supervise::notify_registration_release();
1854                }
1855                return Ok(vec![control_error_frame(
1856                    &frame,
1857                    forwarding_error_code(&err),
1858                    err.to_string(),
1859                )?]);
1860            }
1861            Vec::new()
1862        } else {
1863            // No sink means no forwarding endpoint, so nothing can be routed
1864            // ahead of the ack; it goes out as the reply.
1865            vec![hello_ack]
1866        };
1867
1868        // Exposure over assumption: Concurrency's serde default is pinned to the
1869        // pre-field behavior (ModuleManaged), so a management surface that is
1870        // genuinely Serial and just never declared it inherits concurrent
1871        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
1872        // turns "no module has been bitten yet" into the checkable claim "no
1873        // module is exposed" -- one read of the boot log instead of a fleet
1874        // audit. Detected from the raw HELLO bytes because the serde default
1875        // deliberately erases the absent/declared distinction from the type.
1876        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
1877            info!(
1878                module_id = %registration.manifest.module_id,
1879                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
1880            );
1881        }
1882
1883        self.apply_registration_capabilities(&registration);
1884
1885        info!(
1886            module_id = %registration.manifest.module_id,
1887            module_version = %registration.manifest.module_version,
1888            negotiated_ver,
1889            routable_provider = manifest_provides_routable_role(&registration.manifest),
1890            connection_id = connection_id.get(),
1891            "module registered"
1892        );
1893
1894        Ok(reply)
1895    }
1896
1897    /// Register a HELLO the swap gate admitted into the candidate slot of the
1898    /// registry and of forwarding, where it is reachable over its own
1899    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
1900    /// nothing routes to it until the supervisor cuts over.
1901    ///
1902    /// Registry first, then forwarding, the same order as an ordinary HELLO;
1903    /// a forwarding failure removes the registry entry again. The capability
1904    /// census is not run: it describes routable modules, and this one is not
1905    /// routable until promotion.
1906    #[allow(clippy::too_many_arguments)]
1907    fn register_swap_candidate(
1908        &self,
1909        connection_id: ConnectionId,
1910        sink: Option<crate::FrameSink>,
1911        frame: &Frame,
1912        manifest: ModuleManifest,
1913        negotiated_ver: u8,
1914        control_ops: Vec<String>,
1915        hello_ack: Frame,
1916    ) -> Result<Vec<Frame>, RouterError> {
1917        let module_id = manifest.module_id.clone();
1918        let registration = match self.registry.register_candidate_with_control_ops(
1919            manifest,
1920            negotiated_ver,
1921            connection_id,
1922            control_ops,
1923        ) {
1924            Ok(registration) => registration,
1925            Err(RegistryError::DuplicateModuleId { module_id }) => {
1926                return Ok(vec![control_error_frame(
1927                    frame,
1928                    "duplicate_module_id",
1929                    format!(
1930                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
1931                    ),
1932                )?])
1933            }
1934            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
1935                return Ok(vec![control_error_frame(
1936                    frame,
1937                    "invalid_module_id",
1938                    err.to_string(),
1939                )?])
1940            }
1941            Err(err) => {
1942                return Ok(vec![control_error_frame(
1943                    frame,
1944                    "registry_error",
1945                    err.to_string(),
1946                )?])
1947            }
1948        };
1949        let reply = if let Some(sink) = sink {
1950            // Same ordering as an ordinary HELLO: the forwarding table queues
1951            // the HELLO_ACK before the candidate endpoint is inserted, because
1952            // a module exits if its first frame after HELLO is anything else.
1953            let concurrency = manifest_concurrency(&registration.manifest);
1954            if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
1955                connection_id,
1956                module_id.clone(),
1957                negotiated_ver,
1958                concurrency,
1959                sink,
1960                hello_ack,
1961            ) {
1962                if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
1963                    crate::supervise::notify_registration_release();
1964                }
1965                return Ok(vec![control_error_frame(
1966                    frame,
1967                    forwarding_error_code(&err),
1968                    err.to_string(),
1969                )?]);
1970            }
1971            Vec::new()
1972        } else {
1973            vec![hello_ack]
1974        };
1975        self.supervisor.mark_swap_candidate_admitted(&module_id);
1976        info!(
1977            module_id = %module_id,
1978            module_version = %registration.manifest.module_version,
1979            negotiated_ver,
1980            ready = registration.ready,
1981            connection_id = connection_id.get(),
1982            "swap candidate registered; not routable until cutover"
1983        );
1984        Ok(reply)
1985    }
1986
1987    fn build_hello_ack(
1988        &self,
1989        frame: &Frame,
1990        negotiated_ver: u8,
1991        module_id: &str,
1992    ) -> Result<Frame, RouterError> {
1993        let ack = ModuleHelloAckBody {
1994            negotiated_ver,
1995            subc_ops: module_subc_ops(),
1996            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
1997            storage: self
1998                .storage_config
1999                .as_ref()
2000                .map(|cfg| cfg.descriptor_for(module_id)),
2001            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2002        };
2003        let body = serde_json::to_vec(&ack).map_err(|err| {
2004            RouterError::backend(
2005                0,
2006                frame.header.corr,
2007                format!("failed to encode HELLO_ACK: {err}"),
2008            )
2009        })?;
2010
2011        Frame::build_with_version(
2012            negotiated_ver,
2013            FrameType::HelloAck,
2014            control_flags(),
2015            0,
2016            0,
2017            frame.header.corr,
2018            body,
2019        )
2020        .map_err(RouterError::FrameBuild)
2021    }
2022
2023    async fn handle_client_control_request(
2024        &self,
2025        ctx: &RouteCtx,
2026        frame: Frame,
2027        request: ClientControlRequest,
2028    ) -> Result<Vec<Frame>, RouterError> {
2029        match request {
2030            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2031            ClientControlRequest::CatalogList { module_id } => {
2032                self.handle_catalog_list(frame, module_id)
2033            }
2034            ClientControlRequest::RouteOpen {
2035                target,
2036                identity,
2037                consumer_identity,
2038                consumer_capabilities,
2039                admission_facts,
2040            } => {
2041                self.handle_route_open(
2042                    ctx,
2043                    frame,
2044                    RouteOpenRequest {
2045                        target,
2046                        identity,
2047                        consumer_identity,
2048                        consumer_capabilities,
2049                        admission_facts,
2050                    },
2051                )
2052                .await
2053            }
2054            ClientControlRequest::RoutePoll {
2055                route_channel,
2056                route_epoch,
2057                kind,
2058            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2059            ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2060            ClientControlRequest::SupervisorSpawnSnapshot {} => {
2061                self.handle_supervisor_spawn_snapshot(frame)
2062            }
2063            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2064                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2065            }
2066            ClientControlRequest::SupervisorRestart {
2067                module_id,
2068                drain_timeout_ms,
2069            } => {
2070                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2071                    .await
2072            }
2073            ClientControlRequest::SupervisorSwap {
2074                module_id,
2075                ready_timeout_ms,
2076            } => {
2077                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2078                    .await
2079            }
2080            ClientControlRequest::SupervisorReload { module_id } => {
2081                self.handle_supervisor_reload(frame, module_id).await
2082            }
2083            ClientControlRequest::SupervisorRescan { preview } => {
2084                self.handle_supervisor_rescan(frame, preview).await
2085            }
2086            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2087                self.handle_supervisor_release_reserved(frame, module_id)
2088                    .await
2089            }
2090            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2091                self.handle_supervisor_set_enabled(frame, module_id, enabled)
2092                    .await
2093            }
2094            ClientControlRequest::SupervisorHealthProbe { module_id } => {
2095                self.handle_supervisor_health_probe(frame, module_id).await
2096            }
2097            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2098            ClientControlRequest::SupervisorRoutes { module_id } => {
2099                self.handle_supervisor_routes(frame, module_id)
2100            }
2101            ClientControlRequest::SupervisorProvenance { module_id } => {
2102                self.handle_supervisor_provenance(frame, module_id).await
2103            }
2104            ClientControlRequest::SupervisorStderrTail {
2105                module_id,
2106                max_lines,
2107                max_bytes,
2108            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2109            ClientControlRequest::SupervisorTerminals { module_id } => {
2110                self.handle_supervisor_terminals(frame, module_id).await
2111            }
2112        }
2113    }
2114
2115    fn handle_module_control_request(
2116        &self,
2117        connection_id: ConnectionId,
2118        frame: Frame,
2119        request: ModuleControlRequestFromModule,
2120    ) -> Result<Vec<Frame>, RouterError> {
2121        match request {
2122            ModuleControlRequestFromModule::CatalogUpdate {
2123                provides,
2124                capabilities,
2125                ready,
2126            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2127            ModuleControlRequestFromModule::LiveRoots {} => {
2128                let registered = self
2129                    .registry
2130                    .get_module_by_connection(connection_id)
2131                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2132                let Some(registration) = registered else {
2133                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2134                };
2135                let response = self
2136                    .forwarding
2137                    .live_roots(&registration.manifest.module_id)
2138                    .map_err(RouterError::Forwarding)?;
2139                Ok(vec![control_response_body_frame(
2140                    &frame,
2141                    &response,
2142                    "ModuleControlResponseToModule::LiveRoots",
2143                )?])
2144            }
2145        }
2146    }
2147
2148    fn handle_catalog_update(
2149        &self,
2150        connection_id: ConnectionId,
2151        frame: Frame,
2152        provides: Vec<ProviderRole>,
2153        capabilities: Option<CapabilityDeclarations>,
2154        ready: Option<bool>,
2155    ) -> Result<Vec<Frame>, RouterError> {
2156        self.refresh_capability_requirements();
2157        let Some(registration) = self
2158            .registry
2159            .get_module_by_connection(connection_id)
2160            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2161        else {
2162            return Ok(vec![control_error_frame(
2163                &frame,
2164                "not_registered",
2165                "catalog.update requires an active module registration owned by this connection",
2166            )?]);
2167        };
2168
2169        if let Some(message) =
2170            catalog_update_frozen_field_message(&registration.manifest, &provides)
2171        {
2172            return Ok(vec![control_error_frame(
2173                &frame,
2174                "catalog_update_frozen_field",
2175                message,
2176            )?]);
2177        }
2178
2179        let mut candidate = registration.manifest.clone();
2180        candidate.provides = provides.clone();
2181        candidate.capabilities = capabilities
2182            .clone()
2183            .or_else(|| registration.manifest.capabilities.clone());
2184        if let Err(err) = candidate.validate_capability_grammar() {
2185            return Ok(vec![control_error_frame(
2186                &frame,
2187                "invalid_capability_grammar",
2188                err.to_string(),
2189            )?]);
2190        }
2191
2192        let updated = self
2193            .registry
2194            .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
2195            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2196        if updated.is_none() {
2197            return Ok(vec![control_error_frame(
2198                &frame,
2199                "not_registered",
2200                "catalog.update requires an active module registration owned by this connection",
2201            )?]);
2202        }
2203        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
2204            log_duplicate_claim_events(
2205                self.capability_evaluator
2206                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
2207            );
2208        }
2209        if capability_census_trigger(
2210            registration.manifest.capabilities.as_ref(),
2211            updated
2212                .as_ref()
2213                .and_then(|entry| entry.manifest.capabilities.as_ref()),
2214        ) {
2215            self.enforce_capability_denies();
2216        }
2217        self.refresh_capability_requirements();
2218
2219        let response = ModuleControlResponseToModule::CatalogUpdate {};
2220        control_response_body_frame(
2221            &frame,
2222            &response,
2223            "ModuleControlResponseToModule::CatalogUpdate",
2224        )
2225        .map(|frame| vec![frame])
2226    }
2227
2228    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2229        self.refresh_capability_requirements();
2230        // A bare connection count is ambiguous between many clients holding a
2231        // route each and one client accumulating hundreds, so publish the
2232        // concentration alongside it. Route state is best-effort here: a
2233        // diagnostic endpoint must still answer if the forwarding lock is
2234        // contended.
2235        let mut counters = self.counters.snapshot();
2236        if let (Ok((connections_with_routes, max)), Some(obj)) = (
2237            self.forwarding.client_route_concentration(),
2238            counters.as_object_mut(),
2239        ) {
2240            obj.insert(
2241                "client_connections_with_routes".into(),
2242                connections_with_routes.into(),
2243            );
2244            obj.insert("max_routes_on_one_connection".into(), max.into());
2245        }
2246        // A module that is being fast-refused and a module that is fine look
2247        // identical from a client that retries and succeeds, so name the open
2248        // breakers here. This rides the existing free-form counters object
2249        // rather than a new wire field, so no sibling that deserializes
2250        // `ServerDescribe` has to be rebuilt to keep reading it.
2251        if let (Some(open_breakers), Some(obj)) = (
2252            self.route_bind_breakers.open_snapshot(),
2253            counters.as_object_mut(),
2254        ) {
2255            obj.insert("route_bind_breakers_open".into(), open_breakers);
2256        }
2257        let response = ClientControlResponse::ServerDescribe {
2258            protocol_ver: PROTOCOL_VERSION,
2259            subc_ops: subc_ops(),
2260            capabilities: self.subc_capabilities.as_ref().to_vec(),
2261            connected_clients: self.connected_clients.count(),
2262            counters: Some(counters),
2263            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
2264            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
2265            capability_requirements: self.capability_requirement_statuses(),
2266            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2267        };
2268        Ok(vec![control_response_body_frame(
2269            &frame,
2270            &response,
2271            "ClientControlResponse::ServerDescribe",
2272        )?])
2273    }
2274
2275    fn handle_catalog_list(
2276        &self,
2277        frame: Frame,
2278        module_id: Option<String>,
2279    ) -> Result<Vec<Frame>, RouterError> {
2280        let (generation, modules) = self.registry.list_modules().map_err(|err| {
2281            RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
2282        })?;
2283        let entries = modules
2284            .into_iter()
2285            .filter(|registration| {
2286                module_id
2287                    .as_deref()
2288                    .map(|wanted| registration.manifest.module_id == wanted)
2289                    .unwrap_or(true)
2290            })
2291            .map(|registration| {
2292                let not_ready = self.not_ready_reason(&registration);
2293                let roles = registration.manifest.provides;
2294                CatalogEntry {
2295                    module_id: registration.manifest.module_id,
2296                    ready: not_ready.is_none(),
2297                    not_ready,
2298                    module_version: Some(registration.manifest.module_version),
2299                    roles,
2300                    control_ops: registration.control_ops,
2301                    capabilities: registration.manifest.capabilities,
2302                    self_signals: registration.manifest.self_signals,
2303                }
2304            })
2305            .collect();
2306        let response = ClientControlResponse::CatalogList {
2307            generation,
2308            modules: entries,
2309            subc_ops: subc_ops(),
2310        };
2311        Ok(vec![control_response_body_frame(
2312            &frame,
2313            &response,
2314            "ClientControlResponse::CatalogList",
2315        )?])
2316    }
2317
2318    fn route_open_principal(
2319        &self,
2320        frame: &Frame,
2321        consumer_identity: Option<ConsumerIdentity>,
2322    ) -> Result<Result<Principal, Frame>, RouterError> {
2323        let Some(consumer_identity) = consumer_identity else {
2324            return Ok(Ok(Principal::Direct));
2325        };
2326
2327        if self.supervisor.spawned_consumer_authorized(
2328            &consumer_identity.module_id,
2329            &consumer_identity.launch_nonce,
2330        ) {
2331            return Ok(Ok(Principal::Reserved {
2332                module_id: consumer_identity.module_id,
2333            }));
2334        }
2335
2336        Ok(Err(control_error_frame(
2337            frame,
2338            "bad_consumer_identity",
2339            format!(
2340                "consumer_identity for module_id '{}' did not match a supervised launch nonce",
2341                consumer_identity.module_id
2342            ),
2343        )?))
2344    }
2345
2346    /// Every refusal of a `route.open` goes through here so the daemon can
2347    /// attest which code it sent: without the event, a client's "the daemon
2348    /// refused me" and the daemon's own view could only be reconciled by
2349    /// argument. Malformed input (`invalid_project_root`) does not come here;
2350    /// rejecting a request that was never a valid open is not a refusal of one.
2351    fn route_open_refusal_frame(
2352        &self,
2353        ctx: &RouteCtx,
2354        frame: &Frame,
2355        module_id: &str,
2356        reason: &'static str,
2357        code: &'static str,
2358        message: impl Into<String>,
2359    ) -> Result<Frame, RouterError> {
2360        self.observe_route_open_refusal(ctx, module_id, reason, code);
2361        control_error_frame(frame, code, message.into())
2362    }
2363
2364    /// Refuse a `route.open` because the target module's bind-relay breaker is
2365    /// open, without attempting the relay.
2366    ///
2367    /// The wire code is `module_timeout`, which is the truth (the module has
2368    /// not been answering binds) and which both SDKs already classify as
2369    /// retryable with capped backoff. Reusing it is what keeps this change out
2370    /// of both SDKs; the daemon-side distinction lives in the counter key
2371    /// instead.
2372    ///
2373    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
2374    /// While a breaker is open this fires on every open to that module, and the
2375    /// stall written up in `docs/designs/route-open-head-of-line.md` already
2376    /// produced 261 lines about a single module inside 3000 lines of daemon
2377    /// log. The rare transitions are logged at warn/info instead and the volume
2378    /// is carried by the counter, so the evidence survives without the flood.
2379    /// The debug line keeps a per-refusal record reachable for whoever turns
2380    /// the level up.
2381    fn route_open_breaker_refusal_frame(
2382        &self,
2383        ctx: &RouteCtx,
2384        frame: &Frame,
2385        module_id: &str,
2386        consecutive_timeouts: u32,
2387        retry_in: Duration,
2388        probe_in_flight: bool,
2389    ) -> Result<Frame, RouterError> {
2390        self.counters
2391            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
2392        debug!(
2393            target: "control",
2394            code = "module_timeout",
2395            module_id = ?module_id,
2396            connection_id = ctx.connection_id.get(),
2397            consecutive_timeouts,
2398            retry_in_ms = retry_in.as_millis() as u64,
2399            probe_in_flight,
2400            "route.open refused by open bind-relay breaker"
2401        );
2402        let detail = if probe_in_flight {
2403            "one probe bind is already in flight; retry once it settles".to_string()
2404        } else {
2405            format!("not relaying for another {retry_in:?}")
2406        };
2407        control_error_frame(
2408            frame,
2409            "module_timeout",
2410            format!(
2411                "module_id '{module_id}' failed {consecutive_timeouts} consecutive route.bind \
2412                 relays; {detail}"
2413            ),
2414        )
2415    }
2416
2417    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
2418    /// requester's bytes (an unknown target is whatever the client sent) and
2419    /// is Debug-formatted so control characters land in the log escaped
2420    /// rather than as terminal sequences for whoever tails it.
2421    ///
2422    /// `reason` names the check that refused, because one wire code has
2423    /// several senders: after a module registers, `target_unavailable` can
2424    /// come from a missing role, an inactive registration, a supervisor that
2425    /// has not marked the process live, a missing forwarding connection, or a
2426    /// failed relay, and a log that records only the code cannot say which of
2427    /// them fired. It is a static, daemon-chosen label per branch, so it is
2428    /// safe to print plainly and stays a closed set.
2429    fn observe_route_open_refusal(
2430        &self,
2431        ctx: &RouteCtx,
2432        module_id: &str,
2433        reason: &'static str,
2434        code: &'static str,
2435    ) {
2436        self.counters.increment_route_open_refused(code);
2437        info!(
2438            target: "control",
2439            code,
2440            reason,
2441            module_id = ?module_id,
2442            connection_id = ctx.connection_id.get(),
2443            "route.open refused"
2444        );
2445    }
2446
2447    /// Record an ACCEPTED route.open.
2448    ///
2449    /// Refusals have been logged and counted since the attestation work; accepts
2450    /// were invisible, so the daemon knew every principal it stamped and wrote
2451    /// none of them down. The party that attests the identity was the only party
2452    /// not recording it, which left a credential vault unable to name the sender
2453    /// of a call that reached it (claustrum #43) and left the launch-nonce
2454    /// concurrency question unanswerable from the outside.
2455    ///
2456    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
2457    /// `code`/`module_id`/`connection_id` returns both directions of the same
2458    /// decision rather than two shapes a reader has to join by hand.
2459    ///
2460    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
2461    /// and the difference carries information rather than being an
2462    /// inconsistency. This line is only reachable after a successful bind to a
2463    /// REGISTERED module, so the value has already passed HELLO validation
2464    /// including the path-hazard refusal and cannot contain control bytes. A
2465    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
2466    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
2467    ///
2468    /// Bare is also what every other daemon line already emits (`module
2469    /// registered`, `configured module supervised`). Shipping `?module_id` here
2470    /// made this instrument the only one in the file whose ids did not answer
2471    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
2472    /// line whose whole purpose is being grepped beside its sibling.
2473    ///
2474    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
2475    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
2476    /// forwards every field type through it and a bare `&str` and a `?`-escaped
2477    /// one are recorded identically. A test written against that harness passes
2478    /// either way -- I wrote one, measured it, and deleted it rather than ship a
2479    /// green assertion that cannot fail. The same limit applies to the escaping
2480    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
2481    /// it reads as a guard on the Debug escaping and cannot detect its removal.
2482    /// Fencing either needs the real formatter, not the capture layer.
2483    ///
2484    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
2485    /// unix-socket options and subc is loopback TCP, so there is no peer identity
2486    /// to record. The ephemeral port would decay within minutes and answer only a
2487    /// live question. The identity question is instead answered by counting
2488    /// distinct live connections presenting one module's `consumer_identity` --
2489    /// "is anyone else holding this secret" rather than "is this the right
2490    /// process".
2491    fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
2492        self.counters.increment_route_open_accepted(principal);
2493        info!(
2494            target: "control",
2495            principal,
2496            module_id,
2497            connection_id = ctx.connection_id.get(),
2498            "route.open accepted"
2499        );
2500    }
2501
2502    fn supervised_absent_route_open_refusal_frame(
2503        &self,
2504        ctx: &RouteCtx,
2505        frame: &Frame,
2506        module_id: &str,
2507        code: &'static str,
2508        status: &crate::supervise::ModuleStatus,
2509    ) -> Result<Frame, RouterError> {
2510        self.counters.increment_route_open_refused(code);
2511        info!(
2512            target: "control",
2513            code,
2514            reason = "supervised_not_registered",
2515            module_id = ?module_id,
2516            connection_id = ctx.connection_id.get(),
2517            state = %status.state,
2518            enabled = status.enabled,
2519            live = status.live,
2520            "route.open refused"
2521        );
2522        control_error_frame(
2523            frame,
2524            code,
2525            format!(
2526                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
2527                status.state, status.enabled, status.live
2528            ),
2529        )
2530    }
2531
2532    async fn handle_route_open(
2533        &self,
2534        ctx: &RouteCtx,
2535        frame: Frame,
2536        request: RouteOpenRequest,
2537    ) -> Result<Vec<Frame>, RouterError> {
2538        let RouteOpenRequest {
2539            target,
2540            mut identity,
2541            consumer_identity,
2542            consumer_capabilities,
2543            admission_facts,
2544        } = request;
2545        let target_module_id = target_module_id(&target).to_string();
2546        debug!(
2547            connection_id = ctx.connection_id.get(),
2548            corr = frame.header.corr,
2549            module_id = %target_module_id,
2550            "handling route.open"
2551        );
2552
2553        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
2554        // opposite. Below, a caller learns whether a module is unregistered,
2555        // supervised-but-down (with state/enabled/live), or registered without the
2556        // requested role. Elsewhere that is an enumeration leak: a probe learning
2557        // the shape of a fleet it cannot otherwise see.
2558        //
2559        // It is not one here, and the reason is the ACCESS MODEL rather than
2560        // anything about these errors. Reaching route.open requires the
2561        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
2562        // connection file, so any caller who completes it already runs as this
2563        // user -- and can read subc.jsonc for the module list and `ck module
2564        // status` for live state. The reply discloses nothing the caller cannot
2565        // read more easily from disk, while the precision is load-bearing:
2566        // `unknown_module` is retryable and a missing role is not.
2567        //
2568        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
2569        // remote transport, a sandboxed caller, a shared-host mode -- THAT
2570        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
2571        let Some(registration) = self
2572            .registry
2573            .get_module(&target_module_id)
2574            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2575        else {
2576            if let Some((status, warming)) =
2577                self.supervisor_status(&target_module_id, frame.header.corr)?
2578            {
2579                // BEFORE the two availability codes below, because for a module
2580                // that speaks no subc wire both of them are false comfort: they
2581                // say "not right now" and are retried, and this module will
2582                // never register no matter how long the caller waits. The
2583                // absence here is the declaration being honoured, not a module
2584                // that is late.
2585                if status.protocol == ModuleProtocol::None {
2586                    return Ok(vec![self.route_open_refusal_frame(
2587                        ctx,
2588                        &frame,
2589                        &target_module_id,
2590                        "protocol_none",
2591                        error_codes::MODULE_NO_PROTOCOL,
2592                        format!(
2593                            "module_id '{target_module_id}' is declared protocol: none; \
2594                             it speaks no subc wire and serves no routes"
2595                        ),
2596                    )?]);
2597                }
2598                let code = if warming {
2599                    "module_warming"
2600                } else {
2601                    "target_unavailable"
2602                };
2603                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
2604                    ctx,
2605                    &frame,
2606                    &target_module_id,
2607                    code,
2608                    &status,
2609                )?]);
2610            }
2611            if let Some(removed_ago_ms) =
2612                self.supervisor.removal_tombstone_age_ms(&target_module_id)
2613            {
2614                return Ok(vec![self.route_open_refusal_frame(
2615                    ctx,
2616                    &frame,
2617                    &target_module_id,
2618                    "removed",
2619                    error_codes::MODULE_REMOVED,
2620                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
2621                )?]);
2622            }
2623            return Ok(vec![self.route_open_refusal_frame(
2624                ctx,
2625                &frame,
2626                &target_module_id,
2627                "not_registered",
2628                error_codes::UNKNOWN_MODULE,
2629                format!("module_id '{target_module_id}' is not registered"),
2630            )?]);
2631        };
2632
2633        // Best-effort only: registry readiness and forwarding reservation use
2634        // different locks, so a module can flip readiness between this read and
2635        // the relay. Modules must still tolerate an `on_bind` while not ready.
2636        if !registration.ready {
2637            self.counters
2638                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
2639            info!(
2640                target: "control",
2641                code = error_codes::MODULE_WARMING,
2642                module_id = ?target_module_id,
2643                connection_id = ctx.connection_id.get(),
2644                reason = "declared_not_ready",
2645                "route.open refused"
2646            );
2647            return Ok(vec![control_error_body_frame(
2648                &frame,
2649                ErrorBody {
2650                    code: error_codes::MODULE_WARMING.to_string(),
2651                    message: format!(
2652                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
2653                    ),
2654                    detail: Some(serde_json::json!({
2655                        "reason": "declared_not_ready"
2656                    })),
2657                },
2658            )?]);
2659        }
2660
2661        // Effective readiness, second half: a module that declares a capability
2662        // `need: required` is not routable while that capability has no
2663        // registered provider. It is enforced HERE, as a retryable routing
2664        // refusal, and deliberately not as spawn ordering or a boot block. The
2665        // module is still started and registered and can make its own calls;
2666        // spawn ordering is a promise that cannot be kept once a provider
2667        // crashes at runtime, and refusing to boot would stop the whole
2668        // machine, including the tools needed to fix its configuration.
2669        //
2670        // "Provided" is the evaluator's verdict, which counts a provider as
2671        // soon as it has REGISTERED, not once it is ready. Two modules that
2672        // require each other's capabilities are therefore both routable once
2673        // both register; counting readiness instead would deadlock them.
2674        //
2675        // Only new opens are refused. Routes already bound when a provider
2676        // goes away stay bound: nothing here tears them down, and the module
2677        // answers them as it can. Like the readiness read above this is
2678        // best-effort against a provider registering or leaving concurrently.
2679        if let Some(capability) = self.first_unprovided_required_capability(&registration) {
2680            self.counters
2681                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
2682            info!(
2683                target: "control",
2684                code = error_codes::MODULE_WARMING,
2685                module_id = ?target_module_id,
2686                connection_id = ctx.connection_id.get(),
2687                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
2688                capability = %capability,
2689                "route.open refused"
2690            );
2691            return Ok(vec![control_error_body_frame(
2692                &frame,
2693                ErrorBody {
2694                    code: error_codes::MODULE_WARMING.to_string(),
2695                    message: format!(
2696                        "module_id '{target_module_id}' requires capability '{capability}', \
2697                         which no registered module provides; retry"
2698                    ),
2699                    detail: Some(serde_json::json!({
2700                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
2701                        "capability": capability,
2702                    })),
2703                },
2704            )?]);
2705        }
2706
2707        if !target_has_required_role(&target, &registration.manifest.provides) {
2708            return Ok(vec![self.route_open_refusal_frame(
2709                ctx,
2710                &frame,
2711                &target_module_id,
2712                "role_not_provided",
2713                "target_unavailable",
2714                format!("module_id '{target_module_id}' does not provide the requested target"),
2715            )?]);
2716        }
2717
2718        if registration.state != ChannelState::Active {
2719            return Ok(vec![self.route_open_refusal_frame(
2720                ctx,
2721                &frame,
2722                &target_module_id,
2723                "registration_not_active",
2724                "target_unavailable",
2725                format!("module_id '{target_module_id}' is not active"),
2726            )?]);
2727        }
2728
2729        if self
2730            .forwarding
2731            .module_is_draining(&target_module_id)
2732            .map_err(RouterError::Forwarding)?
2733        {
2734            return Ok(vec![self.route_open_refusal_frame(
2735                ctx,
2736                &frame,
2737                &target_module_id,
2738                "reloading",
2739                "module_reloading",
2740                format!("module_id '{target_module_id}' is reloading"),
2741            )?]);
2742        }
2743
2744        if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
2745            process_liveness.process_live(&target_module_id) == Some(false)
2746        }) {
2747            // A module the supervisor is restarting or reloading can still hold
2748            // a registration: the old process before its connection closes, or
2749            // a new one that registered while the supervisor was draining. The
2750            // forwarding table does not see that as draining, but the consumer
2751            // should still be told to retry soon, exactly as for the drain
2752            // above, rather than that the target is unavailable.
2753            if process_liveness.process_replacing(&target_module_id) {
2754                return Ok(vec![self.route_open_refusal_frame(
2755                    ctx,
2756                    &frame,
2757                    &target_module_id,
2758                    "reloading",
2759                    "module_reloading",
2760                    format!("module_id '{target_module_id}' is reloading"),
2761                )?]);
2762            }
2763            return Ok(vec![self.route_open_refusal_frame(
2764                ctx,
2765                &frame,
2766                &target_module_id,
2767                "supervisor_not_live",
2768                "target_unavailable",
2769                format!("module_id '{target_module_id}' is not live"),
2770            )?]);
2771        }
2772
2773        if !self
2774            .forwarding
2775            .has_live_module_connection(&target_module_id)
2776            .map_err(RouterError::Forwarding)?
2777        {
2778            return Ok(vec![self.route_open_refusal_frame(
2779                ctx,
2780                &frame,
2781                &target_module_id,
2782                "no_forwarding_connection",
2783                "target_unavailable",
2784                format!("module_id '{target_module_id}' has no live forwarding connection"),
2785            )?]);
2786        }
2787
2788        if let Some(error) =
2789            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
2790        {
2791            self.observe_route_open_refusal(
2792                ctx,
2793                &target_module_id,
2794                "op_not_allowed",
2795                "op_not_allowed",
2796            );
2797            return Ok(vec![error]);
2798        }
2799
2800        let principal = match self.route_open_principal(&frame, consumer_identity)? {
2801            Ok(principal) => principal,
2802            Err(error) => {
2803                self.observe_route_open_refusal(
2804                    ctx,
2805                    &target_module_id,
2806                    "bad_consumer_identity",
2807                    "bad_consumer_identity",
2808                );
2809                return Ok(vec![error]);
2810            }
2811        };
2812
2813        // This is attested, control-plane policy for supervised module origins.
2814        // Keep it before route reservation and out of the opaque forwarding hot
2815        // path: data frames must never acquire a per-frame capability check.
2816        if let Principal::Reserved {
2817            module_id: opening_module_id,
2818        } = &principal
2819        {
2820            if let Some(opening_registration) = self
2821                .registry
2822                .get_module(opening_module_id)
2823                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2824            {
2825                if let Some(capability) =
2826                    denied_capability(&opening_registration.manifest, &registration.manifest)
2827                {
2828                    warn!(
2829                        opening_module_id,
2830                        target_module_id,
2831                        capability,
2832                        "refusing route.open because an attested capability deny edge matches"
2833                    );
2834                    return Ok(vec![self.route_open_refusal_frame(
2835                        ctx,
2836                        &frame,
2837                        &target_module_id,
2838                        "capability_deny_edge",
2839                        "capability_forbidden",
2840                        format!(
2841                            "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
2842                        ),
2843                    )?]);
2844                }
2845            }
2846        }
2847
2848        if admission_facts.is_some() {
2849            let carrier_matches = matches!(
2850                &principal,
2851                Principal::Reserved { module_id }
2852                    if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
2853            );
2854            if !carrier_matches {
2855                return Ok(vec![self.route_open_refusal_frame(
2856                    ctx,
2857                    &frame,
2858                    &target_module_id,
2859                    "admission_facts_carrier_not_permitted",
2860                    "admission_facts_not_permitted",
2861                    "admission facts may only be carried by the configured reserved module",
2862                )?]);
2863            }
2864
2865            let target_allowed = self
2866                .admission_facts_targets
2867                .as_ref()
2868                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
2869            if !target_allowed {
2870                return Ok(vec![self.route_open_refusal_frame(
2871                    ctx,
2872                    &frame,
2873                    &target_module_id,
2874                    "admission_facts_target_not_listed",
2875                    "admission_facts_target_not_allowed",
2876                    format!(
2877                        "admission facts are not permitted for target module_id '{target_module_id}'"
2878                    ),
2879                )?]);
2880            }
2881
2882            // Keep the value opaque to subc. The downstream admission validator owns
2883            // schema and semantic checks; this daemon only enforces carrier authority
2884            // and the configured destination allowlist.
2885        }
2886
2887        // Bind admits a root that no longer exists on disk, because refusing here
2888        // closes the only exit from a paused run: cancel needs a bound route, and a
2889        // renamed or reclaimed directory makes that route unopenable forever. The
2890        // run itself is intact and still addressable by its recorded identity.
2891        //
2892        // This does NOT relax the rule the strict constructor protects. That rule is
2893        // that no root is ever aliased into NEW durable state -- a missing component
2894        // can reappear as a symlink elsewhere, which would move the identity and
2895        // split a session's history across two of them. The engine now refuses the
2896        // two operations that create such state (send and import) at admission,
2897        // which is a narrower way to hold the same invariant: reads and terminations
2898        // are admitted, writes are not. That refusal had to ship before this line
2899        // changed, or there is an interval where a send commits under a provisional
2900        // identity -- the exact failure the original policy existed to prevent.
2901        //
2902        // Resolution follows realpath rather than lexical cleanup: the longest
2903        // existing ancestor is canonicalized and the missing tail re-appended, so a
2904        // live root is unchanged and a vanished leaf keeps the identity it was
2905        // admitted under. Lexical cleanup would mint a DIFFERENT identity for the
2906        // same caller the moment the directory vanished, which strands the run more
2907        // quietly than refusing it.
2908        let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
2909            Ok(project_root) => project_root,
2910            Err(err) => {
2911                return Ok(vec![control_error_frame(
2912                    &frame,
2913                    "invalid_project_root",
2914                    err.to_string(),
2915                )?])
2916            }
2917        };
2918        identity.project_root = project_root.as_path().to_path_buf();
2919
2920        // Last gate before any relay work, and deliberately after the cheap
2921        // registry and availability checks above: those name a more precise
2922        // condition (unknown, removed, reloading) and a caller is better served
2923        // by the precise code than by this one.
2924        //
2925        // Everything below this point costs an egress permit, a reserved handle
2926        // pair and, if the module does not answer, the whole relay budget. The
2927        // reader no longer waits for that budget, so cap each target explicitly;
2928        // serial dispatch used to provide the accidental cap of one relay per
2929        // connection. Admission is a mutex-protected count and never waits.
2930        let _concurrency_guard = match self
2931            .route_bind_concurrency
2932            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
2933        {
2934            Ok(guard) => guard,
2935            Err(in_flight) => {
2936                return Ok(vec![self.route_open_admission_refusal_frame(
2937                    ctx,
2938                    &frame,
2939                    &target_module_id,
2940                    format!(
2941                        "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
2942                    ),
2943                )?]);
2944            }
2945        };
2946
2947        // A module that has already burned the whole budget `threshold` times
2948        // in a row does not get to charge it again until a probe says it recovered.
2949        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
2950            RouteBindAdmission::Admitted { guard, probe } => {
2951                if probe {
2952                    info!(
2953                        module_id = %target_module_id,
2954                        connection_id = ctx.connection_id.get(),
2955                        "route.bind breaker half-open: admitting one probe"
2956                    );
2957                }
2958                guard
2959            }
2960            RouteBindAdmission::Refused {
2961                consecutive_timeouts,
2962                retry_in,
2963                probe_in_flight,
2964            } => {
2965                return Ok(vec![self.route_open_breaker_refusal_frame(
2966                    ctx,
2967                    &frame,
2968                    &target_module_id,
2969                    consecutive_timeouts,
2970                    retry_in,
2971                    probe_in_flight,
2972                )?]);
2973            }
2974        };
2975
2976        // Resolve the per-module budget here so the wait matches the operator's
2977        // intent for this specific target. A per-module override in
2978        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
2979        // daemons) wins over the daemon-wide default.
2980        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
2981        let relay_deadline = Instant::now() + route_bind_relay_timeout;
2982        let pending = match self
2983            .forwarding
2984            .begin_route_bind_relay_for(
2985                ctx.connection_id,
2986                ctx.egress.clone(),
2987                response_version(&frame),
2988                frame.header.corr,
2989                &target_module_id,
2990                principal.clone(),
2991                Some(project_root),
2992                relay_deadline,
2993            )
2994            .await
2995        {
2996            Ok(pending) => pending,
2997            Err(err) => {
2998                return Ok(vec![self.route_open_refusal_frame(
2999                    ctx,
3000                    &frame,
3001                    &target_module_id,
3002                    "relay_reservation_failed",
3003                    forwarding_error_code(&err),
3004                    err.to_string(),
3005                )?])
3006            }
3007        };
3008        let crate::forwarding::PendingRouteBindRelay {
3009            endpoint,
3010            module_sink,
3011            negotiated_ver,
3012            client_channel,
3013            client_epoch,
3014            module_channel,
3015            module_epoch,
3016            corr: relay_corr,
3017            receiver,
3018        } = pending;
3019        let mut reservation =
3020            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
3021
3022        debug!(
3023            connection_id = ctx.connection_id.get(),
3024            client_channel,
3025            client_epoch,
3026            module_channel,
3027            module_epoch,
3028            "reserved route handle pair"
3029        );
3030        // Rendered BEFORE the move into the relay, because the accept arm below
3031        // is where it is logged and the principal is gone by then.
3032        let principal_label = match &principal {
3033            Principal::Reserved { module_id } => format!("reserved:{module_id}"),
3034            Principal::Direct => "direct".to_string(),
3035            other => format!("{other:?}"),
3036        };
3037        let relay = ModuleControlRequest::RouteBind {
3038            route_channel: module_channel,
3039            epoch: module_epoch,
3040            target,
3041            identity,
3042            principal: Some(principal),
3043            consumer_capabilities,
3044            admission_facts,
3045        };
3046        let relay_body = serde_json::to_vec(&relay).map_err(|err| {
3047            RouterError::backend(
3048                0,
3049                frame.header.corr,
3050                format!("failed to encode route.bind request: {err}"),
3051            )
3052        })?;
3053        let relay_frame = Frame::build_with_version(
3054            negotiated_ver,
3055            FrameType::Request,
3056            control_flags(),
3057            0,
3058            0,
3059            relay_corr,
3060            relay_body,
3061        )
3062        .map_err(RouterError::FrameBuild)?;
3063
3064        if let Err(err) = module_sink.send(relay_frame).await {
3065            reservation.release_and_disarm();
3066            return Ok(vec![self.route_open_refusal_frame(
3067                ctx,
3068                &frame,
3069                &target_module_id,
3070                "relay_send_failed",
3071                "target_unavailable",
3072                err.to_string(),
3073            )?]);
3074        }
3075
3076        if !self
3077            .forwarding
3078            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
3079            .map_err(RouterError::Forwarding)?
3080        {
3081            self.send_abandoned_route_bind_goodbye(
3082                &module_sink,
3083                negotiated_ver,
3084                module_channel,
3085                module_epoch,
3086            );
3087        }
3088
3089        match timeout_at(relay_deadline, receiver).await {
3090            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
3091                reservation.disarm();
3092                if breaker.record_accepted() {
3093                    info!(
3094                        module_id = %target_module_id,
3095                        "route.bind breaker closed: the probe was accepted"
3096                    );
3097                }
3098                self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
3099                Ok(Vec::new())
3100            }
3101            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
3102                reservation.release_and_disarm();
3103                // A module that says no in microseconds is healthy. Rejection
3104                // is a different condition with its own refusal and must not
3105                // move the breaker.
3106                breaker.record_inconclusive();
3107                self.counters
3108                    .increment_route_open_refused("module_rejected");
3109                info!(
3110                    target: "control",
3111                    code = "module_rejected",
3112                    module_code = ?body.code,
3113                    module_id = ?target_module_id,
3114                    connection_id = ctx.connection_id.get(),
3115                    "route.open refused"
3116                );
3117                Ok(vec![control_error_body_frame(&frame, body)?])
3118            }
3119            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
3120                reservation.release_and_disarm();
3121                breaker.record_inconclusive();
3122                // Fires when the module's connection closes while a relayed
3123                // bind is pending -- typically a caller racing a module restart
3124                // whose bind was relayed BEFORE the drain mark went up. Logged
3125                // because the caller sees only its own error and the fleet has
3126                // already spent one diagnosis round unable to tell this arm
3127                // from a relay timeout without daemon-side evidence.
3128                tracing::warn!(
3129                    module_id = %target_module_id,
3130                    "route.bind relay abandoned: {message}"
3131                );
3132                Ok(vec![self.route_open_refusal_frame(
3133                    ctx,
3134                    &frame,
3135                    &target_module_id,
3136                    "relay_abandoned",
3137                    "target_unavailable",
3138                    message,
3139                )?])
3140            }
3141            Ok(Err(_)) => {
3142                reservation.release_and_disarm();
3143                breaker.record_inconclusive();
3144                Ok(vec![self.route_open_refusal_frame(
3145                    ctx,
3146                    &frame,
3147                    &target_module_id,
3148                    "relay_waiter_canceled",
3149                    "target_unavailable",
3150                    "route.bind relay waiter was canceled before the module responded",
3151                )?])
3152            }
3153            Err(_) => {
3154                reservation.release_and_disarm();
3155                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
3156                // answer at all is the one condition a fast refusal can
3157                // usefully stand in for; every other arm already answered.
3158                if let Some(opened) = breaker.record_timeout(
3159                    self.route_bind_breaker_threshold,
3160                    self.route_bind_breaker_cooldown,
3161                ) {
3162                    warn!(
3163                        module_id = %target_module_id,
3164                        consecutive_timeouts = opened.consecutive_timeouts,
3165                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
3166                        reopened_after_probe = opened.reopened_after_probe,
3167                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
3168                    );
3169                }
3170                // The generous budget just burned to no answer: the module is
3171                // registered and its connection is up, but its bind handler sat
3172                // on the ack for the full budget (warm-on-bind, cold configure,
3173                // or a wedged handler). Every earlier unavailability shape
3174                // fast-refuses BEFORE the relay, so this arm firing means the
3175                // slowness is module-side -- log it so the per-module timeline
3176                // is reconstructable without client audit rows.
3177                tracing::warn!(
3178                    module_id = %target_module_id,
3179                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
3180                    "route.bind relay timed out: module did not ack within budget"
3181                );
3182                Ok(vec![self.route_open_refusal_frame(
3183                    ctx,
3184                    &frame,
3185                    &target_module_id,
3186                    "relay_timed_out",
3187                    "module_timeout",
3188                    format!(
3189                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
3190                        route_bind_relay_timeout
3191                    ),
3192                )?])
3193            }
3194        }
3195    }
3196
3197    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3198        let response = ClientControlResponse::SupervisorSpawnSnapshot {
3199            snapshot: self.supervisor.spawn_snapshot(),
3200        };
3201        Ok(vec![control_response_body_frame(
3202            &frame,
3203            &response,
3204            "ClientControlResponse::SupervisorSpawnSnapshot",
3205        )?])
3206    }
3207
3208    fn handle_supervisor_spawn_subscribe(
3209        &self,
3210        ctx: &RouteCtx,
3211        frame: Frame,
3212        since: Option<SpawnCursor>,
3213    ) -> Result<Vec<Frame>, RouterError> {
3214        match self.supervisor.subscribe_spawns(
3215            ctx.connection_id,
3216            frame.header.corr,
3217            response_version(&frame),
3218            since,
3219            ctx.egress.clone(),
3220        ) {
3221            Ok(()) => Ok(Vec::new()),
3222            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
3223                Ok(vec![control_error_body_frame(
3224                    &frame,
3225                    ErrorBody {
3226                        code: "spawn_cursor_incarnation_mismatch".to_string(),
3227                        message: "spawn cursor belongs to a different daemon incarnation"
3228                            .to_string(),
3229                        detail: Some(serde_json::json!({
3230                            "current_daemon_incarnation": current
3231                        })),
3232                    },
3233                )?])
3234            }
3235            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
3236                &frame,
3237                ErrorBody {
3238                    code: "spawn_cursor_too_old".to_string(),
3239                    message: "spawn cursor predates the retained event ring".to_string(),
3240                    detail: Some(serde_json::json!({
3241                        "oldest_retained_cursor": oldest
3242                    })),
3243                },
3244            )?]),
3245            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
3246                0,
3247                frame.header.corr,
3248                format!("failed to open supervisor spawn subscription: {error}"),
3249            )),
3250        }
3251    }
3252
3253    async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3254        let generation = self
3255            .registry
3256            .generation()
3257            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3258        let mut modules = Vec::new();
3259        for module in self.supervisor.list() {
3260            let status = module.status_for_control("list").map_err(|err| {
3261                RouterError::backend(
3262                    0,
3263                    frame.header.corr,
3264                    format!("failed to read supervisor status: {err}"),
3265                )
3266            })?;
3267            let (configured, _) = module.configuration().map_err(|err| {
3268                RouterError::backend(
3269                    0,
3270                    frame.header.corr,
3271                    format!("failed to read module configuration: {err}"),
3272                )
3273            })?;
3274            // Status and configuration snapshots release their locks before the image probe awaits.
3275            let image = module.running_image_agreement().await;
3276            let pending_reload = Some(reload_verdict(
3277                &configured.program,
3278                status.spawned_from.as_deref(),
3279                image,
3280            ));
3281            modules.push(SupervisorEntry {
3282                module_id: status.module_id,
3283                state: status.state.to_string(),
3284                enabled: status.enabled,
3285                live: status.live,
3286                protocol: status.protocol,
3287                health: status.health.status,
3288                pending_reload,
3289                last_probe_ms: status.health.last_probe_ms,
3290                last_exit_code: status.last_exit.as_ref().and_then(|e| e.code),
3291                last_exit_signal: status.last_exit.as_ref().and_then(|e| e.signal),
3292                last_exit_ms: status.last_exit.as_ref().map(|e| e.at_ms),
3293                last_exit_kind: status.last_exit.as_ref().map(|e| e.kind.into()),
3294                restart_count: Some(status.restart_count),
3295                max_restarts: Some(status.max_restarts),
3296                lifetime_restarts: Some(status.lifetime_restarts),
3297                spawn_generation: Some(status.spawn_generation),
3298                restart_window_secs: Some(status.restart_window.as_secs()),
3299                drain_timeout_ms: Some(status.drain_timeout.as_millis() as u64),
3300                restart_backoff_ms: Some(status.restart_backoff.as_millis() as u64),
3301                restart_max_backoff_ms: Some(status.restart_max_backoff.as_millis() as u64),
3302            });
3303        }
3304        let response = ClientControlResponse::SupervisorList {
3305            generation,
3306            modules,
3307        };
3308        Ok(vec![control_response_body_frame(
3309            &frame,
3310            &response,
3311            "ClientControlResponse::SupervisorList",
3312        )?])
3313    }
3314
3315    fn handle_supervisor_stderr_tail(
3316        &self,
3317        frame: Frame,
3318        module_id: String,
3319        max_lines: Option<u32>,
3320        max_bytes: Option<u32>,
3321    ) -> Result<Vec<Frame>, RouterError> {
3322        let Some(module) = self.supervisor.get(&module_id) else {
3323            return Ok(vec![control_error_frame(
3324                &frame,
3325                "unknown_module",
3326                format!("module_id '{module_id}' is not supervised"),
3327            )?]);
3328        };
3329
3330        let snapshot = module.stderr_tail(
3331            max_lines.map(|value| value as usize),
3332            max_bytes.map(|value| value as usize),
3333        );
3334
3335        let response = ClientControlResponse::SupervisorStderrTail {
3336            module_id,
3337            tail: StderrTail {
3338                capture: match snapshot.capture {
3339                    CaptureState::Captured => StderrCaptureState::Captured,
3340                    CaptureState::Incomplete { reason } => {
3341                        StderrCaptureState::Incomplete { reason }
3342                    }
3343                    CaptureState::NotCaptured { reason } => {
3344                        StderrCaptureState::NotCaptured { reason }
3345                    }
3346                },
3347                entries: snapshot
3348                    .entries
3349                    .into_iter()
3350                    .map(|entry| match entry {
3351                        TailEntry::Line { text, truncated } => {
3352                            StderrTailEntry::Line { text, truncated }
3353                        }
3354                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
3355                    })
3356                    .collect(),
3357                dropped_lines: snapshot.dropped_lines,
3358            },
3359        };
3360        Ok(vec![control_response_body_frame(
3361            &frame,
3362            &response,
3363            "ClientControlResponse::SupervisorStderrTail",
3364        )?])
3365    }
3366
3367    async fn handle_supervisor_terminals(
3368        &self,
3369        frame: Frame,
3370        module_id: String,
3371    ) -> Result<Vec<Frame>, RouterError> {
3372        let Some(module) = self.supervisor.get(&module_id) else {
3373            return Ok(vec![control_error_frame(
3374                &frame,
3375                "unknown_module",
3376                format!("module_id '{module_id}' is not supervised"),
3377            )?]);
3378        };
3379
3380        // The journal read runs on a blocking thread: it can be megabytes of
3381        // file I/O and must not occupy a runtime worker.
3382        let terminals = module
3383            .read_durable_terminal_history()
3384            .await
3385            .map_err(|error| {
3386                RouterError::backend(
3387                    0,
3388                    frame.header.corr,
3389                    format!("failed to read terminal history: {error}"),
3390                )
3391            })?;
3392        let response = ClientControlResponse::SupervisorTerminals {
3393            module_id,
3394            terminals,
3395        };
3396        Ok(vec![control_response_body_frame(
3397            &frame,
3398            &response,
3399            "ClientControlResponse::SupervisorTerminals",
3400        )?])
3401    }
3402
3403    fn handle_supervisor_routes(
3404        &self,
3405        frame: Frame,
3406        module_id: Option<String>,
3407    ) -> Result<Vec<Frame>, RouterError> {
3408        let modules = self
3409            .forwarding
3410            .route_census(module_id.as_deref())
3411            .map_err(RouterError::Forwarding)?
3412            .into_iter()
3413            .map(|(module_id, routes)| SupervisorRouteModule {
3414                module_id,
3415                routes: routes
3416                    .into_iter()
3417                    .map(|route| SupervisorRoute {
3418                        consumer: match route.principal {
3419                            Principal::Reserved { module_id } => {
3420                                SupervisorRouteConsumer::Reserved { module_id }
3421                            }
3422                            Principal::Direct | Principal::Unverified => {
3423                                SupervisorRouteConsumer::Direct {
3424                                    connection_id: route.goodbye_target.connection_id.get(),
3425                                }
3426                            }
3427                        },
3428                        age_ms: Instant::now()
3429                            .saturating_duration_since(route.bound_at)
3430                            .as_millis()
3431                            .try_into()
3432                            .unwrap_or(u64::MAX),
3433                        draining: route.draining,
3434                        drain_reason: route.drain_reason,
3435                    })
3436                    .collect(),
3437            })
3438            .collect();
3439        let response = ClientControlResponse::SupervisorRoutes { modules };
3440        Ok(vec![control_response_body_frame(
3441            &frame,
3442            &response,
3443            "ClientControlResponse::SupervisorRoutes",
3444        )?])
3445    }
3446
3447    async fn handle_supervisor_provenance(
3448        &self,
3449        frame: Frame,
3450        module_id: Option<String>,
3451    ) -> Result<Vec<Frame>, RouterError> {
3452        let mut selected = if let Some(module_id) = module_id {
3453            let Some(module) = self.supervisor.get(&module_id) else {
3454                return Ok(vec![control_error_frame(
3455                    &frame,
3456                    "unknown_module",
3457                    format!("module_id '{module_id}' is not supervised"),
3458                )?]);
3459            };
3460            vec![module]
3461        } else {
3462            self.supervisor.list()
3463        };
3464
3465        let mut modules = Vec::with_capacity(selected.len());
3466        for module in selected.drain(..) {
3467            let status = module.status().map_err(|err| {
3468                RouterError::backend(
3469                    0,
3470                    frame.header.corr,
3471                    format!("failed to read supervisor status: {err}"),
3472                )
3473            })?;
3474            let module_declared = self
3475                .registry
3476                .get_module(&status.module_id)
3477                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3478                .and_then(|registration| registration.manifest.provenance)
3479                .map(|build| ModuleDeclaredProvenance::Reported { build })
3480                .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
3481            #[cfg(test)]
3482            let running_image = match &self.provenance_probe_override {
3483                Some(result) => result.clone(),
3484                None => module.running_image_agreement().await,
3485            };
3486            #[cfg(not(test))]
3487            let running_image = module.running_image_agreement().await;
3488            modules.push(SupervisorModuleProvenance {
3489                module_id: status.module_id,
3490                module_declared,
3491                daemon_observed: SupervisorObservedProcess {
3492                    pid: status.pid,
3493                    spawned_at_ms: status.spawned_at_ms,
3494                    spawned_from: status.spawned_from,
3495                    running_image,
3496                },
3497            });
3498        }
3499        let daemon = SupervisorDaemonProvenance {
3500            daemon_build: self.daemon_provenance.build.clone(),
3501            daemon_observed: DaemonObservedProcess {
3502                pid: self.daemon_provenance.pid,
3503                started_at_ms: self
3504                    .daemon_provenance
3505                    .start_clock
3506                    .map(|clock| clock.started_at_ms())
3507                    .or(self.daemon_provenance.started_at_ms),
3508                running_image: self
3509                    .daemon_provenance
3510                    .probe
3511                    .observe(
3512                        self.daemon_provenance.pid,
3513                        self.daemon_provenance.executable_path.as_deref(),
3514                        self.daemon_provenance.executable_identity,
3515                        self.daemon_provenance.process_start_time,
3516                    )
3517                    .await,
3518            },
3519        };
3520        let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
3521        Ok(vec![control_response_body_frame(
3522            &frame,
3523            &response,
3524            "ClientControlResponse::SupervisorProvenance",
3525        )?])
3526    }
3527
3528    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3529        self.refresh_capability_requirements();
3530        let generation = self
3531            .registry
3532            .generation()
3533            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3534        let modules = self
3535            .supervisor
3536            .list()
3537            .into_iter()
3538            .map(|module| {
3539                let status = module.status_for_control("health").map_err(|err| {
3540                    RouterError::backend(
3541                        0,
3542                        frame.header.corr,
3543                        format!("failed to read supervisor health: {err}"),
3544                    )
3545                })?;
3546                let module_id = status.module_id;
3547                let capability_detail = self
3548                    .capability_evaluator
3549                    .required_problem_detail(&module_id);
3550                Ok(SupervisorHealthEntry {
3551                    module_id,
3552                    status: status.health.status,
3553                    detail: append_capability_problem_detail(
3554                        status.health.detail,
3555                        capability_detail,
3556                    ),
3557                    metrics: status.health.metrics,
3558                    consecutive_failures: status.health.consecutive_failures,
3559                    late_answer_count: status.health.late_answer_count,
3560                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
3561                    last_action: status.health.last_action,
3562                    last_action_ms: status.health.last_action_ms,
3563                    last_probe_ms: status.health.last_probe_ms,
3564                })
3565            })
3566            .collect::<Result<Vec<_>, RouterError>>()?;
3567        let response = ClientControlResponse::SupervisorHealth {
3568            generation,
3569            modules,
3570        };
3571        Ok(vec![control_response_body_frame(
3572            &frame,
3573            &response,
3574            "ClientControlResponse::SupervisorHealth",
3575        )?])
3576    }
3577
3578    async fn handle_supervisor_restart(
3579        &self,
3580        frame: Frame,
3581        module_id: String,
3582        drain_timeout_ms: Option<u64>,
3583    ) -> Result<Vec<Frame>, RouterError> {
3584        let operation_lock = self.supervisor.operation_lock();
3585        let _operation_guard = operation_lock.lock().await;
3586        let Some(module) = self.supervisor.get(&module_id) else {
3587            return Ok(vec![control_error_frame(
3588                &frame,
3589                "unknown_module",
3590                format!("module_id '{module_id}' is not supervised"),
3591            )?]);
3592        };
3593
3594        if let Err(err) = module.restart(drain_timeout_ms).await {
3595            let (code, message) = match err {
3596                crate::supervise::SuperviseError::Disabled { .. } => {
3597                    ("module_disabled", err.to_string())
3598                }
3599                crate::supervise::SuperviseError::SwapInProgress { .. } => {
3600                    ("swap_in_progress", err.to_string())
3601                }
3602                _ => (
3603                    "target_unavailable",
3604                    format!("failed to restart module_id '{module_id}': {err}"),
3605                ),
3606            };
3607            return Ok(vec![control_error_frame(&frame, code, message)?]);
3608        }
3609
3610        let response = ClientControlResponse::SupervisorAck {
3611            module_id,
3612            applied: true,
3613        };
3614        Ok(vec![control_response_body_frame(
3615            &frame,
3616            &response,
3617            "ClientControlResponse::SupervisorAck",
3618        )?])
3619    }
3620
3621    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
3622    /// when the old process has finished draining: a caller whose own lane
3623    /// rides the old process must get its reply before that drain waits on it.
3624    async fn handle_supervisor_swap(
3625        &self,
3626        frame: Frame,
3627        module_id: String,
3628        ready_timeout_ms: Option<u64>,
3629    ) -> Result<Vec<Frame>, RouterError> {
3630        // The daemon-wide operation lock is held only to resolve the handle,
3631        // not across the swap. The swap can take its whole readiness budget,
3632        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
3633        // holding it here would park an operator's stop behind the swap it is
3634        // meant to abort. A rescan or stop that reaches the module during the
3635        // swap is served by the swap itself (see `supervise_swap`).
3636        let module = {
3637            let operation_lock = self.supervisor.operation_lock();
3638            let _operation_guard = operation_lock.lock().await;
3639            self.supervisor.get(&module_id)
3640        };
3641        let Some(module) = module else {
3642            return Ok(vec![control_error_frame(
3643                &frame,
3644                "unknown_module",
3645                format!("module_id '{module_id}' is not supervised"),
3646            )?]);
3647        };
3648
3649        if let Err(err) = module
3650            .swap(ready_timeout_ms.map(Duration::from_millis))
3651            .await
3652        {
3653            use crate::supervise::SuperviseError;
3654            let message = err.to_string();
3655            let error = match err {
3656                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
3657                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
3658                    code: "swap_refused".to_string(),
3659                    message,
3660                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
3661                },
3662                SuperviseError::SwapFailed {
3663                    arm,
3664                    candidate_exit,
3665                    ..
3666                } => ErrorBody {
3667                    code: "swap_failed".to_string(),
3668                    message,
3669                    detail: Some(serde_json::json!({
3670                        "arm": arm.as_str(),
3671                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
3672                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
3673                    })),
3674                },
3675                _ => ErrorBody::new(
3676                    "target_unavailable",
3677                    format!("failed to swap module_id '{module_id}': {message}"),
3678                ),
3679            };
3680            return Ok(vec![control_error_body_frame(&frame, error)?]);
3681        }
3682
3683        let response = ClientControlResponse::SupervisorAck {
3684            module_id,
3685            applied: true,
3686        };
3687        Ok(vec![control_response_body_frame(
3688            &frame,
3689            &response,
3690            "ClientControlResponse::SupervisorAck",
3691        )?])
3692    }
3693
3694    async fn handle_supervisor_reload(
3695        &self,
3696        frame: Frame,
3697        module_id: String,
3698    ) -> Result<Vec<Frame>, RouterError> {
3699        let operation_lock = self.supervisor.operation_lock();
3700        let _operation_guard = operation_lock.lock().await;
3701        let Some(module) = self.supervisor.get(&module_id) else {
3702            return Ok(vec![control_error_frame(
3703                &frame,
3704                "unknown_module",
3705                format!("module_id '{module_id}' is not supervised"),
3706            )?]);
3707        };
3708
3709        if let Err(err) = module.reload().await {
3710            let (code, message) = match err {
3711                crate::supervise::SuperviseError::Disabled { .. } => {
3712                    ("module_disabled", err.to_string())
3713                }
3714                crate::supervise::SuperviseError::SwapInProgress { .. } => {
3715                    ("swap_in_progress", err.to_string())
3716                }
3717                _ => (
3718                    "reload_failed",
3719                    format!("failed to reload module_id '{module_id}': {err}"),
3720                ),
3721            };
3722            return Ok(vec![control_error_frame(&frame, code, message)?]);
3723        }
3724
3725        let response = ClientControlResponse::SupervisorAck {
3726            module_id,
3727            applied: true,
3728        };
3729        Ok(vec![control_response_body_frame(
3730            &frame,
3731            &response,
3732            "ClientControlResponse::SupervisorAck",
3733        )?])
3734    }
3735
3736    async fn handle_supervisor_rescan(
3737        &self,
3738        frame: Frame,
3739        preview: bool,
3740    ) -> Result<Vec<Frame>, RouterError> {
3741        let Some(context) = self.rescan.clone() else {
3742            return Ok(vec![control_error_frame(
3743                &frame,
3744                "rescan_unavailable",
3745                "the daemon was not started with a reloadable config path".to_string(),
3746            )?]);
3747        };
3748
3749        let operation_lock = self.supervisor.operation_lock();
3750        let _operation_guard = operation_lock.lock().await;
3751        let loaded = match crate::daemon_config::load(&context.config_path) {
3752            Ok(config) => config,
3753            Err(err) => {
3754                return Ok(vec![control_error_frame(
3755                    &frame,
3756                    "invalid_daemon_config",
3757                    format!("supervisor rescan rejected daemon config: {err}"),
3758                )?])
3759            }
3760        };
3761        // `load` reports a missing file as Ok(None), which is correct at boot
3762        // (no config, nothing to supervise) and catastrophic here: rescan treats
3763        // "not in the config" as "remove it", so an absent file would read as an
3764        // empty module list and retire the entire running fleet. An editor
3765        // writing via write-new-then-rename, or a half-finished edit, is enough
3766        // to open that window. Refuse instead: a config that cannot be read
3767        // carries no instruction to remove anything.
3768        let Some(config) = loaded else {
3769            return Ok(vec![control_error_frame(
3770                &frame,
3771                "invalid_daemon_config",
3772                format!(
3773                    "daemon config not found at {}; refusing to rescan (an absent config would \
3774                     retire every supervised module)",
3775                    context.config_path.display()
3776                ),
3777            )?]);
3778        };
3779        let (
3780            configured_port,
3781            storage_config,
3782            admission_facts_carrier_module_id,
3783            admission_facts_targets,
3784            modules,
3785            reserved_capabilities,
3786        ) = (
3787            config.port,
3788            config.storage,
3789            config.admission_facts_carrier_module_id,
3790            config.admission_facts_targets,
3791            config.modules,
3792            config.reserved_capabilities,
3793        );
3794
3795        // Collect the sections rescan cannot apply, so the REPLY carries them.
3796        //
3797        // The warning below has always been correct and has always gone only to
3798        // the journal -- addressed to whoever reads logs, while the person who
3799        // just edited the config is looking at the CLI. Naming each section
3800        // individually rather than setting a flag: "something outside modules
3801        // changed" sends the operator back to diffing their own file, which is
3802        // the work this is meant to save.
3803        let mut restart_required = Vec::new();
3804        for section in RestartRequiredSection::ALL {
3805            let changed = match section {
3806                RestartRequiredSection::Port => configured_port != context.configured_port,
3807                RestartRequiredSection::Storage => storage_config != context.storage_config,
3808                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
3809                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
3810                }
3811                RestartRequiredSection::AdmissionFactsTargets => {
3812                    admission_facts_targets != context.admission_facts_targets
3813                }
3814            };
3815            if changed {
3816                restart_required.push(section.label().to_string());
3817            }
3818        }
3819        if !restart_required.is_empty() {
3820            warn!(
3821                config_path = %context.config_path.display(),
3822                sections = %restart_required.join(", "),
3823                "daemon config changed outside the modules section; restart the daemon to apply those changes"
3824            );
3825        }
3826
3827        for configured in &modules {
3828            if let Err(err) = validate_spec(&configured.module_spec()) {
3829                return Ok(vec![control_error_frame(
3830                    &frame,
3831                    "invalid_daemon_config",
3832                    format!("supervisor rescan rejected daemon config: {err}"),
3833                )?]);
3834            }
3835        }
3836
3837        let configured_capabilities = modules
3838            .iter()
3839            .map(|module| (module.module_id.clone(), module.enabled))
3840            .collect::<Vec<_>>();
3841        let preview_capability_warnings = if preview {
3842            let (_, registrations) = self.runtime_capability_snapshot()?;
3843            let current_modules = self
3844                .supervisor
3845                .list()
3846                .into_iter()
3847                .map(|module| module.module_id().to_string())
3848                .collect::<BTreeSet<_>>();
3849            let resulting_modules = configured_capabilities.clone();
3850            let removed = current_modules
3851                .into_iter()
3852                .filter(|module_id| {
3853                    !resulting_modules
3854                        .iter()
3855                        .any(|(configured_id, _)| configured_id == module_id)
3856                })
3857                .collect::<Vec<_>>();
3858            self.capability_evaluator.preview_removal_warnings(
3859                resulting_modules,
3860                &removed,
3861                &registrations,
3862            )
3863        } else {
3864            Vec::new()
3865        };
3866        let result = match self
3867            .reconcile_supervised_modules(&context.supervisor, modules, preview)
3868            .await
3869        {
3870            Ok(result) => result,
3871            Err(message) => {
3872                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
3873            }
3874        };
3875        if !preview {
3876            self.capability_evaluator
3877                .configure(configured_capabilities, reserved_capabilities);
3878            self.capability_evaluator.wake_deadline_loop();
3879            self.refresh_capability_requirements();
3880        }
3881        let mut result = result;
3882        result.restart_required = restart_required;
3883        result.capability_warnings = preview_capability_warnings;
3884        let response = ClientControlResponse::SupervisorRescan { result };
3885        Ok(vec![control_response_body_frame(
3886            &frame,
3887            &response,
3888            "ClientControlResponse::SupervisorRescan",
3889        )?])
3890    }
3891
3892    async fn handle_supervisor_release_reserved(
3893        &self,
3894        frame: Frame,
3895        module_id: String,
3896    ) -> Result<Vec<Frame>, RouterError> {
3897        let Some(context) = self.rescan.clone() else {
3898            return Ok(vec![control_error_frame(
3899                &frame,
3900                "release_unavailable",
3901                "reserved-id release requires a daemon started with a reloadable config path",
3902            )?]);
3903        };
3904        let operation_lock = self.supervisor.operation_lock();
3905        let _operation_guard = operation_lock.lock().await;
3906        let loaded = match crate::daemon_config::load(&context.config_path) {
3907            Ok(Some(config)) => config,
3908            Ok(None) => {
3909                return Ok(vec![control_error_frame(
3910                    &frame,
3911                    "invalid_daemon_config",
3912                    format!(
3913                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
3914                        context.config_path.display()
3915                    ),
3916                )?])
3917            }
3918            Err(err) => {
3919                return Ok(vec![control_error_frame(
3920                    &frame,
3921                    "invalid_daemon_config",
3922                    format!("unable to verify reserved-id release against daemon config: {err}"),
3923                )?])
3924            }
3925        };
3926        if loaded
3927            .modules
3928            .iter()
3929            .any(|configured| configured.module_id == module_id)
3930        {
3931            return Ok(vec![control_error_frame(
3932                &frame,
3933                "reserved_module_configured",
3934                format!(
3935                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
3936                ),
3937            )?]);
3938        }
3939        if !self.supervisor.release_retained_reserved_gate(&module_id) {
3940            return Ok(vec![control_error_frame(
3941                &frame,
3942                "reserved_gate_not_retained",
3943                format!(
3944                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
3945                ),
3946            )?]);
3947        }
3948
3949        let response = ClientControlResponse::SupervisorAck {
3950            module_id,
3951            applied: true,
3952        };
3953        Ok(vec![control_response_body_frame(
3954            &frame,
3955            &response,
3956            "ClientControlResponse::SupervisorAck",
3957        )?])
3958    }
3959
3960    /// Reconcile the running module set against the configured one.
3961    ///
3962    /// With `preview` set, the diff is computed and returned WITHOUT applying any
3963    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
3964    /// deliberately shares this function with the executing path rather than
3965    /// computing the same diff somewhere else -- two implementations of one
3966    /// decision agree until they do not, and the whole value of a preview is that
3967    /// it describes the operation that will actually run.
3968    async fn reconcile_supervised_modules(
3969        &self,
3970        supervisor: &Supervisor,
3971        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
3972        preview: bool,
3973    ) -> Result<SupervisorRescanResult, String> {
3974        let mut current = BTreeMap::new();
3975        for module in self.supervisor.list() {
3976            let (spec, health) = module.configuration().map_err(|err| {
3977                format!(
3978                    "failed to read configuration for module_id '{}': {err}",
3979                    module.module_id()
3980                )
3981            })?;
3982            let enabled = module
3983                .status()
3984                .map_err(|err| {
3985                    format!(
3986                        "failed to read status for module_id '{}': {err}",
3987                        module.module_id()
3988                    )
3989                })?
3990                .enabled;
3991            current.insert(
3992                module.module_id().to_string(),
3993                (module, spec, health, enabled),
3994            );
3995        }
3996        let configured = configured_modules
3997            .into_iter()
3998            .map(|module| (module.module_id.clone(), module))
3999            .collect::<BTreeMap<_, _>>();
4000
4001        let added = configured
4002            .keys()
4003            .filter(|module_id| !current.contains_key(*module_id))
4004            .cloned()
4005            .collect::<Vec<_>>();
4006        let removed = current
4007            .keys()
4008            .filter(|module_id| !configured.contains_key(*module_id))
4009            .cloned()
4010            .collect::<Vec<_>>();
4011        let mut changed_pending_reload = Vec::new();
4012        let mut configuration_changes = BTreeSet::new();
4013        let mut enabled_changes = BTreeSet::new();
4014        let mut unchanged = 0_u32;
4015
4016        for (module_id, configured_module) in &configured {
4017            let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
4018            else {
4019                continue;
4020            };
4021            let configuration_changed = *current_spec != configured_module.module_spec()
4022                || *current_health != configured_module.health;
4023            let enabled_changed = *current_enabled != configured_module.enabled;
4024            if configuration_changed {
4025                configuration_changes.insert(module_id.clone());
4026                changed_pending_reload.push(module_id.clone());
4027            }
4028            if enabled_changed {
4029                enabled_changes.insert(module_id.clone());
4030            }
4031            if !configuration_changed && !enabled_changed {
4032                unchanged = unchanged.saturating_add(1);
4033            }
4034        }
4035
4036        // Everything above this point is pure computation over two snapshots.
4037        // Everything below MUTATES. The preview returns here so the boundary is a
4038        // single early return rather than a condition repeated at each mutation
4039        // site, where one missed guard would apply part of a change the caller was
4040        // told would not happen.
4041        if preview {
4042            return Ok(SupervisorRescanResult {
4043                added,
4044                removed,
4045                changed_pending_reload,
4046                enabled_changes: enabled_changes.iter().cloned().collect(),
4047                unchanged,
4048                preview: true,
4049                // Filled by the caller on both paths, so the preview reports
4050                // restart-required sections identically to an executed rescan --
4051                // the preview is where an operator is most likely to be looking.
4052                restart_required: Vec::new(),
4053                capability_warnings: Vec::new(),
4054            });
4055        }
4056
4057        for module_id in &removed {
4058            let module = &current
4059                .get(module_id)
4060                .expect("removed module came from current supervisor state")
4061                .0;
4062            module.retire().await.map_err(|err| {
4063                format!("failed to retire module_id '{module_id}' during rescan: {err}")
4064            })?;
4065            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
4066            //
4067            // `handle_route_open` resolves an absent module in three steps:
4068            // registry, then supervisor status, then tombstone. Retiring first
4069            // opens a window where ALL THREE ARE ABSENT -- the registry entry
4070            // went with the teardown above, the supervisor entry went with
4071            // `retire`, and the tombstone does not exist yet -- so a route.open
4072            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
4073            // it") for a module that was deliberately removed and whose caller
4074            // should get `module_removed` (TERMINAL, carrying a removal age).
4075            //
4076            // Writing the tombstone first closes it: during the window the
4077            // supervisor entry still answers, so the caller gets
4078            // `target_unavailable` -- retryable, and TRUE, because the module
4079            // is mid-teardown. After both statements it is `module_removed`.
4080            // No instant remains where a removed module reads as one that
4081            // never existed.
4082            //
4083            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
4084            // the absence of a test beside a fix invites deletion: these are
4085            // two sync statements with no await between them, so reaching the
4086            // window needs a second worker thread to land exactly between them
4087            // and there is no hook to force it. MEASURED: the 25 daemon_config
4088            // tests pass identically with the old order and the new one, so
4089            // the existing suite cannot see this and a green run is not
4090            // evidence either way. What the suite does hold is the
4091            // post-condition -- a removed module answers `module_removed` --
4092            // which this preserves.
4093            //
4094            // Found by an Athena panel reading the shipped tree against a
4095            // design note (2026-09-19), as the one concrete instance of that
4096            // note's class that survived contact with source. Direction is
4097            // benign: retryable where terminal was intended, never the reverse.
4098            self.supervisor.record_rescan_removal(module_id);
4099            self.supervisor.retire(module_id);
4100        }
4101
4102        for module_id in configured.keys() {
4103            let Some((module, _, _, _)) = current.get(module_id) else {
4104                continue;
4105            };
4106            let configured_module = configured
4107                .get(module_id)
4108                .expect("configured module id came from configured map");
4109            if configuration_changes.contains(module_id) {
4110                module
4111                    .update_configuration(
4112                        configured_module.module_spec(),
4113                        configured_module.health,
4114                        configured_module.drain_timeout_ms,
4115                    )
4116                    .await
4117                    .map_err(|err| {
4118                        format!(
4119                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
4120                        )
4121                    })?;
4122            }
4123            if enabled_changes.contains(module_id) {
4124                module
4125                    .set_enabled(configured_module.enabled)
4126                    .await
4127                    .map_err(|err| {
4128                        format!(
4129                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
4130                            configured_module.enabled
4131                        )
4132                    })?;
4133            }
4134        }
4135
4136        for module_id in &added {
4137            let configured_module = configured
4138                .get(module_id)
4139                .expect("added module id came from configured map");
4140            supervisor
4141                .supervise_configured_with_health(
4142                    configured_module.module_spec(),
4143                    configured_module.enabled,
4144                    configured_module.health,
4145                    configured_module.drain_timeout_ms,
4146                    configured_module.restart,
4147                )
4148                .map_err(|err| {
4149                    format!("failed to add module_id '{module_id}' during rescan: {err}")
4150                })?;
4151        }
4152
4153        Ok(SupervisorRescanResult {
4154            added,
4155            removed,
4156            changed_pending_reload,
4157            enabled_changes: enabled_changes.iter().cloned().collect(),
4158            unchanged,
4159            preview: false,
4160            // Filled by the caller, which is the only layer that can see the
4161            // previous config to diff against.
4162            restart_required: Vec::new(),
4163            capability_warnings: Vec::new(),
4164        })
4165    }
4166
4167    async fn handle_supervisor_set_enabled(
4168        &self,
4169        frame: Frame,
4170        module_id: String,
4171        enabled: bool,
4172    ) -> Result<Vec<Frame>, RouterError> {
4173        let operation_lock = self.supervisor.operation_lock();
4174        let _operation_guard = operation_lock.lock().await;
4175        let Some(module) = self.supervisor.get(&module_id) else {
4176            return Ok(vec![control_error_frame(
4177                &frame,
4178                "unknown_module",
4179                format!("module_id '{module_id}' is not supervised"),
4180            )?]);
4181        };
4182
4183        let applied = match module.set_enabled(enabled).await {
4184            Ok(applied) => applied,
4185            Err(err) => {
4186                return Ok(vec![control_error_frame(
4187                    &frame,
4188                    "target_unavailable",
4189                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
4190                )?])
4191            }
4192        };
4193
4194        self.capability_evaluator.wake_deadline_loop();
4195        self.refresh_capability_requirements();
4196        let response = ClientControlResponse::SupervisorAck { module_id, applied };
4197        Ok(vec![control_response_body_frame(
4198            &frame,
4199            &response,
4200            "ClientControlResponse::SupervisorAck",
4201        )?])
4202    }
4203
4204    async fn handle_supervisor_health_probe(
4205        &self,
4206        frame: Frame,
4207        module_id: String,
4208    ) -> Result<Vec<Frame>, RouterError> {
4209        self.refresh_capability_requirements();
4210        let Some(registration) = self
4211            .registry
4212            .get_module(&module_id)
4213            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4214        else {
4215            return Ok(vec![control_error_frame(
4216                &frame,
4217                "unknown_module",
4218                format!("module_id '{module_id}' is not registered"),
4219            )?]);
4220        };
4221
4222        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
4223        // named for it. Making `module_registration_grants_op` return false
4224        // unconditionally reddens five tests, and every one is named for something
4225        // else -- capability relay, probe/bind demultiplexing, supervision-only
4226        // probing. They exercise a successful advertisement check on the way to their
4227        // own subject.
4228        //
4229        // Real protection, fragile in a specific way: narrowing any of those tests to
4230        // focus on its stated subject would silently remove coverage nobody knows
4231        // they are carrying. Recorded here rather than as a sixth test, because the
4232        // useful fact is WHICH tests hold the guard up -- a new test would add
4233        // coverage without telling the next person what the existing ones quietly do.
4234        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
4235        {
4236            return Ok(vec![control_error_frame(
4237                &frame,
4238                "health_not_advertised",
4239                format!("module_id '{module_id}' did not advertise health.check"),
4240            )?]);
4241        }
4242
4243        let deadline = Instant::now() + self.health_probe_timeout;
4244        let pending = match self.forwarding.begin_module_control_rpc_for(
4245            &module_id,
4246            MODULE_CONTROL_OP_HEALTH_CHECK,
4247            deadline,
4248        ) {
4249            Ok(pending) => pending,
4250            Err(err) => {
4251                return Ok(vec![control_error_frame(
4252                    &frame,
4253                    forwarding_error_code(&err),
4254                    err.to_string(),
4255                )?])
4256            }
4257        };
4258
4259        let PendingModuleControlRpc {
4260            endpoint,
4261            module_sink,
4262            negotiated_ver,
4263            corr: probe_corr,
4264            receiver,
4265        } = pending;
4266        let mut guard =
4267            ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
4268        let probe_body =
4269            serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
4270                RouterError::backend(
4271                    0,
4272                    frame.header.corr,
4273                    format!("failed to encode health.check request: {err}"),
4274                )
4275            })?;
4276        let probe_frame = Frame::build_with_version(
4277            negotiated_ver,
4278            FrameType::Request,
4279            control_flags(),
4280            0,
4281            0,
4282            probe_corr,
4283            probe_body,
4284        )
4285        .map_err(RouterError::FrameBuild)?;
4286
4287        if let Err(err) = module_sink.send(probe_frame).await {
4288            return Ok(vec![control_error_frame(
4289                &frame,
4290                "target_unavailable",
4291                err.to_string(),
4292            )?]);
4293        }
4294
4295        match timeout_at(deadline, receiver).await {
4296            Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
4297                guard.disarm();
4298                let Some(report) = response.health_report() else {
4299                    return Ok(vec![control_error_frame(
4300                        &frame,
4301                        "invalid_control_body",
4302                        "health.check RPC returned a non-health response",
4303                    )?]);
4304                };
4305                // Metrics go out whole here. The supervisor's cached snapshot
4306                // caps this blob (see truncate_health_metrics), and this path
4307                // exists precisely to answer without that cap -- so applying it
4308                // here would leave no way to see what the cached view drops.
4309                let HealthReport {
4310                    status,
4311                    detail,
4312                    metrics,
4313                } = report;
4314                let capability_detail = self
4315                    .capability_evaluator
4316                    .required_problem_detail(&module_id);
4317                let response = ClientControlResponse::SupervisorHealthProbe {
4318                    module_id,
4319                    status,
4320                    detail: append_capability_problem_detail(detail, capability_detail),
4321                    metrics,
4322                };
4323                Ok(vec![control_response_body_frame(
4324                    &frame,
4325                    &response,
4326                    "ClientControlResponse::SupervisorHealthProbe",
4327                )?])
4328            }
4329            Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
4330                guard.disarm();
4331                Ok(vec![control_error_body_frame(&frame, body)?])
4332            }
4333            Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
4334                guard.disarm();
4335                Ok(vec![control_error_frame(
4336                    &frame,
4337                    "target_unavailable",
4338                    message,
4339                )?])
4340            }
4341            Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
4342                guard.disarm();
4343                Ok(vec![control_error_frame(
4344                    &frame,
4345                    "invalid_control_body",
4346                    message,
4347                )?])
4348            }
4349            Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
4350                guard.disarm();
4351                Ok(vec![control_error_frame(
4352                    &frame,
4353                    "invalid_control_body",
4354                    format!("expected module-control op '{expected}', got '{actual}'"),
4355                )?])
4356            }
4357            Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
4358                guard.disarm();
4359                Ok(vec![control_error_frame(
4360                    &frame,
4361                    "module_timeout",
4362                    format!(
4363                        "module_id '{module_id}' answered health.check after {:?}",
4364                        self.health_probe_timeout
4365                    ),
4366                )?])
4367            }
4368            Ok(Err(_)) => Ok(vec![control_error_frame(
4369                &frame,
4370                "target_unavailable",
4371                "health.check waiter was canceled before the module responded",
4372            )?]),
4373            Err(_) => Ok(vec![control_error_frame(
4374                &frame,
4375                "module_timeout",
4376                format!(
4377                    "module_id '{module_id}' did not answer health.check within {:?}",
4378                    self.health_probe_timeout
4379                ),
4380            )?]),
4381        }
4382    }
4383
4384    fn supervisor_status(
4385        &self,
4386        module_id: &str,
4387        corr: u64,
4388    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
4389        self.supervisor
4390            .get(module_id)
4391            .map(|module| {
4392                let warming = module.is_warming_for_control("status").map_err(|err| {
4393                    RouterError::backend(
4394                        0,
4395                        corr,
4396                        format!(
4397                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
4398                        ),
4399                    )
4400                })?;
4401                module.status_for_control("status").map_err(|err| {
4402                    RouterError::backend(
4403                        0,
4404                        corr,
4405                        format!(
4406                            "failed to read supervisor status for module_id '{module_id}': {err}"
4407                        ),
4408                    )
4409                }).map(|status| (status, warming))
4410            })
4411            .transpose()
4412    }
4413
4414    fn guard_module_control_op(
4415        &self,
4416        frame: &Frame,
4417        module_id: &str,
4418        op: &str,
4419    ) -> Result<Option<Frame>, RouterError> {
4420        if self.module_grants_op(module_id, op, frame.header.corr)? {
4421            return Ok(None);
4422        }
4423
4424        Ok(Some(control_error_frame(
4425            frame,
4426            "op_not_allowed",
4427            format!("module_id '{module_id}' did not grant control op '{op}'"),
4428        )?))
4429    }
4430
4431    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
4432        let Some(registration) = self
4433            .registry
4434            .get_module(module_id)
4435            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
4436        else {
4437            return Ok(false);
4438        };
4439        Ok(module_registration_grants_op(&registration.control_ops, op))
4440    }
4441
4442    fn handle_status_update(
4443        &self,
4444        endpoint: ModuleEndpointId,
4445        frame: Frame,
4446    ) -> Result<Vec<Frame>, RouterError> {
4447        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
4448            Ok(update) => update,
4449            Err(err) => {
4450                // Forward-compat: a newer module may push a channel-0 op this subc
4451                // version doesn't know. The control contract says unknown push ops
4452                // are IGNORED, never answered with an error. Only a malformed body
4453                // for an op we DO know is a real error worth surfacing.
4454                if is_known_module_push_op(&frame.body) {
4455                    return Ok(vec![control_error_frame(
4456                        &frame,
4457                        "invalid_control_body",
4458                        format!("malformed module control push body: {err}"),
4459                    )?]);
4460                }
4461                return Ok(Vec::new());
4462            }
4463        };
4464
4465        match update {
4466            ModuleControlPush::RouteStatus {
4467                route_channel,
4468                route_epoch,
4469                status,
4470            } => {
4471                self.forwarding
4472                    .cache_status(endpoint, route_channel, route_epoch, status)
4473                    .map_err(RouterError::Forwarding)?;
4474            }
4475        }
4476        Ok(Vec::new())
4477    }
4478
4479    fn handle_route_poll(
4480        &self,
4481        ctx: &RouteCtx,
4482        frame: Frame,
4483        route_channel: u16,
4484        route_epoch: u32,
4485        kind: PollKind,
4486    ) -> Result<Vec<Frame>, RouterError> {
4487        let snapshot = self
4488            .forwarding
4489            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
4490            .map_err(RouterError::Forwarding)?;
4491        let response = match (kind, snapshot) {
4492            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
4493                ClientControlResponse::RoutePoll {
4494                    route_channel,
4495                    route_epoch,
4496                    status,
4497                    live: None,
4498                }
4499            }
4500            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
4501                route_channel,
4502                route_epoch,
4503                status: None,
4504                live: None,
4505            },
4506            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
4507                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
4508                // is what makes reporting `true` correct rather than a
4509                // confident guess. `process_live` returns None only when the
4510                // module id has no supervisor snapshot at all -- an
4511                // externally-started module the daemon did not spawn -- and
4512                // for those the supervisor has no opinion to offer, ever. It
4513                // is never None for a supervised module in an unknown state:
4514                // a supervised module always has a snapshot, and the answer
4515                // comes from `state == Running && process_alive`.
4516                //
4517                // The route is Bound, so the module completed a HELLO on a
4518                // live connection; "the process this route points at is
4519                // running" is therefore attested by the binding rather than
4520                // assumed. Reporting `false` for an unsupervised module would
4521                // be the actual lie -- it would tell a client its healthy
4522                // route is dead because the daemon does not manage the
4523                // process.
4524                //
4525                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
4526                // module whose liveness is genuinely unknown, e.g. a snapshot
4527                // that has not been populated yet -- THIS DEFAULT BECOMES
4528                // WRONG and must split: unsupervised stays true, unknown
4529                // becomes null so the client can tell the two apart. The
4530                // response field is already `Option<bool>`, so the wire can
4531                // carry that distinction today.
4532                let live = self
4533                    .process_liveness
4534                    .as_ref()
4535                    .and_then(|source| source.process_live(&module_id))
4536                    .unwrap_or(true);
4537                ClientControlResponse::RoutePoll {
4538                    route_channel,
4539                    route_epoch,
4540                    status: None,
4541                    live: Some(live),
4542                }
4543            }
4544            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
4545                route_channel,
4546                route_epoch,
4547                status: None,
4548                live: Some(false),
4549            },
4550        };
4551
4552        Ok(vec![control_response_body_frame(
4553            &frame,
4554            &response,
4555            "ClientControlResponse::RoutePoll",
4556        )?])
4557    }
4558
4559    pub(crate) fn observe_module_control_completion(
4560        &self,
4561        completion: ModuleControlRpcCompletion,
4562    ) -> bool {
4563        match completion {
4564            ModuleControlRpcCompletion::Unknown => false,
4565            ModuleControlRpcCompletion::Settled => true,
4566            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
4567                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
4568                info!(
4569                    module_id = %module_id,
4570                    latency_ms,
4571                    "late health.check answer proves the module is alive"
4572                );
4573                match self
4574                    .supervisor
4575                    .record_late_health_answer(&module_id, latency_ms)
4576                {
4577                    Ok(true) => {}
4578                    Ok(false) => debug!(
4579                        module_id = %module_id,
4580                        latency_ms,
4581                        "late health.check answer has no active supervisor snapshot"
4582                    ),
4583                    Err(err) => warn!(
4584                        module_id = %module_id,
4585                        latency_ms,
4586                        error = %err,
4587                        "failed to record late health.check answer"
4588                    ),
4589                }
4590                true
4591            }
4592        }
4593    }
4594
4595    /// Decide whether a failure while settling a relayed `route.bind` belongs to
4596    /// the module connection whose frame is being handled, or to the client that
4597    /// relay was opened for.
4598    ///
4599    /// This runs on the MODULE connection's frame handler, where returning `Err`
4600    /// ends that connection -- and a module connection carries every client's
4601    /// routes to that module, so ending it costs the whole fleet its tools.
4602    /// `ConnectionClosing` carries the id of the connection that is closing, and
4603    /// when that id is a CLIENT's, the condition is entirely about that one
4604    /// client's route.open. A client-scoped condition has no authority over a
4605    /// shared module connection, so it is logged and the single relay is dropped:
4606    /// the client is going away, and `complete_pending_relay` already removed the
4607    /// relay before failing, so there is nothing left to settle. Anything that
4608    /// relay still reserved is released by that client's own connection teardown,
4609    /// which is already under way -- that is what "closing" means.
4610    ///
4611    /// Every other failure is a statement about THIS connection and stays fatal:
4612    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
4613    /// id in `ConnectionClosing` all mean this connection cannot keep serving
4614    /// frames correctly.
4615    fn refuse_to_end_module_connection_for_a_client(
4616        &self,
4617        module_connection_id: ConnectionId,
4618        corr: u64,
4619        err: ForwardingError,
4620    ) -> Result<(), RouterError> {
4621        if let ForwardingError::ConnectionClosing { connection_id } = err {
4622            if connection_id != module_connection_id {
4623                warn!(
4624                    module_connection_id = module_connection_id.get(),
4625                    client_connection_id = connection_id.get(),
4626                    corr,
4627                    "dropping a route.bind response for a closing client; the module connection keeps serving"
4628                );
4629                return Ok(());
4630            }
4631        }
4632        Err(RouterError::Forwarding(err))
4633    }
4634
4635    fn handle_module_relay_response(
4636        &self,
4637        connection_id: ConnectionId,
4638        frame: Frame,
4639    ) -> Result<Vec<Frame>, RouterError> {
4640        let mut secondary_error = None;
4641        let outcome = match frame.header.ty {
4642            FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
4643                Ok(probe) if probe.op == "route.bind" => {
4644                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
4645                        Ok(ModuleControlResponse::RouteBindAck {}) => {
4646                            RouteBindRelayOutcome::Accepted
4647                        }
4648                        Ok(other) => {
4649                            let message =
4650                                format!("route.bind response carried unexpected body: {other:?}");
4651                            secondary_error = Some(control_error_frame(
4652                                &frame,
4653                                "invalid_control_body",
4654                                message.clone(),
4655                            )?);
4656                            RouteBindRelayOutcome::ModuleGone(message)
4657                        }
4658                        Err(err) => {
4659                            let message = format!("malformed route.bind response body: {err}");
4660                            secondary_error = Some(control_error_frame(
4661                                &frame,
4662                                "invalid_control_body",
4663                                message.clone(),
4664                            )?);
4665                            RouteBindRelayOutcome::ModuleGone(message)
4666                        }
4667                    }
4668                }
4669                Ok(probe) => {
4670                    let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
4671                    {
4672                        Ok(response) => ModuleControlRpcOutcome::Response(response),
4673                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
4674                            "malformed {} response body: {err}",
4675                            probe.op
4676                        )),
4677                    };
4678                    let completion = self
4679                        .forwarding
4680                        .complete_module_control_rpc(
4681                            connection_id,
4682                            frame.header.corr,
4683                            Some(&probe.op),
4684                            outcome,
4685                        )
4686                        .map_err(RouterError::Forwarding)?;
4687                    if !self.observe_module_control_completion(completion) {
4688                        debug!(
4689                            connection_id = connection_id.get(),
4690                            corr = frame.header.corr,
4691                            op = %probe.op,
4692                            "dropping late or unknown module-control RPC response"
4693                        );
4694                    }
4695                    return Ok(Vec::new());
4696                }
4697                Err(err) => {
4698                    if let Some(expected_op) = self
4699                        .forwarding
4700                        .pending_module_control_op(connection_id, frame.header.corr)
4701                        .map_err(RouterError::Forwarding)?
4702                    {
4703                        let completion = self
4704                            .forwarding
4705                            .complete_module_control_rpc(
4706                                connection_id,
4707                                frame.header.corr,
4708                                None,
4709                                ModuleControlRpcOutcome::MalformedResponse(format!(
4710                                    "malformed {expected_op} response body: {err}"
4711                                )),
4712                            )
4713                            .map_err(RouterError::Forwarding)?;
4714                        if !self.observe_module_control_completion(completion) {
4715                            debug!(
4716                                connection_id = connection_id.get(),
4717                                corr = frame.header.corr,
4718                                "dropping late malformed module-control RPC response"
4719                            );
4720                        }
4721                        return Ok(Vec::new());
4722                    }
4723                    let message = format!("malformed route.bind response body: {err}");
4724                    secondary_error = Some(control_error_frame(
4725                        &frame,
4726                        "invalid_control_body",
4727                        message.clone(),
4728                    )?);
4729                    RouteBindRelayOutcome::ModuleGone(message)
4730                }
4731            },
4732            FrameType::Error => {
4733                if self
4734                    .forwarding
4735                    .pending_module_control_op(connection_id, frame.header.corr)
4736                    .map_err(RouterError::Forwarding)?
4737                    .is_some()
4738                {
4739                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
4740                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
4741                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
4742                            "malformed module-control ERROR body: {err}"
4743                        )),
4744                    };
4745                    let completion = self
4746                        .forwarding
4747                        .complete_module_control_rpc(
4748                            connection_id,
4749                            frame.header.corr,
4750                            None,
4751                            outcome,
4752                        )
4753                        .map_err(RouterError::Forwarding)?;
4754                    if !self.observe_module_control_completion(completion) {
4755                        debug!(
4756                            connection_id = connection_id.get(),
4757                            corr = frame.header.corr,
4758                            "dropping late or unknown module-control RPC error"
4759                        );
4760                    }
4761                    return Ok(Vec::new());
4762                }
4763                match serde_json::from_slice::<ErrorBody>(&frame.body) {
4764                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
4765                    Err(err) => {
4766                        let message = format!("malformed route.bind ERROR body: {err}");
4767                        secondary_error = Some(control_error_frame(
4768                            &frame,
4769                            "invalid_control_body",
4770                            message.clone(),
4771                        )?);
4772                        RouteBindRelayOutcome::ModuleGone(message)
4773                    }
4774                }
4775            }
4776            ty => {
4777                return Ok(vec![control_error_frame(
4778                    &frame,
4779                    "unsupported_control_frame",
4780                    format!("unsupported module channel-0 frame {ty:?}"),
4781                )?])
4782            }
4783        };
4784
4785        let settled =
4786            self.forwarding
4787                .complete_pending_relay(connection_id, frame.header.corr, outcome);
4788        let completion = match settled {
4789            Ok(completion) => completion,
4790            Err(err) => {
4791                self.refuse_to_end_module_connection_for_a_client(
4792                    connection_id,
4793                    frame.header.corr,
4794                    err,
4795                )?;
4796                return Ok(secondary_error.into_iter().collect());
4797            }
4798        };
4799        if let Some(target) = completion.abandoned.as_ref() {
4800            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
4801        }
4802        if !completion.settled {
4803            debug!(
4804                connection_id = connection_id.get(),
4805                corr = frame.header.corr,
4806                frame_type = ?frame.header.ty,
4807                "dropping late or unknown route.bind relay response"
4808            );
4809        }
4810        Ok(secondary_error.into_iter().collect())
4811    }
4812
4813    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
4814        debug!(connection_id = connection_id.get(), "handling GOODBYE");
4815        let registrations = self
4816            .deregister_connection(connection_id)
4817            .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
4818        let released_routes = self
4819            .forwarding
4820            .cleanup_connection(connection_id)
4821            .map_err(RouterError::Forwarding)?;
4822        self.emit_route_goodbyes(released_routes);
4823        // Notify only after forwarding teardown completes (see cleanup_connection).
4824        if !registrations.is_empty() {
4825            crate::supervise::notify_registration_release();
4826        }
4827        Ok(Vec::new())
4828    }
4829}
4830
4831impl Default for ControlHandler {
4832    fn default() -> Self {
4833        Self::new(Arc::new(Registry::default()))
4834    }
4835}
4836
4837impl crate::supervise::SwapPromotionObserver for ControlHandler {
4838    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
4839        self.apply_registration_capabilities(registration);
4840    }
4841}
4842
4843fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
4844    CapabilityRequirementStatus {
4845        consumer: status.consumer,
4846        capability: status.capability,
4847        need: match status.need {
4848            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
4849            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
4850        },
4851        verdict: status.verdict.as_str().to_string(),
4852        episode_seq: status.episode_seq,
4853        config_satisfiable: status.config_satisfiable,
4854        runtime_available: status.runtime_available,
4855        detail: status.detail,
4856    }
4857}
4858
4859fn append_capability_problem_detail(
4860    detail: Option<String>,
4861    capability_detail: Option<String>,
4862) -> Option<String> {
4863    match (detail, capability_detail) {
4864        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
4865        (Some(detail), None) => Some(detail),
4866        (None, Some(capability_detail)) => Some(capability_detail),
4867        (None, None) => None,
4868    }
4869}
4870
4871fn subc_ops() -> Vec<String> {
4872    SUBC_CONTROL_OPS
4873        .iter()
4874        .map(|op| (*op).to_string())
4875        .collect()
4876}
4877
4878fn module_subc_ops() -> Vec<String> {
4879    SUBC_CONTROL_OPS
4880        .iter()
4881        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
4882        .map(|op| (*op).to_string())
4883        .collect()
4884}
4885
4886#[cfg(test)]
4887fn module_baseline_control_ops() -> Vec<String> {
4888    MODULE_BASELINE_CONTROL_OPS
4889        .iter()
4890        .map(|op| (*op).to_string())
4891        .collect()
4892}
4893
4894fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
4895    let mut seen = HashSet::new();
4896    let mut effective = Vec::new();
4897    for op in MODULE_BASELINE_CONTROL_OPS {
4898        if seen.insert((*op).to_string()) {
4899            effective.push((*op).to_string());
4900        }
4901    }
4902    for op in declared.unwrap_or_default() {
4903        if seen.insert(op.clone()) {
4904            effective.push(op);
4905        }
4906    }
4907    effective
4908}
4909
4910fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
4911    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
4912}
4913
4914fn target_module_id(target: &RouteTarget) -> &str {
4915    match target {
4916        RouteTarget::ToolProvider { module_id }
4917        | RouteTarget::ManagementSurface { module_id }
4918        | RouteTarget::InternalService { module_id, .. } => module_id,
4919    }
4920}
4921
4922fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
4923    roles.iter().any(|role| match (target, role) {
4924        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
4925        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
4926        (
4927            RouteTarget::InternalService { service_id, .. },
4928            ProviderRole::InternalService {
4929                service_id: provided,
4930                ..
4931            },
4932        ) => service_id == provided,
4933        _ => false,
4934    })
4935}
4936
4937fn is_routable_role(role: &ProviderRole) -> bool {
4938    matches!(
4939        role,
4940        ProviderRole::ToolProvider { .. }
4941            | ProviderRole::ManagementSurface { .. }
4942            | ProviderRole::InternalService { .. }
4943    )
4944}
4945
4946#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4947enum ControlRequestBodyError {
4948    UnknownOp,
4949    InvalidBody,
4950}
4951
4952#[derive(Debug, Deserialize)]
4953struct ControlOpProbe {
4954    op: String,
4955}
4956
4957/// Channel-0 push ops this subc version understands. A push whose `op` is not in
4958/// this set is treated as a forward-compat unknown and ignored rather than errored.
4959const MODULE_PUSH_OPS: &[&str] = &["route.status"];
4960
4961fn is_known_module_push_op(body: &[u8]) -> bool {
4962    serde_json::from_slice::<ControlOpProbe>(body)
4963        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
4964        .unwrap_or(false)
4965}
4966
4967fn is_known_module_request_op(body: &[u8]) -> bool {
4968    serde_json::from_slice::<ControlOpProbe>(body)
4969        .map(|probe| MODULE_TO_SUBC_CONTROL_OPS.contains(&probe.op.as_str()))
4970        .unwrap_or(false)
4971}
4972
4973fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
4974    debug!(
4975        op = %op,
4976        connection_id = connection_id.get(),
4977        corr,
4978        "control dispatch"
4979    );
4980}
4981
4982fn log_slow_control_dispatch(
4983    dispatch_started_at: Option<StdInstant>,
4984    op: &'static str,
4985    connection_id: ConnectionId,
4986    corr: u64,
4987) {
4988    let Some(dispatch_started_at) = dispatch_started_at else {
4989        return;
4990    };
4991    let elapsed = dispatch_started_at.elapsed();
4992    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
4993        warn!(
4994            op = %op,
4995            connection_id = connection_id.get(),
4996            corr,
4997            elapsed_ms = elapsed.as_millis() as u64,
4998            "slow control dispatch"
4999        );
5000    }
5001}
5002
5003fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
5004    match request {
5005        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
5006        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
5007        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
5008        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
5009        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
5010        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
5011        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
5012        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
5013        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
5014        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
5015        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
5016        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
5017        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
5018        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
5019        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
5020        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
5021        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
5022        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
5023        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
5024    }
5025}
5026
5027fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
5028    match request {
5029        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
5030        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
5031    }
5032}
5033
5034fn parse_client_control_request(
5035    body: &[u8],
5036) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
5037    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
5038        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5039            Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5040                ControlRequestBodyError::InvalidBody
5041            }
5042            Ok(_) => ControlRequestBodyError::UnknownOp,
5043            Err(_) => ControlRequestBodyError::InvalidBody,
5044        };
5045        (err, classification)
5046    })
5047}
5048
5049fn parse_module_control_request_from_module(
5050    body: &[u8],
5051) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
5052    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
5053        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5054            Ok(probe) if MODULE_TO_SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5055                ControlRequestBodyError::InvalidBody
5056            }
5057            Ok(_) => ControlRequestBodyError::UnknownOp,
5058            Err(_) => ControlRequestBodyError::InvalidBody,
5059        };
5060        (err, classification)
5061    })
5062}
5063
5064#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
5065enum ProviderRoleKind {
5066    ToolProvider,
5067    PipelineStage,
5068    ManagementSurface,
5069    InternalService,
5070}
5071
5072fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
5073    match role {
5074        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
5075        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
5076        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
5077        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
5078    }
5079}
5080
5081fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
5082    roles.iter().map(provider_role_kind).collect()
5083}
5084
5085/// Return whether a catalog change can create a newly violating live route.
5086/// Removing an attested claim is intentionally excluded: it makes fewer routes
5087/// forbidden and therefore must leave the existing route census untouched.
5088fn capability_census_trigger(
5089    old: Option<&CapabilityDeclarations>,
5090    new: Option<&CapabilityDeclarations>,
5091) -> bool {
5092    let old_provides = old
5093        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
5094        .unwrap_or_default();
5095    let old_denies = old
5096        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
5097        .unwrap_or_default();
5098    let new = new.cloned().unwrap_or(CapabilityDeclarations {
5099        provides: Vec::new(),
5100        requires: Vec::new(),
5101        must_never_reach: Vec::new(),
5102    });
5103
5104    new.provides
5105        .iter()
5106        .any(|capability| !old_provides.contains(capability))
5107        || new
5108            .must_never_reach
5109            .iter()
5110            .any(|capability| !old_denies.contains(capability))
5111}
5112
5113/// Find the first capability an attested opener denies that an attested target
5114/// claims. Both manifests are live registry records, never cached or client data.
5115fn denied_capability<'a>(
5116    opening_manifest: &'a ModuleManifest,
5117    target_manifest: &ModuleManifest,
5118) -> Option<&'a str> {
5119    let opening_capabilities = opening_manifest.capabilities.as_ref()?;
5120    let target_capabilities = target_manifest.capabilities.as_ref()?;
5121    opening_capabilities
5122        .must_never_reach
5123        .iter()
5124        .find(|denied| {
5125            target_capabilities
5126                .provides
5127                .iter()
5128                .any(|provided| provided == *denied)
5129        })
5130        .map(String::as_str)
5131}
5132
5133fn catalog_update_frozen_field_message(
5134    registered: &ModuleManifest,
5135    provides: &[ProviderRole],
5136) -> Option<String> {
5137    let old_has_provides = !registered.provides.is_empty();
5138    let new_has_provides = !provides.is_empty();
5139    if old_has_provides != new_has_provides {
5140        return Some(format!(
5141            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
5142            registered.module_id
5143        ));
5144    }
5145
5146    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
5147        return Some(format!(
5148            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
5149            registered.module_id
5150        ));
5151    }
5152
5153    let registered_concurrency = manifest_concurrency(registered);
5154    let mut candidate = registered.clone();
5155    candidate.provides = provides.to_vec();
5156    let candidate_concurrency = manifest_concurrency(&candidate);
5157    if candidate_concurrency != registered_concurrency {
5158        return Some(format!(
5159            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
5160            registered.module_id, registered_concurrency, candidate_concurrency
5161        ));
5162    }
5163
5164    // control_ops live beside the manifest in the HELLO body, not inside
5165    // ModuleManifest, so a provides-only catalog.update cannot change them.
5166    None
5167}
5168
5169fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
5170    manifest.provides.iter().any(is_routable_role)
5171}
5172
5173/// Returns the routable-provider concurrency subc should enforce for this manifest.
5174///
5175/// ToolProvider and ManagementSurface store their delivery concurrency directly.
5176/// InternalService has no role-specific concurrency field, so it retains the
5177/// existing ModuleManaged default for backward compatibility.
5178fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
5179    manifest
5180        .provides
5181        .iter()
5182        .find_map(|provider| match provider {
5183            ProviderRole::ToolProvider { concurrency, .. }
5184            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
5185            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
5186        })
5187        .unwrap_or(Concurrency::ModuleManaged)
5188}
5189
5190/// True when the manifest carries a ManagementSurface role whose concurrency
5191/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
5192/// bytes because the typed manifest deliberately erases that distinction: the
5193/// default exists for wire compatibility, and this probe exists so the default
5194/// stays observable. Any parse irregularity returns false -- the caller only
5195/// logs, and a malformed body already failed registration upstream.
5196fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
5197    let has_management_surface = manifest
5198        .provides
5199        .iter()
5200        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
5201    if !has_management_surface {
5202        return false;
5203    }
5204    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
5205        return false;
5206    };
5207    let Some(provides) = raw
5208        .get("manifest")
5209        .and_then(|manifest| manifest.get("provides"))
5210        .and_then(serde_json::Value::as_array)
5211    else {
5212        return false;
5213    };
5214    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
5215    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
5216    // against the management_surface_manifest_without_concurrency golden, not
5217    // recalled (the externally-tagged guess was this function's first bug).
5218    provides.iter().any(|role| {
5219        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
5220            && role.get("concurrency").is_none()
5221    })
5222}
5223
5224fn negotiate_version(peer_version: u8) -> Result<u8, String> {
5225    if peer_version != PROTOCOL_VERSION {
5226        return Err(format!(
5227            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
5228        ));
5229    }
5230    Ok(PROTOCOL_VERSION)
5231}
5232
5233fn pong(frame: &Frame) -> Result<Frame, RouterError> {
5234    Frame::build_with_version(
5235        response_version(frame),
5236        FrameType::Pong,
5237        frame.header.flags,
5238        0,
5239        0,
5240        frame.header.corr,
5241        Vec::new(),
5242    )
5243    .map_err(RouterError::FrameBuild)
5244}
5245
5246fn control_error_frame(
5247    frame: &Frame,
5248    code: &'static str,
5249    message: impl Into<String>,
5250) -> Result<Frame, RouterError> {
5251    control_error_body_frame(
5252        frame,
5253        ErrorBody {
5254            code: code.to_string(),
5255            message: message.into(),
5256            detail: None,
5257        },
5258    )
5259}
5260
5261fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
5262    let body = serde_json::to_vec(&error).map_err(|err| {
5263        RouterError::backend(
5264            0,
5265            frame.header.corr,
5266            format!("failed to encode control ERROR: {err}"),
5267        )
5268    })?;
5269
5270    Frame::build_with_version(
5271        response_version(frame),
5272        FrameType::Error,
5273        control_flags(),
5274        0,
5275        0,
5276        frame.header.corr,
5277        body,
5278    )
5279    .map_err(RouterError::FrameBuild)
5280}
5281
5282fn control_response_body_frame<T: Serialize>(
5283    frame: &Frame,
5284    reply: &T,
5285    label: &'static str,
5286) -> Result<Frame, RouterError> {
5287    let body = serde_json::to_vec(reply).map_err(|err| {
5288        RouterError::backend(
5289            0,
5290            frame.header.corr,
5291            format!("failed to encode {label}: {err}"),
5292        )
5293    })?;
5294
5295    Frame::build_with_version(
5296        response_version(frame),
5297        FrameType::Response,
5298        control_flags(),
5299        0,
5300        0,
5301        frame.header.corr,
5302        body,
5303    )
5304    .map_err(RouterError::FrameBuild)
5305}
5306
5307/// Map a forwarding failure to the wire code a client sees.
5308///
5309/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
5310/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
5311/// chosen here decides whether a caller retries or gives up.
5312///
5313/// That makes attribution the load-bearing property, not merely having a code. A
5314/// permanent fault published as a retryable one produces a fleet-wide retry storm
5315/// against something that can never recover; a transient fault published as
5316/// permanent gives up on work that would have succeeded. Both look correct in a
5317/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
5318/// the mapping per variant rather than merely asserting that some code exists.
5319///
5320/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
5321/// two codes on the same side of the boundary passes it. Measured rather than
5322/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
5323/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
5324/// a test named for something else that happens to assert the string.
5325///
5326/// That accidental coverage is deliberately left alone rather than promoted to a
5327/// named test, because it guards a property this function does not promise.
5328/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
5329/// specific code within a class, so identity is free to change and only the
5330/// partition is a contract. Splitting it out would assert a guarantee nothing
5331/// depends on — and a suite that promises more than the code does is the harder
5332/// thing to correct later, because the next reader cannot tell which assertions
5333/// are load-bearing.
5334///
5335/// Pin identity here the moment a consumer branches on a specific code.
5336fn forwarding_error_code(err: &ForwardingError) -> &'static str {
5337    match err {
5338        ForwardingError::NoModuleConnection => "target_unavailable",
5339        ForwardingError::ModuleReloading { .. } => "module_reloading",
5340        ForwardingError::ClientRouteChannelExhausted { .. }
5341        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
5342        ForwardingError::StaleModuleEndpoint
5343        | ForwardingError::UnknownReservation { .. }
5344        | ForwardingError::ConnectionClosing { .. }
5345        | ForwardingError::ClientEgressClosed { .. }
5346        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
5347        // Only a swap candidate's registration can produce this, and it means
5348        // exactly what a second active HELLO for a live id means.
5349        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
5350        ForwardingError::RelayCorrelationExhausted
5351        | ForwardingError::RouteOpenBuild(_)
5352        | ForwardingError::Poisoned => "forwarding_error",
5353    }
5354}
5355
5356fn response_version(frame: &Frame) -> u8 {
5357    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
5358        frame.header.ver
5359    } else {
5360        PROTOCOL_VERSION
5361    }
5362}
5363
5364fn control_flags() -> Flags {
5365    Flags::new(false, Priority::Passive, false)
5366}
5367
5368/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
5369/// channel. The target is the module (a client never saw the route), so this
5370/// takes the module path: delivered late rather than dropped when the module's
5371/// queue is momentarily full, and never closing its connection.
5372fn send_goodbye_target_best_effort(
5373    counters: &DaemonCounters,
5374    target: &GoodbyeTarget,
5375    context: &'static str,
5376) {
5377    let Ok(frame) = Frame::build_with_version(
5378        target.negotiated_ver,
5379        FrameType::Goodbye,
5380        control_flags(),
5381        target.channel,
5382        target.epoch,
5383        0,
5384        Vec::new(),
5385    ) else {
5386        return;
5387    };
5388    crate::forwarding::send_module_route_goodbye(
5389        counters,
5390        &target.sink,
5391        frame,
5392        target.module_id.as_deref(),
5393        context,
5394    );
5395}
5396
5397pub(crate) fn send_route_control_pushes(
5398    forwarding: &ForwardingTable,
5399    routes: Vec<EndpointRoute>,
5400    push: ClientControlPush,
5401) {
5402    let body = match serde_json::to_vec(&push) {
5403        Ok(body) => body,
5404        Err(err) => {
5405            warn!(error = %err, "failed to serialize route lifecycle control PUSH");
5406            return;
5407        }
5408    };
5409    let mut targets = Vec::new();
5410    for route in routes {
5411        let target = route.goodbye_target;
5412        if let Some(existing) = targets
5413            .iter()
5414            .find(|existing: &&GoodbyeTarget| existing.connection_id == target.connection_id)
5415        {
5416            debug_assert_eq!(
5417                existing.negotiated_ver, target.negotiated_ver,
5418                "one connection cannot negotiate multiple frame versions"
5419            );
5420            continue;
5421        }
5422        targets.push(target);
5423    }
5424    for target in targets {
5425        let frame = match Frame::build_with_version(
5426            target.negotiated_ver,
5427            FrameType::Push,
5428            control_flags(),
5429            0,
5430            0,
5431            0,
5432            body.clone(),
5433        ) {
5434            Ok(frame) => frame,
5435            Err(err) => {
5436                warn!(
5437                    route_channel = target.channel,
5438                    error = %err,
5439                    "failed to build route lifecycle control PUSH frame"
5440                );
5441                continue;
5442            }
5443        };
5444        if let Err(err) = target.sink.try_send(frame) {
5445            if target.close_on_delivery_failure() {
5446                warn!(
5447                    target_connection_id = target.connection_id.get(),
5448                    route_channel = target.channel,
5449                    error = %err,
5450                    "route lifecycle control PUSH was not delivered to client; closing target connection"
5451                );
5452                let _ = forwarding.escalate_client_delivery_failure(
5453                    target.connection_id,
5454                    target.channel,
5455                    target.epoch,
5456                    CloseReason::new(
5457                        "route_lifecycle_push_delivery_failed",
5458                        format!(
5459                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
5460                            target.channel
5461                        ),
5462                    ),
5463                    crate::forwarding::UndeliveredFrame {
5464                        module_id: target.module_id.as_deref(),
5465                        sink: &target.sink,
5466                    },
5467                );
5468            }
5469        }
5470    }
5471}
5472
5473#[cfg(test)]
5474mod tests {
5475    use std::{
5476        collections::BTreeMap,
5477        fmt,
5478        path::PathBuf,
5479        sync::{Arc, Mutex},
5480        time::Duration,
5481    };
5482
5483    use serde_json::{json, Value};
5484    use subc_protocol::{
5485        manifest::{
5486            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
5487            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
5488        },
5489        session::HealthStatus,
5490        FrameType,
5491    };
5492
5493    use super::*;
5494    use crate::{
5495        forwarding::{DataRoute, DataRouteState},
5496        registry::ChannelState,
5497        router::FrameSink,
5498        stderr_tail::DEFAULT_MAX_LINE_BYTES,
5499        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
5500        test_support::TestTempDir,
5501        RouteCtx, Router,
5502    };
5503    use tokio::{
5504        sync::mpsc,
5505        time::{sleep, Instant},
5506    };
5507    use tracing::{
5508        field::{Field, Visit},
5509        Event, Subscriber,
5510    };
5511    use tracing_subscriber::{layer::Context, prelude::*, Layer};
5512
5513    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
5514    ///
5515    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
5516    /// is only populated for `tests/*.rs` integration test binaries -- this file
5517    /// compiles as part of the library target, which gets neither. This test's
5518    /// own executable path is `<target-dir>/<profile>/deps/subc_core-<hash>`,
5519    /// and the sibling binary lives two directories up at
5520    /// `<target-dir>/<profile>/fake-aft-stub`.
5521    ///
5522    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
5523    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
5524    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
5525    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
5526    /// `NotFound`, which reads as a broken test rather than an unbuilt
5527    /// dependency -- so state the cause and the remedy instead. Deliberately a
5528    /// panic and not a silent skip: a test that quietly passes when it could not
5529    /// run is worse than one that fails, because it reports health it never
5530    /// verified.
5531    fn fake_aft_stub_path() -> PathBuf {
5532        let mut path = std::env::current_exe().expect("current_exe available in tests");
5533        path.pop(); // .../deps/
5534        path.pop(); // .../<profile>/
5535        path.push(if cfg!(windows) {
5536            "fake-aft-stub.exe"
5537        } else {
5538            "fake-aft-stub"
5539        });
5540        assert!(
5541            path.exists(),
5542            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
5543             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
5544            path.display()
5545        );
5546        path
5547    }
5548
5549    /// Whether clients retry `code` in place: the predicate itself, never a copy
5550    /// of its set. A copied list breaks silently when a code is added to or
5551    /// removed from the real one, and a stale copy here would let exactly the
5552    /// failure this test exists to catch pass.
5553    fn client_retries(code: &str) -> bool {
5554        subc_protocol::error_codes::is_retryable_route_open(code)
5555    }
5556
5557    /// A code is not a label — clients branch on it, so publishing the wrong KIND
5558    /// of failure is worse than publishing none. A permanent fault dressed as
5559    /// retryable makes every client in the fleet retry forever against something
5560    /// that cannot recover; a transient fault dressed as permanent abandons work
5561    /// that would have succeeded.
5562    ///
5563    /// Asserting "a code exists" cannot catch either, because the string is free
5564    /// to say anything. This enumerates every variant and pins which side of the
5565    /// retry boundary it lands on, so a new variant must be classified here
5566    /// deliberately rather than inheriting whichever arm it was appended to.
5567    #[test]
5568    fn retryability_of_forwarding_codes_matches_the_failure() {
5569        // Transient by nature: the target is booting, reloading, or its endpoint
5570        // was swapped mid-flight. Retrying is how these resolve.
5571        let transient = [
5572            ForwardingError::NoModuleConnection,
5573            ForwardingError::ModuleReloading {
5574                module_id: "m".into(),
5575            },
5576            ForwardingError::StaleModuleEndpoint,
5577            ForwardingError::UnknownReservation {
5578                client_channel: 1,
5579                module_channel: 1,
5580            },
5581            ForwardingError::ConnectionClosing {
5582                connection_id: ConnectionId::new(1),
5583            },
5584            ForwardingError::ClientEgressClosed {
5585                connection_id: ConnectionId::new(1),
5586            },
5587            ForwardingError::ModuleEgressUnavailable {
5588                connection_id: ConnectionId::new(1),
5589            },
5590        ];
5591        for err in transient {
5592            let code = forwarding_error_code(&err);
5593            assert!(
5594                client_retries(code),
5595                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
5596            );
5597        }
5598
5599        // Not fixed by retrying. Channel and correlation exhaustion need the
5600        // caller to close routes, and a poisoned lock is a daemon that cannot
5601        // recover at all — the worst thing to advertise as retryable, since every
5602        // client would storm a daemon that will never answer.
5603        let permanent = [
5604            ForwardingError::ClientRouteChannelExhausted {
5605                connection_id: ConnectionId::new(1),
5606            },
5607            ForwardingError::ModuleRouteChannelExhausted {
5608                endpoint: ModuleEndpointId {
5609                    connection_id: ConnectionId::new(1),
5610                    generation: 1,
5611                },
5612            },
5613            ForwardingError::RelayCorrelationExhausted,
5614            ForwardingError::RouteOpenBuild("x".into()),
5615            ForwardingError::Poisoned,
5616        ];
5617        for err in permanent {
5618            let code = forwarding_error_code(&err);
5619            assert!(
5620                !client_retries(code),
5621                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
5622            );
5623        }
5624    }
5625
5626    /// The principal is the daemon's answer to "who is calling", and modules
5627    /// branch on it: aft gates bash on it, cerebellum gates browser control,
5628    /// plexus gates connector invocation. So a stamp is an authorization input in
5629    /// another process, not a label — and both possible answers SUCCEED, which is
5630    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
5631    /// first-party capability to something that never proved it; a supervised one
5632    /// stamped `Direct` silently strips a module of capability it is entitled to.
5633    ///
5634    /// Neither shows up in a test that only checks the bind succeeded. Before this
5635    /// test the only coverage was accidental —
5636    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
5637    /// stamped principal on its way past, so narrowing that wire-shape test to its
5638    /// stated subject would have deleted the last assertion on this value. It
5639    /// still asserts the stamp, which is now redundancy rather than the only
5640    /// guard: both fail under the same mutation, and this one names the reason.
5641    /// SCOPE: this handler's supervisor has spawned nothing, so
5642    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
5643    /// is unreachable here. Both assertions below are refusals, and a mutant that
5644    /// refuses everything would satisfy them.
5645    ///
5646    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
5647    /// spawns a supervised consumer, reads its live nonce, and asserts the module
5648    /// observed `principal.kind == "reserved"` carrying that module_id — verified
5649    /// at source rather than assumed, since a citation is a claim about another
5650    /// file and ages like one. Recorded because a harness that structurally
5651    /// cannot reach an arm reports "none" for that arm identically to one that
5652    /// covers it and found nothing.
5653    #[tokio::test]
5654    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
5655        let handler = ControlHandler::default();
5656        let frame =
5657            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
5658
5659        // Absent consumer_identity is the ordinary case: a human at a terminal, or
5660        // any process holding the connection file. Nothing was proved, so nothing
5661        // may be granted beyond the unattested floor.
5662        let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
5663        assert_eq!(
5664            stamped,
5665            Principal::Direct,
5666            "a caller that proved nothing must not be stamped as a supervised module"
5667        );
5668
5669        // A claimed module_id with a nonce no supervised child was given is a
5670        // forgery attempt, not a weaker caller: it must be REFUSED rather than
5671        // quietly demoted to Direct, or an impersonation attempt looks identical
5672        // to an ordinary unattested connection.
5673        let forged = handler
5674            .route_open_principal(
5675                &frame,
5676                Some(ConsumerIdentity {
5677                    module_id: "aft".to_string(),
5678                    launch_nonce: "not-a-real-nonce".to_string(),
5679                }),
5680            )
5681            .unwrap();
5682        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
5683        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
5684    }
5685
5686    /// The test above hands `route_open_principal` an identity it built itself,
5687    /// which proves the stamping rule and nothing about where the identity comes
5688    /// from. The real producer is a wire body, and the two are joined by a serde
5689    /// field name that nothing else asserts.
5690    ///
5691    /// That join fails quietly in one specific way: an unrecognised key is simply
5692    /// absent after parsing, so a renamed or misspelled `consumer_identity`
5693    /// yields `None` and every supervised module silently drops to `Direct`.
5694    /// Capability-wise that is the safe direction, but it surfaces far from its
5695    /// cause — as a module mysteriously refused bash — and it would pass every
5696    /// test that builds its own input.
5697    ///
5698    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
5699    /// would break every client the moment the daemon gains a field, trading a
5700    /// quiet demotion for a hard refusal on additive change. Asserting the join
5701    /// instead means a rename breaks a test here rather than the fleet.
5702    #[test]
5703    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
5704        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
5705        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
5706        let ClientControlRequest::RouteOpen {
5707            consumer_identity, ..
5708        } = parsed
5709        else {
5710            panic!("route.open body must parse as RouteOpen");
5711        };
5712        assert_eq!(
5713            consumer_identity,
5714            Some(ConsumerIdentity {
5715                module_id: "aft".to_string(),
5716                launch_nonce: "n".to_string(),
5717            }),
5718            "the wire field name must reach the value route_open_principal reads"
5719        );
5720    }
5721
5722    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
5723        ModuleManifest::builder(module_id, "0.1.0")
5724            .protocol_ver(protocol_ver)
5725            .provides(vec![ProviderRole::ToolProvider {
5726                tools: vec![Tool {
5727                    name: "read".to_string(),
5728                    description: None,
5729                    execution_mode: ExecutionMode::Pure,
5730                    schema: json!({"type": "object"}),
5731                }],
5732                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
5733                concurrency: Concurrency::ModuleManaged,
5734                emits_push: true,
5735                sub_supervises: true,
5736            }])
5737            .build()
5738    }
5739
5740    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
5741        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
5742    }
5743
5744    fn hello_frame_with_control_ops(
5745        module_id: &str,
5746        protocol_ver: u8,
5747        corr: u64,
5748        control_ops: Option<Vec<String>>,
5749    ) -> Frame {
5750        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
5751    }
5752
5753    fn hello_frame_with_nonce(
5754        module_id: &str,
5755        protocol_ver: u8,
5756        corr: u64,
5757        launch_nonce: Option<&str>,
5758    ) -> Frame {
5759        hello_frame_full(
5760            module_id,
5761            protocol_ver,
5762            corr,
5763            None,
5764            launch_nonce.map(ToOwned::to_owned),
5765        )
5766    }
5767
5768    fn hello_frame_full(
5769        module_id: &str,
5770        protocol_ver: u8,
5771        corr: u64,
5772        control_ops: Option<Vec<String>>,
5773        launch_nonce: Option<String>,
5774    ) -> Frame {
5775        let body = serde_json::to_vec(&ModuleHelloBody {
5776            manifest: manifest(module_id, protocol_ver),
5777            protocol_ver,
5778            control_ops,
5779            launch_nonce,
5780        })
5781        .unwrap();
5782        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
5783    }
5784
5785    fn non_routable_hello_frame_with_control_ops(
5786        module_id: &str,
5787        corr: u64,
5788        control_ops: Option<Vec<String>>,
5789    ) -> Frame {
5790        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
5791        manifest.provides.clear();
5792        let body = serde_json::to_vec(&ModuleHelloBody {
5793            manifest,
5794            protocol_ver: PROTOCOL_VERSION,
5795            control_ops,
5796            launch_nonce: None,
5797        })
5798        .unwrap();
5799        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
5800    }
5801
5802    fn capability_grammar_hello_frame(
5803        capabilities: Value,
5804        runtime_computed: Option<Value>,
5805        corr: u64,
5806    ) -> Frame {
5807        let mut body = serde_json::to_value(ModuleHelloBody {
5808            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
5809            protocol_ver: PROTOCOL_VERSION,
5810            control_ops: None,
5811            launch_nonce: None,
5812        })
5813        .expect("HELLO body serializes");
5814        body["manifest"]["capabilities"] = capabilities;
5815        if let Some(runtime_computed) = runtime_computed {
5816            body["runtime_computed"] = runtime_computed;
5817        }
5818        Frame::build(
5819            FrameType::Hello,
5820            control_flags(),
5821            0,
5822            0,
5823            corr,
5824            serde_json::to_vec(&body).expect("HELLO body reserializes"),
5825        )
5826        .expect("HELLO frame builds")
5827    }
5828
5829    fn channel_request(channel: u16, corr: u64) -> Frame {
5830        Frame::build(
5831            FrameType::Request,
5832            Flags::new(true, Priority::Interactive, false),
5833            channel,
5834            0,
5835            corr,
5836            b"opaque".to_vec(),
5837        )
5838        .unwrap()
5839    }
5840
5841    fn route_ctx(
5842        connection_id: ConnectionId,
5843    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
5844        let (tx, rx) = mpsc::channel(8);
5845        (
5846            RouteCtx {
5847                connection_id,
5848                egress: FrameSink::new(tx),
5849            },
5850            rx,
5851        )
5852    }
5853
5854    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
5855        serde_json::from_slice(&frame.body).unwrap()
5856    }
5857
5858    /// Register a module over a connection that has a sink and return the
5859    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
5860    /// module's own sink rather than returning it as a reply, so the ack is
5861    /// taken off `rx` here and whatever the test reads next is what followed it.
5862    async fn hello_via_sink(
5863        handler: &ControlHandler,
5864        ctx: &RouteCtx,
5865        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
5866        hello: Frame,
5867    ) -> Frame {
5868        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
5869        assert!(
5870            replies.is_empty(),
5871            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
5872        );
5873        let ack = rx
5874            .try_recv()
5875            .expect("HELLO_ACK is queued on the module sink")
5876            .frame;
5877        assert_eq!(ack.header.ty, FrameType::HelloAck);
5878        ack
5879    }
5880
5881    fn parse_error(frame: &Frame) -> Value {
5882        serde_json::from_slice(&frame.body).unwrap()
5883    }
5884
5885    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
5886        serde_json::from_slice(&frame.body).unwrap()
5887    }
5888
5889    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
5890        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
5891            route_channel,
5892            route_epoch: 0,
5893            kind,
5894        })
5895        .unwrap();
5896        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5897    }
5898
5899    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
5900        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
5901            module_id: module_id.to_string(),
5902        })
5903        .unwrap();
5904        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5905    }
5906
5907    fn route_open_frame(corr: u64, module_id: &str, project_root: TestTempDir) -> Frame {
5908        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
5909    }
5910
5911    fn route_open_frame_with_consumer_capabilities(
5912        corr: u64,
5913        module_id: &str,
5914        project_root: TestTempDir,
5915        consumer_capabilities: Option<Vec<String>>,
5916    ) -> Frame {
5917        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
5918            target: RouteTarget::ToolProvider {
5919                module_id: module_id.to_string(),
5920            },
5921            identity: BindIdentity::new(
5922                project_root.path().to_path_buf(),
5923                "unit".to_string(),
5924                "session".to_string(),
5925            ),
5926            consumer_identity: None,
5927            consumer_capabilities,
5928            admission_facts: None,
5929        })
5930        .unwrap();
5931        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5932    }
5933
5934    fn route_open_frame_with_admission_facts(
5935        corr: u64,
5936        module_id: &str,
5937        project_root: TestTempDir,
5938        consumer_identity: Option<subc_control::ConsumerIdentity>,
5939        facts: Option<Value>,
5940    ) -> Frame {
5941        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
5942            target: RouteTarget::ToolProvider {
5943                module_id: module_id.to_string(),
5944            },
5945            identity: BindIdentity::new(
5946                project_root.path().to_path_buf(),
5947                "unit".to_string(),
5948                format!("session-{corr}"),
5949            ),
5950            consumer_identity,
5951            consumer_capabilities: None,
5952            admission_facts: facts,
5953        })
5954        .unwrap();
5955        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5956    }
5957
5958    #[derive(Clone, Default)]
5959    struct EventCapture {
5960        events: Arc<Mutex<Vec<CapturedEvent>>>,
5961    }
5962
5963    #[derive(Clone, Debug)]
5964    struct CapturedEvent {
5965        target: String,
5966        fields: BTreeMap<String, String>,
5967    }
5968
5969    impl EventCapture {
5970        fn events(&self) -> Vec<CapturedEvent> {
5971            self.events.lock().unwrap().clone()
5972        }
5973    }
5974
5975    impl<S> Layer<S> for EventCapture
5976    where
5977        S: Subscriber,
5978    {
5979        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
5980            let mut visitor = EventFieldVisitor::default();
5981            event.record(&mut visitor);
5982            self.events.lock().unwrap().push(CapturedEvent {
5983                target: event.metadata().target().to_string(),
5984                fields: visitor.fields,
5985            });
5986        }
5987    }
5988
5989    #[derive(Default)]
5990    struct EventFieldVisitor {
5991        fields: BTreeMap<String, String>,
5992    }
5993
5994    impl Visit for EventFieldVisitor {
5995        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
5996            self.fields
5997                .insert(field.name().to_string(), format!("{value:?}"));
5998        }
5999    }
6000
6001    fn health_response(corr: u64, status: HealthStatus) -> Frame {
6002        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
6003            status,
6004            detail: Some("warming".to_string()),
6005            metrics: Some(json!({"queue_depth": 3})),
6006        })
6007        .unwrap();
6008        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6009    }
6010
6011    fn route_bind_ack(corr: u64) -> Frame {
6012        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
6013        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6014    }
6015
6016    fn unique_project_root(label: &str) -> TestTempDir {
6017        TestTempDir::new(label)
6018    }
6019
6020    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
6021        match parse_route_poll(frame) {
6022            ClientControlResponse::RoutePoll {
6023                status: None,
6024                live: Some(live),
6025                ..
6026            } => assert_eq!(live, expected_live),
6027            other => panic!("unexpected route.poll response: {other:?}"),
6028        }
6029    }
6030
6031    fn bind_liveness_route(
6032        registry: &Registry,
6033        forwarding: &ForwardingTable,
6034        module_id: &str,
6035    ) -> (RouteCtx, u16, u32) {
6036        let module_connection = ConnectionId::new(101);
6037        let client_connection = ConnectionId::new(202);
6038        let registration = registry
6039            .register_with_control_ops(
6040                manifest(module_id, PROTOCOL_VERSION),
6041                PROTOCOL_VERSION,
6042                module_connection,
6043                module_baseline_control_ops(),
6044            )
6045            .unwrap();
6046        let (module_tx, _module_rx) = mpsc::channel(8);
6047        let endpoint = forwarding
6048            .register_module_connection(
6049                module_connection,
6050                module_id.to_string(),
6051                PROTOCOL_VERSION,
6052                manifest_concurrency(&registration.manifest),
6053                FrameSink::new(module_tx),
6054            )
6055            .unwrap();
6056        let (client_ctx, _client_rx) = route_ctx(client_connection);
6057        let pending = forwarding
6058            .begin_route_bind_relay_for_test(
6059                client_connection,
6060                client_ctx.egress.clone(),
6061                1,
6062                module_id,
6063            )
6064            .unwrap();
6065        assert_eq!(pending.endpoint, endpoint);
6066        let route_channel = pending.client_channel;
6067        let route_epoch = pending.client_epoch;
6068        forwarding
6069            .complete_pending_relay(
6070                module_connection,
6071                pending.corr,
6072                RouteBindRelayOutcome::Accepted,
6073            )
6074            .unwrap();
6075        (client_ctx, route_channel, route_epoch)
6076    }
6077
6078    struct FakeProcessLiveness {
6079        live: Option<bool>,
6080    }
6081
6082    impl ModuleProcessLiveness for FakeProcessLiveness {
6083        fn process_live(&self, _module_id: &str) -> Option<bool> {
6084            self.live
6085        }
6086    }
6087
6088    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6089    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
6090    {
6091        let registry = Arc::new(Registry::default());
6092        let supervisor_handle = SupervisorHandle::new();
6093        let supervisor = Supervisor::new(
6094            Arc::clone(&registry),
6095            RestartPolicy::new(1, Duration::from_millis(10)),
6096        )
6097        .with_handle(supervisor_handle.clone());
6098        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
6099        let module = supervisor
6100            .spawn(ModuleSpec {
6101                module_id: "stderr-tail-wire".to_string(),
6102                program: fake_aft_stub_path(),
6103                args: Vec::new(),
6104                env: vec![
6105                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
6106                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
6107                ],
6108                reserved: false,
6109                reserved_prefixes: Vec::new(),
6110                protocol: ModuleProtocol::Subc,
6111                overlap: Default::default(),
6112            })
6113            .unwrap();
6114
6115        let deadline = Instant::now() + Duration::from_secs(5);
6116        loop {
6117            let tail = module.stderr_tail(None, None);
6118            if tail
6119                .entries
6120                .iter()
6121                .any(|entry| matches!(entry, TailEntry::ProcessStart))
6122                && tail.entries.iter().any(|entry| {
6123                    matches!(
6124                        entry,
6125                        TailEntry::Line {
6126                            truncated: true,
6127                            ..
6128                        }
6129                    )
6130                })
6131            {
6132                break;
6133            }
6134            assert!(
6135                Instant::now() < deadline,
6136                "module did not produce a truncated line and restart boundary: {tail:?}"
6137            );
6138            sleep(Duration::from_millis(10)).await;
6139        }
6140
6141        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
6142        let request = ClientControlRequest::SupervisorStderrTail {
6143            module_id: "stderr-tail-wire".to_string(),
6144            max_lines: None,
6145            max_bytes: None,
6146        };
6147        let frame = Frame::build(
6148            FrameType::Request,
6149            control_flags(),
6150            0,
6151            0,
6152            1,
6153            serde_json::to_vec(&request).unwrap(),
6154        )
6155        .unwrap();
6156        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6157        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6158        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
6159            serde_json::from_slice(&responses[0].body).unwrap()
6160        else {
6161            panic!("expected supervisor.stderr_tail response");
6162        };
6163
6164        assert!(
6165            tail.entries
6166                .iter()
6167                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
6168            "the control response lost the restart boundary"
6169        );
6170        let Some(StderrTailEntry::Line { text, truncated }) = tail.entries.iter().find(|entry| {
6171            matches!(
6172                entry,
6173                StderrTailEntry::Line {
6174                    truncated: true,
6175                    ..
6176                }
6177            )
6178        }) else {
6179            panic!("the control response lost the truncated line");
6180        };
6181        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
6182        assert!(*truncated);
6183    }
6184
6185    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
6186    /// read done on the worker thread would stall every other task until it
6187    /// finished; the read must run off the worker so this test's own task keeps
6188    /// running while the read is paused.
6189    #[tokio::test(flavor = "current_thread")]
6190    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
6191        let dir = TestTempDir::new("terminals-off-worker");
6192        let journal_path = dir.join("terminals.jsonl");
6193        let registry = Arc::new(Registry::default());
6194        let supervisor_handle = SupervisorHandle::new();
6195        let supervisor =
6196            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
6197                .with_handle(supervisor_handle.clone())
6198                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
6199        let module = supervisor
6200            .spawn(ModuleSpec {
6201                module_id: "terminal-off-worker".to_string(),
6202                program: fake_aft_stub_path(),
6203                args: Vec::new(),
6204                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6205                reserved: false,
6206                reserved_prefixes: Vec::new(),
6207                protocol: ModuleProtocol::Subc,
6208                overlap: Default::default(),
6209            })
6210            .unwrap();
6211        let deadline = Instant::now() + Duration::from_secs(5);
6212        while module.terminal_history().entries.len() != 2 {
6213            assert!(Instant::now() < deadline, "module did not record two exits");
6214            sleep(Duration::from_millis(10)).await;
6215        }
6216
6217        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
6218        let handler =
6219            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
6220        let frame = Frame::build(
6221            FrameType::Request,
6222            control_flags(),
6223            0,
6224            0,
6225            1,
6226            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
6227                module_id: "terminal-off-worker".to_string(),
6228            })
6229            .unwrap(),
6230        )
6231        .unwrap();
6232        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6233        let spawned_at = std::time::Instant::now();
6234        let read = tokio::spawn({
6235            let handler = Arc::clone(&handler);
6236            async move { handler.handle_control_frame(&ctx, frame).await }
6237        });
6238        // Waiting for the pause from a blocking thread keeps this task pending,
6239        // so the runtime's single worker is free to run the read task.
6240        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
6241            .await
6242            .unwrap()
6243            .expect("the history read reached its pause");
6244        let elapsed = spawned_at.elapsed();
6245        assert!(
6246            elapsed < Duration::from_secs(2) && !read.is_finished(),
6247            "this task could not run while the history read was paused \
6248             (resumed after {elapsed:?}, read finished: {})",
6249            read.is_finished()
6250        );
6251
6252        drop(release);
6253        let responses = read.await.unwrap().unwrap();
6254        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6255        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
6256            panic!("expected supervisor.terminals response");
6257        };
6258        assert_eq!(terminals.entries.len(), 2);
6259        assert_eq!(terminals.journal_skipped_lines, 0);
6260        assert_eq!(terminals.journal_read_errors, 0);
6261    }
6262
6263    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6264    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
6265        let registry = Arc::new(Registry::default());
6266        let supervisor_handle = SupervisorHandle::new();
6267        let supervisor =
6268            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
6269                .with_handle(supervisor_handle.clone());
6270        let module = supervisor
6271            .spawn(ModuleSpec {
6272                module_id: "terminal-golden".to_string(),
6273                program: fake_aft_stub_path(),
6274                args: Vec::new(),
6275                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6276                reserved: false,
6277                reserved_prefixes: Vec::new(),
6278                protocol: ModuleProtocol::Subc,
6279                overlap: Default::default(),
6280            })
6281            .unwrap();
6282
6283        let deadline = Instant::now() + Duration::from_secs(5);
6284        while module.terminal_history().entries.len() != 2 {
6285            assert!(
6286                Instant::now() < deadline,
6287                "module did not retain two terminal exits: {:?}",
6288                module.terminal_history()
6289            );
6290            sleep(Duration::from_millis(10)).await;
6291        }
6292
6293        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
6294        let request = ClientControlRequest::SupervisorTerminals {
6295            module_id: "terminal-golden".to_string(),
6296        };
6297        let frame = Frame::build(
6298            FrameType::Request,
6299            control_flags(),
6300            0,
6301            0,
6302            1,
6303            serde_json::to_vec(&request).unwrap(),
6304        )
6305        .unwrap();
6306        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6307        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6308        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6309        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
6310            panic!("expected supervisor.terminals response");
6311        };
6312        assert_eq!(terminals.entries.len(), 2);
6313        assert_eq!(terminals.dropped, 0);
6314
6315        let mut rendered = serde_json::to_value(response).unwrap();
6316        // Wall-clock fields are the observation contract, but not stable fixture
6317        // bytes; normalize only them after the real handler has shaped the response.
6318        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
6319        for (index, entry) in rendered["entries"]
6320            .as_array_mut()
6321            .expect("terminal response entries array")
6322            .iter_mut()
6323            .enumerate()
6324        {
6325            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
6326        }
6327
6328        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
6329            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
6330        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
6331        if std::env::var_os("UPDATE_GOLDEN").is_some() {
6332            std::fs::write(&golden_path, &serialized).unwrap();
6333        }
6334        let expected: Value =
6335            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
6336        assert_eq!(rendered, expected);
6337    }
6338
6339    #[test]
6340    fn hello_registers_manifest_and_returns_ack() {
6341        let registry = Arc::new(Registry::default());
6342        let handler = ControlHandler::new(Arc::clone(&registry));
6343        let conn = ConnectionId::new(1);
6344
6345        let responses = handler
6346            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
6347            .unwrap();
6348
6349        assert_eq!(responses.len(), 1);
6350        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6351        assert_eq!(responses[0].header.channel, 0);
6352        assert_eq!(responses[0].header.corr, 7);
6353        let ack = parse_ack(&responses[0]);
6354        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
6355        assert!(ack
6356            .subc_capabilities
6357            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
6358        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
6359        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
6360        assert!(ack
6361            .subc_ops
6362            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
6363        assert!(ack
6364            .subc_ops
6365            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
6366
6367        let registration = registry.get_module("aft").unwrap().unwrap();
6368        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
6369        assert_eq!(registration.state, ChannelState::Active);
6370        assert_eq!(registration.connection_id, conn);
6371        assert_eq!(registration.control_ops, module_baseline_control_ops());
6372    }
6373
6374    #[test]
6375    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
6376        let invalid_identifiers = [
6377            ("case_change", "credentials-Provider/v1"),
6378            ("leading_zero", "credentials-provider/v01"),
6379            ("trailing_hyphen", "credentials-provider-/v1"),
6380            ("consecutive_hyphens", "credentials--provider/v1"),
6381            ("uppercase", "Credentials-provider/v1"),
6382            ("missing_v", "credentials-provider/1"),
6383            ("whitespace", "credentials provider/v1"),
6384            ("zero_version", "credentials-provider/v0"),
6385            ("out_of_range_version", "credentials-provider/v4294967296"),
6386            (
6387                "overlength_name",
6388                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
6389            ),
6390        ];
6391        let mut cases = invalid_identifiers
6392            .into_iter()
6393            .map(|(name, identifier)| {
6394                (
6395                    format!("identifier_{name}"),
6396                    "capabilities.provides[0]".to_string(),
6397                    identifier.to_string(),
6398                    json!({ "provides": [identifier] }),
6399                    None,
6400                )
6401            })
6402            .collect::<Vec<_>>();
6403        cases.extend([
6404            (
6405                "unknown_need".to_string(),
6406                "capabilities.requires[0].need".to_string(),
6407                "deferred".to_string(),
6408                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
6409                None,
6410            ),
6411            (
6412                "duplicate_provides".to_string(),
6413                "capabilities.provides[1]".to_string(),
6414                "credentials-provider/v1".to_string(),
6415                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
6416                None,
6417            ),
6418            (
6419                "duplicate_must_never_reach".to_string(),
6420                "capabilities.must_never_reach[1]".to_string(),
6421                "credentials-provider/v1".to_string(),
6422                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
6423                None,
6424            ),
6425            (
6426                "duplicate_requires_same_need".to_string(),
6427                "capabilities.requires[1]".to_string(),
6428                "credentials-provider/v1".to_string(),
6429                json!({ "requires": [
6430                    { "capability": "credentials-provider/v1", "need": "required" },
6431                    { "capability": "credentials-provider/v1", "need": "required" }
6432                ] }),
6433                None,
6434            ),
6435            (
6436                "duplicate_requires_conflicting_need".to_string(),
6437                "capabilities.requires[1]".to_string(),
6438                "credentials-provider/v1".to_string(),
6439                json!({ "requires": [
6440                    { "capability": "credentials-provider/v1", "need": "required" },
6441                    { "capability": "credentials-provider/v1", "need": "optional" }
6442                ] }),
6443                None,
6444            ),
6445            (
6446                "capabilities_root_pointer".to_string(),
6447                "runtime_computed[0]".to_string(),
6448                "/capabilities".to_string(),
6449                json!({}),
6450                Some(json!(["/capabilities"])),
6451            ),
6452            (
6453                "capabilities_descendant_pointer".to_string(),
6454                "runtime_computed[0]".to_string(),
6455                "/capabilities/provides".to_string(),
6456                json!({}),
6457                Some(json!(["/capabilities/provides"])),
6458            ),
6459            (
6460                "malformed_pointer_without_leading_slash".to_string(),
6461                "runtime_computed[0]".to_string(),
6462                "capabilities".to_string(),
6463                json!({}),
6464                Some(json!(["capabilities"])),
6465            ),
6466            (
6467                "malformed_pointer_escape".to_string(),
6468                "runtime_computed[0]".to_string(),
6469                "/roles/~2/tools".to_string(),
6470                json!({}),
6471                Some(json!(["/roles/~2/tools"])),
6472            ),
6473            (
6474                "unknown_capabilities_field".to_string(),
6475                "capabilities.future".to_string(),
6476                "<array>".to_string(),
6477                json!({ "future": [] }),
6478                None,
6479            ),
6480        ]);
6481
6482        for (index, (name, field, value, capabilities, runtime_computed)) in
6483            cases.into_iter().enumerate()
6484        {
6485            let registry = Arc::new(Registry::default());
6486            let handler = ControlHandler::new(Arc::clone(&registry));
6487            let response = handler
6488                .handle_control(
6489                    ConnectionId::new((index + 1) as u64),
6490                    capability_grammar_hello_frame(
6491                        capabilities,
6492                        runtime_computed,
6493                        index as u64 + 1,
6494                    ),
6495                )
6496                .expect("invalid HELLO returns a refusal");
6497
6498            assert_eq!(response.len(), 1, "{name} must emit one refusal");
6499            let error = parse_error(&response[0]);
6500            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
6501            let message = error["message"]
6502                .as_str()
6503                .expect("error message is a string");
6504            assert!(
6505                message.contains(&field),
6506                "{name}: field missing from {message}"
6507            );
6508            assert!(
6509                message.contains(&value),
6510                "{name}: value missing from {message}"
6511            );
6512            assert_eq!(
6513                registry
6514                    .active_registration_count()
6515                    .expect("registry reads"),
6516                0,
6517                "{name}: refused HELLO must not create a catalog entry"
6518            );
6519        }
6520    }
6521
6522    #[test]
6523    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
6524        let registry = Arc::new(Registry::default());
6525        let handler = ControlHandler::new(Arc::clone(&registry));
6526        let capabilities = json!({
6527            "provides": ["credentials-provider/v1"],
6528            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
6529            "must_never_reach": ["federation-transport/v1"]
6530        });
6531        let response = handler
6532            .handle_control(
6533                ConnectionId::new(99),
6534                capability_grammar_hello_frame(
6535                    capabilities.clone(),
6536                    Some(json!(["/roles/0/tools"])),
6537                    99,
6538                ),
6539            )
6540            .expect("valid HELLO registers");
6541        assert_eq!(response[0].header.ty, FrameType::HelloAck);
6542
6543        let request = Frame::build(
6544            FrameType::Request,
6545            control_flags(),
6546            0,
6547            0,
6548            100,
6549            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6550                .expect("catalog request serializes"),
6551        )
6552        .expect("catalog request frame builds");
6553        let response = handler
6554            .handle_catalog_list(request, None)
6555            .expect("catalog list succeeds");
6556        let ClientControlResponse::CatalogList { modules, .. } =
6557            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
6558        else {
6559            panic!("catalog request must return catalog.list");
6560        };
6561        assert_eq!(modules.len(), 1);
6562        assert_eq!(
6563            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
6564            capabilities
6565        );
6566    }
6567
6568    #[test]
6569    fn catalog_list_mirrors_management_operation_description() {
6570        let registry = Arc::new(Registry::default());
6571        let handler = ControlHandler::new(Arc::clone(&registry));
6572        let description = "List managed records and return their identifiers and metadata.";
6573        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
6574        manifest.provides = vec![ProviderRole::ManagementSurface {
6575            operations: vec![ManagementOperation {
6576                name: "records.list".to_string(),
6577                kind: ManagementOperationKind::Query,
6578                description: Some(description.to_string()),
6579            }],
6580            config_schema: json!({"type": "object"}),
6581            observability: vec![ObservabilitySurface {
6582                name: "records.stats".to_string(),
6583                kind: ObservabilityKind::Snapshot,
6584            }],
6585            identity_scope: vec![IdentityScope::Project],
6586            concurrency: Concurrency::ModuleManaged,
6587        }];
6588        registry
6589            .register_with_control_ops(
6590                manifest,
6591                PROTOCOL_VERSION,
6592                ConnectionId::new(99),
6593                Vec::new(),
6594            )
6595            .expect("described management manifest registers");
6596
6597        let request = Frame::build(
6598            FrameType::Request,
6599            control_flags(),
6600            0,
6601            0,
6602            100,
6603            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6604                .expect("catalog request serializes"),
6605        )
6606        .expect("catalog request frame builds");
6607        let response = handler
6608            .handle_catalog_list(request, None)
6609            .expect("catalog list succeeds");
6610        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
6611        assert_eq!(
6612            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
6613            "catalog.list must preserve the declared operation description verbatim"
6614        );
6615    }
6616
6617    #[test]
6618    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
6619        let registry = Arc::new(Registry::default());
6620        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
6621            [("vault".to_string(), true), ("squatter".to_string(), true)],
6622            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
6623        );
6624        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
6625        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
6626            provides: vec!["credentials-provider/v1".to_string()],
6627            requires: Vec::new(),
6628            must_never_reach: Vec::new(),
6629        });
6630        let frame = Frame::build(
6631            FrameType::Hello,
6632            control_flags(),
6633            0,
6634            0,
6635            77,
6636            serde_json::to_vec(&ModuleHelloBody {
6637                manifest: squatter,
6638                protocol_ver: PROTOCOL_VERSION,
6639                control_ops: None,
6640                launch_nonce: None,
6641            })
6642            .expect("HELLO serializes"),
6643        )
6644        .expect("HELLO frame builds");
6645        let response = handler
6646            .handle_control(ConnectionId::new(77), frame)
6647            .expect("reserved claim receives a typed refusal");
6648        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
6649        assert_eq!(
6650            registry
6651                .active_registration_count()
6652                .expect("registry reads"),
6653            0,
6654            "a reserved capability refusal must not leave a catalog entry"
6655        );
6656    }
6657
6658    #[test]
6659    fn server_describe_surfaces_required_capability_verdict_fields() {
6660        let registry = Arc::new(Registry::default());
6661        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
6662            [
6663                ("consumer".to_string(), true),
6664                ("provider".to_string(), false),
6665            ],
6666            BTreeMap::new(),
6667        );
6668        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
6669        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
6670            provides: Vec::new(),
6671            requires: vec![subc_protocol::manifest::CapabilityRequirement {
6672                capability: "credentials-provider/v1".to_string(),
6673                need: subc_protocol::manifest::CapabilityNeed::Required,
6674            }],
6675            must_never_reach: Vec::new(),
6676        });
6677        let hello = Frame::build(
6678            FrameType::Hello,
6679            control_flags(),
6680            0,
6681            0,
6682            78,
6683            serde_json::to_vec(&ModuleHelloBody {
6684                manifest: consumer,
6685                protocol_ver: PROTOCOL_VERSION,
6686                control_ops: None,
6687                launch_nonce: None,
6688            })
6689            .expect("HELLO serializes"),
6690        )
6691        .expect("HELLO frame builds");
6692        handler
6693            .handle_control(ConnectionId::new(78), hello)
6694            .expect("consumer registers");
6695        let describe = Frame::build(
6696            FrameType::Request,
6697            control_flags(),
6698            0,
6699            0,
6700            79,
6701            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
6702                .expect("request serializes"),
6703        )
6704        .expect("describe frame builds");
6705        let response = handler
6706            .handle_server_describe(describe)
6707            .expect("server.describe succeeds");
6708        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
6709        let requirement = &rendered["capability_requirements"][0];
6710        assert_eq!(requirement["consumer"], "consumer");
6711        assert_eq!(requirement["verdict"], "never_provided");
6712        assert_eq!(requirement["episode_seq"], 1);
6713        assert_eq!(requirement["config_satisfiable"], false);
6714        assert_eq!(requirement["runtime_available"], false);
6715        assert!(requirement["detail"]
6716            .as_str()
6717            .expect("detail string")
6718            .contains("credentials-provider/v1"));
6719    }
6720
6721    #[test]
6722    fn catalog_list_omits_capabilities_for_legacy_manifest() {
6723        let registry = Arc::new(Registry::default());
6724        let handler = ControlHandler::new(Arc::clone(&registry));
6725        let hello = handler
6726            .handle_control(
6727                ConnectionId::new(101),
6728                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
6729            )
6730            .expect("legacy HELLO registers");
6731        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
6732
6733        let request = Frame::build(
6734            FrameType::Request,
6735            control_flags(),
6736            0,
6737            0,
6738            102,
6739            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6740                .expect("catalog request serializes"),
6741        )
6742        .expect("catalog request frame builds");
6743        let response = handler
6744            .handle_catalog_list(request, None)
6745            .expect("catalog list succeeds");
6746        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
6747        assert!(
6748            body["modules"][0].get("capabilities").is_none(),
6749            "legacy manifest must retain an absent capabilities field on catalog.list"
6750        );
6751    }
6752
6753    #[test]
6754    fn hello_ack_omits_storage_when_no_storage_config() {
6755        let registry = Arc::new(Registry::default());
6756        let handler = ControlHandler::new(Arc::clone(&registry));
6757        let responses = handler
6758            .handle_control(
6759                ConnectionId::new(1),
6760                hello_frame("aft", PROTOCOL_VERSION, 7),
6761            )
6762            .unwrap();
6763        let ack = parse_ack(&responses[0]);
6764        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
6765        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
6766    }
6767
6768    #[tokio::test]
6769    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
6770        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
6771        let registry = Arc::new(Registry::default());
6772        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
6773        let responses = handler
6774            .handle_control(
6775                ConnectionId::new(1),
6776                hello_frame("aft", PROTOCOL_VERSION, 7),
6777            )
6778            .unwrap();
6779        let ack = parse_ack(&responses[0]);
6780        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
6781
6782        let described = handler
6783            .handle_control_frame(
6784                &route_ctx(ConnectionId::new(2)).0,
6785                Frame::build(
6786                    FrameType::Request,
6787                    control_flags(),
6788                    0,
6789                    0,
6790                    9,
6791                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
6792                )
6793                .unwrap(),
6794            )
6795            .await
6796            .unwrap();
6797        let ClientControlResponse::ServerDescribe { machine_id, .. } =
6798            serde_json::from_slice(&described[0].body).unwrap()
6799        else {
6800            panic!("server.describe answered with another shape");
6801        };
6802        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
6803    }
6804
6805    #[test]
6806    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
6807        // With a central sqlite storage policy, each registering module gets its
6808        // own resolved descriptor in HELLO_ACK, keyed by its module id.
6809        let registry = Arc::new(Registry::default());
6810        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
6811            crate::daemon_config::StorageConfig::Sqlite {
6812                data_home: std::path::PathBuf::from("/data"),
6813            },
6814        ));
6815
6816        let responses = handler
6817            .handle_control(
6818                ConnectionId::new(1),
6819                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
6820            )
6821            .unwrap();
6822        let ack = parse_ack(&responses[0]);
6823        assert_eq!(
6824            ack.storage,
6825            Some(serde_json::json!({
6826                "module_id": "alfonso-routing",
6827                "storage_namespace": "default",
6828                "isolation": { "kind": "module" },
6829                "backend": {
6830                    "backend": "sqlite",
6831                    "path": "/data/cortexkit/alfonso-routing/store.db"
6832                }
6833            })),
6834            "the delivered descriptor is the module's own sqlite store path"
6835        );
6836    }
6837
6838    #[test]
6839    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
6840        let registry = Arc::new(Registry::default());
6841        let handler = ControlHandler::new(Arc::clone(&registry));
6842        let conn = ConnectionId::new(1);
6843        let responses = handler
6844            .handle_control(
6845                conn,
6846                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
6847            )
6848            .unwrap();
6849        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6850        let registration = registry.get_module("aft").unwrap().unwrap();
6851        assert_eq!(registration.control_ops, module_baseline_control_ops());
6852
6853        let frame =
6854            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
6855        assert!(handler
6856            .guard_module_control_op(&frame, "aft", "route.bind")
6857            .unwrap()
6858            .is_none());
6859        let error = handler
6860            .guard_module_control_op(&frame, "aft", "test.synthetic")
6861            .unwrap()
6862            .expect("synthetic ungranted op should be rejected");
6863        assert_eq!(error.header.ty, FrameType::Error);
6864        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
6865    }
6866
6867    #[test]
6868    fn hello_control_ops_some_adds_optional_grants() {
6869        let registry = Arc::new(Registry::default());
6870        let handler = ControlHandler::new(Arc::clone(&registry));
6871        handler
6872            .handle_control(
6873                ConnectionId::new(1),
6874                hello_frame_with_control_ops(
6875                    "aft",
6876                    PROTOCOL_VERSION,
6877                    7,
6878                    Some(vec![
6879                        "future.synthetic".to_string(),
6880                        "route.bind".to_string(),
6881                    ]),
6882                ),
6883            )
6884            .unwrap();
6885        let registration = registry.get_module("aft").unwrap().unwrap();
6886        assert_eq!(
6887            registration.control_ops,
6888            vec![
6889                "route.bind".to_string(),
6890                "route.status".to_string(),
6891                "future.synthetic".to_string(),
6892            ]
6893        );
6894        let frame =
6895            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
6896        assert!(handler
6897            .guard_module_control_op(&frame, "aft", "future.synthetic")
6898            .unwrap()
6899            .is_none());
6900    }
6901
6902    #[tokio::test]
6903    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
6904        let registry = Arc::new(Registry::default());
6905        let forwarding = Arc::new(ForwardingTable::default());
6906        let handler =
6907            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
6908        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
6909        hello_via_sink(
6910            &handler,
6911            &module_ctx,
6912            &mut module_rx,
6913            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
6914        )
6915        .await;
6916
6917        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
6918        let responses = handler
6919            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
6920            .await
6921            .unwrap();
6922        assert_eq!(responses.len(), 1);
6923        assert_eq!(responses[0].header.ty, FrameType::Error);
6924        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
6925        assert!(module_rx.try_recv().is_err());
6926    }
6927
6928    #[tokio::test]
6929    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
6930        let registry = Arc::new(Registry::default());
6931        let forwarding = Arc::new(ForwardingTable::default());
6932        let handler =
6933            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
6934        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
6935        hello_via_sink(
6936            &handler,
6937            &module_ctx,
6938            &mut module_rx,
6939            hello_frame_with_control_ops(
6940                "aft",
6941                PROTOCOL_VERSION,
6942                7,
6943                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
6944            ),
6945        )
6946        .await;
6947
6948        let project_root = unique_project_root("demux");
6949        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
6950        let route_handler = handler.clone();
6951        let route_task = tokio::spawn(async move {
6952            route_handler
6953                .handle_control_frame(
6954                    &route_client_ctx,
6955                    route_open_frame(100, "aft", project_root),
6956                )
6957                .await
6958                .unwrap()
6959        });
6960        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
6961            .await
6962            .unwrap()
6963            .unwrap();
6964        assert!(matches!(
6965            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
6966            ModuleControlRequest::RouteBind { .. }
6967        ));
6968
6969        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
6970        let health_handler = handler.clone();
6971        let health_task = tokio::spawn(async move {
6972            health_handler
6973                .handle_control_frame(
6974                    &health_client_ctx,
6975                    supervisor_health_probe_frame(101, "aft"),
6976                )
6977                .await
6978                .unwrap()
6979        });
6980        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
6981            .await
6982            .unwrap()
6983            .unwrap();
6984        assert_eq!(
6985            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
6986            ModuleControlRequest::HealthCheck {}
6987        );
6988
6989        handler
6990            .handle_control_frame(
6991                &module_ctx,
6992                health_response(health_frame.header.corr, HealthStatus::Degraded),
6993            )
6994            .await
6995            .unwrap();
6996        let health_response = health_task.await.unwrap();
6997        assert_eq!(health_response.len(), 1);
6998        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
6999            ClientControlResponse::SupervisorHealthProbe {
7000                module_id,
7001                status,
7002                detail,
7003                metrics,
7004            } => {
7005                assert_eq!(module_id, "aft");
7006                assert_eq!(status, HealthStatus::Degraded);
7007                assert_eq!(detail.as_deref(), Some("warming"));
7008                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
7009            }
7010            other => panic!("unexpected health response: {other:?}"),
7011        }
7012
7013        handler
7014            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
7015            .await
7016            .unwrap();
7017        let route_response = route_task.await.unwrap();
7018        assert!(route_response.is_empty());
7019        let published = route_client_rx.recv().await.unwrap();
7020        assert!(matches!(
7021            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
7022            ClientControlResponse::RouteOpen { .. }
7023        ));
7024    }
7025
7026    /// Start one `route.open` on `client_connection` and return its still-running
7027    /// handler task together with the `route.bind` the module received for it.
7028    /// The handler blocks until the module answers, so it has to run as a task
7029    /// while the test drives the module side.
7030    async fn relay_route_open(
7031        handler: &ControlHandler,
7032        client_connection: ConnectionId,
7033        client_egress: &FrameSink,
7034        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7035        corr: u64,
7036        module_id: &str,
7037        project_root_label: &str,
7038    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
7039        let ctx = RouteCtx {
7040            connection_id: client_connection,
7041            egress: client_egress.clone(),
7042        };
7043        let handler = handler.clone();
7044        let project_root = unique_project_root(project_root_label);
7045        let module_id = module_id.to_string();
7046        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
7047        let task = tokio::spawn(async move {
7048            let _guard = tracing::dispatcher::set_default(&dispatch);
7049            handler
7050                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
7051                .await
7052                .unwrap()
7053        });
7054        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
7055            .await
7056            .expect("module receives the relayed route.bind")
7057            .expect("module egress is open");
7058        (task, bind.frame)
7059    }
7060
7061    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
7062        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
7063            ModuleControlRequest::RouteBind {
7064                route_channel,
7065                epoch,
7066                ..
7067            } => (route_channel, epoch),
7068            other => panic!("expected a route.bind request, got {other:?}"),
7069        }
7070    }
7071
7072    fn published_route(frame: &Frame) -> (u16, u32) {
7073        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
7074            ClientControlResponse::RouteOpen {
7075                route_channel,
7076                route_epoch,
7077            } => (route_channel, route_epoch),
7078            other => panic!("expected a route.open response, got {other:?}"),
7079        }
7080    }
7081
7082    /// Reproduction of a production outage. A client had `route.open`s in
7083    /// flight to a module and was already marked closing -- its egress had refused a
7084    /// module frame, so the daemon asked its connection to end -- while its sink
7085    /// was still open. When the module acked those binds, the daemon refused to
7086    /// commit a route for a closing client, and that refusal was returned from
7087    /// the MODULE connection's frame handler, where a router error that has no
7088    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
7089    /// the supervisor correctly did not respawn a clean exit, and every seat lost
7090    /// its tools for hours -- one client's teardown took down a connection
7091    /// carrying ~170 other routes.
7092    ///
7093    /// The window is opened here by calling the production path that opens it
7094    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
7095    /// state that matters is "in `closing_connections`, sink still open, relay
7096    /// still pending", and it lasts only from the close request until the
7097    /// connection loop reacts to it; a socket-level test can flood a client into
7098    /// that escalation but cannot pin the module's ack inside the window. Closing
7099    /// the socket instead takes the other path entirely -- connection teardown
7100    /// removes the pending relay under the same lock, so the ack finds nothing.
7101    #[tokio::test]
7102    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
7103        let registry = Arc::new(Registry::default());
7104        let forwarding = Arc::new(ForwardingTable::default());
7105        let handler =
7106            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
7107
7108        let module_connection = ConnectionId::new(30);
7109        let (module_ctx, mut module_rx) = route_ctx(module_connection);
7110        hello_via_sink(
7111            &handler,
7112            &module_ctx,
7113            &mut module_rx,
7114            hello_frame("aft", PROTOCOL_VERSION, 7),
7115        )
7116        .await;
7117
7118        let dying_client = ConnectionId::new(31);
7119        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
7120
7121        // A published route on the dying client. The escalation below only marks
7122        // a connection closing for a route it has already published.
7123        let (first_task, first_bind) = relay_route_open(
7124            &handler,
7125            dying_client,
7126            &dying_ctx.egress,
7127            &mut module_rx,
7128            100,
7129            "aft",
7130            "closing-first",
7131        )
7132        .await;
7133        handler
7134            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
7135            .await
7136            .unwrap();
7137        assert!(first_task.await.unwrap().is_empty());
7138        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
7139
7140        // A second route.open from the same client, relayed and awaiting its ack.
7141        let (second_task, second_bind) = relay_route_open(
7142            &handler,
7143            dying_client,
7144            &dying_ctx.egress,
7145            &mut module_rx,
7146            101,
7147            "aft",
7148            "closing-second",
7149        )
7150        .await;
7151        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
7152
7153        // The window: the client is closing, its sink is still open, and its
7154        // second bind is still pending.
7155        assert!(forwarding
7156            .escalate_client_delivery_failure(
7157                dying_client,
7158                first_channel,
7159                first_epoch,
7160                CloseReason::new(
7161                    "module_to_client_delivery_failed",
7162                    "client egress refused a module frame",
7163                ),
7164                crate::forwarding::UndeliveredFrame {
7165                    module_id: None,
7166                    sink: &dying_ctx.egress,
7167                },
7168            )
7169            .unwrap());
7170        assert!(!dying_ctx.egress.is_closed());
7171
7172        // The frame that used to end the module connection.
7173        let ack = handler
7174            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
7175            .await;
7176        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
7177        if module_loop_error.is_some() {
7178            // What the server's connection loop does with a router error that has
7179            // no ERROR-frame translation: end the connection, which releases the
7180            // module's registration and every route on it.
7181            handler.cleanup_connection(module_connection).unwrap();
7182        }
7183        // Read the module's next frame before opening the co-tenant's route, so
7184        // the GOODBYE assertion below is about THIS ack and not about later
7185        // traffic. `None` means the module was told nothing.
7186        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7187            .await
7188            .ok()
7189            .flatten();
7190
7191        // 1. The module connection is still registered.
7192        assert!(
7193            registry
7194                .get_module_by_connection(module_connection)
7195                .unwrap()
7196                .is_some(),
7197            "one client's closing connection ended the shared module connection: \
7198             {module_loop_error:?}"
7199        );
7200        // ...and still serving: another client can open and use a route on it.
7201        let cotenant = ConnectionId::new(32);
7202        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
7203        let (cotenant_task, cotenant_bind) = relay_route_open(
7204            &handler,
7205            cotenant,
7206            &cotenant_ctx.egress,
7207            &mut module_rx,
7208            102,
7209            "aft",
7210            "closing-cotenant",
7211        )
7212        .await;
7213        handler
7214            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
7215            .await
7216            .unwrap();
7217        assert!(cotenant_task.await.unwrap().is_empty());
7218        let (cotenant_channel, cotenant_epoch) =
7219            published_route(&cotenant_rx.recv().await.unwrap());
7220        assert!(matches!(
7221            forwarding
7222                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
7223                .unwrap(),
7224            DataRoute::Client(DataRouteState::Bound(_))
7225        ));
7226
7227        // 2. The module was told to drop the binding it created for the route
7228        //    that will never be published.
7229        let goodbye = post_ack_module_frame
7230            .expect("module receives a GOODBYE for the abandoned route channel");
7231        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
7232        assert_eq!(goodbye.header.channel, abandoned_channel);
7233        assert_eq!(goodbye.header.epoch, abandoned_epoch);
7234
7235        // 3. The dying client received nothing: no route was ever published to
7236        //    it. Its route.open is answered as unavailable, which the connection
7237        //    loop would write to a socket that is already going away.
7238        assert!(dying_rx.try_recv().is_err());
7239        let second_response = second_task.await.unwrap();
7240        assert_eq!(second_response.len(), 1);
7241        assert_eq!(
7242            parse_error(&second_response[0])["code"],
7243            "target_unavailable"
7244        );
7245    }
7246
7247    /// The fence at the module-loop boundary, stated as its own contract: which
7248    /// forwarding failures are allowed to end the module connection that is being
7249    /// served. A `ConnectionClosing` naming some client is about that client, and
7250    /// a module connection is shared; the same error naming the module's own
7251    /// connection is about this connection and must stay fatal, as must failures
7252    /// that are about the forwarding table itself.
7253    #[test]
7254    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
7255        let handler = ControlHandler::default();
7256        let module_connection = ConnectionId::new(30);
7257        let client_connection = ConnectionId::new(31);
7258
7259        handler
7260            .refuse_to_end_module_connection_for_a_client(
7261                module_connection,
7262                77,
7263                ForwardingError::ConnectionClosing {
7264                    connection_id: client_connection,
7265                },
7266            )
7267            .expect("a closing client must never end the module connection");
7268
7269        assert!(matches!(
7270            handler.refuse_to_end_module_connection_for_a_client(
7271                module_connection,
7272                78,
7273                ForwardingError::ConnectionClosing {
7274                    connection_id: module_connection,
7275                },
7276            ),
7277            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
7278                connection_id
7279            })) if connection_id == module_connection
7280        ));
7281        assert!(matches!(
7282            handler.refuse_to_end_module_connection_for_a_client(
7283                module_connection,
7284                79,
7285                ForwardingError::Poisoned,
7286            ),
7287            Err(RouterError::Forwarding(ForwardingError::Poisoned))
7288        ));
7289        assert!(matches!(
7290            handler.refuse_to_end_module_connection_for_a_client(
7291                module_connection,
7292                80,
7293                ForwardingError::StaleModuleEndpoint,
7294            ),
7295            Err(RouterError::Forwarding(
7296                ForwardingError::StaleModuleEndpoint
7297            ))
7298        ));
7299    }
7300
7301    /// The spawn-attestation guard is what stops a connected module from claiming
7302    /// another module's identity and being stamped `Reserved` for it. Every other
7303    /// test that supplies a consumer_identity supplies a CORRECT one, because a
7304    /// correct one is what the rest of the flow needs -- so the guard's rejection
7305    /// branch was never the subject of an assertion, only its acceptance branch.
7306    ///
7307    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
7308    /// whole subc-core library suite green; only the forwarding integration tests
7309    /// notice, and they notice for unrelated reasons. This test exists so the
7310    /// refusal itself is asserted where the guard lives: it fails if the identity
7311    /// check stops refusing, which is the direction that matters, since a guard
7312    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
7313    #[tokio::test]
7314    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
7315        let registry = Arc::new(Registry::default());
7316        let forwarding = Arc::new(ForwardingTable::default());
7317        let supervisor = SupervisorHandle::new();
7318        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7319        let handler =
7320            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
7321                .with_supervisor(supervisor);
7322
7323        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
7324        hello_via_sink(
7325            &handler,
7326            &target_ctx,
7327            &mut target_rx,
7328            hello_frame("target", PROTOCOL_VERSION, 1),
7329        )
7330        .await;
7331
7332        // A real supervised module id presenting the wrong nonce. This is the
7333        // impersonation case: the attacker knows a privileged module_id, which is
7334        // public, and guesses at the nonce, which is not.
7335        let wrong_nonce = handler
7336            .handle_control_frame(
7337                &route_ctx(ConnectionId::new(91)).0,
7338                route_open_frame_with_admission_facts(
7339                    20,
7340                    "target",
7341                    unique_project_root("admission-facts"),
7342                    Some(subc_control::ConsumerIdentity {
7343                        module_id: "fed".to_string(),
7344                        launch_nonce: "not-the-real-nonce".to_string(),
7345                    }),
7346                    None,
7347                ),
7348            )
7349            .await
7350            .unwrap();
7351        assert_eq!(
7352            parse_error(&wrong_nonce[0])["code"],
7353            "bad_consumer_identity",
7354            "a mismatched launch nonce must be refused, not stamped Reserved"
7355        );
7356
7357        // A module id the supervisor never spawned at all, so no nonce exists to
7358        // compare against. An implementation that treats "no record" as "nothing
7359        // to check" fails open here while passing the case above.
7360        let never_spawned = handler
7361            .handle_control_frame(
7362                &route_ctx(ConnectionId::new(92)).0,
7363                route_open_frame_with_admission_facts(
7364                    21,
7365                    "target",
7366                    unique_project_root("admission-facts"),
7367                    Some(subc_control::ConsumerIdentity {
7368                        module_id: "never-spawned".to_string(),
7369                        launch_nonce: "any-nonce".to_string(),
7370                    }),
7371                    None,
7372                ),
7373            )
7374            .await
7375            .unwrap();
7376        assert_eq!(
7377            parse_error(&never_spawned[0])["code"],
7378            "bad_consumer_identity",
7379            "an unspawned module_id must be refused rather than accepted for lack of a record"
7380        );
7381    }
7382
7383    /// The refusal test above proves the guard says NO. Nothing proved it can say
7384    /// YES, and the difference is not academic: replacing the whole authorization
7385    /// with `false` -- admitting no consumer identity at all, revoking Reserved
7386    /// standing for every supervised module in the fleet -- leaves 110 of the 111
7387    /// library tests GREEN. The one that notices does so by HANGING, because it
7388    /// waits for a bind that can no longer happen.
7389    ///
7390    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
7391    /// or flaky test, invites a RETRY rather than an investigation, and the retry
7392    /// hangs too and gets blamed on the runner. So a total revocation of the
7393    /// daemon's trust grant would have shipped behind a symptom nobody attributes
7394    /// to code.
7395    ///
7396    /// The bias is structural rather than accidental. A REFUSAL looks like a
7397    /// failure someone writes a test for; a GRANT looks like the happy path. Every
7398    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
7399    /// suite for that reason, and this one is the purest case in the daemon.
7400    ///
7401    /// This test asserts the EFFECT rather than the absence of an error: the module
7402    /// receives a RouteBind and it carries `Reserved` naming the attested module.
7403    /// A guard that admitted nobody would produce no bind at all; one that admitted
7404    /// everybody would stamp the wrong principal, which the refusal test catches.
7405    #[tokio::test]
7406    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
7407        let registry = Arc::new(Registry::default());
7408        let forwarding = Arc::new(ForwardingTable::default());
7409        let supervisor = SupervisorHandle::new();
7410        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7411        let handler =
7412            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
7413                .with_supervisor(supervisor);
7414
7415        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
7416        hello_via_sink(
7417            &handler,
7418            &target_ctx,
7419            &mut target_rx,
7420            hello_frame("target", PROTOCOL_VERSION, 1),
7421        )
7422        .await;
7423
7424        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
7425        let route_handler = handler.clone();
7426        let route_task = tokio::spawn(async move {
7427            route_handler
7428                .handle_control_frame(
7429                    &client_ctx,
7430                    route_open_frame_with_admission_facts(
7431                        30,
7432                        "target",
7433                        unique_project_root("admission-facts"),
7434                        Some(subc_control::ConsumerIdentity {
7435                            module_id: "fed".to_string(),
7436                            launch_nonce: "fed-nonce".to_string(),
7437                        }),
7438                        None,
7439                    ),
7440                )
7441                .await
7442                .unwrap()
7443        });
7444
7445        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
7446        // the very mutation it exists to catch -- a guard that admits nobody -- no
7447        // bind is ever sent, so it HUNG rather than failing. That reproduces the
7448        // exact defect being fixed: a total revocation detected only as a stalled
7449        // suite, which reads as flakiness and invites a retry. An acceptance test
7450        // that waits for an effect must bound the wait, or a red becomes a hang.
7451        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7452            .await
7453            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
7454            .expect("module control channel closed before route.bind");
7455        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
7456        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
7457            panic!("expected route.bind")
7458        };
7459        assert_eq!(
7460            principal,
7461            Some(Principal::Reserved {
7462                module_id: "fed".to_string()
7463            }),
7464            "a correctly attested consumer must be stamped Reserved for its own id"
7465        );
7466
7467        handler
7468            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
7469            .await
7470            .unwrap();
7471        assert!(route_task.await.unwrap().is_empty());
7472        assert!(
7473            matches!(
7474                serde_json::from_slice::<ClientControlResponse>(
7475                    &client_rx.recv().await.unwrap().body
7476                )
7477                .unwrap(),
7478                ClientControlResponse::RouteOpen { .. }
7479            ),
7480            "the route must actually open, not merely avoid an error"
7481        );
7482    }
7483
7484    #[tokio::test(start_paused = true)]
7485    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
7486        let registry = Arc::new(Registry::default());
7487        let forwarding = Arc::new(ForwardingTable::default());
7488        let supervisor = SupervisorHandle::new();
7489        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7490        let handler =
7491            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
7492                .with_supervisor(supervisor);
7493
7494        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
7495        hello_via_sink(
7496            &handler,
7497            &target_ctx,
7498            &mut target_rx,
7499            hello_frame("target", PROTOCOL_VERSION, 1),
7500        )
7501        .await;
7502
7503        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
7504        let direct_handler = handler.clone();
7505        let direct_open = tokio::spawn(async move {
7506            direct_handler
7507                .handle_control_frame(
7508                    &direct_ctx,
7509                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
7510                )
7511                .await
7512                .unwrap()
7513        });
7514        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7515            .await
7516            .expect("no direct route.bind within 5s")
7517            .expect("target control channel closed before direct route.bind");
7518        handler
7519            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
7520            .await
7521            .unwrap();
7522        assert!(direct_open.await.unwrap().is_empty());
7523        let _ = direct_rx.recv().await.unwrap();
7524
7525        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
7526        let reserved_handler = handler.clone();
7527        let reserved_open = tokio::spawn(async move {
7528            reserved_handler
7529                .handle_control_frame(
7530                    &reserved_ctx,
7531                    route_open_frame_with_admission_facts(
7532                        3,
7533                        "target",
7534                        unique_project_root("admission-facts"),
7535                        Some(ConsumerIdentity {
7536                            module_id: "fed".to_string(),
7537                            launch_nonce: "fed-nonce".to_string(),
7538                        }),
7539                        None,
7540                    ),
7541                )
7542                .await
7543                .unwrap()
7544        });
7545        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7546            .await
7547            .expect("no reserved route.bind within 5s")
7548            .expect("target control channel closed before reserved route.bind");
7549        handler
7550            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
7551            .await
7552            .unwrap();
7553        assert!(reserved_open.await.unwrap().is_empty());
7554        let _ = reserved_rx.recv().await.unwrap();
7555
7556        forwarding
7557            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
7558            .unwrap();
7559        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
7560        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
7561            module_id: Some("target".to_string()),
7562        })
7563        .unwrap();
7564        let census_frame =
7565            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
7566        let response = handler
7567            .handle_control_frame(&census_ctx, census_frame)
7568            .await
7569            .unwrap()
7570            .pop()
7571            .unwrap();
7572        let actual: Value = serde_json::from_slice(&response.body).unwrap();
7573        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
7574        assert!(matches!(
7575            decoded,
7576            ClientControlResponse::SupervisorRoutes { .. }
7577        ));
7578        let routes = actual["modules"][0]["routes"].as_array().unwrap();
7579        assert_eq!(routes.len(), 2);
7580        assert!(routes.iter().all(|route| route["draining"] == true));
7581        // The census carries WHY: the reason the drain was begun with, in the
7582        // route.closing vocabulary, on every draining route this drain marked.
7583        assert!(
7584            routes.iter().all(|route| route["drain_reason"] == "reload"),
7585            "draining routes must name the drain's reason: {routes:?}"
7586        );
7587        assert!(routes.iter().any(|route| {
7588            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
7589        }));
7590        assert!(routes.iter().any(|route| {
7591            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
7592        }));
7593
7594        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7595            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
7596        if std::env::var_os("UPDATE_GOLDEN").is_some() {
7597            std::fs::write(
7598                &golden_path,
7599                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
7600            )
7601            .unwrap();
7602        }
7603        let expected: Value =
7604            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
7605        assert_eq!(actual, expected);
7606    }
7607
7608    async fn query_live_roots(
7609        handler: &ControlHandler,
7610        module_ctx: &RouteCtx,
7611    ) -> ModuleControlResponseToModule {
7612        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
7613        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
7614        let response = handler
7615            .handle_control_frame(module_ctx, frame)
7616            .await
7617            .unwrap()
7618            .pop()
7619            .unwrap();
7620        serde_json::from_slice(&response.body).unwrap()
7621    }
7622
7623    #[tokio::test(start_paused = true)]
7624    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
7625        let registry = Arc::new(Registry::default());
7626        let forwarding = Arc::new(ForwardingTable::default());
7627        let handler = ControlHandler::with_forwarding(registry, forwarding);
7628        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
7629        hello_via_sink(
7630            &handler,
7631            &target_ctx,
7632            &mut target_rx,
7633            hello_frame("target", PROTOCOL_VERSION, 1),
7634        )
7635        .await;
7636        let root = unique_project_root("live-roots-known");
7637        let path = ProjectRootId::from_path_allowing_missing(root.path())
7638            .unwrap()
7639            .as_path()
7640            .to_path_buf();
7641        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
7642        let open_handler = handler.clone();
7643        let opened = tokio::spawn(async move {
7644            open_handler
7645                .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
7646                .await
7647                .unwrap()
7648        });
7649        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7650            .await
7651            .unwrap()
7652            .unwrap();
7653        handler
7654            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
7655            .await
7656            .unwrap();
7657        assert!(opened.await.unwrap().is_empty());
7658        let _ = client_rx.recv().await.unwrap();
7659
7660        let root = unique_project_root("live-roots-pending");
7661        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
7662            .unwrap()
7663            .as_path()
7664            .to_path_buf();
7665        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
7666        let open_handler = handler.clone();
7667        let pending = tokio::spawn(async move {
7668            open_handler
7669                .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
7670                .await
7671                .unwrap()
7672        });
7673        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7674            .await
7675            .unwrap()
7676            .unwrap();
7677        let actual = query_live_roots(&handler, &target_ctx).await;
7678        let ModuleControlResponseToModule::LiveRoots {
7679            roots,
7680            unknown_root_bindings,
7681            total_bindings,
7682        } = actual
7683        else {
7684            panic!("expected live roots")
7685        };
7686        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
7687        assert_eq!(unknown_root_bindings, 0);
7688        assert_eq!(
7689            roots.len(),
7690            2,
7691            "root-known arm must retain each canonical root"
7692        );
7693        assert_eq!(
7694            total_bindings,
7695            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7696        );
7697        let counts = roots
7698            .iter()
7699            .map(|root| (root.project_root.clone(), root.bound, root.pending))
7700            .collect::<Vec<_>>();
7701        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
7702        expected.sort_by(|a, b| a.0.cmp(&b.0));
7703        assert_eq!(
7704            counts, expected,
7705            "roots must sort by path and count pending separately"
7706        );
7707        handler
7708            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
7709            .await
7710            .unwrap();
7711        assert!(pending.await.unwrap().is_empty());
7712    }
7713
7714    #[tokio::test(start_paused = true)]
7715    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
7716        let forwarding = Arc::new(ForwardingTable::default());
7717        let handler =
7718            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
7719        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
7720        hello_via_sink(
7721            &handler,
7722            &target_ctx,
7723            &mut target_rx,
7724            hello_frame("target", PROTOCOL_VERSION, 1),
7725        )
7726        .await;
7727        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
7728        let pending = forwarding
7729            .begin_route_bind_relay_for_test(
7730                client_ctx.connection_id,
7731                client_ctx.egress.clone(),
7732                2,
7733                "target",
7734            )
7735            .unwrap();
7736        forwarding
7737            .complete_pending_relay(
7738                target_ctx.connection_id,
7739                pending.corr,
7740                RouteBindRelayOutcome::Accepted,
7741            )
7742            .unwrap();
7743        let actual = query_live_roots(&handler, &target_ctx).await;
7744        let ModuleControlResponseToModule::LiveRoots {
7745            roots,
7746            unknown_root_bindings,
7747            total_bindings,
7748        } = actual
7749        else {
7750            panic!("expected live roots")
7751        };
7752        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
7753        assert_eq!(
7754            unknown_root_bindings, 1,
7755            "unknown-root arm must not read as no bindings"
7756        );
7757        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
7758        assert_eq!(
7759            total_bindings,
7760            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7761        );
7762    }
7763
7764    /// A module reads HELLO_ACK as its first frame and exits on anything else,
7765    /// so the ack has to be on its outbound queue before the module is
7766    /// routable. The connection loop writes a handler's replies only after the
7767    /// handler returns; this test stops in exactly that gap, runs a real
7768    /// route.open from another connection, and only then writes whatever the
7769    /// HELLO handler returned, the way the loop would. If the ack were still a
7770    /// reply, the route.bind request would reach the module first.
7771    #[tokio::test(start_paused = true)]
7772    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
7773        let forwarding = Arc::new(ForwardingTable::default());
7774        let handler =
7775            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
7776        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
7777        let replies = handler
7778            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
7779            .await
7780            .unwrap();
7781        let queued_by_hello = module_rx.len();
7782
7783        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
7784        let open_handler = handler.clone();
7785        let open = tokio::spawn(async move {
7786            open_handler
7787                .handle_control_frame(
7788                    &client_ctx,
7789                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
7790                )
7791                .await
7792                .unwrap()
7793        });
7794        // Let the route.open run until its route.bind is on the module's queue.
7795        let mut spins = 0;
7796        while module_rx.len() == queued_by_hello {
7797            spins += 1;
7798            assert!(spins < 10_000, "route.open never queued a route.bind");
7799            tokio::task::yield_now().await;
7800        }
7801
7802        // Now the connection loop's half: write the HELLO handler's replies.
7803        for reply in replies {
7804            module_ctx.egress.send(reply).await.unwrap();
7805        }
7806
7807        let first = module_rx.recv().await.unwrap().frame;
7808        assert_eq!(
7809            first.header.ty,
7810            FrameType::HelloAck,
7811            "the first frame a registering module reads must be its HELLO_ACK"
7812        );
7813        assert_eq!(first.header.corr, 7);
7814        let second = module_rx.recv().await.unwrap().frame;
7815        assert_eq!(second.header.ty, FrameType::Request);
7816        assert!(
7817            matches!(
7818                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
7819                ModuleControlRequest::RouteBind { .. }
7820            ),
7821            "the route.bind follows the ack"
7822        );
7823        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
7824
7825        handler
7826            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
7827            .await
7828            .unwrap();
7829        assert!(open.await.unwrap().is_empty());
7830        let _ = client_rx.recv().await.unwrap();
7831    }
7832
7833    #[tokio::test(start_paused = true)]
7834    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
7835        let handler = ControlHandler::with_forwarding(
7836            Arc::new(Registry::default()),
7837            Arc::new(ForwardingTable::default()),
7838        );
7839        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
7840        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
7841        hello_via_sink(
7842            &handler,
7843            &first_ctx,
7844            &mut first_rx,
7845            hello_frame("first", PROTOCOL_VERSION, 1),
7846        )
7847        .await;
7848        hello_via_sink(
7849            &handler,
7850            &second_ctx,
7851            &mut second_rx,
7852            hello_frame("second", PROTOCOL_VERSION, 2),
7853        )
7854        .await;
7855        let root = unique_project_root("second-only");
7856        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
7857        let cloned = handler.clone();
7858        let open = tokio::spawn(async move {
7859            cloned
7860                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
7861                .await
7862                .unwrap()
7863        });
7864        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
7865            .await
7866            .unwrap()
7867            .unwrap();
7868        let first = query_live_roots(&handler, &first_ctx).await;
7869        let second = query_live_roots(&handler, &second_ctx).await;
7870        assert!(
7871            matches!(
7872                first,
7873                ModuleControlResponseToModule::LiveRoots {
7874                    total_bindings: 0,
7875                    ..
7876                }
7877            ),
7878            "cross-module scope must not expose another module's roots"
7879        );
7880        assert!(
7881            matches!(
7882                second,
7883                ModuleControlResponseToModule::LiveRoots {
7884                    total_bindings: 1,
7885                    ..
7886                }
7887            ),
7888            "second module must see its pending route"
7889        );
7890        handler
7891            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
7892            .await
7893            .unwrap();
7894        assert!(open.await.unwrap().is_empty());
7895    }
7896
7897    #[tokio::test(start_paused = true)]
7898    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
7899        let handler = ControlHandler::with_forwarding(
7900            Arc::new(Registry::default()),
7901            Arc::new(ForwardingTable::default()),
7902        );
7903        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
7904        hello_via_sink(
7905            &handler,
7906            &target_ctx,
7907            &mut target_rx,
7908            hello_frame("target", PROTOCOL_VERSION, 1),
7909        )
7910        .await;
7911        let actual = query_live_roots(&handler, &target_ctx).await;
7912        let ModuleControlResponseToModule::LiveRoots {
7913            roots,
7914            unknown_root_bindings,
7915            total_bindings,
7916        } = actual
7917        else {
7918            panic!("expected live roots")
7919        };
7920        assert!(roots.is_empty());
7921        assert_eq!(unknown_root_bindings, 0);
7922        assert_eq!(total_bindings, 0);
7923        assert_eq!(
7924            total_bindings,
7925            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7926        );
7927    }
7928
7929    /// Read the vendored fed corpus rather than hand-building a package.
7930    ///
7931    /// A hand-built object encodes what the test author believed the carrier
7932    /// emits. These vectors are what it actually emits, and one of them exists
7933    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
7934    /// ignores additive unknown fields at the traversal emit terminus."
7935    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
7936        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7937            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
7938        let text = std::fs::read_to_string(&path)
7939            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
7940        let vectors: Vec<(String, Value)> = text
7941            .lines()
7942            .filter(|line| !line.trim().is_empty())
7943            .map(|line| {
7944                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
7945                let id = entry["corpus_id"]
7946                    .as_str()
7947                    .expect("every vector carries a corpus_id")
7948                    .to_string();
7949                (id, entry["package"].clone())
7950            })
7951            .collect();
7952        // Pin the count: a corpus that silently shrinks would take its coverage
7953        // with it, and a suite reading N-1 vectors reports the same clean pass
7954        // as one reading N.
7955        assert_eq!(
7956            vectors.len(),
7957            3,
7958            "vendored fed corpus changed size; re-sync from subc-federation"
7959        );
7960
7961        // Pin what makes the corpus DISCRIMINATING, not just present.
7962        //
7963        // The relay test below takes its expected value from the corpus, so the
7964        // corpus supplies the test's power to detect a lossy relay rather than
7965        // its correctness. A relay that dropped unrecognised fields would still
7966        // be caught -- but only by a package carrying fields it does not know.
7967        // Shrink every package to the handful of keys any implementation would
7968        // recognise and the test keeps passing over an input that can no longer
7969        // fail, which is the same clean green as a corpus that shrank away.
7970        //
7971        // So assert the precondition rather than duplicating the packages here:
7972        // at least one vector must carry a field beyond the small common set.
7973        // That is one claim to maintain instead of nine, and it fails loudly if
7974        // a re-sync ever flattens the corpus.
7975        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
7976        let richest = vectors
7977            .iter()
7978            .filter_map(|(_, package)| package.as_object())
7979            .map(|object| {
7980                object
7981                    .keys()
7982                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
7983                    .count()
7984            })
7985            .max()
7986            .unwrap_or(0);
7987        assert!(
7988            richest >= 2,
7989            "vendored corpus no longer carries a package with unmodelled fields, \
7990             so the relay test can no longer distinguish a verbatim relay from a lossy one"
7991        );
7992
7993        vectors
7994    }
7995
7996    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
7997    ///
7998    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
7999    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
8000    /// three-key object, so a relay that quietly dropped fields it did not
8001    /// recognise would satisfy it. These vectors carry nine keys including ones
8002    /// this crate has no type for, so a typed relay fails here and only here.
8003    #[tokio::test]
8004    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
8005        for (corpus_id, package) in fed_admission_facts_vectors() {
8006            let registry = Arc::new(Registry::default());
8007            let forwarding = Arc::new(ForwardingTable::default());
8008            let supervisor = SupervisorHandle::new();
8009            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8010            let handler =
8011                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8012                    .with_supervisor(supervisor)
8013                    .with_admission_facts_config(
8014                        Some("fed".to_string()),
8015                        Some(vec!["target".to_string()]),
8016                    );
8017
8018            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
8019            hello_via_sink(
8020                &handler,
8021                &target_ctx,
8022                &mut target_rx,
8023                hello_frame("target", PROTOCOL_VERSION, 1),
8024            )
8025            .await;
8026
8027            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
8028            let route_handler = handler.clone();
8029            let expected = package.clone();
8030            let route_task = tokio::spawn(async move {
8031                route_handler
8032                    .handle_control_frame(
8033                        &client_ctx,
8034                        route_open_frame_with_admission_facts(
8035                            20,
8036                            "target",
8037                            unique_project_root("admission-facts"),
8038                            Some(subc_control::ConsumerIdentity {
8039                                module_id: "fed".to_string(),
8040                                launch_nonce: "fed-nonce".to_string(),
8041                            }),
8042                            Some(package),
8043                        ),
8044                    )
8045                    .await
8046                    .unwrap()
8047            });
8048
8049            let bind_frame = target_rx.recv().await.unwrap();
8050            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8051            let ModuleControlRequest::RouteBind {
8052                admission_facts, ..
8053            } = bind
8054            else {
8055                panic!("{corpus_id}: expected route.bind")
8056            };
8057            assert_eq!(
8058                admission_facts,
8059                Some(expected),
8060                "{corpus_id}: relay must not add, drop or reshape any field"
8061            );
8062
8063            handler
8064                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8065                .await
8066                .unwrap();
8067            route_task.await.unwrap();
8068        }
8069    }
8070
8071    #[tokio::test]
8072    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
8073        let registry = Arc::new(Registry::default());
8074        let forwarding = Arc::new(ForwardingTable::default());
8075        let supervisor = SupervisorHandle::new();
8076        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8077        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
8078        let handler =
8079            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8080                .with_supervisor(supervisor)
8081                .with_admission_facts_config(
8082                    Some("fed".to_string()),
8083                    Some(vec!["target".to_string()]),
8084                );
8085
8086        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
8087        hello_via_sink(
8088            &handler,
8089            &target_ctx,
8090            &mut target_rx,
8091            hello_frame("target", PROTOCOL_VERSION, 1),
8092        )
8093        .await;
8094        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
8095        hello_via_sink(
8096            &handler,
8097            &other_ctx,
8098            &mut other_rx,
8099            hello_frame("other", PROTOCOL_VERSION, 2),
8100        )
8101        .await;
8102
8103        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
8104        let expected_facts = facts.clone();
8105        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
8106        let route_handler = handler.clone();
8107        let route_task = tokio::spawn(async move {
8108            route_handler
8109                .handle_control_frame(
8110                    &client_ctx,
8111                    route_open_frame_with_admission_facts(
8112                        10,
8113                        "target",
8114                        unique_project_root("admission-facts"),
8115                        Some(subc_control::ConsumerIdentity {
8116                            module_id: "fed".to_string(),
8117                            launch_nonce: "fed-nonce".to_string(),
8118                        }),
8119                        Some(facts.clone()),
8120                    ),
8121                )
8122                .await
8123                .unwrap()
8124        });
8125        let bind_frame = target_rx.recv().await.unwrap();
8126        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8127        let ModuleControlRequest::RouteBind {
8128            admission_facts, ..
8129        } = bind
8130        else {
8131            panic!("expected route.bind")
8132        };
8133        assert_eq!(admission_facts, Some(expected_facts));
8134        handler
8135            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8136            .await
8137            .unwrap();
8138        assert!(route_task.await.unwrap().is_empty());
8139        assert!(matches!(
8140            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
8141                .unwrap(),
8142            ClientControlResponse::RouteOpen { .. }
8143        ));
8144
8145        let direct = handler
8146            .handle_control_frame(
8147                &route_ctx(ConnectionId::new(73)).0,
8148                route_open_frame_with_admission_facts(
8149                    11,
8150                    "target",
8151                    unique_project_root("admission-facts"),
8152                    None,
8153                    Some(json!({"x": 1})),
8154                ),
8155            )
8156            .await
8157            .unwrap();
8158        assert_eq!(
8159            parse_error(&direct[0])["code"],
8160            "admission_facts_not_permitted"
8161        );
8162
8163        let different_reserved = handler
8164            .handle_control_frame(
8165                &route_ctx(ConnectionId::new(77)).0,
8166                route_open_frame_with_admission_facts(
8167                    15,
8168                    "target",
8169                    unique_project_root("admission-facts"),
8170                    Some(subc_control::ConsumerIdentity {
8171                        module_id: "other".to_string(),
8172                        launch_nonce: "other-nonce".to_string(),
8173                    }),
8174                    Some(json!({"x": 1})),
8175                ),
8176            )
8177            .await
8178            .unwrap();
8179        assert_eq!(
8180            parse_error(&different_reserved[0])["code"],
8181            "admission_facts_not_permitted"
8182        );
8183
8184        let other_target = handler
8185            .handle_control_frame(
8186                &route_ctx(ConnectionId::new(74)).0,
8187                route_open_frame_with_admission_facts(
8188                    12,
8189                    "other",
8190                    unique_project_root("admission-facts"),
8191                    Some(subc_control::ConsumerIdentity {
8192                        module_id: "fed".to_string(),
8193                        launch_nonce: "fed-nonce".to_string(),
8194                    }),
8195                    Some(json!({"x": 1})),
8196                ),
8197            )
8198            .await
8199            .unwrap();
8200        assert_eq!(
8201            parse_error(&other_target[0])["code"],
8202            "admission_facts_target_not_allowed"
8203        );
8204
8205        let nonexistent = handler
8206            .handle_control_frame(
8207                &route_ctx(ConnectionId::new(75)).0,
8208                route_open_frame_with_admission_facts(
8209                    13,
8210                    "missing",
8211                    unique_project_root("admission-facts"),
8212                    None,
8213                    Some(json!({"x": 1})),
8214                ),
8215            )
8216            .await
8217            .unwrap();
8218        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
8219
8220        let described = handler
8221            .handle_control_frame(
8222                &route_ctx(ConnectionId::new(76)).0,
8223                Frame::build(
8224                    FrameType::Request,
8225                    control_flags(),
8226                    0,
8227                    0,
8228                    14,
8229                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8230                )
8231                .unwrap(),
8232            )
8233            .await
8234            .unwrap();
8235        let ClientControlResponse::ServerDescribe { capabilities, .. } =
8236            serde_json::from_slice(&described[0].body).unwrap()
8237        else {
8238            panic!("expected server.describe response")
8239        };
8240        assert!(capabilities
8241            .iter()
8242            .any(|cap| cap == "admission_facts_relay_v1"));
8243    }
8244
8245    #[tokio::test]
8246    async fn admission_facts_without_configured_carrier_are_rejected() {
8247        let registry = Arc::new(Registry::default());
8248        let forwarding = Arc::new(ForwardingTable::default());
8249        let handler = ControlHandler::with_forwarding(registry, forwarding);
8250        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
8251        hello_via_sink(
8252            &handler,
8253            &target_ctx,
8254            &mut target_rx,
8255            hello_frame("target", PROTOCOL_VERSION, 1),
8256        )
8257        .await;
8258
8259        let responses = handler
8260            .handle_control_frame(
8261                &route_ctx(ConnectionId::new(79)).0,
8262                route_open_frame_with_admission_facts(
8263                    16,
8264                    "target",
8265                    unique_project_root("admission-facts"),
8266                    None,
8267                    Some(json!({"x": 1})),
8268                ),
8269            )
8270            .await
8271            .unwrap();
8272        assert_eq!(
8273            parse_error(&responses[0])["code"],
8274            "admission_facts_not_permitted"
8275        );
8276    }
8277
8278    #[tokio::test]
8279    async fn route_open_relays_consumer_capabilities_verbatim() {
8280        let registry = Arc::new(Registry::default());
8281        let forwarding = Arc::new(ForwardingTable::default());
8282        let handler =
8283            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
8284        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
8285        hello_via_sink(
8286            &handler,
8287            &module_ctx,
8288            &mut module_rx,
8289            hello_frame("aft", PROTOCOL_VERSION, 7),
8290        )
8291        .await;
8292
8293        let expected = vec!["elicitation".to_string(), "roots".to_string()];
8294        let expected_for_request = expected.clone();
8295        let project_root = unique_project_root("consumer-capabilities-present");
8296        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
8297        let route_handler = handler.clone();
8298        let route_task = tokio::spawn(async move {
8299            route_handler
8300                .handle_control_frame(
8301                    &client_ctx,
8302                    route_open_frame_with_consumer_capabilities(
8303                        401,
8304                        "aft",
8305                        project_root,
8306                        Some(expected_for_request),
8307                    ),
8308                )
8309                .await
8310                .unwrap()
8311        });
8312        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8313            .await
8314            .unwrap()
8315            .unwrap();
8316        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8317        let ModuleControlRequest::RouteBind {
8318            consumer_capabilities,
8319            ..
8320        } = bind
8321        else {
8322            panic!("expected route.bind request, got {bind:?}");
8323        };
8324        assert_eq!(consumer_capabilities, Some(expected.clone()));
8325
8326        handler
8327            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8328            .await
8329            .unwrap();
8330        let route_response = route_task.await.unwrap();
8331        assert!(route_response.is_empty());
8332        let published = client_rx.recv().await.unwrap();
8333        assert!(matches!(
8334            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8335            ClientControlResponse::RouteOpen { .. }
8336        ));
8337    }
8338
8339    #[tokio::test]
8340    async fn route_open_without_consumer_capabilities_relays_none() {
8341        let registry = Arc::new(Registry::default());
8342        let forwarding = Arc::new(ForwardingTable::default());
8343        let handler =
8344            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
8345        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
8346        hello_via_sink(
8347            &handler,
8348            &module_ctx,
8349            &mut module_rx,
8350            hello_frame("aft", PROTOCOL_VERSION, 7),
8351        )
8352        .await;
8353
8354        let project_root = unique_project_root("consumer-capabilities-absent");
8355        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
8356        let route_handler = handler.clone();
8357        let route_task = tokio::spawn(async move {
8358            route_handler
8359                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
8360                .await
8361                .unwrap()
8362        });
8363        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8364            .await
8365            .unwrap()
8366            .unwrap();
8367        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8368        let ModuleControlRequest::RouteBind {
8369            consumer_capabilities,
8370            ..
8371        } = bind
8372        else {
8373            panic!("expected route.bind request, got {bind:?}");
8374        };
8375        assert_eq!(consumer_capabilities, None);
8376
8377        handler
8378            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8379            .await
8380            .unwrap();
8381        let route_response = route_task.await.unwrap();
8382        assert!(route_response.is_empty());
8383        let published = client_rx.recv().await.unwrap();
8384        assert!(matches!(
8385            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8386            ClientControlResponse::RouteOpen { .. }
8387        ));
8388    }
8389
8390    #[tokio::test]
8391    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
8392        let registry = Arc::new(Registry::default());
8393        let forwarding = Arc::new(ForwardingTable::default());
8394        let handler =
8395            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8396                .with_health_probe_timeout(Duration::from_secs(5));
8397        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
8398        hello_via_sink(
8399            &handler,
8400            &module_ctx,
8401            &mut module_rx,
8402            non_routable_hello_frame_with_control_ops(
8403                "mcp",
8404                300,
8405                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8406            ),
8407        )
8408        .await;
8409        assert!(registry
8410            .get_module("mcp")
8411            .unwrap()
8412            .unwrap()
8413            .manifest
8414            .provides
8415            .is_empty());
8416
8417        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
8418        let route_response = handler
8419            .handle_control_frame(
8420                &route_client_ctx,
8421                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
8422            )
8423            .await
8424            .unwrap();
8425        assert_eq!(route_response[0].header.ty, FrameType::Error);
8426        assert_eq!(
8427            parse_error(&route_response[0])["code"],
8428            "target_unavailable"
8429        );
8430        assert!(parse_error(&route_response[0])["message"]
8431            .as_str()
8432            .unwrap()
8433            .contains("does not provide the requested target"));
8434        assert!(module_rx.try_recv().is_err());
8435
8436        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
8437        let health_handler = handler.clone();
8438        let health_task = tokio::spawn(async move {
8439            health_handler
8440                .handle_control_frame(
8441                    &health_client_ctx,
8442                    supervisor_health_probe_frame(302, "mcp"),
8443                )
8444                .await
8445                .unwrap()
8446        });
8447        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8448            .await
8449            .unwrap()
8450            .unwrap();
8451        assert_eq!(
8452            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
8453            ModuleControlRequest::HealthCheck {}
8454        );
8455        handler
8456            .handle_control_frame(
8457                &module_ctx,
8458                health_response(health_frame.header.corr, HealthStatus::Ok),
8459            )
8460            .await
8461            .unwrap();
8462        let health_response = health_task.await.unwrap();
8463        assert_eq!(health_response[0].header.ty, FrameType::Response);
8464        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
8465            ClientControlResponse::SupervisorHealthProbe {
8466                module_id, status, ..
8467            } => {
8468                assert_eq!(module_id, "mcp");
8469                assert_eq!(status, HealthStatus::Ok);
8470            }
8471            other => panic!("unexpected health response: {other:?}"),
8472        }
8473
8474        // Exercise the forwarding cleanup path directly while leaving the registry
8475        // advertisement in place. If cleanup leaves a stale control sink behind,
8476        // the next probe will enqueue onto it and wait for the long probe timeout
8477        // instead of returning an immediate no-connection error.
8478        forwarding
8479            .cleanup_connection(module_ctx.connection_id)
8480            .unwrap();
8481        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
8482        let cleanup_response = tokio::time::timeout(
8483            Duration::from_millis(200),
8484            handler.handle_control_frame(
8485                &cleanup_probe_ctx,
8486                supervisor_health_probe_frame(303, "mcp"),
8487            ),
8488        )
8489        .await
8490        .expect("probe should fail immediately when the control lane is gone")
8491        .unwrap();
8492        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
8493        assert_eq!(
8494            parse_error(&cleanup_response[0])["code"],
8495            "target_unavailable"
8496        );
8497        assert!(parse_error(&cleanup_response[0])["message"]
8498            .as_str()
8499            .unwrap()
8500            .contains("no module connection"));
8501
8502        handler
8503            .cleanup_connection(module_ctx.connection_id)
8504            .unwrap();
8505    }
8506
8507    #[tokio::test]
8508    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
8509        let registry = Arc::new(Registry::default());
8510        let supervisor_handle = SupervisorHandle::new();
8511        let supervisor =
8512            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
8513                .with_handle(supervisor_handle.clone())
8514                .with_connection_file_path(
8515                    std::env::temp_dir()
8516                        .join(format!("subc-route-open-warming-{}", std::process::id())),
8517                );
8518        let module = supervisor
8519            .supervise_configured(
8520                ModuleSpec {
8521                    module_id: "warming".to_string(),
8522                    program: fake_aft_stub_path(),
8523                    args: Vec::new(),
8524                    env: Vec::new(),
8525                    reserved: false,
8526                    reserved_prefixes: Vec::new(),
8527                    protocol: ModuleProtocol::Subc,
8528                    overlap: Default::default(),
8529                },
8530                true,
8531            )
8532            .unwrap();
8533        assert_eq!(module.state().unwrap(), ModuleState::Running);
8534
8535        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8536        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
8537        let response = handler
8538            .handle_control_frame(
8539                &ctx,
8540                route_open_frame(304, "warming", unique_project_root("warming")),
8541            )
8542            .await
8543            .unwrap();
8544        module.stop().await.unwrap();
8545
8546        assert_eq!(response[0].header.ty, FrameType::Error);
8547        let error = parse_error(&response[0]);
8548        assert_eq!(error["code"], "module_warming");
8549        assert!(error["message"]
8550            .as_str()
8551            .unwrap()
8552            .contains("state=running, enabled=true, live=false"));
8553    }
8554
8555    /// One wire code has several senders, so the refusal line names the check
8556    /// that refused. This drives the shared refusal path (every non-supervised
8557    /// refusal goes through `route_open_refusal_frame`) with an unregistered
8558    /// target and requires the branch label on the event.
8559    #[tokio::test]
8560    async fn route_open_refusal_names_the_check_that_refused() {
8561        let handler = ControlHandler::new(Arc::new(Registry::default()));
8562        let capture = EventCapture::default();
8563        let _subscriber =
8564            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8565        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
8566        let response = handler
8567            .handle_control_frame(
8568                &ctx,
8569                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
8570            )
8571            .await
8572            .unwrap();
8573
8574        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
8575        let event = capture
8576            .events()
8577            .into_iter()
8578            .find(|event| {
8579                event.target == "control"
8580                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
8581            })
8582            .expect("route.open refusal event");
8583        assert_eq!(
8584            event.fields.get("reason"),
8585            Some(&"\"not_registered\"".to_string())
8586        );
8587    }
8588
8589    #[tokio::test]
8590    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
8591        let registry = Arc::new(Registry::default());
8592        let supervisor_handle = SupervisorHandle::new();
8593        let supervisor =
8594            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
8595                .with_handle(supervisor_handle.clone())
8596                .with_connection_file_path(std::env::temp_dir().join(format!(
8597                    "subc-route-open-refusal-info-{}",
8598                    std::process::id()
8599                )));
8600        let module = supervisor
8601            .supervise_configured(
8602                ModuleSpec {
8603                    module_id: "warming".to_string(),
8604                    program: fake_aft_stub_path(),
8605                    args: Vec::new(),
8606                    env: Vec::new(),
8607                    reserved: false,
8608                    reserved_prefixes: Vec::new(),
8609                    protocol: ModuleProtocol::Subc,
8610                    overlap: Default::default(),
8611                },
8612                true,
8613            )
8614            .unwrap();
8615        assert_eq!(module.state().unwrap(), ModuleState::Running);
8616
8617        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8618        assert!(handler
8619            .counters()
8620            .snapshot()
8621            .get("route_open_refused_by_code")
8622            .is_none());
8623        let capture = EventCapture::default();
8624        let _subscriber =
8625            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8626        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
8627        let response = handler
8628            .handle_control_frame(
8629                &ctx,
8630                route_open_frame(394, "warming", unique_project_root("refusal-info")),
8631            )
8632            .await
8633            .unwrap();
8634        module.stop().await.unwrap();
8635
8636        assert_eq!(parse_error(&response[0])["code"], "module_warming");
8637        let event = capture
8638            .events()
8639            .into_iter()
8640            .find(|event| {
8641                event.target == "control"
8642                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
8643            })
8644            .expect("route.open refusal event");
8645        assert_eq!(
8646            event.fields.get("module_id"),
8647            Some(&"\"warming\"".to_string())
8648        );
8649        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
8650        assert_eq!(
8651            event.fields.get("reason"),
8652            Some(&"\"supervised_not_registered\"".to_string())
8653        );
8654        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
8655        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
8656        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
8657        assert_eq!(
8658            handler.counters().snapshot()["route_open_refused_by_code"],
8659            json!({ "module_warming": 1 })
8660        );
8661    }
8662
8663    #[tokio::test(flavor = "current_thread")]
8664    async fn route_open_unknown_module_escapes_target_module_id() {
8665        let handler = ControlHandler::new(Arc::new(Registry::default()));
8666        let capture = EventCapture::default();
8667        let _subscriber =
8668            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8669        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
8670        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
8671        let response = handler
8672            .handle_control_frame(
8673                &ctx,
8674                route_open_frame(
8675                    395,
8676                    hostile_module_id,
8677                    unique_project_root("hostile-target-module-id"),
8678                ),
8679            )
8680            .await
8681            .unwrap();
8682
8683        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
8684        let event = capture
8685            .events()
8686            .into_iter()
8687            .find(|event| {
8688                event.target == "control"
8689                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
8690            })
8691            .expect("route.open unknown-module refusal event");
8692        let logged = event.fields.get("module_id").expect("module_id field");
8693        assert!(!logged.bytes().any(|byte| byte < 0x20));
8694        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
8695    }
8696
8697    #[tokio::test(flavor = "current_thread")]
8698    async fn route_open_module_rejection_uses_daemon_counter_key() {
8699        let registry = Arc::new(Registry::default());
8700        let forwarding = Arc::new(ForwardingTable::default());
8701        let handler =
8702            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
8703        let module_connection = ConnectionId::new(95);
8704        let (module_ctx, mut module_rx) = route_ctx(module_connection);
8705        hello_via_sink(
8706            &handler,
8707            &module_ctx,
8708            &mut module_rx,
8709            hello_frame("aft", PROTOCOL_VERSION, 395),
8710        )
8711        .await;
8712
8713        let client_connection = ConnectionId::new(96);
8714        let (client_ctx, _client_rx) = route_ctx(client_connection);
8715        let capture = EventCapture::default();
8716        let _subscriber =
8717            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8718        let (route_task, bind) = relay_route_open(
8719            &handler,
8720            client_connection,
8721            &client_ctx.egress,
8722            &mut module_rx,
8723            396,
8724            "aft",
8725            "hostile-module-code",
8726        )
8727        .await;
8728        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
8729        let rejection = Frame::build(
8730            FrameType::Error,
8731            control_flags(),
8732            0,
8733            0,
8734            bind.header.corr,
8735            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
8736        )
8737        .unwrap();
8738        handler
8739            .handle_control_frame(&module_ctx, rejection)
8740            .await
8741            .unwrap();
8742
8743        let response = route_task.await.unwrap();
8744        assert_eq!(parse_error(&response[0])["code"], hostile_code);
8745        let counters = handler.counters().snapshot();
8746        assert_eq!(
8747            counters["route_open_refused_by_code"],
8748            json!({ "module_rejected": 1 })
8749        );
8750        assert!(counters["route_open_refused_by_code"]
8751            .get(hostile_code)
8752            .is_none());
8753
8754        let event = capture
8755            .events()
8756            .into_iter()
8757            .find(|event| {
8758                event.target == "control"
8759                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
8760            })
8761            .expect("route.open module-rejection refusal event");
8762        let logged = event.fields.get("module_code").expect("module_code field");
8763        assert!(!logged.bytes().any(|byte| byte < 0x20));
8764        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
8765    }
8766
8767    #[tokio::test]
8768    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
8769        let registry = Arc::new(Registry::default());
8770        let supervisor_handle = SupervisorHandle::new();
8771        let missing_program = std::env::temp_dir().join(format!(
8772            "subc-route-open-missing-program-{}",
8773            std::process::id()
8774        ));
8775        let supervisor =
8776            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
8777                .with_handle(supervisor_handle.clone());
8778        let module = supervisor
8779            .supervise_configured(
8780                ModuleSpec {
8781                    module_id: "failed".to_string(),
8782                    program: missing_program,
8783                    args: Vec::new(),
8784                    env: Vec::new(),
8785                    reserved: false,
8786                    reserved_prefixes: Vec::new(),
8787                    protocol: ModuleProtocol::Subc,
8788                    overlap: Default::default(),
8789                },
8790                true,
8791            )
8792            .unwrap();
8793        assert_eq!(module.state().unwrap(), ModuleState::Failed);
8794
8795        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
8796        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
8797        let response = handler
8798            .handle_control_frame(
8799                &ctx,
8800                route_open_frame(305, "failed", unique_project_root("failed")),
8801            )
8802            .await
8803            .unwrap();
8804
8805        assert_eq!(response[0].header.ty, FrameType::Error);
8806        let error = parse_error(&response[0]);
8807        assert_eq!(error["code"], "target_unavailable");
8808        assert!(error["message"]
8809            .as_str()
8810            .unwrap()
8811            .contains("state=failed, enabled=true, live=false"));
8812    }
8813
8814    #[tokio::test]
8815    async fn route_open_role_mismatch_remains_target_unavailable() {
8816        let registry = Arc::new(Registry::default());
8817        let handler = ControlHandler::new(Arc::clone(&registry));
8818        handler
8819            .handle_control(
8820                ConnectionId::new(41),
8821                non_routable_hello_frame_with_control_ops("health-only", 306, None),
8822            )
8823            .unwrap();
8824
8825        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
8826        let response = handler
8827            .handle_control_frame(
8828                &ctx,
8829                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
8830            )
8831            .await
8832            .unwrap();
8833
8834        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8835        assert!(parse_error(&response[0])["message"]
8836            .as_str()
8837            .unwrap()
8838            .contains("does not provide the requested target"));
8839    }
8840
8841    #[tokio::test]
8842    async fn route_open_inactive_registration_remains_target_unavailable() {
8843        let registry = Arc::new(Registry::default());
8844        let handler = ControlHandler::new(Arc::clone(&registry));
8845        handler
8846            .handle_control(
8847                ConnectionId::new(43),
8848                hello_frame("inactive", PROTOCOL_VERSION, 308),
8849            )
8850            .unwrap();
8851        assert!(registry
8852            .set_module_state_for_test("inactive", ChannelState::Closed)
8853            .unwrap());
8854
8855        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
8856        let response = handler
8857            .handle_control_frame(
8858                &ctx,
8859                route_open_frame(309, "inactive", unique_project_root("inactive")),
8860            )
8861            .await
8862            .unwrap();
8863
8864        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8865        assert!(parse_error(&response[0])["message"]
8866            .as_str()
8867            .unwrap()
8868            .contains("is not active"));
8869    }
8870
8871    #[tokio::test]
8872    async fn late_health_reply_is_recorded_through_the_module_response_path() {
8873        let registry = Arc::new(Registry::default());
8874        let forwarding = Arc::new(ForwardingTable::default());
8875        let supervisor_handle = SupervisorHandle::new();
8876        let supervisor = Supervisor::new(Arc::clone(&registry), crate::RestartPolicy::default())
8877            .with_forwarding(Arc::clone(&forwarding))
8878            .with_handle(supervisor_handle.clone());
8879        let module = supervisor
8880            .supervise_configured(
8881                crate::ModuleSpec {
8882                    module_id: "late-health-response".to_string(),
8883                    program: PathBuf::from("disabled-module"),
8884                    args: Vec::new(),
8885                    env: Vec::new(),
8886                    reserved: false,
8887                    reserved_prefixes: Vec::new(),
8888                    protocol: ModuleProtocol::Subc,
8889                    overlap: Default::default(),
8890                },
8891                false,
8892            )
8893            .unwrap();
8894        let handler =
8895            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8896                .with_supervisor(supervisor_handle);
8897        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
8898        handler
8899            .handle_control_frame(
8900                &module_ctx,
8901                hello_frame_with_control_ops(
8902                    "late-health-response",
8903                    PROTOCOL_VERSION,
8904                    7,
8905                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8906                ),
8907            )
8908            .await
8909            .unwrap();
8910        let probe_started_at = Instant::now() - Duration::from_millis(80);
8911        let pending = forwarding
8912            .begin_health_probe_rpc_for(
8913                "late-health-response",
8914                MODULE_CONTROL_OP_HEALTH_CHECK,
8915                probe_started_at,
8916                Instant::now() - Duration::from_millis(1),
8917            )
8918            .unwrap();
8919        assert!(forwarding
8920            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
8921            .unwrap());
8922
8923        let responses = handler
8924            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
8925            .await
8926            .unwrap();
8927
8928        assert!(responses.is_empty());
8929        let health = module.status().unwrap().health;
8930        assert_eq!(health.late_answer_count, 1);
8931        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
8932    }
8933
8934    #[tokio::test]
8935    async fn health_probe_timeout_and_module_death_are_typed() {
8936        let registry = Arc::new(Registry::default());
8937        let forwarding = Arc::new(ForwardingTable::default());
8938        let handler =
8939            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8940                .with_health_probe_timeout(Duration::from_millis(50));
8941        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
8942        hello_via_sink(
8943            &handler,
8944            &module_ctx,
8945            &mut module_rx,
8946            hello_frame_with_control_ops(
8947                "aft",
8948                PROTOCOL_VERSION,
8949                7,
8950                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8951            ),
8952        )
8953        .await;
8954
8955        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
8956        let responses = handler
8957            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
8958            .await
8959            .unwrap();
8960        assert_eq!(responses[0].header.ty, FrameType::Error);
8961        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
8962        let _ = module_rx.try_recv();
8963
8964        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
8965        let health_handler = handler.clone();
8966        let death_task = tokio::spawn(async move {
8967            health_handler
8968                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
8969                .await
8970                .unwrap()
8971        });
8972        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8973            .await
8974            .unwrap()
8975            .unwrap();
8976        handler
8977            .cleanup_connection(module_ctx.connection_id)
8978            .unwrap();
8979        let responses = death_task.await.unwrap();
8980        assert_eq!(responses[0].header.ty, FrameType::Error);
8981        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
8982    }
8983
8984    #[test]
8985    fn hello_requires_exact_protocol_version() {
8986        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
8987            let registry = Arc::new(Registry::default());
8988            let handler = ControlHandler::new(Arc::clone(&registry));
8989            let responses = handler
8990                .handle_control(
8991                    ConnectionId::new(connection),
8992                    hello_frame("aft", offered, 9),
8993                )
8994                .unwrap();
8995
8996            assert_eq!(responses.len(), 1);
8997            assert_eq!(responses[0].header.ty, FrameType::Error);
8998            let error = parse_error(&responses[0]);
8999            assert_eq!(error["code"], "version_unsupported");
9000            assert!(registry.get_module("aft").unwrap().is_none());
9001            assert_eq!(registry.active_registration_count().unwrap(), 0);
9002        }
9003    }
9004
9005    #[test]
9006    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
9007        let registry = Arc::new(Registry::default());
9008        let forwarding = Arc::new(ForwardingTable::default());
9009        let handler =
9010            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9011        let module_connection = ConnectionId::new(301);
9012        let registration = registry
9013            .register_with_control_ops(
9014                manifest("aft-push", PROTOCOL_VERSION),
9015                PROTOCOL_VERSION,
9016                module_connection,
9017                module_baseline_control_ops(),
9018            )
9019            .unwrap();
9020        let (module_tx, _module_rx) = mpsc::channel(8);
9021        let endpoint = forwarding
9022            .register_module_connection(
9023                module_connection,
9024                "aft-push".to_string(),
9025                PROTOCOL_VERSION,
9026                manifest_concurrency(&registration.manifest),
9027                FrameSink::new(module_tx),
9028            )
9029            .unwrap();
9030
9031        // A push op this version does not know is ignored (forward-compat), not errored.
9032        let unknown = Frame::build(
9033            FrameType::Push,
9034            control_flags(),
9035            0,
9036            0,
9037            5,
9038            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
9039        )
9040        .unwrap();
9041        let out = handler.handle_status_update(endpoint, unknown).unwrap();
9042        assert!(
9043            out.is_empty(),
9044            "unknown push op must be ignored, got {out:?}"
9045        );
9046
9047        // A malformed body for a KNOWN op is a real error worth surfacing.
9048        let malformed = Frame::build(
9049            FrameType::Push,
9050            control_flags(),
9051            0,
9052            0,
9053            6,
9054            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
9055        )
9056        .unwrap();
9057        let out = handler.handle_status_update(endpoint, malformed).unwrap();
9058        assert_eq!(out.len(), 1);
9059        assert_eq!(out[0].header.ty, FrameType::Error);
9060        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
9061    }
9062
9063    #[test]
9064    fn hello_rejected_when_connection_already_owns_client_routes() {
9065        let registry = Arc::new(Registry::default());
9066        let forwarding = Arc::new(ForwardingTable::default());
9067        let handler =
9068            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9069        // Commits a client route on connection 202 (bound to a module on conn 101).
9070        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
9071        let client_connection = ConnectionId::new(202);
9072
9073        // That same connection now tries to register as a module: rejected, so one
9074        // connection never holds both client-route and module-endpoint state.
9075        let responses = handler
9076            .handle_control(
9077                client_connection,
9078                hello_frame("aft-second", PROTOCOL_VERSION, 9),
9079            )
9080            .unwrap();
9081        assert_eq!(responses[0].header.ty, FrameType::Error);
9082        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
9083        assert!(registry.get_module("aft-second").unwrap().is_none());
9084    }
9085
9086    #[test]
9087    fn reserved_module_hello_requires_matching_launch_nonce() {
9088        let registry = Arc::new(Registry::default());
9089        let supervisor = SupervisorHandle::new();
9090        // The supervisor recorded the nonce it injected when it spawned the reserved
9091        // module; the HELLO verifier checks against the same shared handle.
9092        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9093        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
9094
9095        // A HELLO with NO nonce is rejected.
9096        let no_nonce = handler
9097            .handle_control(
9098                ConnectionId::new(1),
9099                hello_frame("vault", PROTOCOL_VERSION, 1),
9100            )
9101            .unwrap();
9102        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
9103        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
9104        assert!(registry.get_module("vault").unwrap().is_none());
9105
9106        // A HELLO with the WRONG nonce is rejected.
9107        let wrong = handler
9108            .handle_control(
9109                ConnectionId::new(2),
9110                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
9111            )
9112            .unwrap();
9113        assert_eq!(wrong[0].header.ty, FrameType::Error);
9114        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
9115        assert!(registry.get_module("vault").unwrap().is_none());
9116
9117        // A HELLO with the CORRECT nonce registers.
9118        let ok = handler
9119            .handle_control(
9120                ConnectionId::new(3),
9121                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
9122            )
9123            .unwrap();
9124        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
9125        assert!(registry.get_module("vault").unwrap().is_some());
9126    }
9127
9128    #[test]
9129    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
9130        let registry = Arc::new(Registry::default());
9131        let supervisor = SupervisorHandle::new();
9132        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
9133        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
9134        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
9135
9136        let squat = handler
9137            .handle_control(
9138                ConnectionId::new(1),
9139                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
9140            )
9141            .unwrap();
9142        assert_eq!(squat[0].header.ty, FrameType::Error);
9143        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
9144        assert!(parse_error(&squat[0])["message"]
9145            .as_str()
9146            .unwrap()
9147            .contains("fed:"));
9148
9149        let accepted_peer = handler
9150            .handle_control(
9151                ConnectionId::new(2),
9152                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
9153            )
9154            .unwrap();
9155        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
9156
9157        let accepted_short = handler
9158            .handle_control(
9159                ConnectionId::new(3),
9160                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
9161            )
9162            .unwrap();
9163        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
9164
9165        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
9166            let response = handler
9167                .handle_control(
9168                    ConnectionId::new(conn),
9169                    hello_frame(module_id, PROTOCOL_VERSION, conn),
9170                )
9171                .unwrap();
9172            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
9173        }
9174    }
9175
9176    #[test]
9177    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
9178        let registry = Arc::new(Registry::default());
9179        let supervisor = SupervisorHandle::new();
9180        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
9181        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
9182        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
9183        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
9184
9185        let owner_nonce = handler
9186            .handle_control(
9187                ConnectionId::new(1),
9188                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
9189            )
9190            .unwrap();
9191        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
9192        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
9193        assert!(registry.get_module("fed:special").unwrap().is_none());
9194
9195        let exact_nonce = handler
9196            .handle_control(
9197                ConnectionId::new(2),
9198                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
9199            )
9200            .unwrap();
9201        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
9202        assert!(registry.get_module("fed:special").unwrap().is_some());
9203    }
9204
9205    #[test]
9206    fn non_reserved_module_ignores_launch_nonce() {
9207        let registry = Arc::new(Registry::default());
9208        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
9209        // registration succeeds whether a spawned process echoes a nonce or not.
9210        let handler = ControlHandler::new(Arc::clone(&registry));
9211        let no_nonce = handler
9212            .handle_control(
9213                ConnectionId::new(1),
9214                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
9215            )
9216            .unwrap();
9217        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
9218        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
9219
9220        let echoed_nonce = handler
9221            .handle_control(
9222                ConnectionId::new(2),
9223                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
9224            )
9225            .unwrap();
9226        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
9227        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
9228    }
9229
9230    #[test]
9231    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
9232        let handler = ControlHandler::default();
9233        let conn = ConnectionId::new(1);
9234        let malformed = Frame::build(
9235            FrameType::Hello,
9236            control_flags(),
9237            0,
9238            0,
9239            3,
9240            b"{not json".to_vec(),
9241        )
9242        .unwrap();
9243
9244        let error = handler.handle_control(conn, malformed).unwrap();
9245        assert_eq!(error[0].header.ty, FrameType::Error);
9246        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
9247
9248        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
9249        let pong = handler.handle_control(conn, ping).unwrap();
9250        assert_eq!(pong[0].header.ty, FrameType::Pong);
9251        assert_eq!(pong[0].header.corr, 4);
9252    }
9253
9254    #[test]
9255    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
9256        let registry = Arc::new(Registry::default());
9257        let handler = ControlHandler::new(Arc::clone(&registry));
9258
9259        handler
9260            .handle_control(
9261                ConnectionId::new(1),
9262                hello_frame("aft", PROTOCOL_VERSION, 1),
9263            )
9264            .unwrap();
9265        let duplicate = handler
9266            .handle_control(
9267                ConnectionId::new(2),
9268                hello_frame("aft", PROTOCOL_VERSION, 2),
9269            )
9270            .unwrap();
9271
9272        assert_eq!(duplicate[0].header.ty, FrameType::Error);
9273        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
9274        let registration = registry.get_module("aft").unwrap().unwrap();
9275        assert_eq!(registration.connection_id, ConnectionId::new(1));
9276    }
9277
9278    #[test]
9279    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
9280        let registry = Arc::new(Registry::default());
9281        let forwarding = Arc::new(ForwardingTable::default());
9282        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
9283        let handler =
9284            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9285                .with_process_liveness(process_liveness);
9286        let (ctx, route_channel, route_epoch) =
9287            bind_liveness_route(&registry, &forwarding, "aft-dead");
9288        let responses = handler
9289            .handle_route_poll(
9290                &ctx,
9291                route_poll_frame(41, PollKind::Liveness, route_channel),
9292                route_channel,
9293                route_epoch,
9294                PollKind::Liveness,
9295            )
9296            .unwrap();
9297
9298        assert_eq!(responses.len(), 1);
9299        assert_eq!(responses[0].header.ty, FrameType::Response);
9300        assert_route_poll_liveness(&responses[0], false);
9301    }
9302
9303    #[test]
9304    fn liveness_poll_without_process_source_uses_bound_route() {
9305        let registry = Arc::new(Registry::default());
9306        let forwarding = Arc::new(ForwardingTable::default());
9307        let handler =
9308            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9309        let (ctx, route_channel, route_epoch) =
9310            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
9311        let responses = handler
9312            .handle_route_poll(
9313                &ctx,
9314                route_poll_frame(42, PollKind::Liveness, route_channel),
9315                route_channel,
9316                route_epoch,
9317                PollKind::Liveness,
9318            )
9319            .unwrap();
9320
9321        assert_route_poll_liveness(&responses[0], true);
9322    }
9323
9324    #[test]
9325    fn liveness_poll_untracked_process_source_uses_bound_route() {
9326        let registry = Arc::new(Registry::default());
9327        let forwarding = Arc::new(ForwardingTable::default());
9328        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
9329        let handler =
9330            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9331                .with_process_liveness(process_liveness);
9332        let (ctx, route_channel, route_epoch) =
9333            bind_liveness_route(&registry, &forwarding, "aft-untracked");
9334        let responses = handler
9335            .handle_route_poll(
9336                &ctx,
9337                route_poll_frame(43, PollKind::Liveness, route_channel),
9338                route_channel,
9339                route_epoch,
9340                PollKind::Liveness,
9341            )
9342            .unwrap();
9343
9344        assert_route_poll_liveness(&responses[0], true);
9345    }
9346
9347    #[tokio::test]
9348    async fn unknown_op_returns_unknown_control_op() {
9349        let handler = ControlHandler::default();
9350        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
9351        let request = Frame::build(
9352            FrameType::Request,
9353            control_flags(),
9354            0,
9355            0,
9356            55,
9357            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
9358        )
9359        .unwrap();
9360
9361        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9362
9363        assert_eq!(response.len(), 1);
9364        assert_eq!(response[0].header.ty, FrameType::Error);
9365        assert_eq!(response[0].header.corr, 55);
9366        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
9367    }
9368
9369    #[tokio::test]
9370    async fn supervisor_provenance_rejects_unknown_exact_module() {
9371        let handler = ControlHandler::default();
9372        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
9373        let request = Frame::build(
9374            FrameType::Request,
9375            control_flags(),
9376            0,
9377            0,
9378            57,
9379            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
9380        )
9381        .unwrap();
9382
9383        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9384
9385        assert_eq!(response.len(), 1);
9386        assert_eq!(response[0].header.ty, FrameType::Error);
9387        assert_eq!(response[0].header.corr, 57);
9388        let error = parse_error(&response[0]);
9389        assert_eq!(error["code"], "unknown_module");
9390        assert_eq!(error["message"], "module_id 'missing' is not supervised");
9391    }
9392
9393    #[test]
9394    fn provenance_probe_override_keeps_handler_tests_deterministic() {
9395        let expected = subc_control::RunningImageAgreement::Unavailable {
9396            reason: subc_control::RunningImageUnavailableReason::HashFailed,
9397        };
9398        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
9399        assert_eq!(handler.provenance_probe_override, Some(expected));
9400    }
9401
9402    #[test]
9403    fn reload_verdict_detects_configured_program_different_from_spawned_path() {
9404        let verdict = reload_verdict(
9405            std::path::Path::new("/bin/new"),
9406            Some(std::path::Path::new("/bin/old")),
9407            subc_control::RunningImageAgreement::Unavailable {
9408                reason: subc_control::RunningImageUnavailableReason::HashFailed,
9409            },
9410        );
9411        assert!(matches!(
9412            verdict.path,
9413            subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
9414                if configured == std::path::Path::new("/bin/new")
9415                    && spawned_from == std::path::Path::new("/bin/old")
9416        ));
9417    }
9418
9419    #[test]
9420    fn reload_verdict_detects_replaced_image_at_same_path() {
9421        let image = subc_control::RunningImageAgreement::Mismatch {
9422            running: subc_control::RunningImageEvidence::LinuxProcSha256 {
9423                digest: "old".into(),
9424            },
9425            disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
9426                digest: "new".into(),
9427            },
9428        };
9429        let verdict = reload_verdict(
9430            std::path::Path::new("/bin/same"),
9431            Some(std::path::Path::new("/bin/same")),
9432            image.clone(),
9433        );
9434        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
9435        assert_eq!(verdict.image, image);
9436    }
9437
9438    #[test]
9439    fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
9440        let image = subc_control::RunningImageAgreement::Unavailable {
9441            reason: subc_control::RunningImageUnavailableReason::NotRunning,
9442        };
9443        let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
9444        assert_eq!(
9445            verdict.path,
9446            subc_control::ReloadPathAgreement::Unavailable {
9447                reason: subc_control::ReloadPathUnavailableReason::NotRunning,
9448            }
9449        );
9450        assert_eq!(verdict.image, image);
9451
9452        let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
9453            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
9454        };
9455        let verdict = reload_verdict(
9456            std::path::Path::new("/bin/same"),
9457            Some(std::path::Path::new("/bin/same")),
9458            unconfirmed.clone(),
9459        );
9460        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
9461        assert_eq!(verdict.image, unconfirmed);
9462    }
9463
9464    #[test]
9465    fn reload_verdict_preserves_each_image_unavailability_reason() {
9466        use subc_control::RunningImageUnavailableReason as Reason;
9467
9468        for reason in [
9469            Reason::NotRunning,
9470            Reason::UnsupportedPlatform,
9471            Reason::RunningExecutableUnreadable,
9472            Reason::SpawnedPathUnreadable,
9473            Reason::HashFailed,
9474            Reason::ProcessIdentityUnconfirmed,
9475            Reason::Unknown("future_probe_reason".to_string()),
9476        ] {
9477            let image = subc_control::RunningImageAgreement::Unavailable {
9478                reason: reason.clone(),
9479            };
9480            let verdict = reload_verdict(
9481                std::path::Path::new("/bin/same"),
9482                Some(std::path::Path::new("/bin/same")),
9483                image.clone(),
9484            );
9485            assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
9486            assert_eq!(verdict.image, image, "{reason:?}");
9487        }
9488    }
9489
9490    #[tokio::test]
9491    async fn malformed_control_bodies_return_invalid_control_body() {
9492        let handler = ControlHandler::default();
9493        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
9494
9495        for (corr, body) in [
9496            (56, br#"{"route_channel":1}"#.as_slice()),
9497            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
9498            (
9499                58,
9500                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
9501            ),
9502        ] {
9503            let request = Frame::build(
9504                FrameType::Request,
9505                control_flags(),
9506                0,
9507                0,
9508                corr,
9509                body.to_vec(),
9510            )
9511            .unwrap();
9512            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9513
9514            assert_eq!(response.len(), 1);
9515            assert_eq!(response[0].header.ty, FrameType::Error);
9516            assert_eq!(response[0].header.corr, corr);
9517            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
9518        }
9519    }
9520
9521    #[tokio::test]
9522    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
9523        let registry = Arc::new(Registry::default());
9524        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
9525        let router = Router::with_control_handler(Arc::clone(&control));
9526        let connection = router.begin_connection();
9527        let (ctx, mut rx) = route_ctx(connection.id());
9528
9529        router
9530            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
9531            .await
9532            .unwrap();
9533        let response = rx.recv().await.unwrap();
9534        let ack = parse_ack(&response);
9535        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9536        let channel = 1;
9537
9538        let goodbye =
9539            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
9540        router.route_for_connection(&ctx, goodbye).await.unwrap();
9541        assert!(rx.try_recv().is_err());
9542        assert!(registry.get_module("aft").unwrap().is_none());
9543
9544        router
9545            .route_for_connection(&ctx, channel_request(channel, 13))
9546            .await
9547            .unwrap();
9548        let error_frame = rx.recv().await.unwrap();
9549        assert_eq!(error_frame.header.ty, FrameType::Error);
9550        assert_eq!(error_frame.header.channel, channel);
9551    }
9552
9553    #[tokio::test]
9554    async fn dropping_router_connection_releases_registration() {
9555        let registry = Arc::new(Registry::default());
9556        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
9557        let router = Router::with_control_handler(control);
9558        let connection = router.begin_connection();
9559        let (ctx, mut rx) = route_ctx(connection.id());
9560
9561        router
9562            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
9563            .await
9564            .unwrap();
9565        let response = rx.recv().await.unwrap();
9566        let ack = parse_ack(&response);
9567        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9568        assert!(registry.get_module("aft").unwrap().is_some());
9569
9570        drop(connection);
9571
9572        assert!(registry.get_module("aft").unwrap().is_none());
9573        assert_eq!(registry.active_registration_count().unwrap(), 0);
9574    }
9575
9576    fn capability_manifest(
9577        module_id: &str,
9578        provides: &[&str],
9579        must_never_reach: &[&str],
9580    ) -> ModuleManifest {
9581        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
9582        manifest.capabilities = Some(CapabilityDeclarations {
9583            provides: provides
9584                .iter()
9585                .map(|capability| (*capability).to_string())
9586                .collect(),
9587            requires: Vec::new(),
9588            must_never_reach: must_never_reach
9589                .iter()
9590                .map(|capability| (*capability).to_string())
9591                .collect(),
9592        });
9593        manifest
9594    }
9595
9596    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
9597        Frame::build(
9598            FrameType::Hello,
9599            control_flags(),
9600            0,
9601            0,
9602            corr,
9603            serde_json::to_vec(&ModuleHelloBody {
9604                protocol_ver: manifest.protocol_ver,
9605                manifest,
9606                control_ops: None,
9607                launch_nonce: None,
9608            })
9609            .expect("capability test HELLO serializes"),
9610        )
9611        .expect("capability test HELLO frame builds")
9612    }
9613
9614    fn catalog_update_with_capabilities_frame(
9615        corr: u64,
9616        capabilities: CapabilityDeclarations,
9617    ) -> Frame {
9618        Frame::build(
9619            FrameType::Request,
9620            control_flags(),
9621            0,
9622            0,
9623            corr,
9624            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
9625                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
9626                capabilities: Some(capabilities),
9627                ready: None,
9628            })
9629            .expect("capability catalog.update serializes"),
9630        )
9631        .expect("capability catalog.update frame builds")
9632    }
9633
9634    async fn register_capability_manifest(
9635        handler: &ControlHandler,
9636        ctx: &RouteCtx,
9637        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9638        manifest: ModuleManifest,
9639        corr: u64,
9640    ) {
9641        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
9642    }
9643
9644    async fn open_route_for_capability_test(
9645        handler: &ControlHandler,
9646        target_ctx: &RouteCtx,
9647        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9648        client_connection_id: u64,
9649        corr: u64,
9650        target_module_id: &str,
9651        consumer_identity: Option<ConsumerIdentity>,
9652    ) -> (
9653        mpsc::Receiver<crate::router::OutboundFrame>,
9654        ModuleControlRequest,
9655    ) {
9656        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
9657        let route_handler = handler.clone();
9658        let target_module_id = target_module_id.to_string();
9659        let route_task = tokio::spawn(async move {
9660            route_handler
9661                .handle_control_frame(
9662                    &client_ctx,
9663                    route_open_frame_with_admission_facts(
9664                        corr,
9665                        &target_module_id,
9666                        unique_project_root("admission-facts"),
9667                        consumer_identity,
9668                        None,
9669                    ),
9670                )
9671                .await
9672                .expect("capability test route.open succeeds")
9673        });
9674        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
9675            .await
9676            .expect("capability test route.open must reach route.bind")
9677            .expect("target control receiver stays open");
9678        let bind_request: ModuleControlRequest =
9679            serde_json::from_slice(&bind.body).expect("route.bind decodes");
9680        handler
9681            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
9682            .await
9683            .expect("capability test route.bind ACK succeeds");
9684        assert!(route_task.await.expect("route.open task joins").is_empty());
9685        let opened = client_rx
9686            .recv()
9687            .await
9688            .expect("successful route.open publishes a response");
9689        assert!(matches!(
9690            serde_json::from_slice::<ClientControlResponse>(&opened.body),
9691            Ok(ClientControlResponse::RouteOpen { .. })
9692        ));
9693        (client_rx, bind_request)
9694    }
9695
9696    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
9697        assert_eq!(frame.header.ty, FrameType::Push);
9698        assert_eq!(frame.header.channel, 0);
9699        assert_eq!(
9700            serde_json::from_slice::<ClientControlPush>(&frame.body)
9701                .expect("route.closed control push decodes"),
9702            ClientControlPush::RouteClosed {
9703                module_id: target_module_id.to_string(),
9704                reason: RouteCloseReason::CapabilityDenied,
9705                drained: false,
9706                abandoned: 0,
9707                excluded_subscriptions: 0,
9708                terminal: Some(false),
9709            }
9710        );
9711    }
9712
9713    #[tokio::test]
9714    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
9715        let registry = Arc::new(Registry::default());
9716        let forwarding = Arc::new(ForwardingTable::default());
9717        let supervisor = SupervisorHandle::new();
9718        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9719        let handler =
9720            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9721                .with_supervisor(supervisor);
9722        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
9723        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
9724        register_capability_manifest(
9725            &handler,
9726            &target_ctx,
9727            &mut target_rx,
9728            capability_manifest("target", &["credentials-provider/v1"], &[]),
9729            1,
9730        )
9731        .await;
9732        register_capability_manifest(
9733            &handler,
9734            &opener_ctx,
9735            &mut opener_rx,
9736            capability_manifest("opener", &[], &["credentials-provider/v1"]),
9737            2,
9738        )
9739        .await;
9740
9741        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
9742        let replies = handler
9743            .handle_control_frame(
9744                &client_ctx,
9745                route_open_frame_with_admission_facts(
9746                    3,
9747                    "target",
9748                    unique_project_root("admission-facts"),
9749                    Some(ConsumerIdentity {
9750                        module_id: "opener".to_string(),
9751                        launch_nonce: "opener-nonce".to_string(),
9752                    }),
9753                    None,
9754                ),
9755            )
9756            .await
9757            .expect("denied route.open returns a typed frame");
9758        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
9759        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9760        assert!(
9761            target_rx.try_recv().is_err(),
9762            "forbidden route.open must not relay route.bind"
9763        );
9764    }
9765
9766    #[tokio::test]
9767    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
9768        let registry = Arc::new(Registry::default());
9769        let forwarding = Arc::new(ForwardingTable::default());
9770        let supervisor = SupervisorHandle::new();
9771        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9772        let handler =
9773            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9774                .with_supervisor(supervisor);
9775        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
9776        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
9777        register_capability_manifest(
9778            &handler,
9779            &target_ctx,
9780            &mut target_rx,
9781            capability_manifest("target", &["credentials-provider/v1"], &[]),
9782            1,
9783        )
9784        .await;
9785        register_capability_manifest(
9786            &handler,
9787            &old_opener_ctx,
9788            &mut old_opener_rx,
9789            capability_manifest("opener", &[], &[]),
9790            2,
9791        )
9792        .await;
9793        let (mut client_rx, _) = open_route_for_capability_test(
9794            &handler,
9795            &target_ctx,
9796            &mut target_rx,
9797            712,
9798            3,
9799            "target",
9800            Some(ConsumerIdentity {
9801                module_id: "opener".to_string(),
9802                launch_nonce: "opener-nonce".to_string(),
9803            }),
9804        )
9805        .await;
9806        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9807
9808        handler
9809            .cleanup_connection(old_opener_ctx.connection_id)
9810            .expect("old opener registration cleans up");
9811        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
9812        register_capability_manifest(
9813            &handler,
9814            &new_opener_ctx,
9815            &mut new_opener_rx,
9816            capability_manifest("opener", &[], &["credentials-provider/v1"]),
9817            4,
9818        )
9819        .await;
9820
9821        assert_capability_denied_push(
9822            client_rx
9823                .try_recv()
9824                .expect("HELLO deny addition must emit route.closed")
9825                .frame,
9826            "target",
9827        );
9828        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9829        assert!(matches!(
9830            target_rx.try_recv(),
9831            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
9832        ));
9833    }
9834
9835    #[tokio::test]
9836    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
9837        let registry = Arc::new(Registry::default());
9838        let forwarding = Arc::new(ForwardingTable::default());
9839        let supervisor = SupervisorHandle::new();
9840        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9841        let handler =
9842            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9843                .with_supervisor(supervisor);
9844        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
9845        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
9846        register_capability_manifest(
9847            &handler,
9848            &target_ctx,
9849            &mut target_rx,
9850            capability_manifest("target", &[], &[]),
9851            1,
9852        )
9853        .await;
9854        register_capability_manifest(
9855            &handler,
9856            &opener_ctx,
9857            &mut opener_rx,
9858            capability_manifest("opener", &[], &["credentials-provider/v1"]),
9859            2,
9860        )
9861        .await;
9862        let (mut client_rx, _) = open_route_for_capability_test(
9863            &handler,
9864            &target_ctx,
9865            &mut target_rx,
9866            722,
9867            3,
9868            "target",
9869            Some(ConsumerIdentity {
9870                module_id: "opener".to_string(),
9871                launch_nonce: "opener-nonce".to_string(),
9872            }),
9873        )
9874        .await;
9875        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9876
9877        let replies = handler
9878            .handle_control_frame(
9879                &target_ctx,
9880                catalog_update_with_capabilities_frame(
9881                    4,
9882                    CapabilityDeclarations {
9883                        provides: vec!["credentials-provider/v1".to_string()],
9884                        requires: Vec::new(),
9885                        must_never_reach: Vec::new(),
9886                    },
9887                ),
9888            )
9889            .await
9890            .expect("claim catalog.update succeeds");
9891        assert!(matches!(
9892            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
9893            Ok(ModuleControlResponseToModule::CatalogUpdate {})
9894        ));
9895        assert_capability_denied_push(
9896            client_rx
9897                .try_recv()
9898                .expect("claim addition must emit route.closed")
9899                .frame,
9900            "target",
9901        );
9902        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9903        assert!(matches!(
9904            target_rx.try_recv(),
9905            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
9906        ));
9907    }
9908
9909    #[tokio::test]
9910    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
9911        let registry = Arc::new(Registry::default());
9912        let forwarding = Arc::new(ForwardingTable::default());
9913        let supervisor = SupervisorHandle::new();
9914        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9915        let handler =
9916            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9917                .with_supervisor(supervisor);
9918        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
9919        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
9920        register_capability_manifest(
9921            &handler,
9922            &target_ctx,
9923            &mut target_rx,
9924            capability_manifest("target", &["credentials-provider/v1"], &[]),
9925            1,
9926        )
9927        .await;
9928        register_capability_manifest(
9929            &handler,
9930            &opener_ctx,
9931            &mut opener_rx,
9932            capability_manifest("opener", &[], &[]),
9933            2,
9934        )
9935        .await;
9936        let (mut client_rx, _) = open_route_for_capability_test(
9937            &handler,
9938            &target_ctx,
9939            &mut target_rx,
9940            732,
9941            3,
9942            "target",
9943            Some(ConsumerIdentity {
9944                module_id: "opener".to_string(),
9945                launch_nonce: "opener-nonce".to_string(),
9946            }),
9947        )
9948        .await;
9949        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9950
9951        handler
9952            .handle_control_frame(
9953                &target_ctx,
9954                catalog_update_with_capabilities_frame(
9955                    4,
9956                    CapabilityDeclarations {
9957                        provides: Vec::new(),
9958                        requires: Vec::new(),
9959                        must_never_reach: Vec::new(),
9960                    },
9961                ),
9962            )
9963            .await
9964            .expect("claim removal catalog.update succeeds");
9965        assert_eq!(
9966            forwarding.active_binding_count().unwrap(),
9967            1,
9968            "removing an attested target claim must leave the route census unchanged"
9969        );
9970        assert!(
9971            client_rx.try_recv().is_err(),
9972            "claim removal must not emit route.closed capability_denied"
9973        );
9974        assert!(
9975            target_rx.try_recv().is_err(),
9976            "claim removal must not send the target a route GOODBYE"
9977        );
9978    }
9979
9980    /// A direct client may open a route to a denied capability provider; this
9981    /// policy applies only to attested supervised module origins, not to direct clients.
9982    #[tokio::test]
9983    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
9984        let registry = Arc::new(Registry::default());
9985        let forwarding = Arc::new(ForwardingTable::default());
9986        let supervisor = SupervisorHandle::new();
9987        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9988        let handler =
9989            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9990                .with_supervisor(supervisor);
9991        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
9992        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
9993        register_capability_manifest(
9994            &handler,
9995            &target_ctx,
9996            &mut target_rx,
9997            capability_manifest("target", &["credentials-provider/v1"], &[]),
9998            1,
9999        )
10000        .await;
10001        register_capability_manifest(
10002            &handler,
10003            &opener_ctx,
10004            &mut opener_rx,
10005            capability_manifest("opener", &[], &["credentials-provider/v1"]),
10006            2,
10007        )
10008        .await;
10009
10010        let (_client_rx, bind) = open_route_for_capability_test(
10011            &handler,
10012            &target_ctx,
10013            &mut target_rx,
10014            742,
10015            3,
10016            "target",
10017            None,
10018        )
10019        .await;
10020        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
10021            panic!("direct scope-honesty route must bind");
10022        };
10023        assert_eq!(principal, Some(Principal::Direct));
10024        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10025    }
10026
10027    /// A module that denies a capability receives no self-route exemption when it
10028    /// also attestedly provides that capability.
10029    #[tokio::test]
10030    async fn must_never_reach_self_route_is_capability_forbidden() {
10031        let registry = Arc::new(Registry::default());
10032        let forwarding = Arc::new(ForwardingTable::default());
10033        let supervisor = SupervisorHandle::new();
10034        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
10035        let handler =
10036            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10037                .with_supervisor(supervisor);
10038        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
10039        register_capability_manifest(
10040            &handler,
10041            &self_ctx,
10042            &mut self_rx,
10043            capability_manifest(
10044                "self-provider",
10045                &["credentials-provider/v1"],
10046                &["credentials-provider/v1"],
10047            ),
10048            1,
10049        )
10050        .await;
10051
10052        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
10053        let replies = handler
10054            .handle_control_frame(
10055                &client_ctx,
10056                route_open_frame_with_admission_facts(
10057                    2,
10058                    "self-provider",
10059                    unique_project_root("admission-facts"),
10060                    Some(ConsumerIdentity {
10061                        module_id: "self-provider".to_string(),
10062                        launch_nonce: "self-nonce".to_string(),
10063                    }),
10064                    None,
10065                ),
10066            )
10067            .await
10068            .expect("self-route refusal returns a typed frame");
10069        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
10070        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10071        assert!(
10072            self_rx.try_recv().is_err(),
10073            "self denial must not relay route.bind"
10074        );
10075    }
10076
10077    #[test]
10078    fn unsupported_channel_zero_frame_returns_error() {
10079        let handler = ControlHandler::default();
10080        let request = Frame::build(
10081            FrameType::Request,
10082            control_flags(),
10083            0,
10084            0,
10085            21,
10086            b"opaque".to_vec(),
10087        )
10088        .unwrap();
10089
10090        let response = handler
10091            .handle_control(ConnectionId::new(1), request)
10092            .unwrap();
10093
10094        assert_eq!(response[0].header.ty, FrameType::Error);
10095        assert_eq!(
10096            parse_error(&response[0])["code"],
10097            "unsupported_control_frame"
10098        );
10099    }
10100
10101    /// Blue/green swap at the control-plane boundary. The supervisor that opens
10102    /// a swap is not wired yet, so the candidate is registered here directly
10103    /// into the registry and forwarding candidate slots, the way the swap's
10104    /// HELLO admission will.
10105    mod swap {
10106        use super::*;
10107
10108        const INCUMBENT: ConnectionId = ConnectionId::new(30);
10109        const CANDIDATE: ConnectionId = ConnectionId::new(40);
10110
10111        struct Swap {
10112            registry: Arc<Registry>,
10113            forwarding: Arc<ForwardingTable>,
10114            handler: ControlHandler,
10115            incumbent_ctx: RouteCtx,
10116            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
10117            candidate_ctx: RouteCtx,
10118            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
10119        }
10120
10121        async fn swap_with_incumbent() -> Swap {
10122            let registry = Arc::new(Registry::default());
10123            let forwarding = Arc::new(ForwardingTable::default());
10124            let handler =
10125                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10126            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
10127            hello_via_sink(
10128                &handler,
10129                &incumbent_ctx,
10130                &mut incumbent_rx,
10131                hello_frame("aft", PROTOCOL_VERSION, 7),
10132            )
10133            .await;
10134            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
10135            Swap {
10136                registry,
10137                forwarding,
10138                handler,
10139                incumbent_ctx,
10140                incumbent_rx,
10141                candidate_ctx,
10142                candidate_rx,
10143            }
10144        }
10145
10146        fn register_candidate(swap: &Swap, ready: Option<bool>) {
10147            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
10148            candidate_manifest.ready = ready;
10149            let registration = swap
10150                .registry
10151                .register_candidate_with_control_ops(
10152                    candidate_manifest,
10153                    PROTOCOL_VERSION,
10154                    CANDIDATE,
10155                    module_baseline_control_ops(),
10156                )
10157                .unwrap();
10158            swap.forwarding
10159                .register_candidate_module_connection(
10160                    CANDIDATE,
10161                    "aft".to_string(),
10162                    PROTOCOL_VERSION,
10163                    manifest_concurrency(&registration.manifest),
10164                    swap.candidate_ctx.egress.clone(),
10165                )
10166                .unwrap();
10167        }
10168
10169        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
10170            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
10171            swap.registry.promote_candidate("aft").unwrap().unwrap();
10172            cutover.incumbent.unwrap()
10173        }
10174
10175        fn keyed_total(counters: &Value, key: &str) -> u64 {
10176            counters[key]
10177                .as_object()
10178                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
10179                .unwrap_or(0)
10180        }
10181
10182        /// An ack from the incumbent for a bind it was sent before cutover,
10183        /// arriving before the incumbent is drained. The incumbent is the live
10184        /// connection carrying every other client's routes, so the ack must
10185        /// not end it: the waiting client is told to retry, the reservation is
10186        /// given back, and the incumbent is told to drop just that binding.
10187        #[tokio::test]
10188        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
10189            let mut swap = swap_with_incumbent().await;
10190            let handler = swap.handler.clone();
10191
10192            // A co-tenant route, bound on the incumbent before the swap.
10193            let cotenant = ConnectionId::new(31);
10194            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
10195            let (cotenant_task, cotenant_bind) = relay_route_open(
10196                &handler,
10197                cotenant,
10198                &cotenant_ctx.egress,
10199                &mut swap.incumbent_rx,
10200                100,
10201                "aft",
10202                "swap-cotenant",
10203            )
10204            .await;
10205            handler
10206                .handle_control_frame(
10207                    &swap.incumbent_ctx,
10208                    route_bind_ack(cotenant_bind.header.corr),
10209                )
10210                .await
10211                .unwrap();
10212            assert!(cotenant_task.await.unwrap().is_empty());
10213            let (cotenant_channel, cotenant_epoch) =
10214                published_route(&cotenant_rx.recv().await.unwrap());
10215
10216            // A second route.open, relayed to the incumbent and not yet acked.
10217            let caller = ConnectionId::new(32);
10218            let (caller_ctx, mut caller_rx) = route_ctx(caller);
10219            let (caller_task, caller_bind) = relay_route_open(
10220                &handler,
10221                caller,
10222                &caller_ctx.egress,
10223                &mut swap.incumbent_rx,
10224                101,
10225                "aft",
10226                "swap-caller",
10227            )
10228            .await;
10229            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
10230
10231            register_candidate(&swap, None);
10232            cutover(&swap);
10233
10234            // The incumbent acks after promotion and before any drain.
10235            let ack = handler
10236                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
10237                .await;
10238            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
10239            if module_loop_error.is_some() {
10240                // What the connection loop does with an untranslated router
10241                // error: end the connection, releasing every route on it.
10242                handler.cleanup_connection(INCUMBENT).unwrap();
10243            }
10244
10245            // 1. The incumbent's other routes survive.
10246            assert!(
10247                cotenant_rx.try_recv().is_err(),
10248                "the co-tenant route on the incumbent was torn down by one late ack: \
10249                 {module_loop_error:?}"
10250            );
10251            assert!(matches!(
10252                swap.forwarding
10253                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
10254                    .unwrap(),
10255                DataRoute::Client(DataRouteState::Bound(_))
10256            ));
10257            assert_eq!(module_loop_error, None);
10258            assert!(swap
10259                .registry
10260                .get_module_by_connection(INCUMBENT)
10261                .unwrap()
10262                .is_some());
10263
10264            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
10265            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
10266                .await
10267                .expect("the incumbent is told to drop the abandoned binding")
10268                .unwrap()
10269                .frame;
10270            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10271            assert_eq!(goodbye.header.channel, abandoned_channel);
10272            assert_eq!(goodbye.header.epoch, abandoned_epoch);
10273            assert!(swap.incumbent_rx.try_recv().is_err());
10274
10275            // 3. The waiting client gets a retryable refusal and no route.
10276            let response = caller_task.await.unwrap();
10277            assert_eq!(response.len(), 1);
10278            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
10279            assert!(caller_rx.try_recv().is_err());
10280
10281            // 4. The reservation pair is given back, and the pending bind
10282            //    settled exactly once: one accepted open (the co-tenant) and one
10283            //    refused open (the caller), nothing counted twice.
10284            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
10285            let counters = handler.counters().snapshot();
10286            assert_eq!(
10287                keyed_total(&counters, "route_open_accepted_by_principal"),
10288                1
10289            );
10290            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
10291            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
10292        }
10293
10294        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
10295        /// id would resolve to the promoted candidate and every new route.open
10296        /// would be refused as reloading, leaving neither process routable.
10297        #[tokio::test]
10298        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
10299            let mut swap = swap_with_incumbent().await;
10300            register_candidate(&swap, None);
10301            let incumbent = cutover(&swap);
10302            swap.forwarding
10303                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
10304                .unwrap()
10305                .expect("the incumbent is still registered");
10306
10307            let client = ConnectionId::new(33);
10308            let (client_ctx, mut client_rx) = route_ctx(client);
10309            let route_handler = swap.handler.clone();
10310            let open_ctx = RouteCtx {
10311                connection_id: client,
10312                egress: client_ctx.egress.clone(),
10313            };
10314            let mut route_task = tokio::spawn(async move {
10315                route_handler
10316                    .handle_control_frame(
10317                        &open_ctx,
10318                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
10319                    )
10320                    .await
10321                    .unwrap()
10322            });
10323            let bind = tokio::select! {
10324                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
10325                response = &mut route_task => {
10326                    let response = response.unwrap();
10327                    panic!(
10328                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
10329                        parse_error(&response[0])["code"]
10330                    );
10331                }
10332            };
10333            swap.handler
10334                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
10335                .await
10336                .unwrap();
10337            assert!(route_task.await.unwrap().is_empty());
10338            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
10339            match swap
10340                .forwarding
10341                .lookup_data_route(client, channel, epoch)
10342                .unwrap()
10343            {
10344                DataRoute::Client(DataRouteState::Bound(route)) => {
10345                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
10346                }
10347                other => panic!("expected a bound route on the candidate, got {other:?}"),
10348            }
10349            assert!(swap.incumbent_rx.try_recv().is_err());
10350        }
10351
10352        /// A candidate declares itself ready with `catalog.update` on its own
10353        /// connection. If the connection-keyed registry lookups searched only the
10354        /// active slot, this would answer `not_registered` and the candidate
10355        /// would never become ready.
10356        #[tokio::test]
10357        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
10358            let swap = swap_with_incumbent().await;
10359            register_candidate(&swap, Some(false));
10360            let update = Frame::build(
10361                FrameType::Request,
10362                control_flags(),
10363                0,
10364                0,
10365                55,
10366                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
10367                    provides: manifest("aft", PROTOCOL_VERSION).provides,
10368                    capabilities: None,
10369                    ready: Some(true),
10370                })
10371                .unwrap(),
10372            )
10373            .unwrap();
10374
10375            let replies = swap
10376                .handler
10377                .handle_control_frame(&swap.candidate_ctx, update)
10378                .await
10379                .unwrap();
10380
10381            assert_eq!(replies.len(), 1);
10382            assert_eq!(
10383                replies[0].header.ty,
10384                FrameType::Response,
10385                "candidate catalog.update was refused: {:?}",
10386                serde_json::from_slice::<Value>(&replies[0].body).ok()
10387            );
10388            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
10389            assert_eq!(
10390                swap.registry
10391                    .get_module("aft")
10392                    .unwrap()
10393                    .unwrap()
10394                    .connection_id,
10395                INCUMBENT
10396            );
10397        }
10398    }
10399
10400    /// The HELLO gate while the supervisor has a swap open: only the nonce it
10401    /// minted for the candidate admits a second process, into the candidate
10402    /// slot, and that check runs ahead of the reserved-module gate.
10403    mod swap_admission {
10404        use super::*;
10405
10406        const INCUMBENT_NONCE: &str = "incumbent-nonce";
10407        const CANDIDATE_NONCE: &str = "candidate-nonce";
10408
10409        fn handler_with_incumbent(
10410            module_id: &str,
10411            reserved: bool,
10412        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
10413            let registry = Arc::new(Registry::default());
10414            let supervisor = SupervisorHandle::new();
10415            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
10416            if reserved {
10417                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
10418            }
10419            let handler =
10420                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
10421            let incumbent = handler
10422                .handle_control(
10423                    ConnectionId::new(1),
10424                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
10425                )
10426                .unwrap();
10427            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
10428            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
10429            (registry, supervisor, handler)
10430        }
10431
10432        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
10433        /// admits every nonce, so while a swap is open the swap gate is the only
10434        /// thing between a key-holder and the candidate slot. A nonce the
10435        /// supervisor did not mint, or none at all, is refused, and neither the
10436        /// incumbent's registration nor the candidate slot moves.
10437        #[test]
10438        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
10439            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
10440
10441            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
10442                let replies = handler
10443                    .handle_control(
10444                        ConnectionId::new(connection),
10445                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
10446                    )
10447                    .unwrap();
10448                assert_eq!(replies[0].header.ty, FrameType::Error);
10449                assert_eq!(
10450                    parse_error(&replies[0])["code"],
10451                    "swap_token_invalid",
10452                    "nonce {nonce:?}"
10453                );
10454            }
10455            assert!(registry.get_candidate("aft").unwrap().is_none());
10456            assert_eq!(
10457                registry.get_module("aft").unwrap().unwrap().connection_id,
10458                ConnectionId::new(1)
10459            );
10460
10461            // Control: the minted token is admitted, into the candidate slot,
10462            // and only once.
10463            let admitted = handler
10464                .handle_control(
10465                    ConnectionId::new(4),
10466                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
10467                )
10468                .unwrap();
10469            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
10470            assert_eq!(
10471                registry
10472                    .get_candidate("aft")
10473                    .unwrap()
10474                    .unwrap()
10475                    .connection_id,
10476                ConnectionId::new(4)
10477            );
10478            assert_eq!(
10479                registry.get_module("aft").unwrap().unwrap().connection_id,
10480                ConnectionId::new(1),
10481                "the candidate must not take the active slot"
10482            );
10483            let replayed = handler
10484                .handle_control(
10485                    ConnectionId::new(5),
10486                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
10487                )
10488                .unwrap();
10489            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
10490
10491            // The case only this gate covers: the incumbent has died mid-swap,
10492            // so its duplicate refusal is gone too, and without the gate a
10493            // key-holder would take the id's ACTIVE slot.
10494            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
10495            let squatter = handler
10496                .handle_control(
10497                    ConnectionId::new(6),
10498                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
10499                )
10500                .unwrap();
10501            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
10502            assert!(
10503                registry.get_module("aft").unwrap().is_none(),
10504                "a squatter took the active slot of an id being swapped"
10505            );
10506        }
10507
10508        /// Design mutation arm (iii). A reserved module's candidate presents a
10509        /// nonce the reserved gate has never seen (that gate holds the
10510        /// incumbent's), so the swap gate must run first or the candidate is
10511        /// refused `reserved_module` and a reserved module can never be swapped.
10512        #[test]
10513        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
10514            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
10515
10516            let replies = handler
10517                .handle_control(
10518                    ConnectionId::new(2),
10519                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
10520                )
10521                .unwrap();
10522
10523            assert_eq!(
10524                replies[0].header.ty,
10525                FrameType::HelloAck,
10526                "reserved candidate refused: {:?}",
10527                serde_json::from_slice::<Value>(&replies[0].body).ok()
10528            );
10529            assert_eq!(
10530                registry
10531                    .get_candidate("vault")
10532                    .unwrap()
10533                    .unwrap()
10534                    .connection_id,
10535                ConnectionId::new(2)
10536            );
10537        }
10538
10539        /// With no swap open the gate is inert: the incumbent's reserved gate
10540        /// and duplicate refusal behave exactly as before.
10541        #[test]
10542        fn without_an_open_swap_the_ordinary_gates_decide() {
10543            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
10544            supervisor.close_swap("vault");
10545
10546            let candidate = handler
10547                .handle_control(
10548                    ConnectionId::new(2),
10549                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
10550                )
10551                .unwrap();
10552            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
10553            let duplicate = handler
10554                .handle_control(
10555                    ConnectionId::new(3),
10556                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
10557                )
10558                .unwrap();
10559            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10560            assert!(registry.get_candidate("vault").unwrap().is_none());
10561        }
10562    }
10563}
10564
10565#[cfg(test)]
10566mod concurrency_default_exposure_tests {
10567    use super::*;
10568
10569    fn hello_body(role_json: &str) -> Vec<u8> {
10570        format!(
10571            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
10572        )
10573        .into_bytes()
10574    }
10575
10576    fn manifest_from(body: &[u8]) -> ModuleManifest {
10577        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
10578        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
10579            .expect("manifest parses")
10580    }
10581
10582    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
10583
10584    #[test]
10585    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
10586        let body = hello_body(&format!(
10587            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
10588        ));
10589        let manifest = manifest_from(&body);
10590        // Precondition: serde really resolved it to the default, so the typed
10591        // manifest alone cannot answer the question this probe exists for.
10592        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
10593        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
10594    }
10595
10596    #[test]
10597    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
10598        let body = hello_body(&format!(
10599            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
10600        ));
10601        let manifest = manifest_from(&body);
10602        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
10603        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
10604    }
10605
10606    #[test]
10607    fn non_management_roles_are_never_reported() {
10608        let body = hello_body(
10609            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
10610        );
10611        let manifest = manifest_from(&body);
10612        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
10613    }
10614}