Skip to main content

subc_daemon/
control.rs

1use std::{
2    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3    fmt,
4    path::{Path, PathBuf},
5    sync::{Arc, Mutex, RwLock},
6    time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14    ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20    error_codes,
21    manifest::{
22        validate_hello_capability_grammar, validate_hello_event_declarations,
23        validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
24        Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
25    },
26    scope::{
27        ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
28        CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_DESCRIBE_OP, SCOPE_SYNC_OP,
29    },
30    session::{
31        validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
32        ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
33        OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
34        ROLE_VERSIONS_FIELD,
35    },
36    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
37    Priority, RouteTarget, PROTOCOL_VERSION,
38};
39use tokio::time::{timeout_at, Instant};
40use tracing::{debug, info, warn};
41
42use crate::{
43    capability_requirements::{
44        log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
45        CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
46        RuntimeModule,
47    },
48    daemon_config::RestartRequiredSection,
49    forwarding::{
50        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
51        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
52        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
53    },
54    observability::{
55        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
56    },
57    provenance::{
58        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
59    },
60    registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
61    router::{RouteCtx, RouterError},
62    scopes::{BoundScope, HelloLaunchNonces, ScopeTable},
63    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
64    stderr_tail::{CaptureState, TailEntry},
65    supervise::{
66        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
67        SupervisorHandle, SwapHelloAdmission,
68    },
69    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
70};
71
72/// Lowest envelope version this subc build will negotiate.
73///
74/// Module HELLO negotiation is exact: peers must use the daemon's locked
75/// protocol version. Older and newer peers receive `version_unsupported` and
76/// are not registered.
77pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
78
79const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
80const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
81const CAP_PING_PONG: &str = "ping_pong_v1";
82const CAP_SESSION_ATTACH: &str = "session_attach_v1";
83const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
84
85const SUBC_CONTROL_OPS: &[&str] = &[
86    ops::SERVER_DESCRIBE,
87    ops::CATALOG_LIST,
88    ops::ROUTE_OPEN,
89    ops::ROUTE_POLL,
90    ops::ROUTE_CLOSING,
91    ops::ROUTE_CLOSED,
92    ops::SUPERVISOR_LIST,
93    ops::SUPERVISOR_RESTART,
94    ops::SUPERVISOR_SWAP,
95    ops::SUPERVISOR_RELOAD,
96    ops::SUPERVISOR_RESCAN,
97    ops::SUPERVISOR_RELEASE_RESERVED,
98    ops::SUPERVISOR_SET_ENABLED,
99    ops::SUPERVISOR_HEALTH_PROBE,
100    ops::SUPERVISOR_HEALTH,
101    ops::SUPERVISOR_STDERR_TAIL,
102    ops::SUPERVISOR_TERMINALS,
103    ops::SUPERVISOR_ROUTES,
104    ops::SUPERVISOR_PROVENANCE,
105    ops::SUPERVISOR_SPAWN_SNAPSHOT,
106    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
107];
108
109const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
110    MODULE_TO_SUBC_OP_CATALOG_UPDATE,
111    "supervisor.live_roots",
112    SCOPE_SYNC_OP,
113    SCOPE_DESCRIBE_OP,
114    "operator.confirm",
115];
116
117/// Module-originated ops the daemon answers but does not advertise in
118/// `HELLO_ACK`. Empty today; an op is served from here while the feature it
119/// belongs to is incomplete, so no module is told it works before it does.
120const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
121
122const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
123
124/// How long subc waits for a module to ack a relayed route.bind before returning
125/// `module_timeout`. The ack waits on the module's own configure, which for AFT
126/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
127/// and DB-open work — on a cold page cache or a large repo that legitimately
128/// exceeds a couple of seconds. The default is generous because rejecting a VALID
129/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
130/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
131pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
132
133/// How many CONSECUTIVE full-budget relay timeouts against one target module
134/// open that module's bind-relay breaker.
135///
136/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
137/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
138/// so three consecutive full-budget timeouts take ~36s to observe: every client
139/// whose open contributed to opening the breaker had already given up on its
140/// own. That is what makes opening the breaker unable to turn a call that would
141/// have succeeded into a refusal — it can only make an already-failing module
142/// fail faster.
143///
144/// Two would be reachable inside one default deadline. One would convict a
145/// module on a single cold-cache bind, which is exactly the valid-but-slow case
146/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
147pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
148
149/// How long a module's bind-relay breaker stays open before exactly one
150/// `route.open` is let through as a probe.
151///
152/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
153/// re-pays a full-budget stall almost continuously, and the breaker stops being
154/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
155/// deadline: a client that starts retrying after the module recovers has to get
156/// a probe opportunity inside its own deadline, or the breaker converts a
157/// recovered module into a failed call — the failure it exists to prevent,
158/// pointed the other way.
159///
160/// 20s sits between those with room on both sides, and it caps what a wedged
161/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
162/// rather than one per `route.open` per connection. The stall that motivated
163/// this, with its measurements, is written up in
164/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
165/// waited the whole budget out.
166pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
167
168const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
169const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
170
171fn reload_verdict(
172    configured: &Path,
173    spawned_from: Option<&Path>,
174    image: subc_control::RunningImageAgreement,
175) -> PendingReloadVerdict {
176    let path = match spawned_from {
177        Some(spawned_from) if configured == spawned_from => ReloadPathAgreement::Match,
178        Some(spawned_from) => ReloadPathAgreement::Mismatch {
179            configured: configured.to_path_buf(),
180            spawned_from: spawned_from.to_path_buf(),
181        },
182        None => ReloadPathAgreement::Unavailable {
183            reason: if matches!(
184                image,
185                subc_control::RunningImageAgreement::Unavailable {
186                    reason: subc_control::RunningImageUnavailableReason::NotRunning
187                }
188            ) {
189                ReloadPathUnavailableReason::NotRunning
190            } else {
191                ReloadPathUnavailableReason::SpawnedPathUnavailable
192            },
193        },
194    };
195    PendingReloadVerdict { path, image }
196}
197
198#[derive(Clone)]
199struct DaemonProvenanceFacts {
200    build: DaemonBuildProvenance,
201    pid: Option<u32>,
202    started_at_ms: Option<u64>,
203    start_clock: Option<crate::clock::StartClock>,
204    executable_path: Option<PathBuf>,
205    executable_identity: Option<SpawnedFileIdentity>,
206    process_start_time: Option<u64>,
207    probe: ExecutableIdentityProbe,
208}
209
210impl Default for DaemonProvenanceFacts {
211    fn default() -> Self {
212        Self {
213            build: DaemonBuildProvenance {
214                build_git_sha: None,
215                build_lock_digest: None,
216            },
217            pid: None,
218            started_at_ms: None,
219            start_clock: None,
220            executable_path: None,
221            executable_identity: None,
222            process_start_time: None,
223            probe: ExecutableIdentityProbe::default(),
224        }
225    }
226}
227
228#[derive(Debug, Clone)]
229struct SupervisorRescanContext {
230    supervisor: Supervisor,
231    config_path: PathBuf,
232    configured_port: Option<u16>,
233    storage_config: Option<crate::daemon_config::StorageConfig>,
234    admission_facts_carrier_module_id: Option<String>,
235    admission_facts_targets: Option<Vec<String>>,
236    scope_authority_owners: Vec<String>,
237}
238
239/// Refusal labels passed to `observe_route_open_refusal` that mean the target
240/// module is not serving right now, and so open or extend an outage in the
241/// route outage tracker. Every one of them is only reachable after the target
242/// was found in the registry, which is what keeps an arbitrary client-chosen
243/// id from ever creating tracker state.
244///
245/// Deliberately absent: `not_registered` and `removed` (the id may be
246/// anything a client sent, and a removed module is gone on purpose),
247/// `protocol_none` (such a module never serves routes, so nothing is out),
248/// `role_not_provided`, `op_not_allowed`, `bad_consumer_identity`, the
249/// capability and admission-facts refusals (they refuse the caller, not a
250/// module outage), and `relay_reservation_failed` (its code ranges over
251/// capacity limits as well as a vanished connection). Capacity, breaker,
252/// relay-timeout and module-rejection refusals do not pass through that
253/// function at all; the breaker logs its own transitions.
254///
255/// The two not-serving refusals that bypass that function record themselves
256/// at their own sites: `supervised_not_registered` and `declared_not_ready`.
257/// `required_capability_unprovided` is not tracked: the module itself is up,
258/// and the outage belongs to the missing provider.
259const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
260    "reloading",
261    "supervisor_not_live",
262    "registration_not_active",
263    "no_forwarding_connection",
264    "relay_send_failed",
265];
266
267/// Real channel-0 control handler for subc itself.
268#[derive(Clone)]
269pub struct ControlHandler {
270    registry: Arc<Registry>,
271    forwarding: Arc<ForwardingTable>,
272    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
273    supervisor: SupervisorHandle,
274    subc_capabilities: Arc<[String]>,
275    /// Daemon-wide route.bind relay budget. Used as the fallback when the
276    /// target module has no per-module override in
277    /// `route_bind_relay_timeouts`.
278    route_bind_relay_timeout: Duration,
279    /// Per-module route.bind relay budget overrides, keyed by module id. When
280    /// `handle_route_open` resolves the deadline for a target module, a
281    /// per-module entry wins over the daemon-wide value above.
282    route_bind_relay_timeouts: BTreeMap<String, Duration>,
283    /// Per-target-module bind-relay breaker state. Shared with the forwarding
284    /// table, which is where a new module connection resets it.
285    route_bind_breakers: RouteBindBreakers,
286    /// Live relay admissions keyed by target module. Shared through the
287    /// forwarding table so cloned or separately built handlers enforce one cap.
288    route_bind_concurrency: RouteBindConcurrency,
289    /// Start and end of each module's not-serving period as seen by
290    /// `route.open`, so an outage gets one line at each edge instead of only
291    /// the per-refusal INFO lines. Taken from the forwarding table, so every
292    /// handler built over one table shares it.
293    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
294    /// Consecutive relay timeouts that open a module's breaker.
295    route_bind_breaker_threshold: u32,
296    /// How long a breaker stays open before one probe is admitted.
297    route_bind_breaker_cooldown: Duration,
298    health_probe_timeout: Duration,
299    /// Central storage policy. When set, each registering module receives its
300    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
301    storage_config: Option<crate::daemon_config::StorageConfig>,
302    /// The machine id established at boot, served on every HELLO_ACK and on
303    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
304    /// changes the file, never this value. `None` serves no id.
305    machine_id: Option<crate::machine_id::MachineId>,
306    admission_facts_carrier_module_id: Option<String>,
307    admission_facts_targets: Option<Vec<String>>,
308    /// Scope records with their sync authorities and tombstones; see
309    /// `crate::scopes`. Shared by clones of this handler, so every connection
310    /// reads and writes one table.
311    scopes: Arc<RwLock<ScopeTable>>,
312    /// The configured `scope_authority_owners`, kept so a rescan can report a
313    /// changed value as needing a daemon restart; rescan never applies it.
314    scope_authority_owners: Vec<String>,
315    /// The launch nonce each module connection presented at HELLO, which is how
316    /// a `scope.sync` is matched to the owner's current launch.
317    hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
318    rescan: Option<SupervisorRescanContext>,
319    connected_clients: ConnectedClients,
320    counters: DaemonCounters,
321    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
322    daemon_provenance: DaemonProvenanceFacts,
323    #[cfg(test)]
324    control_dispatch_delay: Option<Duration>,
325    #[cfg(test)]
326    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
327}
328
329impl fmt::Debug for ControlHandler {
330    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
331        f.debug_struct("ControlHandler")
332            .field("registry", &self.registry)
333            .field("forwarding", &self.forwarding)
334            .field("process_liveness", &self.process_liveness.is_some())
335            .field("supervisor", &self.supervisor)
336            .field("subc_capabilities", &self.subc_capabilities)
337            .finish()
338    }
339}
340
341struct RouteOpenRequest {
342    target: RouteTarget,
343    identity: BindIdentity,
344    consumer_identity: Option<ConsumerIdentity>,
345    consumer_capabilities: Option<Vec<String>>,
346    role_versions: Option<BTreeMap<String, String>>,
347    admission_facts: Option<serde_json::Value>,
348    scope: Option<ScopeSelector>,
349}
350
351struct RouteBindReservationGuard {
352    forwarding: Arc<ForwardingTable>,
353    endpoint: ModuleEndpointId,
354    relay_corr: u64,
355    armed: bool,
356}
357
358struct ModuleControlRpcGuard {
359    forwarding: Arc<ForwardingTable>,
360    endpoint: ModuleEndpointId,
361    corr: u64,
362    armed: bool,
363}
364
365impl ModuleControlRpcGuard {
366    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
367        Self {
368            forwarding,
369            endpoint,
370            corr,
371            armed: true,
372        }
373    }
374
375    fn disarm(&mut self) {
376        self.armed = false;
377    }
378}
379
380impl Drop for ModuleControlRpcGuard {
381    fn drop(&mut self) {
382        if self.armed {
383            let _ = self
384                .forwarding
385                .cancel_module_control_rpc(self.endpoint, self.corr);
386        }
387    }
388}
389
390impl RouteBindReservationGuard {
391    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
392        Self {
393            forwarding,
394            endpoint,
395            relay_corr,
396            armed: true,
397        }
398    }
399
400    fn release_and_disarm(&mut self) {
401        if !self.armed {
402            return;
403        }
404        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
405            self.endpoint,
406            self.relay_corr,
407            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
408        ) {
409            send_goodbye_target_best_effort(
410                &self.forwarding.counters(),
411                &target,
412                "canceled route.bind",
413            );
414        }
415        self.armed = false;
416    }
417
418    fn disarm(&mut self) {
419        self.armed = false;
420    }
421}
422
423impl Drop for RouteBindReservationGuard {
424    fn drop(&mut self) {
425        self.release_and_disarm();
426    }
427}
428
429/// Per-target-module circuit breaker around the `route.bind` relay.
430///
431/// The connection reader is serial per connection, so a module whose `on_bind`
432/// sits on the ack blocks every LATER frame on the connections that call it,
433/// including calls to unrelated modules. This does not make any module's bind
434/// fast; it stops the daemon paying the full budget again and again for a
435/// condition it has already observed.
436///
437/// State is keyed by TARGET MODULE and shared by every connection: a wedged
438/// module wedges everyone, so what one connection learned should protect the
439/// rest.
440///
441/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
442/// relay to that module has actually timed out, and is removed again when a
443/// relay is accepted or the module reconnects, so it cannot grow with traffic
444/// or with modules that behave.
445///
446/// # Why a `std` mutex here is not the head-of-line defect again
447///
448/// Acquisition never awaits. The critical section is a hash lookup plus a few
449/// integer updates, with no I/O and no `.await` inside it, so a reader task
450/// cannot be descheduled behind it the way it can behind
451/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
452/// primitive, held for the same kind of work, as the refusal counter this very
453/// path already increments.
454///
455/// It is also NOT on the data-plane splice path: only `route.open` and module
456/// registration touch it, so bound-route frames gain no state check and no
457/// contention.
458#[derive(Debug, Clone, Default)]
459pub(crate) struct RouteBindBreakers {
460    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
461}
462
463#[derive(Debug, Clone, Default)]
464pub(crate) struct RouteBindConcurrency {
465    modules: Arc<Mutex<HashMap<String, usize>>>,
466}
467
468struct RouteBindConcurrencyGuard {
469    concurrency: RouteBindConcurrency,
470    module_id: String,
471}
472
473impl RouteBindConcurrency {
474    /// Admit without waiting. Waiting here would move the bind stall from the
475    /// module reply to a semaphore and restore reader head-of-line blocking.
476    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
477        let mut modules = self
478            .modules
479            .lock()
480            .expect("route.bind concurrency mutex poisoned");
481        let in_flight = modules.entry(module_id.to_string()).or_default();
482        if *in_flight >= limit {
483            return Err(*in_flight);
484        }
485        *in_flight += 1;
486        Ok(RouteBindConcurrencyGuard {
487            concurrency: self.clone(),
488            module_id: module_id.to_string(),
489        })
490    }
491}
492
493impl Drop for RouteBindConcurrencyGuard {
494    fn drop(&mut self) {
495        let mut modules = self
496            .concurrency
497            .modules
498            .lock()
499            .expect("route.bind concurrency mutex poisoned");
500        let remove = {
501            let in_flight = modules
502                .get_mut(&self.module_id)
503                .expect("admitted route.bind has a concurrency entry");
504            *in_flight -= 1;
505            *in_flight == 0
506        };
507        if remove {
508            modules.remove(&self.module_id);
509        }
510    }
511}
512
513#[derive(Debug, Default)]
514struct ModuleBreakerState {
515    /// Relay timeouts observed with no accepted relay in between.
516    consecutive_timeouts: u32,
517    /// `Some` while the breaker is open: the instant the cooldown expires and
518    /// the next arrival may probe. `None` means closed.
519    cooldown_until: Option<Instant>,
520    /// A half-open probe has been admitted and has not settled yet. This is
521    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
522    /// that read the cooldown, so concurrent opens arriving at the moment the
523    /// cooldown expires cannot all decide that they are the probe.
524    probe_in_flight: Option<Arc<()>>,
525}
526
527/// What the breaker decided for one `route.open`, before any relay work.
528enum RouteBindAdmission<'a> {
529    Admitted {
530        guard: RouteBindBreakerGuard<'a>,
531        /// This open is the single half-open probe, so the transition is worth
532        /// one log line.
533        probe: bool,
534    },
535    Refused {
536        consecutive_timeouts: u32,
537        /// What is left of the cooldown. Zero when the refusal is because the
538        /// one probe is already in flight rather than because the cooldown has
539        /// not elapsed.
540        retry_in: Duration,
541        probe_in_flight: bool,
542    },
543}
544
545/// An outstanding admission, which must be told how its relay settled.
546///
547/// `Drop` settles it as inconclusive, so an early return between admission and
548/// the relay -- or the whole handler being cancelled when the client
549/// disconnects -- releases a half-open probe slot instead of leaving the
550/// breaker wedged half-open with no further probes.
551struct RouteBindBreakerGuard<'a> {
552    breakers: RouteBindBreakers,
553    module_id: &'a str,
554    probe_token: Option<Arc<()>>,
555    settled: bool,
556}
557
558impl RouteBindBreakerGuard<'_> {
559    /// The module answered within the budget and took the bind. THE ONLY
560    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
561    /// breaker, which is a transition worth logging.
562    fn record_accepted(&mut self) -> bool {
563        self.settled = true;
564        self.breakers.record_accepted(self.module_id)
565    }
566
567    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
568    /// COUNTS TOWARD OPENING.
569    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
570        self.settled = true;
571        self.breakers.record_timeout(
572            self.module_id,
573            self.probe_token.as_ref(),
574            threshold,
575            cooldown,
576        )
577    }
578
579    /// Everything else: the module REJECTED the bind, its connection went away
580    /// mid-relay, or the waiter was cancelled.
581    ///
582    /// None of these is evidence that a module is slow, and each already has
583    /// its own refusal with its own code. A module that rejects a bind in
584    /// microseconds is healthy and must never be convicted for it; a module
585    /// that died has said nothing about the module that replaces it. So these
586    /// neither increment nor reset the count -- they only release a probe slot.
587    fn record_inconclusive(&mut self) {
588        self.settled = true;
589        self.breakers
590            .record_inconclusive(self.module_id, self.probe_token.as_ref());
591    }
592}
593
594impl Drop for RouteBindBreakerGuard<'_> {
595    fn drop(&mut self) {
596        if !self.settled {
597            self.breakers
598                .record_inconclusive(self.module_id, self.probe_token.as_ref());
599        }
600    }
601}
602
603/// The breaker moved to open, reported so the caller can log it outside the
604/// lock. Opening is rare and load-bearing; the refusals that follow are
605/// frequent and are counted rather than logged.
606struct BreakerOpened {
607    consecutive_timeouts: u32,
608    /// True when a failed probe re-opened an already-open breaker, which reads
609    /// very differently in a log from a first opening.
610    reopened_after_probe: bool,
611}
612
613impl RouteBindBreakers {
614    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
615        self.modules
616            .lock()
617            .expect("route.bind breaker mutex poisoned")
618    }
619
620    /// Decide whether this `route.open` may attempt its relay. Takes the map
621    /// lock and nothing else, and never awaits.
622    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
623        let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
624            probe: probe_token.is_some(),
625            guard: RouteBindBreakerGuard {
626                breakers: self.clone(),
627                module_id,
628                probe_token,
629                settled: false,
630            },
631        };
632
633        let mut modules = self.lock();
634        let Some(state) = modules.get_mut(module_id) else {
635            return admitted(None);
636        };
637        let Some(cooldown_until) = state.cooldown_until else {
638            return admitted(None);
639        };
640        if state.probe_in_flight.is_some() {
641            return RouteBindAdmission::Refused {
642                consecutive_timeouts: state.consecutive_timeouts,
643                retry_in: Duration::ZERO,
644                probe_in_flight: true,
645            };
646        }
647        let now = Instant::now();
648        if now < cooldown_until {
649            return RouteBindAdmission::Refused {
650                consecutive_timeouts: state.consecutive_timeouts,
651                retry_in: cooldown_until - now,
652                probe_in_flight: false,
653            };
654        }
655        let token = Arc::new(());
656        state.probe_in_flight = Some(Arc::clone(&token));
657        admitted(Some(token))
658    }
659
660    fn record_accepted(&self, module_id: &str) -> bool {
661        self.lock()
662            .remove(module_id)
663            .is_some_and(|state| state.cooldown_until.is_some())
664    }
665
666    fn record_timeout(
667        &self,
668        module_id: &str,
669        probe_token: Option<&Arc<()>>,
670        threshold: u32,
671        cooldown: Duration,
672    ) -> Option<BreakerOpened> {
673        let mut modules = self.lock();
674        let state = modules.entry(module_id.to_string()).or_default();
675        let was_open = state.cooldown_until.is_some();
676        let was_probe = Self::owns_probe(state, probe_token);
677        if was_probe {
678            state.probe_in_flight = None;
679        }
680        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
681        if state.consecutive_timeouts < threshold {
682            return None;
683        }
684        state.cooldown_until = Some(Instant::now() + cooldown);
685        Some(BreakerOpened {
686            consecutive_timeouts: state.consecutive_timeouts,
687            reopened_after_probe: was_open && was_probe,
688        })
689    }
690
691    fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
692        // A relay can finish after the breaker was reset or after it opened
693        // again and started a new probe. Only the guard whose token matches the
694        // active probe may release it, so a late relay never frees a newer probe.
695        state
696            .probe_in_flight
697            .as_ref()
698            .zip(token)
699            .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
700    }
701
702    fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
703        if let Some(state) = self.lock().get_mut(module_id) {
704            if Self::owns_probe(state, probe_token) {
705                state.probe_in_flight = None;
706            }
707        }
708    }
709
710    /// Discard what was learned about a module, because the process it was
711    /// learned about is gone. Returns the discarded count when it was non-zero.
712    ///
713    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
714    /// `module_id` is a configuration identity that outlives any particular
715    /// child; what the breaker observed was the process behind the module
716    /// connection of the moment. When a new connection registers under that id
717    /// the verdict's subject no longer exists, so the verdict is stale by
718    /// construction rather than merely likely to be wrong. Keeping it would
719    /// apply a dead process's record to a live one, which is the same defect
720    /// class this breaker exists to stop the daemon committing.
721    ///
722    /// A half-open probe in flight is discarded with the rest: it was a
723    /// question about the old process.
724    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
725        self.lock()
726            .remove(module_id)
727            .map(|state| state.consecutive_timeouts)
728            .filter(|discarded| *discarded > 0)
729    }
730
731    /// Open breakers, for the `server.describe` counters object. `None` when
732    /// none is open, so the key stays absent rather than present-and-empty.
733    ///
734    /// This is the operator's answer to "is this module refusing instantly or
735    /// is it fine?", which look identical from a client that retries and then
736    /// succeeds.
737    fn open_snapshot(&self) -> Option<serde_json::Value> {
738        let now = Instant::now();
739        let modules = self.lock();
740        let open = modules
741            .iter()
742            .filter_map(|(module_id, state)| {
743                let cooldown_until = state.cooldown_until?;
744                Some((
745                    module_id.clone(),
746                    serde_json::json!({
747                        "consecutive_timeouts": state.consecutive_timeouts,
748                        "cooldown_remaining_ms":
749                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
750                        "probe_in_flight": state.probe_in_flight.is_some(),
751                    }),
752                ))
753            })
754            .collect::<serde_json::Map<String, serde_json::Value>>();
755        (!open.is_empty()).then_some(serde_json::Value::Object(open))
756    }
757}
758
759impl ControlHandler {
760    pub fn new(registry: Arc<Registry>) -> Self {
761        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
762    }
763
764    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
765        let counters = forwarding.counters();
766        // Taken from the forwarding table rather than created here, so that the
767        // breaker a `route.open` consults is the same one a module's
768        // registration resets, however many handlers are built over one table.
769        let route_bind_breakers = forwarding.route_bind_breakers();
770        let route_bind_concurrency = forwarding.route_bind_concurrency();
771        let route_outages = forwarding.route_outages();
772        Self {
773            registry,
774            forwarding,
775            process_liveness: None,
776            supervisor: SupervisorHandle::new(),
777            subc_capabilities: Arc::from([
778                CAP_MANIFEST_REGISTRATION.to_string(),
779                CAP_CHANNEL_LIFECYCLE.to_string(),
780                CAP_PING_PONG.to_string(),
781                CAP_SESSION_ATTACH.to_string(),
782                CAP_ADMISSION_FACTS_RELAY.to_string(),
783                CAP_SCOPES_V1.to_string(),
784                CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
785            ]),
786            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
787            route_bind_relay_timeouts: BTreeMap::new(),
788            route_bind_breakers,
789            route_bind_concurrency,
790            route_outages,
791            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
792            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
793            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
794            storage_config: None,
795            machine_id: None,
796            admission_facts_carrier_module_id: None,
797            admission_facts_targets: None,
798            scopes: Arc::new(RwLock::new(ScopeTable::new(
799                crate::daemon_config::default_scope_authority_owners(),
800            ))),
801            scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
802            hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
803            rescan: None,
804            connected_clients: ConnectedClients::new(),
805            counters,
806            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
807            daemon_provenance: DaemonProvenanceFacts::default(),
808            #[cfg(test)]
809            control_dispatch_delay: None,
810            #[cfg(test)]
811            provenance_probe_override: None,
812        }
813    }
814
815    /// Set the central storage policy: registering modules then receive their
816    /// resolved storage descriptor in HELLO_ACK.
817    pub fn with_storage_config(
818        mut self,
819        storage_config: Option<crate::daemon_config::StorageConfig>,
820    ) -> Self {
821        self.storage_config = storage_config;
822        self
823    }
824
825    /// Set the machine id served to every registering module (HELLO_ACK) and on
826    /// `server.describe`.
827    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
828        self.machine_id = machine_id;
829        self
830    }
831
832    /// Configure the exact reserved module and target ids permitted to relay
833    /// opaque admission facts. Config-file loading validates this authority;
834    /// this builder keeps the same policy available to embedded test daemons.
835    pub fn with_admission_facts_config(
836        mut self,
837        carrier_module_id: Option<String>,
838        targets: Option<Vec<String>>,
839    ) -> Self {
840        self.admission_facts_carrier_module_id = carrier_module_id;
841        self.admission_facts_targets = targets;
842        self
843    }
844
845    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
846    /// Replaces the scope table with an empty one under the new list, so call it
847    /// while building the handler, before any module can sync.
848    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
849        self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
850        self.scope_authority_owners = owners;
851        self
852    }
853
854    /// Override the route.bind relay timeout. Used by tests that assert the
855    /// timeout path so they don't block on the production-safe default.
856    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
857        self.route_bind_relay_timeout = timeout;
858        self
859    }
860
861    /// Install per-module route.bind relay budget overrides. A module id
862    /// listed here wins over the daemon-wide default set via
863    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
864    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
865    /// the same `Duration` the bind path will use.
866    pub fn with_route_bind_relay_timeouts(
867        mut self,
868        timeouts: impl IntoIterator<Item = (String, Duration)>,
869    ) -> Self {
870        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
871        self
872    }
873
874    /// Resolve the route.bind relay budget for a specific target module id.
875    /// Per-module overrides win; the daemon-wide value (set via
876    /// `with_route_bind_relay_timeout` or the built-in default) is the
877    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
878    /// the same resolution `handle_route_open` will use.
879    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
880        self.route_bind_relay_timeouts
881            .get(module_id)
882            .copied()
883            .unwrap_or(self.route_bind_relay_timeout)
884    }
885
886    /// Override the per-module bind-relay breaker policy.
887    ///
888    /// Used by tests, which cannot spend three production budgets opening a
889    /// breaker or twenty seconds waiting for its cooldown. The production
890    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
891    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
892    /// reasoning for the numbers.
893    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
894        self.route_bind_breaker_threshold = threshold.max(1);
895        self.route_bind_breaker_cooldown = cooldown;
896        self
897    }
898
899    #[cfg(test)]
900    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
901        self.health_probe_timeout = timeout;
902        self
903    }
904
905    #[cfg(test)]
906    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
907        self.control_dispatch_delay = Some(delay);
908        self
909    }
910
911    pub fn with_process_liveness(
912        mut self,
913        process_liveness: Arc<dyn ModuleProcessLiveness>,
914    ) -> Self {
915        self.process_liveness = Some(process_liveness);
916        self
917    }
918
919    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
920        self.supervisor = supervisor;
921        self
922    }
923
924    pub fn with_daemon_provenance(
925        mut self,
926        pid: u32,
927        started_at_ms: u64,
928        executable_path: Option<PathBuf>,
929        build_git_sha: Option<String>,
930        build_lock_digest: Option<String>,
931    ) -> Self {
932        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
933        let process_start_time = process_start_time(pid);
934        self.daemon_provenance = DaemonProvenanceFacts {
935            build: DaemonBuildProvenance {
936                build_git_sha,
937                build_lock_digest,
938            },
939            pid: Some(pid),
940            started_at_ms: Some(started_at_ms),
941            start_clock: None,
942            executable_path,
943            executable_identity,
944            process_start_time,
945            probe: ExecutableIdentityProbe::default(),
946        };
947        self
948    }
949
950    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
951        self.daemon_provenance.start_clock = Some(clock);
952        self
953    }
954
955    #[cfg(test)]
956    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
957        self.provenance_probe_override = Some(result);
958        self
959    }
960
961    /// Install the configured module set and its reserved capability bindings.
962    /// Bindings are configuration-scoped and may point at a provider that has not
963    /// been installed yet, so this does not require the bound module to exist.
964    pub fn with_capability_config(
965        self,
966        modules: impl IntoIterator<Item = (String, bool)>,
967        reserved_capabilities: BTreeMap<String, String>,
968    ) -> Self {
969        self.capability_evaluator
970            .configure(modules, reserved_capabilities);
971        self
972    }
973
974    pub fn with_supervisor_rescan(
975        mut self,
976        supervisor: Supervisor,
977        config_path: impl Into<PathBuf>,
978        configured_port: Option<u16>,
979    ) -> Self {
980        self.rescan = Some(SupervisorRescanContext {
981            supervisor,
982            config_path: config_path.into(),
983            configured_port,
984            storage_config: self.storage_config.clone(),
985            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
986            admission_facts_targets: self.admission_facts_targets.clone(),
987            scope_authority_owners: self.scope_authority_owners.clone(),
988        });
989        self
990    }
991
992    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
993        self.connected_clients = connected_clients;
994        self
995    }
996
997    pub fn forwarding(&self) -> Arc<ForwardingTable> {
998        Arc::clone(&self.forwarding)
999    }
1000
1001    pub(crate) fn counters(&self) -> DaemonCounters {
1002        self.counters.clone()
1003    }
1004
1005    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
1006    /// requirement event without depending on an operator polling a status command.
1007    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1008        tokio::spawn(async move {
1009            loop {
1010                self.capability_evaluator
1011                    .wait_for_change_or_deadline()
1012                    .await;
1013                self.refresh_capability_requirements();
1014            }
1015        });
1016    }
1017
1018    fn runtime_capability_snapshot(
1019        &self,
1020    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1021        let runtime = self
1022            .supervisor
1023            .list()
1024            .into_iter()
1025            .map(|module| {
1026                let status = module.status().map_err(|err| {
1027                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1028                })?;
1029                Ok(RuntimeModule {
1030                    module_id: status.module_id,
1031                    state: status.state,
1032                    enabled: status.enabled,
1033                })
1034            })
1035            .collect::<Result<Vec<_>, RouterError>>()?;
1036        let (_, registrations) = self.registry.list_modules().map_err(|err| {
1037            RouterError::backend(
1038                0,
1039                0,
1040                format!("failed to list capability registrations: {err}"),
1041            )
1042        })?;
1043        let registrations = registrations
1044            .into_iter()
1045            .map(|registration| RegisteredModule {
1046                module_id: registration.manifest.module_id,
1047                module_version: registration.manifest.module_version,
1048                capabilities: registration.manifest.capabilities,
1049            })
1050            .collect();
1051        Ok((runtime, registrations))
1052    }
1053
1054    /// The capability side effects of a module becoming the active registration
1055    /// for its id: cache its manifest (warning if its claims drifted), run the
1056    /// deny census when its declarations call for one, and recompute the
1057    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
1058    /// candidate's does not, and the supervisor does it at promotion instead,
1059    /// through [`crate::supervise::SwapPromotionObserver`].
1060    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1061        let cached_registration = RegisteredModule {
1062            module_id: registration.manifest.module_id.clone(),
1063            module_version: registration.manifest.module_version.clone(),
1064            capabilities: registration.manifest.capabilities.clone(),
1065        };
1066        if self.capability_evaluator.record_hello(&cached_registration) {
1067            warn!(
1068                module_id = %cached_registration.module_id,
1069                "capability claims drifted from the cached manifest"
1070            );
1071        }
1072        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1073            self.enforce_capability_denies();
1074        }
1075        self.refresh_capability_requirements();
1076    }
1077
1078    /// Point the shared supervisor handle at this handler for swap promotions.
1079    /// Called wherever a handler is put behind the `Arc` the router serves, so
1080    /// it can be held weakly.
1081    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1082        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1083            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1084        self.supervisor.set_swap_promotion_observer(observer);
1085    }
1086
1087    pub fn refresh_capability_requirements(&self) {
1088        match self.runtime_capability_snapshot() {
1089            Ok((runtime, registrations)) => {
1090                log_requirement_events(
1091                    self.capability_evaluator
1092                        .evaluate_now(&runtime, &registrations),
1093                );
1094            }
1095            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1096        }
1097    }
1098
1099    /// Reconcile only live, attested route bindings after a capability deny edge
1100    /// or target claim was added. This is deliberately a control-plane census:
1101    /// the opaque forwarding hot path must not grow a per-frame capability check.
1102    fn enforce_capability_denies(&self) {
1103        let (_, registrations) = match self.registry.list_modules() {
1104            Ok(snapshot) => snapshot,
1105            Err(err) => {
1106                warn!(error = %err, "failed to read registrations for capability deny census");
1107                return;
1108            }
1109        };
1110        let manifests = registrations
1111            .into_iter()
1112            .map(|registration| {
1113                (
1114                    registration.manifest.module_id.clone(),
1115                    registration.manifest,
1116                )
1117            })
1118            .collect::<BTreeMap<_, _>>();
1119        let census = match self.forwarding.route_census(None) {
1120            Ok(census) => census,
1121            Err(err) => {
1122                warn!(error = %err, "failed to read route census for capability deny enforcement");
1123                return;
1124            }
1125        };
1126
1127        for (target_module_id, routes) in census {
1128            let Some(target_manifest) = manifests.get(&target_module_id) else {
1129                continue;
1130            };
1131            let mut closed_routes = Vec::new();
1132            let mut module_goodbyes = Vec::new();
1133            for route in routes {
1134                let Principal::Reserved {
1135                    module_id: opening_module_id,
1136                } = &route.principal
1137                else {
1138                    continue;
1139                };
1140                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1141                    continue;
1142                };
1143                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1144                    continue;
1145                };
1146
1147                match self.forwarding.release_client_route(
1148                    route.goodbye_target.connection_id,
1149                    route.goodbye_target.channel,
1150                    route.goodbye_target.epoch,
1151                ) {
1152                    Ok(RouteRelease::Removed(module_goodbye)) => {
1153                        warn!(
1154                            opening_module_id,
1155                            target_module_id,
1156                            capability,
1157                            "force-closing route because an attested capability deny edge now matches"
1158                        );
1159                        closed_routes.push(route);
1160                        module_goodbyes.push(module_goodbye);
1161                    }
1162                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1163                    Err(err) => warn!(
1164                        opening_module_id,
1165                        target_module_id,
1166                        capability,
1167                        error = %err,
1168                        "failed to force-close capability-denied route"
1169                    ),
1170                }
1171            }
1172
1173            if closed_routes.is_empty() {
1174                continue;
1175            }
1176            send_route_control_pushes(
1177                &self.forwarding,
1178                closed_routes,
1179                ClientControlPush::RouteClosed {
1180                    module_id: target_module_id,
1181                    channels: Vec::new(),
1182                    reason: RouteCloseReason::CapabilityDenied,
1183                    drained: false,
1184                    abandoned: 0,
1185                    excluded_subscriptions: 0,
1186                    terminal: Some(false),
1187                },
1188            );
1189            self.emit_route_goodbyes(module_goodbyes);
1190        }
1191    }
1192
1193    /// Why a registered module is not accepting new route binds, or `None` when
1194    /// it is. This is the module's effective readiness: its declared readiness
1195    /// first, then every `need: required` capability it declares evaluating to
1196    /// `provided`. `route.open` and `catalog.list` both read it here so the
1197    /// catalog never reports a module routable that `route.open` would refuse.
1198    fn not_ready_reason(
1199        &self,
1200        registration: &crate::registry::ModuleRegistration,
1201    ) -> Option<NotReadyReason> {
1202        if !registration.ready {
1203            return Some(NotReadyReason {
1204                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1205                capability: None,
1206            });
1207        }
1208        self.first_unprovided_required_capability(registration)
1209            .map(|capability| NotReadyReason {
1210                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1211                capability: Some(capability),
1212            })
1213    }
1214
1215    /// The lexicographically first capability this registration declares
1216    /// `need: required` whose evaluator verdict is not `provided`.
1217    ///
1218    /// The verdicts are the capability evaluator's own; nothing here decides
1219    /// what "provided" means. The evaluator counts a capability provided as
1220    /// soon as a module claiming it has REGISTERED, not once that module is
1221    /// ready. That distinction is what keeps two modules that require each
1222    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1223    /// is ready", each would wait for the other to become ready first and
1224    /// neither ever would. Do not tighten it to readiness.
1225    ///
1226    /// A required capability with no verdict at all means this registration's
1227    /// HELLO or catalog.update landed after the last recompute; recompute once
1228    /// rather than let a missing verdict read as either answer. If it is still
1229    /// missing (the recompute itself failed) the capability counts as
1230    /// unprovided: the refusal is retryable, and routing a module whose
1231    /// required provider is unknown is the outcome this check exists to stop.
1232    fn first_unprovided_required_capability(
1233        &self,
1234        registration: &crate::registry::ModuleRegistration,
1235    ) -> Option<String> {
1236        let required = registration
1237            .manifest
1238            .capabilities
1239            .iter()
1240            .flat_map(|declarations| declarations.requires.iter())
1241            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1242            .map(|requirement| requirement.capability.as_str())
1243            .collect::<BTreeSet<_>>();
1244        if required.is_empty() {
1245            return None;
1246        }
1247        let module_id = registration.manifest.module_id.as_str();
1248        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1249        if required
1250            .iter()
1251            .any(|capability| verdict(capability).is_none())
1252        {
1253            self.refresh_capability_requirements();
1254        }
1255        required
1256            .into_iter()
1257            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1258            .map(str::to_string)
1259    }
1260
1261    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1262        self.capability_evaluator
1263            .statuses()
1264            .into_iter()
1265            .map(capability_requirement_status)
1266            .collect()
1267    }
1268
1269    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1270    /// registration-release watch. The signal is what the supervisor waits on
1271    /// before spawning a replacement, so it must only fire once forwarding
1272    /// teardown is also done (see [`Self::cleanup_connection`] /
1273    /// [`Self::handle_goodbye`]). Used directly only where a registry entry was
1274    /// admitted but its forwarding endpoint could not be installed.
1275    fn deregister_connection(
1276        &self,
1277        connection_id: ConnectionId,
1278        reason: RegistrationEndReason,
1279    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1280        self.registry
1281            .deregister_connection_with_reason(connection_id, reason)
1282    }
1283
1284    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1285        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1286            return None;
1287        }
1288        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1289            parse_client_control_request(&frame.body)
1290        else {
1291            return None;
1292        };
1293        Some(target_module_id(&target).to_string())
1294    }
1295
1296    pub(crate) fn route_open_capacity_refusal(
1297        &self,
1298        ctx: &RouteCtx,
1299        frame: &Frame,
1300        target_module_id: &str,
1301        in_flight: usize,
1302        limit: usize,
1303    ) -> Result<Frame, RouterError> {
1304        self.route_open_admission_refusal_frame(
1305            ctx,
1306            frame,
1307            target_module_id,
1308            "open_admission_full",
1309            (in_flight, limit),
1310            format!(
1311                "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1312            ),
1313        )
1314    }
1315
1316    fn route_open_target_capacity_refusal(
1317        &self,
1318        ctx: &RouteCtx,
1319        frame: &Frame,
1320        target_module_id: &str,
1321        in_flight: usize,
1322    ) -> Result<Frame, RouterError> {
1323        self.route_open_admission_refusal_frame(
1324            ctx,
1325            frame,
1326            target_module_id,
1327            "target_binds_full",
1328            (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1329            format!(
1330                "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1331            ),
1332        )
1333    }
1334
1335    /// Admission pressure clears as existing binds settle, so its refusal must
1336    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
1337    /// `module_reloading`, `module_warming`, `target_unavailable`, or
1338    /// `module_timeout`. `target_unavailable` is honest for an attempt that
1339    /// cannot currently reach its target; `module_timeout` would falsely claim
1340    /// that a wait expired. A new, cleaner code would be terminal to deployed
1341    /// clients, so it requires a client-tolerance rollout before daemon emission.
1342    fn route_open_admission_refusal_frame(
1343        &self,
1344        ctx: &RouteCtx,
1345        frame: &Frame,
1346        target_module_id: &str,
1347        reason: &'static str,
1348        (in_flight, limit): (usize, usize),
1349        message: impl Into<String>,
1350    ) -> Result<Frame, RouterError> {
1351        let code = error_codes::TARGET_UNAVAILABLE;
1352        self.counters.increment_route_open_refused(code);
1353        info!(
1354            target: "control",
1355            code,
1356            reason,
1357            module_id = ?target_module_id,
1358            connection_id = ctx.connection_id.get(),
1359            in_flight,
1360            limit,
1361            "route.open refused"
1362        );
1363        control_error_frame(frame, code, message.into())
1364    }
1365
1366    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
1367    ///
1368    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
1369    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
1370    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
1371    #[cfg(test)]
1372    pub fn handle_control(
1373        &self,
1374        connection_id: ConnectionId,
1375        frame: Frame,
1376    ) -> Result<Vec<Frame>, RouterError> {
1377        match frame.header.ty {
1378            FrameType::Ping => Ok(vec![pong(&frame)?]),
1379            FrameType::Hello => self.handle_hello(connection_id, None, frame),
1380            FrameType::Goodbye => self.handle_goodbye(connection_id),
1381            ty => Ok(vec![control_error_frame(
1382                &frame,
1383                "unsupported_control_frame",
1384                format!("unsupported channel-0 frame {ty:?}"),
1385            )?]),
1386        }
1387    }
1388
1389    pub async fn handle_control_frame(
1390        &self,
1391        ctx: &RouteCtx,
1392        frame: Frame,
1393    ) -> Result<Vec<Frame>, RouterError> {
1394        self.handle_control_frame_timed(ctx, frame, None).await
1395    }
1396
1397    pub(crate) async fn handle_control_frame_timed(
1398        &self,
1399        ctx: &RouteCtx,
1400        frame: Frame,
1401        dispatch_started_at: Option<StdInstant>,
1402    ) -> Result<Vec<Frame>, RouterError> {
1403        match frame.header.ty {
1404            FrameType::Ping => Ok(vec![pong(&frame)?]),
1405            FrameType::Hello => {
1406                self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1407            }
1408            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1409            FrameType::Cancel => {
1410                // A Cancel on channel 0 names either a waiting operator.confirm
1411                // or a spawn-event subscription; both answer nothing on success.
1412                if self
1413                    .forwarding
1414                    .operator_confirms()
1415                    .cancel(ctx.connection_id, frame.header.corr)
1416                    || self
1417                        .supervisor
1418                        .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1419                {
1420                    Ok(Vec::new())
1421                } else {
1422                    Ok(vec![control_error_frame(
1423                        &frame,
1424                        "unknown_subscription",
1425                        "no supervisor spawn subscription has this correlation id",
1426                    )?])
1427                }
1428            }
1429            FrameType::Request => {
1430                // This additive operation is not part of the existing exhaustive
1431                // module-control enum. Probe the op before decoding that enum.
1432                let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1433                if op
1434                    .as_ref()
1435                    .is_some_and(|probe| probe.op == "operator.confirm")
1436                {
1437                    return self.handle_operator_confirm(ctx, frame);
1438                }
1439                if self
1440                    .forwarding
1441                    .module_endpoint_for_connection(ctx.connection_id)
1442                    .map_err(RouterError::Forwarding)?
1443                    .is_some()
1444                {
1445                    if !is_known_module_request_op(&frame.body) {
1446                        return Ok(vec![control_error_frame(
1447                            &frame,
1448                            "unsupported_control_frame",
1449                            "module-originated channel-0 REQUEST is not supported",
1450                        )?]);
1451                    }
1452                    let request = match parse_module_control_request_from_module(&frame.body) {
1453                        Ok(request) => request,
1454                        Err((err, ControlRequestBodyError::UnknownOp)) => {
1455                            return Ok(vec![control_error_frame(
1456                                &frame,
1457                                "unsupported_control_frame",
1458                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
1459                            )?])
1460                        }
1461                        Err((err, ControlRequestBodyError::InvalidBody)) => {
1462                            return Ok(vec![control_error_frame(
1463                                &frame,
1464                                "invalid_control_body",
1465                                format!("malformed module control body: {err}"),
1466                            )?])
1467                        }
1468                    };
1469                    let op = module_control_request_op(&request);
1470                    let corr = frame.header.corr;
1471                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
1472                    let result =
1473                        self.handle_module_control_request(ctx.connection_id, frame, request);
1474                    log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1475                    return result;
1476                }
1477
1478                if is_known_module_request_op(&frame.body) {
1479                    return Ok(vec![control_error_frame(
1480                        &frame,
1481                        "not_registered",
1482                        "catalog.update requires an active module registration owned by this connection",
1483                    )?]);
1484                }
1485
1486                let request = match parse_client_control_request(&frame.body) {
1487                    Ok(request) => request,
1488                    Err((err, ControlRequestBodyError::UnknownOp)) => {
1489                        return Ok(vec![control_error_frame(
1490                            &frame,
1491                            "unknown_control_op",
1492                            format!("unknown client control op: {err}"),
1493                        )?])
1494                    }
1495                    Err((err, ControlRequestBodyError::InvalidBody)) => {
1496                        return Ok(vec![control_error_frame(
1497                            &frame,
1498                            "invalid_control_body",
1499                            format!("malformed client control body: {err}"),
1500                        )?])
1501                    }
1502                };
1503                let op = client_control_request_op(&request);
1504                let corr = frame.header.corr;
1505                log_control_dispatch_arrival(op, ctx.connection_id, corr);
1506                #[cfg(test)]
1507                if let Some(delay) = self.control_dispatch_delay {
1508                    tokio::time::sleep(delay).await;
1509                }
1510                let result = self
1511                    .handle_client_control_request(ctx, frame, request)
1512                    .await;
1513                log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1514                result
1515            }
1516            FrameType::Push => {
1517                let Some(endpoint) = self
1518                    .forwarding
1519                    .module_endpoint_for_connection(ctx.connection_id)
1520                    .map_err(RouterError::Forwarding)?
1521                else {
1522                    return Ok(vec![control_error_frame(
1523                        &frame,
1524                        "unsupported_control_frame",
1525                        "client-originated channel-0 PUSH is not supported",
1526                    )?]);
1527                };
1528                self.handle_status_update(endpoint, frame)
1529            }
1530            FrameType::Response | FrameType::Error
1531                if self
1532                    .forwarding
1533                    .module_endpoint_for_connection(ctx.connection_id)
1534                    .map_err(RouterError::Forwarding)?
1535                    .is_some() =>
1536            {
1537                self.handle_module_relay_response(ctx.connection_id, frame)
1538            }
1539            ty => Ok(vec![control_error_frame(
1540                &frame,
1541                "unsupported_control_frame",
1542                format!("unsupported channel-0 frame {ty:?}"),
1543            )?]),
1544        }
1545    }
1546
1547    pub fn cleanup_connection(
1548        &self,
1549        connection_id: ConnectionId,
1550    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1551        self.cleanup_connection_with_end_reason(
1552            connection_id,
1553            RegistrationEndReason::ConnectionClosed,
1554        )
1555    }
1556
1557    fn cleanup_connection_with_end_reason(
1558        &self,
1559        connection_id: ConnectionId,
1560        requested_reason: RegistrationEndReason,
1561    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1562        let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
1563            self.registry
1564                .get_module_by_connection(connection_id)?
1565                .and_then(|registration| self.supervisor.get(&registration.manifest.module_id))
1566                .and_then(|module| module.registration_end_reason().ok().flatten())
1567                .unwrap_or(requested_reason)
1568        } else {
1569            requested_reason
1570        };
1571        let crash_closed = self
1572            .registry
1573            .get_module_by_connection(connection_id)?
1574            .and_then(|registration| {
1575                self.forwarding
1576                    .module_endpoint_for_connection(connection_id)
1577                    .ok()
1578                    .flatten()
1579                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1580                    .map(|routes| (registration.manifest.module_id, routes))
1581            });
1582        let crash_closed = crash_closed.map(|(module_id, routes)| {
1583            let terminal = match self.supervisor.get(&module_id) {
1584                None => false,
1585                Some(module) => match module.will_recover_after_connection_loss() {
1586                    Ok(will_recover) => !will_recover,
1587                    Err(err) => {
1588                        warn!(
1589                            %module_id,
1590                            error = %err,
1591                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
1592                        );
1593                        false
1594                    }
1595                },
1596            };
1597            // The forwarding table gates all providers at the start of daemon
1598            // shutdown, before their connections are closed. An ordinary
1599            // module disconnect still reports crash if that gate is not set.
1600            let reason = match self.forwarding.is_daemon_draining() {
1601                Ok(true) => RouteCloseReason::Restart,
1602                Ok(false) => RouteCloseReason::Crash,
1603                Err(err) => {
1604                    warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1605                    RouteCloseReason::Crash
1606                }
1607            };
1608            (module_id, routes, reason, terminal)
1609        });
1610        let registrations = self.deregister_connection(connection_id, end_reason);
1611        let cleanup = if crash_closed.is_some() {
1612            self.forwarding.cleanup_connection_counted(connection_id)
1613        } else {
1614            // A module connection's teardown needs the count of abandoned
1615            // route.bind relays for its route.closed notice below. Any other
1616            // connection, such as a client's, sends no such notice and needs
1617            // only its routes released, so it uses the route-only wrapper and
1618            // reports zero.
1619            self.forwarding
1620                .cleanup_connection(connection_id)
1621                .map(|released| crate::forwarding::ConnectionCleanup {
1622                    released,
1623                    abandoned_relays: 0,
1624                })
1625        };
1626        // The route.closed push waits for forwarding teardown because only
1627        // teardown knows how many pending route.bind relays it aborted. It still
1628        // goes out before the GOODBYEs for the released routes, and its targets
1629        // were captured above, before teardown removed those routes.
1630        if let Some((module_id, routes, reason, terminal)) = crash_closed {
1631            let abandoned = cleanup
1632                .as_ref()
1633                .map_or(0, |cleanup| cleanup.abandoned_relays);
1634            send_route_control_pushes(
1635                &self.forwarding,
1636                routes,
1637                ClientControlPush::RouteClosed {
1638                    module_id,
1639                    channels: Vec::new(),
1640                    reason,
1641                    drained: false,
1642                    abandoned,
1643                    excluded_subscriptions: 0,
1644                    terminal: Some(terminal),
1645                },
1646            );
1647        }
1648        if let Ok(cleanup) = cleanup {
1649            self.emit_route_goodbyes(cleanup.released);
1650        }
1651        // Signal the registration-release watch only now that BOTH registry and
1652        // forwarding teardown are done, so a supervisor waiting to spawn a
1653        // replacement never observes release while old routes still exist.
1654        if matches!(&registrations, Ok(r) if !r.is_empty()) {
1655            crate::supervise::notify_registration_release();
1656            self.capability_evaluator.wake_deadline_loop();
1657            self.refresh_capability_requirements();
1658        }
1659        self.supervisor.remove_spawn_subscribers(connection_id);
1660        // Sync authority dies with its connection, so the owner's next
1661        // connection can take it; the owner's scopes stay as they are.
1662        self.hello_launch_nonces
1663            .lock()
1664            .unwrap_or_else(|poisoned| poisoned.into_inner())
1665            .forget(connection_id);
1666        self.scopes
1667            .write()
1668            .unwrap_or_else(|poisoned| poisoned.into_inner())
1669            .release_connection(connection_id);
1670        registrations
1671    }
1672
1673    pub(crate) fn handle_route_goodbye(
1674        &self,
1675        connection_id: ConnectionId,
1676        route_channel: u16,
1677        route_epoch: u32,
1678    ) -> Result<bool, RouterError> {
1679        debug!(
1680            connection_id = connection_id.get(),
1681            route_channel, route_epoch, "handling route GOODBYE"
1682        );
1683        let RouteRelease::Removed(released_route) = self
1684            .forwarding
1685            .release_client_route(connection_id, route_channel, route_epoch)
1686            .map_err(RouterError::Forwarding)?
1687        else {
1688            return Ok(false);
1689        };
1690        self.emit_route_goodbyes(vec![released_route]);
1691        Ok(true)
1692    }
1693
1694    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1695        for released in released_routes {
1696            let frame = match Frame::build_with_version(
1697                released.negotiated_ver,
1698                FrameType::Goodbye,
1699                control_flags(),
1700                released.channel,
1701                released.epoch,
1702                0,
1703                Vec::new(),
1704            ) {
1705                Ok(frame) => frame,
1706                Err(err) => {
1707                    warn!(
1708                        route_channel = released.channel,
1709                        error = %err,
1710                        "failed to build route GOODBYE frame"
1711                    );
1712                    continue;
1713                }
1714            };
1715            if !released.close_on_delivery_failure() {
1716                crate::forwarding::send_module_route_goodbye(
1717                    &self.counters,
1718                    &released.sink,
1719                    frame,
1720                    released.module_id.as_deref(),
1721                    "client route released",
1722                );
1723                continue;
1724            }
1725            if let Err(err) = released.sink.try_send(frame) {
1726                warn!(
1727                    target_connection_id = released.connection_id.get(),
1728                    route_channel = released.channel,
1729                    error = %err,
1730                    "route GOODBYE was not delivered to client; closing target connection"
1731                );
1732                if self
1733                    .forwarding
1734                    .escalate_client_delivery_failure(
1735                        released.connection_id,
1736                        released.channel,
1737                        released.epoch,
1738                        CloseReason::new(
1739                            "route_goodbye_delivery_failed",
1740                            format!(
1741                                "failed to enqueue route GOODBYE for channel {}: {err}",
1742                                released.channel
1743                            ),
1744                        ),
1745                        crate::forwarding::UndeliveredFrame {
1746                            module_id: released.module_id.as_deref(),
1747                            sink: &released.sink,
1748                        },
1749                    )
1750                    .unwrap_or(false)
1751                {
1752                    self.counters.increment_goodbye_relay_client_failed();
1753                }
1754            }
1755        }
1756    }
1757
1758    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
1759    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
1760    /// own commit failed after the module had already accepted). Without this, a
1761    /// module that accepts late keeps a binding subc has torn down, so a later
1762    /// frame on that module channel could misdeliver if the channel is reused.
1763    ///
1764    /// Never closes the shared module connection on failure: a dropped notification
1765    /// only wastes a bounded amount of warm module-side state, which the module's
1766    /// own idle reaper reclaims. Only call this once the route.bind relay was
1767    /// actually enqueued to the module — if the relay send itself failed, the
1768    /// module never created a binding and there is nothing to tear down.
1769    fn send_abandoned_route_bind_goodbye(
1770        &self,
1771        module_sink: &crate::FrameSink,
1772        negotiated_ver: u8,
1773        module_channel: u16,
1774        module_epoch: u32,
1775    ) {
1776        let frame = match Frame::build_with_version(
1777            negotiated_ver,
1778            FrameType::Goodbye,
1779            control_flags(),
1780            module_channel,
1781            module_epoch,
1782            0,
1783            Vec::new(),
1784        ) {
1785            Ok(frame) => frame,
1786            Err(err) => {
1787                warn!(
1788                    route_channel = module_channel,
1789                    error = %err,
1790                    "failed to build GOODBYE for abandoned route.bind"
1791                );
1792                return;
1793            }
1794        };
1795        crate::forwarding::send_module_route_goodbye(
1796            &self.counters,
1797            module_sink,
1798            frame,
1799            None,
1800            "abandoned route.bind",
1801        );
1802    }
1803
1804    fn handle_hello(
1805        &self,
1806        connection_id: ConnectionId,
1807        sink: Option<crate::FrameSink>,
1808        frame: Frame,
1809    ) -> Result<Vec<Frame>, RouterError> {
1810        debug!(
1811            connection_id = connection_id.get(),
1812            corr = frame.header.corr,
1813            "handling HELLO"
1814        );
1815        // A module connection has one identity for its entire lifetime. A second
1816        // registration would leave the old registry owner behind while replacing
1817        // its forwarding endpoint and launch nonce.
1818        if self
1819            .registry
1820            .get_module_by_connection(connection_id)
1821            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
1822            .is_some()
1823        {
1824            return Ok(vec![control_error_frame(
1825                &frame,
1826                "invalid_hello",
1827                "connection is already registered as a module",
1828            )?]);
1829        }
1830        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1831            Ok(value) => value,
1832            Err(err) => {
1833                return Ok(vec![control_error_frame(
1834                    &frame,
1835                    "invalid_hello",
1836                    format!("malformed HELLO body: {err}"),
1837                )?])
1838            }
1839        };
1840        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1841            return Ok(vec![control_error_frame(
1842                &frame,
1843                "invalid_capability_grammar",
1844                err.to_string(),
1845            )?]);
1846        }
1847        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
1848            return Ok(vec![control_error_frame(
1849                &frame,
1850                "invalid_manifest",
1851                err.to_string(),
1852            )?]);
1853        }
1854        if let Err(err) = validate_hello_event_declarations(&hello_value) {
1855            return Ok(vec![control_error_frame(
1856                &frame,
1857                "invalid_event_declaration",
1858                err.to_string(),
1859            )?]);
1860        }
1861        if let Some(provenance) = hello_value
1862            .get("manifest")
1863            .and_then(|manifest| manifest.get("provenance"))
1864        {
1865            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
1866                return Ok(vec![control_error_frame(
1867                    &frame,
1868                    "invalid_manifest",
1869                    format!("malformed manifest provenance: {err}"),
1870                )?]);
1871            }
1872        }
1873        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
1874            Ok(hello) => hello,
1875            Err(err) => {
1876                return Ok(vec![control_error_frame(
1877                    &frame,
1878                    "invalid_hello",
1879                    format!("malformed HELLO body: {err}"),
1880                )?])
1881            }
1882        };
1883
1884        if hello.protocol_ver != hello.manifest.protocol_ver {
1885            return Ok(vec![control_error_frame(
1886                &frame,
1887                "invalid_manifest",
1888                format!(
1889                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
1890                    hello.protocol_ver, hello.manifest.protocol_ver
1891                ),
1892            )?]);
1893        }
1894
1895        if hello.manifest.module_id.trim().is_empty() {
1896            return Ok(vec![control_error_frame(
1897                &frame,
1898                "invalid_manifest",
1899                "manifest module_id must not be empty",
1900            )?]);
1901        }
1902
1903        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
1904            Ok(negotiated_ver) => negotiated_ver,
1905            Err(message) => {
1906                return Ok(vec![control_error_frame(
1907                    &frame,
1908                    "version_unsupported",
1909                    message,
1910                )?])
1911            }
1912        };
1913
1914        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
1915        // swap is open for this id, the only HELLO admitted as a second process
1916        // is the one carrying the candidate's launch nonce (the swap token), and
1917        // it registers into the candidate slot rather than being refused as a
1918        // duplicate. Run after the reserved gate, a reserved module's candidate
1919        // would be refused `reserved_module` for presenting a nonce that gate
1920        // does not know. See `SupervisorHandle::swap_hello_admission`.
1921        let swap_admission = self
1922            .supervisor
1923            .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
1924        if swap_admission == SwapHelloAdmission::Refused {
1925            warn!(
1926                module_id = %hello.manifest.module_id,
1927                connection_id = connection_id.get(),
1928                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
1929            );
1930            return Ok(vec![control_error_frame(
1931                &frame,
1932                "swap_token_invalid",
1933                format!(
1934                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
1935                    hello.manifest.module_id
1936                ),
1937            )?]);
1938        }
1939        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
1940
1941        // Reserved-module identity gate: a module_id configured `reserved` may be
1942        // registered ONLY by the process subc spawned for it, proven by echoing the
1943        // one-time launch nonce subc injected. A non-reserved id has no recorded
1944        // nonce and always passes. This blocks a key-holder from impersonating a
1945        // security-boundary module (e.g. the credential vault) while the real one is
1946        // down/restarting and its registration slot is momentarily free. A swap
1947        // candidate has already proven the same thing with its own nonce above.
1948        if let Some(rejection) = (!swap_candidate)
1949            .then(|| {
1950                self.supervisor.reserved_hello_rejection(
1951                    &hello.manifest.module_id,
1952                    hello.launch_nonce.as_deref(),
1953                )
1954            })
1955            .flatten()
1956        {
1957            let message = match rejection {
1958                ReservedHelloRejection::Exact { module_id } => format!(
1959                    "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
1960                ),
1961                ReservedHelloRejection::Prefix {
1962                    prefix,
1963                    owner_module_id,
1964                } => format!(
1965                    "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
1966                    hello.manifest.module_id
1967                ),
1968            };
1969            return Ok(vec![control_error_frame(
1970                &frame,
1971                "reserved_module",
1972                message,
1973            )?]);
1974        }
1975
1976        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
1977            &hello.manifest.module_id,
1978            hello.manifest.capabilities.as_ref(),
1979        );
1980        if let Some(refusal) = reserved_capability_refusals.first() {
1981            let capability = refusal.capability.clone();
1982            let bound_module = refusal.claimants[0].clone();
1983            log_duplicate_claim_events(reserved_capability_refusals);
1984            return Ok(vec![control_error_frame(
1985                &frame,
1986                "reserved_capability",
1987                format!(
1988                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
1989                    capability, bound_module, hello.manifest.module_id
1990                ),
1991            )?]);
1992        }
1993
1994        // A connection that already opened client routes must not also register as
1995        // a module: cleanup would then release only one side and leak the other.
1996        if self
1997            .forwarding
1998            .connection_has_client_routes(connection_id)
1999            .map_err(RouterError::Forwarding)?
2000        {
2001            return Ok(vec![control_error_frame(
2002                &frame,
2003                "invalid_hello",
2004                "connection has open client routes and cannot also register as a module",
2005            )?]);
2006        }
2007
2008        // Kept for scope sync authority, which goes only to the connection that
2009        // presented the module's current launch nonce. Recorded before the
2010        // registration is attempted: a connection whose registration then fails
2011        // has no registration, so it cannot sync anyway, and cleanup forgets it.
2012        self.hello_launch_nonces
2013            .lock()
2014            .unwrap_or_else(|poisoned| poisoned.into_inner())
2015            .record(connection_id, hello.launch_nonce.as_deref());
2016        let control_ops = effective_module_control_ops(hello.control_ops);
2017        // Built before anything is registered so an encoding failure leaves no
2018        // registry or forwarding state behind.
2019        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2020        if swap_candidate {
2021            return self.register_swap_candidate(
2022                connection_id,
2023                sink,
2024                &frame,
2025                hello.manifest,
2026                negotiated_ver,
2027                control_ops,
2028                hello_ack,
2029            );
2030        }
2031        let registration = match self.registry.register_with_control_ops(
2032            hello.manifest,
2033            negotiated_ver,
2034            connection_id,
2035            control_ops,
2036        ) {
2037            Ok(registration) => registration,
2038            Err(RegistryError::DuplicateModuleId { module_id }) => {
2039                return Ok(vec![control_error_frame(
2040                    &frame,
2041                    "duplicate_module_id",
2042                    format!(
2043                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2044                    ),
2045                )?])
2046            }
2047            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2048                return Ok(vec![control_error_frame(
2049                    &frame,
2050                    "invalid_module_id",
2051                    err.to_string(),
2052                )?])
2053            }
2054            Err(err) => {
2055                return Ok(vec![control_error_frame(
2056                    &frame,
2057                    "registry_error",
2058                    err.to_string(),
2059                )?])
2060            }
2061        };
2062
2063        let reply = if let Some(sink) = sink {
2064            // The forwarding table's module store is also the daemon-to-module
2065            // control-RPC lane, so every HELLO gets a live endpoint even when the
2066            // manifest has no routable provider role. Non-routable modules still
2067            // cannot receive route.bind in production: `handle_route_open` checks
2068            // the registry manifest with `target_has_required_role` before the
2069            // only production call to `begin_route_bind_relay_for` below that
2070            // route.open path. The remaining direct relay callers are unit tests
2071            // and benchmark harnesses that construct forwarding state explicitly.
2072            //
2073            // The HELLO_ACK is queued by the forwarding table itself, before the
2074            // endpoint becomes visible, and is NOT returned as a reply. A module
2075            // reads HELLO_ACK first and exits on anything else; a reply is only
2076            // written after this handler returns, by which time a route.open on
2077            // another connection could already have queued a route.bind request
2078            // for this module ahead of it.
2079            let concurrency = manifest_concurrency(&registration.manifest);
2080            if let Err(err) = self.forwarding.register_module_connection_acked(
2081                connection_id,
2082                registration.manifest.module_id.clone(),
2083                negotiated_ver,
2084                concurrency,
2085                sink,
2086                hello_ack,
2087            ) {
2088                // Forwarding registration failed, so there is no forwarding
2089                // state to tear down. Remove the registry entry and signal the
2090                // release watch directly.
2091                if matches!(
2092                    self.deregister_connection(
2093                        connection_id,
2094                        RegistrationEndReason::RegistrationFailed,
2095                    ),
2096                    Ok(r) if !r.is_empty()
2097                ) {
2098                    crate::supervise::notify_registration_release();
2099                }
2100                return Ok(vec![control_error_frame(
2101                    &frame,
2102                    if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2103                        "invalid_hello"
2104                    } else {
2105                        forwarding_error_code(&err)
2106                    },
2107                    err.to_string(),
2108                )?]);
2109            }
2110            Vec::new()
2111        } else {
2112            // No sink means no forwarding endpoint, so nothing can be routed
2113            // ahead of the ack; it goes out as the reply.
2114            vec![hello_ack]
2115        };
2116
2117        // Exposure over assumption: Concurrency's serde default is pinned to the
2118        // pre-field behavior (ModuleManaged), so a management surface that is
2119        // genuinely Serial and just never declared it inherits concurrent
2120        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
2121        // turns "no module has been bitten yet" into the checkable claim "no
2122        // module is exposed" -- one read of the boot log instead of a fleet
2123        // audit. Detected from the raw HELLO bytes because the serde default
2124        // deliberately erases the absent/declared distinction from the type.
2125        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
2126            info!(
2127                module_id = %registration.manifest.module_id,
2128                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2129            );
2130        }
2131
2132        self.apply_registration_capabilities(&registration);
2133
2134        info!(
2135            module_id = %registration.manifest.module_id,
2136            module_version = %registration.manifest.module_version,
2137            negotiated_ver,
2138            routable_provider = manifest_provides_routable_role(&registration.manifest),
2139            connection_id = connection_id.get(),
2140            "module registered"
2141        );
2142
2143        Ok(reply)
2144    }
2145
2146    /// Register a HELLO the swap gate admitted into the candidate slot of the
2147    /// registry and of forwarding, where it is reachable over its own
2148    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
2149    /// nothing routes to it until the supervisor cuts over.
2150    ///
2151    /// Registry first, then forwarding, the same order as an ordinary HELLO;
2152    /// a forwarding failure removes the registry entry again. The capability
2153    /// census is not run: it describes routable modules, and this one is not
2154    /// routable until promotion.
2155    #[allow(clippy::too_many_arguments)]
2156    fn register_swap_candidate(
2157        &self,
2158        connection_id: ConnectionId,
2159        sink: Option<crate::FrameSink>,
2160        frame: &Frame,
2161        manifest: ModuleManifest,
2162        negotiated_ver: u8,
2163        control_ops: Vec<String>,
2164        hello_ack: Frame,
2165    ) -> Result<Vec<Frame>, RouterError> {
2166        let module_id = manifest.module_id.clone();
2167        let registration = match self.registry.register_candidate_with_control_ops(
2168            manifest,
2169            negotiated_ver,
2170            connection_id,
2171            control_ops,
2172        ) {
2173            Ok(registration) => registration,
2174            Err(RegistryError::DuplicateModuleId { module_id }) => {
2175                return Ok(vec![control_error_frame(
2176                    frame,
2177                    "duplicate_module_id",
2178                    format!(
2179                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2180                    ),
2181                )?])
2182            }
2183            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2184                return Ok(vec![control_error_frame(
2185                    frame,
2186                    "invalid_module_id",
2187                    err.to_string(),
2188                )?])
2189            }
2190            Err(err) => {
2191                return Ok(vec![control_error_frame(
2192                    frame,
2193                    "registry_error",
2194                    err.to_string(),
2195                )?])
2196            }
2197        };
2198        let reply = if let Some(sink) = sink {
2199            // Same ordering as an ordinary HELLO: the forwarding table queues
2200            // the HELLO_ACK before the candidate endpoint is inserted, because
2201            // a module exits if its first frame after HELLO is anything else.
2202            let concurrency = manifest_concurrency(&registration.manifest);
2203            if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2204                connection_id,
2205                module_id.clone(),
2206                negotiated_ver,
2207                concurrency,
2208                sink,
2209                hello_ack,
2210            ) {
2211                if matches!(
2212                    self.deregister_connection(
2213                        connection_id,
2214                        RegistrationEndReason::RegistrationFailed,
2215                    ),
2216                    Ok(r) if !r.is_empty()
2217                ) {
2218                    crate::supervise::notify_registration_release();
2219                }
2220                return Ok(vec![control_error_frame(
2221                    frame,
2222                    forwarding_error_code(&err),
2223                    err.to_string(),
2224                )?]);
2225            }
2226            Vec::new()
2227        } else {
2228            vec![hello_ack]
2229        };
2230        self.supervisor.mark_swap_candidate_admitted(&module_id);
2231        info!(
2232            module_id = %module_id,
2233            module_version = %registration.manifest.module_version,
2234            negotiated_ver,
2235            ready = registration.ready,
2236            connection_id = connection_id.get(),
2237            "swap candidate registered; not routable until cutover"
2238        );
2239        Ok(reply)
2240    }
2241
2242    fn build_hello_ack(
2243        &self,
2244        frame: &Frame,
2245        negotiated_ver: u8,
2246        module_id: &str,
2247    ) -> Result<Frame, RouterError> {
2248        let ack = ModuleHelloAckBody {
2249            negotiated_ver,
2250            subc_ops: module_subc_ops(),
2251            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2252            storage: self
2253                .storage_config
2254                .as_ref()
2255                .map(|cfg| cfg.descriptor_for(module_id)),
2256            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2257        };
2258        let body = serde_json::to_vec(&ack).map_err(|err| {
2259            RouterError::backend(
2260                0,
2261                frame.header.corr,
2262                format!("failed to encode HELLO_ACK: {err}"),
2263            )
2264        })?;
2265
2266        Frame::build_with_version(
2267            negotiated_ver,
2268            FrameType::HelloAck,
2269            control_flags(),
2270            0,
2271            0,
2272            frame.header.corr,
2273            body,
2274        )
2275        .map_err(RouterError::FrameBuild)
2276    }
2277
2278    async fn handle_client_control_request(
2279        &self,
2280        ctx: &RouteCtx,
2281        frame: Frame,
2282        request: ClientControlRequest,
2283    ) -> Result<Vec<Frame>, RouterError> {
2284        match request {
2285            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2286            ClientControlRequest::CatalogList { module_id } => {
2287                self.handle_catalog_list(frame, module_id)
2288            }
2289            ClientControlRequest::RouteOpen {
2290                target,
2291                identity,
2292                consumer_identity,
2293                consumer_capabilities,
2294                role_versions,
2295                admission_facts,
2296                scope,
2297            } => {
2298                self.handle_route_open(
2299                    ctx,
2300                    frame,
2301                    RouteOpenRequest {
2302                        target,
2303                        identity,
2304                        consumer_identity,
2305                        consumer_capabilities,
2306                        role_versions,
2307                        admission_facts,
2308                        scope,
2309                    },
2310                )
2311                .await
2312            }
2313            ClientControlRequest::RoutePoll {
2314                route_channel,
2315                route_epoch,
2316                kind,
2317            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2318            ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2319            ClientControlRequest::SupervisorSpawnSnapshot {} => {
2320                self.handle_supervisor_spawn_snapshot(frame)
2321            }
2322            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2323                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2324            }
2325            ClientControlRequest::SupervisorRestart {
2326                module_id,
2327                drain_timeout_ms,
2328            } => {
2329                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2330                    .await
2331            }
2332            ClientControlRequest::SupervisorSwap {
2333                module_id,
2334                ready_timeout_ms,
2335            } => {
2336                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2337                    .await
2338            }
2339            ClientControlRequest::SupervisorReload { module_id } => {
2340                self.handle_supervisor_reload(frame, module_id).await
2341            }
2342            ClientControlRequest::SupervisorRescan { preview } => {
2343                self.handle_supervisor_rescan(frame, preview).await
2344            }
2345            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2346                self.handle_supervisor_release_reserved(frame, module_id)
2347                    .await
2348            }
2349            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2350                self.handle_supervisor_set_enabled(frame, module_id, enabled)
2351                    .await
2352            }
2353            ClientControlRequest::SupervisorHealthProbe { module_id } => {
2354                self.handle_supervisor_health_probe(frame, module_id).await
2355            }
2356            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2357            ClientControlRequest::SupervisorRoutes { module_id } => {
2358                self.handle_supervisor_routes(frame, module_id)
2359            }
2360            ClientControlRequest::SupervisorProvenance { module_id } => {
2361                self.handle_supervisor_provenance(frame, module_id).await
2362            }
2363            ClientControlRequest::SupervisorStderrTail {
2364                module_id,
2365                max_lines,
2366                max_bytes,
2367            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2368            ClientControlRequest::SupervisorTerminals { module_id } => {
2369                self.handle_supervisor_terminals(frame, module_id).await
2370            }
2371        }
2372    }
2373
2374    fn handle_module_control_request(
2375        &self,
2376        connection_id: ConnectionId,
2377        frame: Frame,
2378        request: ModuleControlRequestFromModule,
2379    ) -> Result<Vec<Frame>, RouterError> {
2380        match request {
2381            ModuleControlRequestFromModule::CatalogUpdate {
2382                provides,
2383                capabilities,
2384                ready,
2385            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2386            ModuleControlRequestFromModule::LiveRoots {} => {
2387                let registered = self
2388                    .registry
2389                    .get_module_by_connection(connection_id)
2390                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2391                let Some(registration) = registered else {
2392                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2393                };
2394                let response = self
2395                    .forwarding
2396                    .live_roots(&registration.manifest.module_id)
2397                    .map_err(RouterError::Forwarding)?;
2398                Ok(vec![control_response_body_frame(
2399                    &frame,
2400                    &response,
2401                    "ModuleControlResponseToModule::LiveRoots",
2402                )?])
2403            }
2404            ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2405                self.handle_scope_sync(connection_id, frame, generation, scopes)
2406            }
2407            ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2408                self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2409            }
2410        }
2411    }
2412
2413    fn handle_operator_confirm(
2414        &self,
2415        ctx: &RouteCtx,
2416        frame: Frame,
2417    ) -> Result<Vec<Frame>, RouterError> {
2418        use crate::operator_confirm::{audit, Outcome};
2419        let registration = self
2420            .registry
2421            .get_module_by_connection(ctx.connection_id)
2422            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2423        let Some(registration) = registration else {
2424            let outcome = Outcome::refusal("not_registered");
2425            audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2426            return Ok(vec![outcome.frame(&frame)]);
2427        };
2428        let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
2429            Ok(request) => request,
2430            Err(_) => {
2431                let outcome = Outcome::refusal("invalid_control_body");
2432                audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2433                return Ok(vec![outcome.frame(&frame)]);
2434            }
2435        };
2436        let module_id = registration.manifest.module_id;
2437        // Read the launch nonce (under its own lock) before taking the forwarding
2438        // table's lock below: holding forwarding while waiting on another daemon
2439        // lock risks a lock-order deadlock with paths that take them the other way.
2440        let nonce = self
2441            .hello_launch_nonces
2442            .lock()
2443            .unwrap_or_else(|p| p.into_inner())
2444            .nonce(ctx.connection_id)
2445            .map(str::to_owned);
2446        let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
2447            self.supervisor
2448                .spawned_consumer_authorized(&module_id, nonce)
2449        });
2450        let confirms = self.forwarding.operator_confirms();
2451        self.forwarding
2452            .with_operator_route(
2453                ctx.connection_id,
2454                request.route_channel,
2455                request.route_epoch,
2456                |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
2457            )
2458            .map_err(RouterError::Forwarding)
2459    }
2460
2461    /// `scope.sync`: the owner is the module registered on this connection.
2462    /// A connection with no registration (every client connection, `direct`
2463    /// included) is refused `not_registered` before the table is consulted.
2464    fn handle_scope_sync(
2465        &self,
2466        connection_id: ConnectionId,
2467        frame: Frame,
2468        generation: u64,
2469        scopes: Vec<ScopeRecord>,
2470    ) -> Result<Vec<Frame>, RouterError> {
2471        let Some(registration) = self
2472            .registry
2473            .get_module_by_connection(connection_id)
2474            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2475        else {
2476            return Ok(vec![control_error_frame(
2477                &frame,
2478                "not_registered",
2479                "scope.sync requires an active module registration owned by this connection",
2480            )?]);
2481        };
2482        let owner = registration.manifest.module_id;
2483        let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
2484        let is_current_launch = |connection: ConnectionId| {
2485            self.hello_launch_nonces
2486                .lock()
2487                .unwrap_or_else(|poisoned| poisoned.into_inner())
2488                .presented(connection, current_nonce.as_deref())
2489        };
2490        // Lock order is the scope table, then the forwarding table: the new
2491        // tags are published, and the routes the change closes are selected,
2492        // while the scope table is still write-locked, so no admission can read
2493        // a record whose tag is not yet published.
2494        let mut table = self
2495            .scopes
2496            .write()
2497            .unwrap_or_else(|poisoned| poisoned.into_inner());
2498        let outcome = table.sync(&owner, connection_id, is_current_launch, generation, scopes);
2499        let drained = match &outcome {
2500            Ok(applied) => self
2501                .forwarding
2502                .publish_scope_changes(&applied.tag_changes)
2503                .map_err(RouterError::Forwarding)?,
2504            Err(_) => Vec::new(),
2505        };
2506        drop(table);
2507        match outcome {
2508            Ok(applied) => {
2509                let counts = ScopeOutcomeCounts::of(&applied.results);
2510                info!(
2511                    owner = %owner,
2512                    generation,
2513                    records = applied.results.len(),
2514                    created = counts.created,
2515                    replaced = counts.replaced,
2516                    updated = counts.updated,
2517                    unchanged = counts.unchanged,
2518                    refused = counts.refused,
2519                    ended = applied.ended.len(),
2520                    tag_changes = applied.tag_changes.len(),
2521                    routes_closed = drained.len(),
2522                    "scope sync accepted"
2523                );
2524                // An accepted sync can still refuse individual records, and the
2525                // owner is the only party that sees the reply. Name them here so
2526                // an operator can tell a refused session from a missing one
2527                // without the owner's logs. Capped so a sync that refuses
2528                // thousands cannot flood the log; the count above is complete.
2529                for refused in applied
2530                    .results
2531                    .iter()
2532                    .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
2533                    .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
2534                {
2535                    warn!(
2536                        owner = %owner,
2537                        generation,
2538                        scope_ref = %refused.scope_ref,
2539                        scope_epoch = refused.scope_epoch,
2540                        code = refused.code.as_deref().unwrap_or(""),
2541                        "scope record refused"
2542                    );
2543                }
2544                self.close_scope_drained_routes(drained);
2545                let response = ModuleControlResponseToModule::ScopeSync {
2546                    generation,
2547                    results: applied.results,
2548                    ended: applied.ended,
2549                };
2550                Ok(vec![control_response_body_frame(
2551                    &frame,
2552                    &response,
2553                    "ModuleControlResponseToModule::ScopeSync",
2554                )?])
2555            }
2556            Err(refusal) => {
2557                info!(
2558                    owner = %owner,
2559                    generation,
2560                    code = refusal.code,
2561                    "scope sync refused"
2562                );
2563                Ok(vec![control_error_frame(
2564                    &frame,
2565                    refusal.code,
2566                    refusal.message,
2567                )?])
2568            }
2569        }
2570    }
2571
2572    /// Tell both ends of each route a scope change closed. The module gets a
2573    /// channel-scoped GOODBYE and so does the client: the GOODBYE is what ends
2574    /// the client's route handle. The client also gets `route.closed` with the
2575    /// scope reason, one push per module and reason, so it can tell a revoked
2576    /// route from an ordinary close and not reopen it.
2577    fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
2578        if drained.is_empty() {
2579            return;
2580        }
2581        let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
2582            BTreeMap::new();
2583        let mut goodbyes = Vec::with_capacity(drained.len() * 2);
2584        for route in drained {
2585            warn!(
2586                module_id = %route.module_id,
2587                reason = ?route.reason,
2588                client_connection_id = route.client.connection_id.get(),
2589                route_channel = route.client.channel,
2590                "closing route because its scope changed"
2591            );
2592            pushes
2593                .entry((route.module_id.clone(), format!("{:?}", route.reason)))
2594                .or_insert_with(|| (route.reason, Vec::new()))
2595                .1
2596                .push(EndpointRoute {
2597                    goodbye_target: route.client.clone(),
2598                    principal: Principal::Unverified,
2599                    bound_at: Instant::now(),
2600                    draining: false,
2601                    drain_reason: None,
2602                });
2603            goodbyes.push(route.module);
2604            goodbyes.push(route.client);
2605        }
2606        for ((module_id, _), (reason, routes)) in pushes {
2607            send_route_control_pushes(
2608                &self.forwarding,
2609                routes,
2610                ClientControlPush::RouteClosed {
2611                    module_id,
2612                    channels: Vec::new(),
2613                    reason,
2614                    drained: false,
2615                    abandoned: 0,
2616                    excluded_subscriptions: 0,
2617                    terminal: Some(false),
2618                },
2619            );
2620        }
2621        self.emit_route_goodbyes(goodbyes);
2622    }
2623
2624    /// `scope.describe`: any registered module may read any scope, because a
2625    /// provider must read the scope a route it serves is stamped with.
2626    fn handle_scope_describe(
2627        &self,
2628        connection_id: ConnectionId,
2629        frame: Frame,
2630        owner: Principal,
2631        scope_ref: String,
2632    ) -> Result<Vec<Frame>, RouterError> {
2633        let registered = self
2634            .registry
2635            .get_module_by_connection(connection_id)
2636            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2637        if registered.is_none() {
2638            return Ok(vec![control_error_frame(
2639                &frame,
2640                "not_registered",
2641                "scope.describe requires an active module registration owned by this connection",
2642            )?]);
2643        }
2644        let description = self
2645            .scopes
2646            .read()
2647            .unwrap_or_else(|poisoned| poisoned.into_inner())
2648            .describe(&owner, &scope_ref);
2649        let owner_configured = match &owner {
2650            // Ask whether the owner is configured (`is_configured`), not
2651            // whether it is on the roster (`get(..).is_some()`): a supervised
2652            // module's process can register and describe a scope before the
2653            // supervisor has put it on the roster.
2654            Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
2655            _ => false,
2656        };
2657        let response = ModuleControlResponseToModule::ScopeDescribe {
2658            status: description.status,
2659            scope_epoch: description.scope_epoch,
2660            daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
2661            owner_synced: description.owner_synced,
2662            owner_configured,
2663            scope: description.stamp,
2664        };
2665        Ok(vec![control_response_body_frame(
2666            &frame,
2667            &response,
2668            "ModuleControlResponseToModule::ScopeDescribe",
2669        )?])
2670    }
2671
2672    fn handle_catalog_update(
2673        &self,
2674        connection_id: ConnectionId,
2675        frame: Frame,
2676        provides: Vec<ProviderRole>,
2677        capabilities: Option<CapabilityDeclarations>,
2678        ready: Option<bool>,
2679    ) -> Result<Vec<Frame>, RouterError> {
2680        self.refresh_capability_requirements();
2681        let Some(registration) = self
2682            .registry
2683            .get_module_by_connection(connection_id)
2684            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2685        else {
2686            return Ok(vec![control_error_frame(
2687                &frame,
2688                "not_registered",
2689                "catalog.update requires an active module registration owned by this connection",
2690            )?]);
2691        };
2692
2693        if let Some(message) =
2694            catalog_update_frozen_field_message(&registration.manifest, &provides)
2695        {
2696            return Ok(vec![control_error_frame(
2697                &frame,
2698                "catalog_update_frozen_field",
2699                message,
2700            )?]);
2701        }
2702
2703        let mut candidate = registration.manifest.clone();
2704        candidate.provides = provides.clone();
2705        candidate.capabilities = capabilities
2706            .clone()
2707            .or_else(|| registration.manifest.capabilities.clone());
2708        if let Err(err) = candidate.validate_capability_grammar() {
2709            return Ok(vec![control_error_frame(
2710                &frame,
2711                "invalid_capability_grammar",
2712                err.to_string(),
2713            )?]);
2714        }
2715
2716        // Updates must honor the same reserved owner as initial registration;
2717        // otherwise an empty HELLO could acquire the claim after admission.
2718        let mut conflicts = self
2719            .capability_evaluator
2720            .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
2721        if let Some(conflict) = conflicts.first() {
2722            let message = format!(
2723                "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2724                conflict.capability, conflict.claimants[0], candidate.module_id
2725            );
2726            for conflict in &mut conflicts {
2727                conflict.source = DuplicateClaimSource::CatalogUpdate;
2728            }
2729            log_duplicate_claim_events(conflicts);
2730            return Ok(vec![control_error_frame(
2731                &frame,
2732                "reserved_capability",
2733                message,
2734            )?]);
2735        }
2736
2737        let updated = self
2738            .registry
2739            .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
2740            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2741        if updated.is_none() {
2742            return Ok(vec![control_error_frame(
2743                &frame,
2744                "not_registered",
2745                "catalog.update requires an active module registration owned by this connection",
2746            )?]);
2747        }
2748        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
2749            log_duplicate_claim_events(
2750                self.capability_evaluator
2751                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
2752            );
2753        }
2754        if capability_census_trigger(
2755            registration.manifest.capabilities.as_ref(),
2756            updated
2757                .as_ref()
2758                .and_then(|entry| entry.manifest.capabilities.as_ref()),
2759        ) {
2760            self.enforce_capability_denies();
2761        }
2762        self.refresh_capability_requirements();
2763
2764        let response = ModuleControlResponseToModule::CatalogUpdate {};
2765        control_response_body_frame(
2766            &frame,
2767            &response,
2768            "ModuleControlResponseToModule::CatalogUpdate",
2769        )
2770        .map(|frame| vec![frame])
2771    }
2772
2773    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2774        self.refresh_capability_requirements();
2775        // A bare connection count is ambiguous between many clients holding a
2776        // route each and one client accumulating hundreds, so publish the
2777        // concentration alongside it. Route state is best-effort here: a
2778        // diagnostic endpoint must still answer if the forwarding lock is
2779        // contended.
2780        let mut counters = self.counters.snapshot();
2781        if let (Ok((connections_with_routes, max)), Some(obj)) = (
2782            self.forwarding.client_route_concentration(),
2783            counters.as_object_mut(),
2784        ) {
2785            obj.insert(
2786                "client_connections_with_routes".into(),
2787                connections_with_routes.into(),
2788            );
2789            obj.insert("max_routes_on_one_connection".into(), max.into());
2790        }
2791        // A module that is being fast-refused and a module that is fine look
2792        // identical from a client that retries and succeeds, so name the open
2793        // breakers here. This rides the existing free-form counters object
2794        // rather than a new wire field, so no sibling that deserializes
2795        // `ServerDescribe` has to be rebuilt to keep reading it.
2796        if let (Some(open_breakers), Some(obj)) = (
2797            self.route_bind_breakers.open_snapshot(),
2798            counters.as_object_mut(),
2799        ) {
2800            obj.insert("route_bind_breakers_open".into(), open_breakers);
2801        }
2802        let response = ClientControlResponse::ServerDescribe {
2803            protocol_ver: PROTOCOL_VERSION,
2804            subc_ops: subc_ops(),
2805            capabilities: self.subc_capabilities.as_ref().to_vec(),
2806            connected_clients: self.connected_clients.count(),
2807            counters: Some(counters),
2808            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
2809            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
2810            capability_requirements: self.capability_requirement_statuses(),
2811            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2812        };
2813        Ok(vec![control_response_body_frame(
2814            &frame,
2815            &response,
2816            "ClientControlResponse::ServerDescribe",
2817        )?])
2818    }
2819
2820    fn handle_catalog_list(
2821        &self,
2822        frame: Frame,
2823        module_id: Option<String>,
2824    ) -> Result<Vec<Frame>, RouterError> {
2825        let (generation, modules) = self.registry.list_modules().map_err(|err| {
2826            RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
2827        })?;
2828        let entries = modules
2829            .into_iter()
2830            .filter(|registration| {
2831                module_id
2832                    .as_deref()
2833                    .map(|wanted| registration.manifest.module_id == wanted)
2834                    .unwrap_or(true)
2835            })
2836            .map(|registration| {
2837                let not_ready = self.not_ready_reason(&registration);
2838                let roles = registration.manifest.provides;
2839                CatalogEntry {
2840                    module_id: registration.manifest.module_id,
2841                    ready: not_ready.is_none(),
2842                    not_ready,
2843                    module_version: Some(registration.manifest.module_version),
2844                    roles,
2845                    control_ops: registration.control_ops,
2846                    capabilities: registration.manifest.capabilities,
2847                    self_signals: registration.manifest.self_signals,
2848                }
2849            })
2850            .collect();
2851        let response = ClientControlResponse::CatalogList {
2852            generation,
2853            modules: entries,
2854            subc_ops: subc_ops(),
2855        };
2856        Ok(vec![control_response_body_frame(
2857            &frame,
2858            &response,
2859            "ClientControlResponse::CatalogList",
2860        )?])
2861    }
2862
2863    fn route_open_principal(
2864        &self,
2865        frame: &Frame,
2866        consumer_identity: Option<ConsumerIdentity>,
2867    ) -> Result<Result<Principal, Frame>, RouterError> {
2868        let Some(consumer_identity) = consumer_identity else {
2869            return Ok(Ok(Principal::Direct));
2870        };
2871
2872        if self.supervisor.spawned_consumer_authorized(
2873            &consumer_identity.module_id,
2874            &consumer_identity.launch_nonce,
2875        ) {
2876            return Ok(Ok(Principal::Reserved {
2877                module_id: consumer_identity.module_id,
2878            }));
2879        }
2880
2881        Ok(Err(control_error_frame(
2882            frame,
2883            "bad_consumer_identity",
2884            format!(
2885                "consumer_identity for module_id '{}' did not match a supervised launch nonce",
2886                consumer_identity.module_id
2887            ),
2888        )?))
2889    }
2890
2891    /// Ordinary `route.open` refusals go through here; admission and breaker
2892    /// refusals log separately with their capacity or breaker state. The daemon can
2893    /// attest which code it sent: without the event, a client's "the daemon
2894    /// refused me" and the daemon's own view could only be reconciled by
2895    /// argument. Malformed input (`invalid_project_root`) does not come here;
2896    /// rejecting a request that was never a valid open is not a refusal of one.
2897    fn route_open_refusal_frame(
2898        &self,
2899        ctx: &RouteCtx,
2900        frame: &Frame,
2901        module_id: &str,
2902        reason: &'static str,
2903        code: &'static str,
2904        message: impl Into<String>,
2905    ) -> Result<Frame, RouterError> {
2906        self.observe_route_open_refusal(ctx, module_id, reason, code);
2907        control_error_frame(frame, code, message.into())
2908    }
2909
2910    /// Refuse a `route.open` because the target module's bind-relay breaker is
2911    /// open, without attempting the relay.
2912    ///
2913    /// The wire code is `module_timeout`, which is the truth (the module has
2914    /// not been answering binds) and which both SDKs already classify as
2915    /// retryable with capped backoff. Reusing it is what keeps this change out
2916    /// of both SDKs; the daemon-side distinction lives in the counter key
2917    /// instead.
2918    ///
2919    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
2920    /// While a breaker is open this fires on every open to that module, and the
2921    /// stall written up in `docs/designs/route-open-head-of-line.md` already
2922    /// produced 261 lines about a single module inside 3000 lines of daemon
2923    /// log. The rare transitions are logged at warn/info instead and the volume
2924    /// is carried by the counter, so the evidence survives without the flood.
2925    /// The debug line keeps a per-refusal record reachable for whoever turns
2926    /// the level up.
2927    fn route_open_breaker_refusal_frame(
2928        &self,
2929        ctx: &RouteCtx,
2930        frame: &Frame,
2931        module_id: &str,
2932        consecutive_timeouts: u32,
2933        retry_in: Duration,
2934        probe_in_flight: bool,
2935    ) -> Result<Frame, RouterError> {
2936        self.counters
2937            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
2938        debug!(
2939            target: "control",
2940            code = "module_timeout",
2941            module_id = ?module_id,
2942            connection_id = ctx.connection_id.get(),
2943            consecutive_timeouts,
2944            retry_in_ms = retry_in.as_millis() as u64,
2945            probe_in_flight,
2946            "route.open refused by open bind-relay breaker"
2947        );
2948        // Say what a caller can act on. An open bind-relay breaker means the
2949        // module timed out accepting several new routes in a row. The module
2950        // is still running and its established routes keep working; only new
2951        // route.open requests are refused until the cooldown ends and one
2952        // test route (the probe) gets through. A message that only counts
2953        // failed relays reads as "the module is down" to a worker that sees it.
2954        let detail = if probe_in_flight {
2955            "one test route is already being tried; retry once it settles".to_string()
2956        } else {
2957            format!("retrying new routes in {}s", retry_in.as_secs().max(1))
2958        };
2959        control_error_frame(
2960            frame,
2961            "module_timeout",
2962            format!(
2963                "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
2964                 timed out in a row); {detail}; its established routes are unaffected"
2965            ),
2966        )
2967    }
2968
2969    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
2970    /// requester's bytes (an unknown target is whatever the client sent) and
2971    /// is Debug-formatted so control characters land in the log escaped
2972    /// rather than as terminal sequences for whoever tails it.
2973    ///
2974    /// `reason` names the check that refused, because one wire code has
2975    /// several senders: after a module registers, `target_unavailable` can
2976    /// come from a missing role, an inactive registration, a supervisor that
2977    /// has not marked the process live, a missing forwarding connection, or a
2978    /// failed relay, and a log that records only the code cannot say which of
2979    /// them fired. It is a static, daemon-chosen label per branch, so it is
2980    /// safe to print plainly and stays a closed set.
2981    fn observe_route_open_refusal(
2982        &self,
2983        ctx: &RouteCtx,
2984        module_id: &str,
2985        reason: &'static str,
2986        code: &'static str,
2987    ) {
2988        self.counters.increment_route_open_refused(code);
2989        info!(
2990            target: "control",
2991            code,
2992            reason,
2993            module_id = ?module_id,
2994            connection_id = ctx.connection_id.get(),
2995            "route.open refused"
2996        );
2997        if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
2998            self.route_outages.record_not_serving(module_id, reason);
2999        }
3000    }
3001
3002    /// Record an ACCEPTED route.open.
3003    ///
3004    /// Refusals have been logged and counted since the attestation work; accepts
3005    /// were invisible, so the daemon knew every principal it stamped and wrote
3006    /// none of them down. The party that attests the identity was the only party
3007    /// not recording it, which left a credential vault unable to name the sender
3008    /// of a call that reached it (claustrum #43) and left the launch-nonce
3009    /// concurrency question unanswerable from the outside.
3010    ///
3011    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
3012    /// `code`/`module_id`/`connection_id` returns both directions of the same
3013    /// decision rather than two shapes a reader has to join by hand.
3014    ///
3015    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
3016    /// and the difference carries information rather than being an
3017    /// inconsistency. This line is only reachable after a successful bind to a
3018    /// REGISTERED module, so the value has already passed HELLO validation
3019    /// including the path-hazard refusal and cannot contain control bytes. A
3020    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
3021    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
3022    ///
3023    /// Bare is also what every other daemon line already emits (`module
3024    /// registered`, `configured module supervised`). Shipping `?module_id` here
3025    /// made this instrument the only one in the file whose ids did not answer
3026    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
3027    /// line whose whole purpose is being grepped beside its sibling.
3028    ///
3029    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
3030    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
3031    /// forwards every field type through it and a bare `&str` and a `?`-escaped
3032    /// one are recorded identically. A test written against that harness passes
3033    /// either way -- I wrote one, measured it, and deleted it rather than ship a
3034    /// green assertion that cannot fail. The same limit applies to the escaping
3035    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
3036    /// it reads as a guard on the Debug escaping and cannot detect its removal.
3037    /// Fencing either needs the real formatter, not the capture layer.
3038    ///
3039    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
3040    /// unix-socket options and subc is loopback TCP, so there is no peer identity
3041    /// to record. The ephemeral port would decay within minutes and answer only a
3042    /// live question. The identity question is instead answered by counting
3043    /// distinct live connections presenting one module's `consumer_identity` --
3044    /// "is anyone else holding this secret" rather than "is this the right
3045    /// process".
3046    fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
3047        self.route_outages.record_accepted(module_id);
3048        self.counters.increment_route_open_accepted(principal);
3049        info!(
3050            target: "control",
3051            principal,
3052            module_id,
3053            connection_id = ctx.connection_id.get(),
3054            "route.open accepted"
3055        );
3056    }
3057
3058    fn supervised_absent_route_open_refusal_frame(
3059        &self,
3060        ctx: &RouteCtx,
3061        frame: &Frame,
3062        module_id: &str,
3063        code: &'static str,
3064        status: &crate::supervise::ModuleStatus,
3065    ) -> Result<Frame, RouterError> {
3066        self.counters.increment_route_open_refused(code);
3067        info!(
3068            target: "control",
3069            code,
3070            reason = "supervised_not_registered",
3071            module_id = ?module_id,
3072            connection_id = ctx.connection_id.get(),
3073            state = %status.state,
3074            enabled = status.enabled,
3075            live = status.live,
3076            "route.open refused"
3077        );
3078        // A supervised module whose process has not registered is not
3079        // serving, whatever the reason; the supervisor knows this id, so it is
3080        // safe to track.
3081        self.route_outages
3082            .record_not_serving(module_id, "supervised_not_registered");
3083        control_error_frame(
3084            frame,
3085            code,
3086            format!(
3087                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
3088                status.state, status.enabled, status.live
3089            ),
3090        )
3091    }
3092
3093    async fn handle_route_open(
3094        &self,
3095        ctx: &RouteCtx,
3096        frame: Frame,
3097        request: RouteOpenRequest,
3098    ) -> Result<Vec<Frame>, RouterError> {
3099        let RouteOpenRequest {
3100            target,
3101            mut identity,
3102            consumer_identity,
3103            consumer_capabilities,
3104            role_versions,
3105            admission_facts,
3106            scope,
3107        } = request;
3108        let target_module_id = target_module_id(&target).to_string();
3109        if self
3110            .registry
3111            .get_module_by_connection(ctx.connection_id)
3112            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3113            .is_some()
3114        {
3115            return Ok(vec![control_error_frame(
3116                &frame,
3117                "invalid_request",
3118                "module connections cannot open client routes",
3119            )?]);
3120        }
3121        debug!(
3122            connection_id = ctx.connection_id.get(),
3123            corr = frame.header.corr,
3124            module_id = %target_module_id,
3125            "handling route.open"
3126        );
3127
3128        // A malformed declaration is refused first, before anything about the
3129        // target is looked up: the same body would be refused against any
3130        // module, so the caller learns nothing by retrying or waiting. An empty
3131        // map declares nothing and travels as no field at all, so a provider
3132        // only ever sees a missing field or a non-empty one.
3133        let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
3134        if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
3135            self.observe_route_open_refusal(
3136                ctx,
3137                &target_module_id,
3138                "invalid_role_versions",
3139                error_codes::INVALID_REQUEST,
3140            );
3141            return Ok(vec![control_error_body_frame(
3142                &frame,
3143                ErrorBody {
3144                    code: error_codes::INVALID_REQUEST.to_string(),
3145                    message: error.to_string(),
3146                    detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
3147                },
3148            )?]);
3149        }
3150
3151        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
3152        // opposite. Below, a caller learns whether a module is unregistered,
3153        // supervised-but-down (with state/enabled/live), or registered without the
3154        // requested role. Elsewhere that is an enumeration leak: a probe learning
3155        // the shape of a fleet it cannot otherwise see.
3156        //
3157        // It is not one here, and the reason is the ACCESS MODEL rather than
3158        // anything about these errors. Reaching route.open requires the
3159        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
3160        // connection file, so any caller who completes it already runs as this
3161        // user -- and can read subc.jsonc for the module list and `ck module
3162        // status` for live state. The reply discloses nothing the caller cannot
3163        // read more easily from disk, while the precision is load-bearing:
3164        // `unknown_module` is retryable and a missing role is not.
3165        //
3166        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
3167        // remote transport, a sandboxed caller, a shared-host mode -- THAT
3168        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
3169        let Some(registration) = self
3170            .registry
3171            .get_module(&target_module_id)
3172            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3173        else {
3174            if let Some((status, warming)) =
3175                self.supervisor_status(&target_module_id, frame.header.corr)?
3176            {
3177                // BEFORE the two availability codes below, because for a module
3178                // that speaks no subc wire both of them are false comfort: they
3179                // say "not right now" and are retried, and this module will
3180                // never register no matter how long the caller waits. The
3181                // absence here is the declaration being honoured, not a module
3182                // that is late.
3183                if status.protocol == ModuleProtocol::None {
3184                    return Ok(vec![self.route_open_refusal_frame(
3185                        ctx,
3186                        &frame,
3187                        &target_module_id,
3188                        "protocol_none",
3189                        error_codes::MODULE_NO_PROTOCOL,
3190                        format!(
3191                            "module_id '{target_module_id}' is declared protocol: none; \
3192                             it speaks no subc wire and serves no routes"
3193                        ),
3194                    )?]);
3195                }
3196                let code = if warming {
3197                    "module_warming"
3198                } else {
3199                    "target_unavailable"
3200                };
3201                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
3202                    ctx,
3203                    &frame,
3204                    &target_module_id,
3205                    code,
3206                    &status,
3207                )?]);
3208            }
3209            if let Some(removed_ago_ms) =
3210                self.supervisor.removal_tombstone_age_ms(&target_module_id)
3211            {
3212                return Ok(vec![self.route_open_refusal_frame(
3213                    ctx,
3214                    &frame,
3215                    &target_module_id,
3216                    "removed",
3217                    error_codes::MODULE_REMOVED,
3218                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
3219                )?]);
3220            }
3221            return Ok(vec![self.route_open_refusal_frame(
3222                ctx,
3223                &frame,
3224                &target_module_id,
3225                "not_registered",
3226                error_codes::UNKNOWN_MODULE,
3227                format!("module_id '{target_module_id}' is not registered"),
3228            )?]);
3229        };
3230
3231        // Best-effort only: registry readiness and forwarding reservation use
3232        // different locks, so a module can flip readiness between this read and
3233        // the relay. Modules must still tolerate an `on_bind` while not ready.
3234        if !registration.ready {
3235            self.counters
3236                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
3237            info!(
3238                target: "control",
3239                code = error_codes::MODULE_WARMING,
3240                module_id = ?target_module_id,
3241                connection_id = ctx.connection_id.get(),
3242                reason = "declared_not_ready",
3243                "route.open refused"
3244            );
3245            // The module is registered but says it cannot take work, which is
3246            // an outage from the caller's side even though its process is up.
3247            self.route_outages
3248                .record_not_serving(&target_module_id, "declared_not_ready");
3249            return Ok(vec![control_error_body_frame(
3250                &frame,
3251                ErrorBody {
3252                    code: error_codes::MODULE_WARMING.to_string(),
3253                    message: format!(
3254                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
3255                    ),
3256                    detail: Some(serde_json::json!({
3257                        "reason": "declared_not_ready"
3258                    })),
3259                },
3260            )?]);
3261        }
3262
3263        // Effective readiness, second half: a module that declares a capability
3264        // `need: required` is not routable while that capability has no
3265        // registered provider. It is enforced HERE, as a retryable routing
3266        // refusal, and deliberately not as spawn ordering or a boot block. The
3267        // module is still started and registered and can make its own calls;
3268        // spawn ordering is a promise that cannot be kept once a provider
3269        // crashes at runtime, and refusing to boot would stop the whole
3270        // machine, including the tools needed to fix its configuration.
3271        //
3272        // "Provided" is the evaluator's verdict, which counts a provider as
3273        // soon as it has REGISTERED, not once it is ready. Two modules that
3274        // require each other's capabilities are therefore both routable once
3275        // both register; counting readiness instead would deadlock them.
3276        //
3277        // Only new opens are refused. Routes already bound when a provider
3278        // goes away stay bound: nothing here tears them down, and the module
3279        // answers them as it can. Like the readiness read above this is
3280        // best-effort against a provider registering or leaving concurrently.
3281        if let Some(capability) = self.first_unprovided_required_capability(&registration) {
3282            self.counters
3283                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
3284            info!(
3285                target: "control",
3286                code = error_codes::MODULE_WARMING,
3287                module_id = ?target_module_id,
3288                connection_id = ctx.connection_id.get(),
3289                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3290                capability = %capability,
3291                "route.open refused"
3292            );
3293            return Ok(vec![control_error_body_frame(
3294                &frame,
3295                ErrorBody {
3296                    code: error_codes::MODULE_WARMING.to_string(),
3297                    message: format!(
3298                        "module_id '{target_module_id}' requires capability '{capability}', \
3299                         which no registered module provides; retry"
3300                    ),
3301                    detail: Some(serde_json::json!({
3302                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3303                        "capability": capability,
3304                    })),
3305                },
3306            )?]);
3307        }
3308
3309        if !target_has_required_role(&target, &registration.manifest.provides) {
3310            return Ok(vec![self.route_open_refusal_frame(
3311                ctx,
3312                &frame,
3313                &target_module_id,
3314                "role_not_provided",
3315                "target_unavailable",
3316                format!("module_id '{target_module_id}' does not provide the requested target"),
3317            )?]);
3318        }
3319
3320        if registration.state != ChannelState::Active {
3321            return Ok(vec![self.route_open_refusal_frame(
3322                ctx,
3323                &frame,
3324                &target_module_id,
3325                "registration_not_active",
3326                "target_unavailable",
3327                format!("module_id '{target_module_id}' is not active"),
3328            )?]);
3329        }
3330
3331        if self
3332            .forwarding
3333            .module_is_draining(&target_module_id)
3334            .map_err(RouterError::Forwarding)?
3335        {
3336            return Ok(vec![self.route_open_refusal_frame(
3337                ctx,
3338                &frame,
3339                &target_module_id,
3340                "reloading",
3341                "module_reloading",
3342                format!("module_id '{target_module_id}' is reloading"),
3343            )?]);
3344        }
3345
3346        if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
3347            process_liveness.process_live(&target_module_id) == Some(false)
3348        }) {
3349            // A module the supervisor is restarting or reloading can still hold
3350            // a registration: the old process before its connection closes, or
3351            // a new one that registered while the supervisor was draining. The
3352            // forwarding table does not see that as draining, but the consumer
3353            // should still be told to retry soon, exactly as for the drain
3354            // above, rather than that the target is unavailable.
3355            if process_liveness.process_replacing(&target_module_id) {
3356                return Ok(vec![self.route_open_refusal_frame(
3357                    ctx,
3358                    &frame,
3359                    &target_module_id,
3360                    "reloading",
3361                    "module_reloading",
3362                    format!("module_id '{target_module_id}' is reloading"),
3363                )?]);
3364            }
3365            return Ok(vec![self.route_open_refusal_frame(
3366                ctx,
3367                &frame,
3368                &target_module_id,
3369                "supervisor_not_live",
3370                "target_unavailable",
3371                format!("module_id '{target_module_id}' is not live"),
3372            )?]);
3373        }
3374
3375        if !self
3376            .forwarding
3377            .has_live_module_connection(&target_module_id)
3378            .map_err(RouterError::Forwarding)?
3379        {
3380            return Ok(vec![self.route_open_refusal_frame(
3381                ctx,
3382                &frame,
3383                &target_module_id,
3384                "no_forwarding_connection",
3385                "target_unavailable",
3386                format!("module_id '{target_module_id}' has no live forwarding connection"),
3387            )?]);
3388        }
3389
3390        if let Some(error) =
3391            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
3392        {
3393            self.observe_route_open_refusal(
3394                ctx,
3395                &target_module_id,
3396                "op_not_allowed",
3397                "op_not_allowed",
3398            );
3399            return Ok(vec![error]);
3400        }
3401
3402        let principal = match self.route_open_principal(&frame, consumer_identity)? {
3403            Ok(principal) => principal,
3404            Err(error) => {
3405                self.observe_route_open_refusal(
3406                    ctx,
3407                    &target_module_id,
3408                    "bad_consumer_identity",
3409                    "bad_consumer_identity",
3410                );
3411                return Ok(vec![error]);
3412            }
3413        };
3414
3415        // This is attested, control-plane policy for supervised module origins.
3416        // Keep it before route reservation and out of the opaque forwarding hot
3417        // path: data frames must never acquire a per-frame capability check.
3418        if let Principal::Reserved {
3419            module_id: opening_module_id,
3420        } = &principal
3421        {
3422            if let Some(opening_registration) = self
3423                .registry
3424                .get_module(opening_module_id)
3425                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3426            {
3427                if let Some(capability) =
3428                    denied_capability(&opening_registration.manifest, &registration.manifest)
3429                {
3430                    warn!(
3431                        opening_module_id,
3432                        target_module_id,
3433                        capability,
3434                        "refusing route.open because an attested capability deny edge matches"
3435                    );
3436                    return Ok(vec![self.route_open_refusal_frame(
3437                        ctx,
3438                        &frame,
3439                        &target_module_id,
3440                        "capability_deny_edge",
3441                        "capability_forbidden",
3442                        format!(
3443                            "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
3444                        ),
3445                    )?]);
3446                }
3447            }
3448        }
3449
3450        if admission_facts.is_some() {
3451            let carrier_matches = matches!(
3452                &principal,
3453                Principal::Reserved { module_id }
3454                    if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
3455            );
3456            if !carrier_matches {
3457                return Ok(vec![self.route_open_refusal_frame(
3458                    ctx,
3459                    &frame,
3460                    &target_module_id,
3461                    "admission_facts_carrier_not_permitted",
3462                    "admission_facts_not_permitted",
3463                    "admission facts may only be carried by the configured reserved module",
3464                )?]);
3465            }
3466
3467            let target_allowed = self
3468                .admission_facts_targets
3469                .as_ref()
3470                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
3471            if !target_allowed {
3472                return Ok(vec![self.route_open_refusal_frame(
3473                    ctx,
3474                    &frame,
3475                    &target_module_id,
3476                    "admission_facts_target_not_listed",
3477                    "admission_facts_target_not_allowed",
3478                    format!(
3479                        "admission facts are not permitted for target module_id '{target_module_id}'"
3480                    ),
3481                )?]);
3482            }
3483
3484            // Keep the value opaque to subc. The downstream admission validator owns
3485            // schema and semantic checks; this daemon only enforces carrier authority
3486            // and the configured destination allowlist.
3487        }
3488
3489        // Scope admission, on the attested principal above and never on the
3490        // request body. The tag read here travels with the pending bind and is
3491        // compared with the published one at commit, so a sync between here
3492        // and the module's ack refuses the open instead of binding a stamp
3493        // that is no longer true.
3494        let (bound_scope, scope_stamp) = match scope {
3495            None => (None, None),
3496            Some(selector) => {
3497                let owner_configured = match &selector.owner {
3498                    // A reserved owner counts as configured from before its
3499                    // process is spawned (see `SupervisorHandle::is_configured`).
3500                    // So an owner that has not synced its scopes yet is refused
3501                    // as retryable (`scope_not_synced`), not as one that will
3502                    // never sync.
3503                    Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3504                    _ => false,
3505                };
3506                let admitted = self
3507                    .scopes
3508                    .read()
3509                    .unwrap_or_else(|poisoned| poisoned.into_inner())
3510                    .admit(&principal, &target_module_id, &selector, owner_configured)
3511                    .and_then(|admission| {
3512                        crate::scopes::check_target_flow_support(
3513                            &admission.stamp,
3514                            &target_module_id,
3515                            registration.manifest.capabilities.as_ref(),
3516                        )?;
3517                        Ok(admission)
3518                    });
3519                match admitted {
3520                    Ok(admission) => (
3521                        Some(BoundScope {
3522                            owner: admission.owner,
3523                            scope_ref: admission.stamp.scope_ref.clone(),
3524                            tag: admission.tag,
3525                        }),
3526                        Some(admission.stamp),
3527                    ),
3528                    Err(refusal) => {
3529                        return Ok(vec![self.route_open_refusal_frame(
3530                            ctx,
3531                            &frame,
3532                            &target_module_id,
3533                            refusal.code,
3534                            refusal.code,
3535                            refusal.message,
3536                        )?]);
3537                    }
3538                }
3539            }
3540        };
3541
3542        // Bind admits a root that no longer exists on disk, because refusing here
3543        // closes the only exit from a paused run: cancel needs a bound route, and a
3544        // renamed or reclaimed directory makes that route unopenable forever. The
3545        // run itself is intact and still addressable by its recorded identity.
3546        //
3547        // This does NOT relax the rule the strict constructor protects. That rule is
3548        // that no root is ever aliased into NEW durable state -- a missing component
3549        // can reappear as a symlink elsewhere, which would move the identity and
3550        // split a session's history across two of them. The engine now refuses the
3551        // two operations that create such state (send and import) at admission,
3552        // which is a narrower way to hold the same invariant: reads and terminations
3553        // are admitted, writes are not. That refusal had to ship before this line
3554        // changed, or there is an interval where a send commits under a provisional
3555        // identity -- the exact failure the original policy existed to prevent.
3556        //
3557        // Resolution follows realpath rather than lexical cleanup: the longest
3558        // existing ancestor is canonicalized and the missing tail re-appended, so a
3559        // live root is unchanged and a vanished leaf keeps the identity it was
3560        // admitted under. Lexical cleanup would mint a DIFFERENT identity for the
3561        // same caller the moment the directory vanished, which strands the run more
3562        // quietly than refusing it.
3563        let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
3564            Ok(project_root) => project_root,
3565            Err(err) => {
3566                return Ok(vec![control_error_frame(
3567                    &frame,
3568                    "invalid_project_root",
3569                    err.to_string(),
3570                )?])
3571            }
3572        };
3573        identity.project_root = project_root.as_path().to_path_buf();
3574
3575        // Last gate before any relay work, and deliberately after the cheap
3576        // registry and availability checks above: those name a more precise
3577        // condition (unknown, removed, reloading) and a caller is better served
3578        // by the precise code than by this one.
3579        //
3580        // Everything below this point costs an egress permit, a reserved handle
3581        // pair and, if the module does not answer, the whole relay budget. The
3582        // reader no longer waits for that budget, so cap each target explicitly;
3583        // serial dispatch used to provide the accidental cap of one relay per
3584        // connection. Admission is a mutex-protected count and never waits.
3585        let _concurrency_guard = match self
3586            .route_bind_concurrency
3587            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
3588        {
3589            Ok(guard) => guard,
3590            Err(in_flight) => {
3591                return Ok(vec![self.route_open_target_capacity_refusal(
3592                    ctx,
3593                    &frame,
3594                    &target_module_id,
3595                    in_flight,
3596                )?]);
3597            }
3598        };
3599
3600        // A module that has already burned the whole budget `threshold` times
3601        // in a row does not get to charge it again until a probe says it recovered.
3602        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
3603            RouteBindAdmission::Admitted { guard, probe } => {
3604                if probe {
3605                    info!(
3606                        module_id = %target_module_id,
3607                        connection_id = ctx.connection_id.get(),
3608                        "route.bind breaker half-open: admitting one probe"
3609                    );
3610                }
3611                guard
3612            }
3613            RouteBindAdmission::Refused {
3614                consecutive_timeouts,
3615                retry_in,
3616                probe_in_flight,
3617            } => {
3618                return Ok(vec![self.route_open_breaker_refusal_frame(
3619                    ctx,
3620                    &frame,
3621                    &target_module_id,
3622                    consecutive_timeouts,
3623                    retry_in,
3624                    probe_in_flight,
3625                )?]);
3626            }
3627        };
3628
3629        // Resolve the per-module budget here so the wait matches the operator's
3630        // intent for this specific target. A per-module override in
3631        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
3632        // daemons) wins over the daemon-wide default.
3633        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
3634        let relay_deadline = Instant::now() + route_bind_relay_timeout;
3635        let pending = match self
3636            .forwarding
3637            .begin_route_bind_relay_for(
3638                ctx.connection_id,
3639                ctx.egress.clone(),
3640                response_version(&frame),
3641                frame.header.corr,
3642                &target_module_id,
3643                principal.clone(),
3644                bound_scope,
3645                Some(project_root),
3646                relay_deadline,
3647            )
3648            .await
3649        {
3650            Ok(pending) => pending,
3651            Err(err) => {
3652                return Ok(vec![self.route_open_refusal_frame(
3653                    ctx,
3654                    &frame,
3655                    &target_module_id,
3656                    "relay_reservation_failed",
3657                    forwarding_error_code(&err),
3658                    err.to_string(),
3659                )?])
3660            }
3661        };
3662        let crate::forwarding::PendingRouteBindRelay {
3663            endpoint,
3664            module_sink,
3665            negotiated_ver,
3666            client_channel,
3667            client_epoch,
3668            module_channel,
3669            module_epoch,
3670            corr: relay_corr,
3671            receiver,
3672        } = pending;
3673        let mut reservation =
3674            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
3675
3676        // Reserving egress can wait while a module reconnects or a swap cuts
3677        // over. Check the connection the relay actually captured, not the
3678        // earlier by-id lookup: a flow-aware module must not vouch for a
3679        // replacement. The captured sink cannot turn into another connection.
3680        if let Some(stamp) = scope_stamp
3681            .as_ref()
3682            .filter(|stamp| stamp.attributes.flow_id.is_some())
3683        {
3684            let relay_registration = self
3685                .registry
3686                .get_module_by_connection(endpoint.connection_id)
3687                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3688            if let Err(refusal) = crate::scopes::check_target_flow_support(
3689                stamp,
3690                &target_module_id,
3691                relay_registration
3692                    .as_ref()
3693                    .and_then(|registration| registration.manifest.capabilities.as_ref()),
3694            ) {
3695                reservation.release_and_disarm();
3696                return Ok(vec![self.route_open_refusal_frame(
3697                    ctx,
3698                    &frame,
3699                    &target_module_id,
3700                    refusal.code,
3701                    refusal.code,
3702                    refusal.message,
3703                )?]);
3704            }
3705        }
3706
3707        debug!(
3708            connection_id = ctx.connection_id.get(),
3709            client_channel,
3710            client_epoch,
3711            module_channel,
3712            module_epoch,
3713            "reserved route handle pair"
3714        );
3715        // Rendered BEFORE the move into the relay, because the accept arm below
3716        // is where it is logged and the principal is gone by then.
3717        let principal_label = match &principal {
3718            Principal::Reserved { module_id } => format!("reserved:{module_id}"),
3719            Principal::Direct => "direct".to_string(),
3720            other => format!("{other:?}"),
3721        };
3722        let relay = ModuleControlRequest::RouteBind {
3723            route_channel: module_channel,
3724            epoch: module_epoch,
3725            target,
3726            identity,
3727            principal: Some(principal),
3728            consumer_capabilities,
3729            role_versions,
3730            admission_facts,
3731            scope: scope_stamp,
3732        };
3733        let relay_body = serde_json::to_vec(&relay).map_err(|err| {
3734            RouterError::backend(
3735                0,
3736                frame.header.corr,
3737                format!("failed to encode route.bind request: {err}"),
3738            )
3739        })?;
3740        let relay_frame = Frame::build_with_version(
3741            negotiated_ver,
3742            FrameType::Request,
3743            control_flags(),
3744            0,
3745            0,
3746            relay_corr,
3747            relay_body,
3748        )
3749        .map_err(RouterError::FrameBuild)?;
3750
3751        if let Err(err) = module_sink.send(relay_frame).await {
3752            reservation.release_and_disarm();
3753            return Ok(vec![self.route_open_refusal_frame(
3754                ctx,
3755                &frame,
3756                &target_module_id,
3757                "relay_send_failed",
3758                "target_unavailable",
3759                err.to_string(),
3760            )?]);
3761        }
3762
3763        if !self
3764            .forwarding
3765            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
3766            .map_err(RouterError::Forwarding)?
3767        {
3768            self.send_abandoned_route_bind_goodbye(
3769                &module_sink,
3770                negotiated_ver,
3771                module_channel,
3772                module_epoch,
3773            );
3774        }
3775
3776        match timeout_at(relay_deadline, receiver).await {
3777            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
3778                reservation.disarm();
3779                if breaker.record_accepted() {
3780                    info!(
3781                        module_id = %target_module_id,
3782                        "route.bind breaker closed: the probe was accepted"
3783                    );
3784                }
3785                self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
3786                Ok(Vec::new())
3787            }
3788            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
3789                reservation.release_and_disarm();
3790                // A module that says no in microseconds is healthy. Rejection
3791                // is a different condition with its own refusal and must not
3792                // move the breaker.
3793                breaker.record_inconclusive();
3794                // The daemon's own commit re-check refused the bind because the
3795                // scope ended or changed after admission. The module accepted;
3796                // counting it as a module rejection would blame the module.
3797                let scope_code = match body.code.as_str() {
3798                    error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
3799                    error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
3800                    _ => None,
3801                };
3802                if let Some(code) = scope_code {
3803                    self.observe_route_open_refusal(
3804                        ctx,
3805                        &target_module_id,
3806                        "scope_changed_before_commit",
3807                        code,
3808                    );
3809                    return Ok(vec![control_error_body_frame(&frame, body)?]);
3810                }
3811                self.counters
3812                    .increment_route_open_refused("module_rejected");
3813                info!(
3814                    target: "control",
3815                    code = "module_rejected",
3816                    module_code = ?body.code,
3817                    module_id = ?target_module_id,
3818                    connection_id = ctx.connection_id.get(),
3819                    "route.open refused"
3820                );
3821                Ok(vec![control_error_body_frame(&frame, body)?])
3822            }
3823            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
3824                reservation.release_and_disarm();
3825                breaker.record_inconclusive();
3826                // Fires when the module's connection closes while a relayed
3827                // bind is pending -- typically a caller racing a module restart
3828                // whose bind was relayed BEFORE the drain mark went up. Logged
3829                // because the caller sees only its own error and the fleet has
3830                // already spent one diagnosis round unable to tell this arm
3831                // from a relay timeout without daemon-side evidence.
3832                tracing::warn!(
3833                    module_id = %target_module_id,
3834                    "route.bind relay abandoned: {message}"
3835                );
3836                Ok(vec![self.route_open_refusal_frame(
3837                    ctx,
3838                    &frame,
3839                    &target_module_id,
3840                    "relay_abandoned",
3841                    "target_unavailable",
3842                    message,
3843                )?])
3844            }
3845            Ok(Err(_)) => {
3846                reservation.release_and_disarm();
3847                breaker.record_inconclusive();
3848                Ok(vec![self.route_open_refusal_frame(
3849                    ctx,
3850                    &frame,
3851                    &target_module_id,
3852                    "relay_waiter_canceled",
3853                    "target_unavailable",
3854                    "route.bind relay waiter was canceled before the module responded",
3855                )?])
3856            }
3857            Err(_) => {
3858                reservation.release_and_disarm();
3859                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
3860                // answer at all is the one condition a fast refusal can
3861                // usefully stand in for; every other arm already answered.
3862                if let Some(opened) = breaker.record_timeout(
3863                    self.route_bind_breaker_threshold,
3864                    self.route_bind_breaker_cooldown,
3865                ) {
3866                    warn!(
3867                        module_id = %target_module_id,
3868                        consecutive_timeouts = opened.consecutive_timeouts,
3869                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
3870                        reopened_after_probe = opened.reopened_after_probe,
3871                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
3872                    );
3873                }
3874                // The generous budget just burned to no answer: the module is
3875                // registered and its connection is up, but its bind handler sat
3876                // on the ack for the full budget (warm-on-bind, cold configure,
3877                // or a wedged handler). Every earlier unavailability shape
3878                // fast-refuses BEFORE the relay, so this arm firing means the
3879                // slowness is module-side -- log it so the per-module timeline
3880                // is reconstructable without client audit rows.
3881                tracing::warn!(
3882                    module_id = %target_module_id,
3883                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
3884                    "route.bind relay timed out: module did not ack within budget"
3885                );
3886                Ok(vec![self.route_open_refusal_frame(
3887                    ctx,
3888                    &frame,
3889                    &target_module_id,
3890                    "relay_timed_out",
3891                    "module_timeout",
3892                    format!(
3893                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
3894                        route_bind_relay_timeout
3895                    ),
3896                )?])
3897            }
3898        }
3899    }
3900
3901    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3902        let response = ClientControlResponse::SupervisorSpawnSnapshot {
3903            snapshot: self.supervisor.spawn_snapshot(),
3904        };
3905        Ok(vec![control_response_body_frame(
3906            &frame,
3907            &response,
3908            "ClientControlResponse::SupervisorSpawnSnapshot",
3909        )?])
3910    }
3911
3912    fn handle_supervisor_spawn_subscribe(
3913        &self,
3914        ctx: &RouteCtx,
3915        frame: Frame,
3916        since: Option<SpawnCursor>,
3917    ) -> Result<Vec<Frame>, RouterError> {
3918        match self.supervisor.subscribe_spawns(
3919            ctx.connection_id,
3920            frame.header.corr,
3921            response_version(&frame),
3922            since,
3923            ctx.egress.clone(),
3924        ) {
3925            Ok(()) => Ok(Vec::new()),
3926            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
3927                Ok(vec![control_error_body_frame(
3928                    &frame,
3929                    ErrorBody {
3930                        code: "spawn_cursor_incarnation_mismatch".to_string(),
3931                        message: "spawn cursor belongs to a different daemon incarnation"
3932                            .to_string(),
3933                        detail: Some(serde_json::json!({
3934                            "current_daemon_incarnation": current
3935                        })),
3936                    },
3937                )?])
3938            }
3939            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
3940                &frame,
3941                ErrorBody {
3942                    code: "spawn_cursor_too_old".to_string(),
3943                    message: "spawn cursor predates the retained event ring".to_string(),
3944                    detail: Some(serde_json::json!({
3945                        "oldest_retained_cursor": oldest
3946                    })),
3947                },
3948            )?]),
3949            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
3950                0,
3951                frame.header.corr,
3952                format!("failed to open supervisor spawn subscription: {error}"),
3953            )),
3954        }
3955    }
3956
3957    async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3958        let generation = self
3959            .registry
3960            .generation()
3961            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3962        let mut modules = Vec::new();
3963        for module in self.supervisor.list() {
3964            let status = module.status_for_control("list").map_err(|err| {
3965                RouterError::backend(
3966                    0,
3967                    frame.header.corr,
3968                    format!("failed to read supervisor status: {err}"),
3969                )
3970            })?;
3971            let (configured, _) = module.configuration().map_err(|err| {
3972                RouterError::backend(
3973                    0,
3974                    frame.header.corr,
3975                    format!("failed to read module configuration: {err}"),
3976                )
3977            })?;
3978            // Status and configuration snapshots release their locks before the image probe awaits.
3979            let image = module.running_image_agreement().await;
3980            // Read per request so the figure is current when the operator asks;
3981            // the daemon samples nothing in between.
3982            let resources = Some(module.child_resource_usage());
3983            let pending_reload = Some(reload_verdict(
3984                &configured.program,
3985                status.spawned_from.as_deref(),
3986                image,
3987            ));
3988            modules.push(SupervisorEntry {
3989                // Keep the retired policy field on the wire for one release so
3990                // existing status consumers still receive the platform policy.
3991                launch_nonce_env: Some(!cfg!(unix)),
3992                module_id: status.module_id,
3993                state: status.state.to_string(),
3994                enabled: status.enabled,
3995                live: status.live,
3996                protocol: status.protocol,
3997                health: status.health.status,
3998                pending_reload,
3999                last_probe_ms: status.health.last_probe_ms,
4000                last_exit_code: status.last_exit.as_ref().and_then(|e| e.code),
4001                last_exit_signal: status.last_exit.as_ref().and_then(|e| e.signal),
4002                last_exit_ms: status.last_exit.as_ref().map(|e| e.at_ms),
4003                last_exit_kind: status.last_exit.as_ref().map(|e| e.kind.into()),
4004                restart_count: Some(status.restart_count),
4005                max_restarts: Some(status.max_restarts),
4006                lifetime_restarts: Some(status.lifetime_restarts),
4007                spawn_generation: Some(status.spawn_generation),
4008                restart_window_secs: Some(status.restart_window.as_secs()),
4009                drain_timeout_ms: Some(status.drain_timeout.as_millis() as u64),
4010                restart_backoff_ms: Some(status.restart_backoff.as_millis() as u64),
4011                restart_max_backoff_ms: Some(status.restart_max_backoff.as_millis() as u64),
4012                resources,
4013            });
4014        }
4015        let response = ClientControlResponse::SupervisorList {
4016            generation,
4017            modules,
4018        };
4019        Ok(vec![control_response_body_frame(
4020            &frame,
4021            &response,
4022            "ClientControlResponse::SupervisorList",
4023        )?])
4024    }
4025
4026    fn handle_supervisor_stderr_tail(
4027        &self,
4028        frame: Frame,
4029        module_id: String,
4030        max_lines: Option<u32>,
4031        max_bytes: Option<u32>,
4032    ) -> Result<Vec<Frame>, RouterError> {
4033        let Some(module) = self.supervisor.get(&module_id) else {
4034            return Ok(vec![control_error_frame(
4035                &frame,
4036                "unknown_module",
4037                format!("module_id '{module_id}' is not supervised"),
4038            )?]);
4039        };
4040
4041        let snapshot = module.stderr_tail(
4042            max_lines.map(|value| value as usize),
4043            max_bytes.map(|value| value as usize),
4044        );
4045
4046        let response = ClientControlResponse::SupervisorStderrTail {
4047            module_id,
4048            tail: StderrTail {
4049                capture: match snapshot.capture {
4050                    CaptureState::Captured => StderrCaptureState::Captured,
4051                    CaptureState::Incomplete { reason } => {
4052                        StderrCaptureState::Incomplete { reason }
4053                    }
4054                    CaptureState::NotCaptured { reason } => {
4055                        StderrCaptureState::NotCaptured { reason }
4056                    }
4057                },
4058                entries: snapshot
4059                    .entries
4060                    .into_iter()
4061                    .map(|entry| match entry {
4062                        TailEntry::Line {
4063                            text,
4064                            truncated,
4065                            at_ms,
4066                        } => StderrTailEntry::Line {
4067                            text,
4068                            truncated,
4069                            at_ms,
4070                        },
4071                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
4072                    })
4073                    .collect(),
4074                dropped_lines: snapshot.dropped_lines,
4075            },
4076        };
4077        Ok(vec![control_response_body_frame(
4078            &frame,
4079            &response,
4080            "ClientControlResponse::SupervisorStderrTail",
4081        )?])
4082    }
4083
4084    async fn handle_supervisor_terminals(
4085        &self,
4086        frame: Frame,
4087        module_id: String,
4088    ) -> Result<Vec<Frame>, RouterError> {
4089        let Some(module) = self.supervisor.get(&module_id) else {
4090            return Ok(vec![control_error_frame(
4091                &frame,
4092                "unknown_module",
4093                format!("module_id '{module_id}' is not supervised"),
4094            )?]);
4095        };
4096
4097        // The journal read runs on a blocking thread: it can be megabytes of
4098        // file I/O and must not occupy a runtime worker.
4099        let terminals = module
4100            .read_durable_terminal_history()
4101            .await
4102            .map_err(|error| {
4103                RouterError::backend(
4104                    0,
4105                    frame.header.corr,
4106                    format!("failed to read terminal history: {error}"),
4107                )
4108            })?;
4109        let response = ClientControlResponse::SupervisorTerminals {
4110            module_id,
4111            terminals,
4112        };
4113        Ok(vec![control_response_body_frame(
4114            &frame,
4115            &response,
4116            "ClientControlResponse::SupervisorTerminals",
4117        )?])
4118    }
4119
4120    fn handle_supervisor_routes(
4121        &self,
4122        frame: Frame,
4123        module_id: Option<String>,
4124    ) -> Result<Vec<Frame>, RouterError> {
4125        let modules = self
4126            .forwarding
4127            .route_census(module_id.as_deref())
4128            .map_err(RouterError::Forwarding)?
4129            .into_iter()
4130            .map(|(module_id, routes)| SupervisorRouteModule {
4131                module_id,
4132                routes: routes
4133                    .into_iter()
4134                    .map(|route| SupervisorRoute {
4135                        consumer: match route.principal {
4136                            Principal::Reserved { module_id } => {
4137                                SupervisorRouteConsumer::Reserved { module_id }
4138                            }
4139                            Principal::Direct | Principal::Unverified => {
4140                                SupervisorRouteConsumer::Direct {
4141                                    connection_id: route.goodbye_target.connection_id.get(),
4142                                }
4143                            }
4144                        },
4145                        age_ms: Instant::now()
4146                            .saturating_duration_since(route.bound_at)
4147                            .as_millis()
4148                            .try_into()
4149                            .unwrap_or(u64::MAX),
4150                        draining: route.draining,
4151                        drain_reason: route.drain_reason,
4152                    })
4153                    .collect(),
4154            })
4155            .collect();
4156        let response = ClientControlResponse::SupervisorRoutes { modules };
4157        Ok(vec![control_response_body_frame(
4158            &frame,
4159            &response,
4160            "ClientControlResponse::SupervisorRoutes",
4161        )?])
4162    }
4163
4164    async fn handle_supervisor_provenance(
4165        &self,
4166        frame: Frame,
4167        module_id: Option<String>,
4168    ) -> Result<Vec<Frame>, RouterError> {
4169        let mut selected = if let Some(module_id) = module_id {
4170            let Some(module) = self.supervisor.get(&module_id) else {
4171                return Ok(vec![control_error_frame(
4172                    &frame,
4173                    "unknown_module",
4174                    format!("module_id '{module_id}' is not supervised"),
4175                )?]);
4176            };
4177            vec![module]
4178        } else {
4179            self.supervisor.list()
4180        };
4181
4182        let mut modules = Vec::with_capacity(selected.len());
4183        for module in selected.drain(..) {
4184            let (status, observed_image) = module
4185                .status_and_running_image_agreement()
4186                .await
4187                .map_err(|err| {
4188                    RouterError::backend(
4189                        0,
4190                        frame.header.corr,
4191                        format!("failed to read supervisor status: {err}"),
4192                    )
4193                })?;
4194            let module_declared = self
4195                .registry
4196                .get_module(&status.module_id)
4197                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4198                .and_then(|registration| registration.manifest.provenance)
4199                .map(|build| ModuleDeclaredProvenance::Reported { build })
4200                .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
4201            #[cfg(test)]
4202            let running_image = match &self.provenance_probe_override {
4203                Some(result) => result.clone(),
4204                None => observed_image,
4205            };
4206            #[cfg(not(test))]
4207            let running_image = observed_image;
4208            modules.push(SupervisorModuleProvenance {
4209                module_id: status.module_id,
4210                module_declared,
4211                daemon_observed: SupervisorObservedProcess {
4212                    pid: status.pid,
4213                    spawned_at_ms: status.spawned_at_ms,
4214                    spawned_from: status.spawned_from,
4215                    running_image,
4216                },
4217            });
4218        }
4219        let daemon = SupervisorDaemonProvenance {
4220            daemon_build: self.daemon_provenance.build.clone(),
4221            daemon_observed: DaemonObservedProcess {
4222                pid: self.daemon_provenance.pid,
4223                started_at_ms: self
4224                    .daemon_provenance
4225                    .start_clock
4226                    .map(|clock| clock.started_at_ms())
4227                    .or(self.daemon_provenance.started_at_ms),
4228                running_image: self
4229                    .daemon_provenance
4230                    .probe
4231                    .observe(
4232                        self.daemon_provenance.pid,
4233                        self.daemon_provenance.executable_path.as_deref(),
4234                        self.daemon_provenance.executable_identity,
4235                        self.daemon_provenance.process_start_time,
4236                    )
4237                    .await,
4238            },
4239        };
4240        let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
4241        Ok(vec![control_response_body_frame(
4242            &frame,
4243            &response,
4244            "ClientControlResponse::SupervisorProvenance",
4245        )?])
4246    }
4247
4248    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4249        self.refresh_capability_requirements();
4250        let generation = self
4251            .registry
4252            .generation()
4253            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4254        let modules = self
4255            .supervisor
4256            .list()
4257            .into_iter()
4258            .map(|module| {
4259                let status = module.status_for_control("health").map_err(|err| {
4260                    RouterError::backend(
4261                        0,
4262                        frame.header.corr,
4263                        format!("failed to read supervisor health: {err}"),
4264                    )
4265                })?;
4266                let module_id = status.module_id;
4267                let capability_detail = self
4268                    .capability_evaluator
4269                    .required_problem_detail(&module_id);
4270                Ok(SupervisorHealthEntry {
4271                    module_id,
4272                    status: status.health.status,
4273                    detail: append_capability_problem_detail(
4274                        status.health.detail,
4275                        capability_detail,
4276                    ),
4277                    metrics: status.health.metrics,
4278                    consecutive_failures: status.health.consecutive_failures,
4279                    late_answer_count: status.health.late_answer_count,
4280                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
4281                    last_action: status.health.last_action,
4282                    last_action_ms: status.health.last_action_ms,
4283                    last_probe_ms: status.health.last_probe_ms,
4284                })
4285            })
4286            .collect::<Result<Vec<_>, RouterError>>()?;
4287        let response = ClientControlResponse::SupervisorHealth {
4288            generation,
4289            modules,
4290        };
4291        Ok(vec![control_response_body_frame(
4292            &frame,
4293            &response,
4294            "ClientControlResponse::SupervisorHealth",
4295        )?])
4296    }
4297
4298    async fn handle_supervisor_restart(
4299        &self,
4300        frame: Frame,
4301        module_id: String,
4302        drain_timeout_ms: Option<u64>,
4303    ) -> Result<Vec<Frame>, RouterError> {
4304        let operation_lock = self.supervisor.operation_lock();
4305        let _operation_guard = operation_lock.lock().await;
4306        let Some(module) = self.supervisor.get(&module_id) else {
4307            return Ok(vec![control_error_frame(
4308                &frame,
4309                "unknown_module",
4310                format!("module_id '{module_id}' is not supervised"),
4311            )?]);
4312        };
4313
4314        self.route_outages.mark_operator_action(&module_id);
4315        if let Err(err) = module.restart(drain_timeout_ms).await {
4316            self.route_outages
4317                .operator_action_ended_unrefused(&module_id);
4318            let (code, message) = match err {
4319                crate::supervise::SuperviseError::Disabled { .. } => {
4320                    ("module_disabled", err.to_string())
4321                }
4322                crate::supervise::SuperviseError::SwapInProgress { .. } => {
4323                    ("swap_in_progress", err.to_string())
4324                }
4325                _ => (
4326                    "target_unavailable",
4327                    format!("failed to restart module_id '{module_id}': {err}"),
4328                ),
4329            };
4330            return Ok(vec![control_error_frame(&frame, code, message)?]);
4331        }
4332
4333        let response = ClientControlResponse::SupervisorAck {
4334            module_id,
4335            applied: true,
4336        };
4337        Ok(vec![control_response_body_frame(
4338            &frame,
4339            &response,
4340            "ClientControlResponse::SupervisorAck",
4341        )?])
4342    }
4343
4344    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
4345    /// when the old process has finished draining: a caller whose own lane
4346    /// rides the old process must get its reply before that drain waits on it.
4347    async fn handle_supervisor_swap(
4348        &self,
4349        frame: Frame,
4350        module_id: String,
4351        ready_timeout_ms: Option<u64>,
4352    ) -> Result<Vec<Frame>, RouterError> {
4353        // The daemon-wide operation lock is held only to resolve the handle,
4354        // not across the swap. The swap can take its whole readiness budget,
4355        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
4356        // holding it here would park an operator's stop behind the swap it is
4357        // meant to abort. A rescan or stop that reaches the module during the
4358        // swap is served by the swap itself (see `supervise_swap`).
4359        let module = {
4360            let operation_lock = self.supervisor.operation_lock();
4361            let _operation_guard = operation_lock.lock().await;
4362            self.supervisor.get(&module_id)
4363        };
4364        let Some(module) = module else {
4365            return Ok(vec![control_error_frame(
4366                &frame,
4367                "unknown_module",
4368                format!("module_id '{module_id}' is not supervised"),
4369            )?]);
4370        };
4371
4372        self.route_outages.mark_operator_action(&module_id);
4373        if let Err(err) = module
4374            .swap(ready_timeout_ms.map(Duration::from_millis))
4375            .await
4376        {
4377            self.route_outages
4378                .operator_action_ended_unrefused(&module_id);
4379            use crate::supervise::SuperviseError;
4380            let message = err.to_string();
4381            let error = match err {
4382                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
4383                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
4384                    code: "swap_refused".to_string(),
4385                    message,
4386                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
4387                },
4388                SuperviseError::SwapFailed {
4389                    arm,
4390                    candidate_exit,
4391                    ..
4392                } => ErrorBody {
4393                    code: "swap_failed".to_string(),
4394                    message,
4395                    detail: Some(serde_json::json!({
4396                        "arm": arm.as_str(),
4397                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
4398                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
4399                    })),
4400                },
4401                _ => ErrorBody::new(
4402                    "target_unavailable",
4403                    format!("failed to swap module_id '{module_id}': {message}"),
4404                ),
4405            };
4406            return Ok(vec![control_error_body_frame(&frame, error)?]);
4407        }
4408        // A completed swap kept the incumbent serving until cutover, so it
4409        // usually opened no outage; a mark left behind would make the next,
4410        // unrelated outage read as requested.
4411        self.route_outages
4412            .operator_action_ended_unrefused(&module_id);
4413
4414        let response = ClientControlResponse::SupervisorAck {
4415            module_id,
4416            applied: true,
4417        };
4418        Ok(vec![control_response_body_frame(
4419            &frame,
4420            &response,
4421            "ClientControlResponse::SupervisorAck",
4422        )?])
4423    }
4424
4425    async fn handle_supervisor_reload(
4426        &self,
4427        frame: Frame,
4428        module_id: String,
4429    ) -> Result<Vec<Frame>, RouterError> {
4430        let operation_lock = self.supervisor.operation_lock();
4431        let _operation_guard = operation_lock.lock().await;
4432        let Some(module) = self.supervisor.get(&module_id) else {
4433            return Ok(vec![control_error_frame(
4434                &frame,
4435                "unknown_module",
4436                format!("module_id '{module_id}' is not supervised"),
4437            )?]);
4438        };
4439
4440        self.route_outages.mark_operator_action(&module_id);
4441        if let Err(err) = module.reload().await {
4442            self.route_outages
4443                .operator_action_ended_unrefused(&module_id);
4444            let (code, message) = match err {
4445                crate::supervise::SuperviseError::Disabled { .. } => {
4446                    ("module_disabled", err.to_string())
4447                }
4448                crate::supervise::SuperviseError::SwapInProgress { .. } => {
4449                    ("swap_in_progress", err.to_string())
4450                }
4451                _ => (
4452                    "reload_failed",
4453                    format!("failed to reload module_id '{module_id}': {err}"),
4454                ),
4455            };
4456            return Ok(vec![control_error_frame(&frame, code, message)?]);
4457        }
4458
4459        let response = ClientControlResponse::SupervisorAck {
4460            module_id,
4461            applied: true,
4462        };
4463        Ok(vec![control_response_body_frame(
4464            &frame,
4465            &response,
4466            "ClientControlResponse::SupervisorAck",
4467        )?])
4468    }
4469
4470    async fn handle_supervisor_rescan(
4471        &self,
4472        frame: Frame,
4473        preview: bool,
4474    ) -> Result<Vec<Frame>, RouterError> {
4475        let Some(context) = self.rescan.clone() else {
4476            return Ok(vec![control_error_frame(
4477                &frame,
4478                "rescan_unavailable",
4479                "the daemon was not started with a reloadable config path".to_string(),
4480            )?]);
4481        };
4482
4483        let operation_lock = self.supervisor.operation_lock();
4484        let _operation_guard = operation_lock.lock().await;
4485        let loaded = match crate::daemon_config::load(&context.config_path) {
4486            Ok(config) => config,
4487            Err(err) => {
4488                return Ok(vec![control_error_frame(
4489                    &frame,
4490                    "invalid_daemon_config",
4491                    format!("supervisor rescan rejected daemon config: {err}"),
4492                )?])
4493            }
4494        };
4495        // `load` reports a missing file as Ok(None), which is correct at boot
4496        // (no config, nothing to supervise) and catastrophic here: rescan treats
4497        // "not in the config" as "remove it", so an absent file would read as an
4498        // empty module list and retire the entire running fleet. An editor
4499        // writing via write-new-then-rename, or a half-finished edit, is enough
4500        // to open that window. Refuse instead: a config that cannot be read
4501        // carries no instruction to remove anything.
4502        let Some(config) = loaded else {
4503            return Ok(vec![control_error_frame(
4504                &frame,
4505                "invalid_daemon_config",
4506                format!(
4507                    "daemon config not found at {}; refusing to rescan (an absent config would \
4508                     retire every supervised module)",
4509                    context.config_path.display()
4510                ),
4511            )?]);
4512        };
4513        let (
4514            configured_port,
4515            storage_config,
4516            admission_facts_carrier_module_id,
4517            admission_facts_targets,
4518            scope_authority_owners,
4519            modules,
4520            reserved_capabilities,
4521        ) = (
4522            config.port,
4523            config.storage,
4524            config.admission_facts_carrier_module_id,
4525            config.admission_facts_targets,
4526            config.scope_authority_owners,
4527            config.modules,
4528            config.reserved_capabilities,
4529        );
4530
4531        // Collect the sections rescan cannot apply, so the REPLY carries them.
4532        //
4533        // The warning below has always been correct and has always gone only to
4534        // the journal -- addressed to whoever reads logs, while the person who
4535        // just edited the config is looking at the CLI. Naming each section
4536        // individually rather than setting a flag: "something outside modules
4537        // changed" sends the operator back to diffing their own file, which is
4538        // the work this is meant to save.
4539        let mut restart_required = Vec::new();
4540        for section in RestartRequiredSection::ALL {
4541            let changed = match section {
4542                RestartRequiredSection::Port => configured_port != context.configured_port,
4543                RestartRequiredSection::Storage => storage_config != context.storage_config,
4544                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
4545                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
4546                }
4547                RestartRequiredSection::AdmissionFactsTargets => {
4548                    admission_facts_targets != context.admission_facts_targets
4549                }
4550                RestartRequiredSection::ScopeAuthorityOwners => {
4551                    scope_authority_owners != context.scope_authority_owners
4552                }
4553            };
4554            if changed {
4555                restart_required.push(section.label().to_string());
4556            }
4557        }
4558        if !restart_required.is_empty() {
4559            warn!(
4560                config_path = %context.config_path.display(),
4561                sections = %restart_required.join(", "),
4562                "daemon config changed outside the modules section; restart the daemon to apply those changes"
4563            );
4564        }
4565
4566        for configured in &modules {
4567            if let Err(err) = validate_spec(&configured.module_spec()) {
4568                return Ok(vec![control_error_frame(
4569                    &frame,
4570                    "invalid_daemon_config",
4571                    format!("supervisor rescan rejected daemon config: {err}"),
4572                )?]);
4573            }
4574        }
4575
4576        let configured_capabilities = modules
4577            .iter()
4578            .map(|module| (module.module_id.clone(), module.enabled))
4579            .collect::<Vec<_>>();
4580        let preview_capability_warnings = if preview {
4581            let (_, registrations) = self.runtime_capability_snapshot()?;
4582            let current_modules = self
4583                .supervisor
4584                .list()
4585                .into_iter()
4586                .map(|module| module.module_id().to_string())
4587                .collect::<BTreeSet<_>>();
4588            let resulting_modules = configured_capabilities.clone();
4589            let removed = current_modules
4590                .into_iter()
4591                .filter(|module_id| {
4592                    !resulting_modules
4593                        .iter()
4594                        .any(|(configured_id, _)| configured_id == module_id)
4595                })
4596                .collect::<Vec<_>>();
4597            self.capability_evaluator.preview_removal_warnings(
4598                resulting_modules,
4599                &removed,
4600                &registrations,
4601            )
4602        } else {
4603            Vec::new()
4604        };
4605        let result = match self
4606            .reconcile_supervised_modules(&context.supervisor, modules, preview)
4607            .await
4608        {
4609            Ok(result) => result,
4610            Err(message) => {
4611                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
4612            }
4613        };
4614        if !preview {
4615            self.capability_evaluator
4616                .configure(configured_capabilities, reserved_capabilities);
4617            self.capability_evaluator.wake_deadline_loop();
4618            self.refresh_capability_requirements();
4619        }
4620        let mut result = result;
4621        result.restart_required = restart_required;
4622        result.capability_warnings = preview_capability_warnings;
4623        let response = ClientControlResponse::SupervisorRescan { result };
4624        Ok(vec![control_response_body_frame(
4625            &frame,
4626            &response,
4627            "ClientControlResponse::SupervisorRescan",
4628        )?])
4629    }
4630
4631    async fn handle_supervisor_release_reserved(
4632        &self,
4633        frame: Frame,
4634        module_id: String,
4635    ) -> Result<Vec<Frame>, RouterError> {
4636        let Some(context) = self.rescan.clone() else {
4637            return Ok(vec![control_error_frame(
4638                &frame,
4639                "release_unavailable",
4640                "reserved-id release requires a daemon started with a reloadable config path",
4641            )?]);
4642        };
4643        let operation_lock = self.supervisor.operation_lock();
4644        let _operation_guard = operation_lock.lock().await;
4645        let loaded = match crate::daemon_config::load(&context.config_path) {
4646            Ok(Some(config)) => config,
4647            Ok(None) => {
4648                return Ok(vec![control_error_frame(
4649                    &frame,
4650                    "invalid_daemon_config",
4651                    format!(
4652                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
4653                        context.config_path.display()
4654                    ),
4655                )?])
4656            }
4657            Err(err) => {
4658                return Ok(vec![control_error_frame(
4659                    &frame,
4660                    "invalid_daemon_config",
4661                    format!("unable to verify reserved-id release against daemon config: {err}"),
4662                )?])
4663            }
4664        };
4665        if loaded
4666            .modules
4667            .iter()
4668            .any(|configured| configured.module_id == module_id)
4669        {
4670            return Ok(vec![control_error_frame(
4671                &frame,
4672                "reserved_module_configured",
4673                format!(
4674                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
4675                ),
4676            )?]);
4677        }
4678        if !self.supervisor.release_retained_reserved_gate(&module_id) {
4679            return Ok(vec![control_error_frame(
4680                &frame,
4681                "reserved_gate_not_retained",
4682                format!(
4683                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
4684                ),
4685            )?]);
4686        }
4687
4688        let response = ClientControlResponse::SupervisorAck {
4689            module_id,
4690            applied: true,
4691        };
4692        Ok(vec![control_response_body_frame(
4693            &frame,
4694            &response,
4695            "ClientControlResponse::SupervisorAck",
4696        )?])
4697    }
4698
4699    /// Reconcile the running module set against the configured one.
4700    ///
4701    /// With `preview` set, the diff is computed and returned WITHOUT applying any
4702    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
4703    /// deliberately shares this function with the executing path rather than
4704    /// computing the same diff somewhere else -- two implementations of one
4705    /// decision agree until they do not, and the whole value of a preview is that
4706    /// it describes the operation that will actually run.
4707    async fn reconcile_supervised_modules(
4708        &self,
4709        supervisor: &Supervisor,
4710        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
4711        preview: bool,
4712    ) -> Result<SupervisorRescanResult, String> {
4713        let mut current = BTreeMap::new();
4714        for module in self.supervisor.list() {
4715            let (spec, health) = module.configuration().map_err(|err| {
4716                format!(
4717                    "failed to read configuration for module_id '{}': {err}",
4718                    module.module_id()
4719                )
4720            })?;
4721            let enabled = module
4722                .status()
4723                .map_err(|err| {
4724                    format!(
4725                        "failed to read status for module_id '{}': {err}",
4726                        module.module_id()
4727                    )
4728                })?
4729                .enabled;
4730            current.insert(
4731                module.module_id().to_string(),
4732                (module, spec, health, enabled),
4733            );
4734        }
4735        let configured = configured_modules
4736            .into_iter()
4737            .map(|module| (module.module_id.clone(), module))
4738            .collect::<BTreeMap<_, _>>();
4739
4740        let added = configured
4741            .keys()
4742            .filter(|module_id| !current.contains_key(*module_id))
4743            .cloned()
4744            .collect::<Vec<_>>();
4745        let removed = current
4746            .keys()
4747            .filter(|module_id| !configured.contains_key(*module_id))
4748            .cloned()
4749            .collect::<Vec<_>>();
4750        let mut changed_pending_reload = Vec::new();
4751        let mut configuration_changes = BTreeSet::new();
4752        let mut enabled_changes = BTreeSet::new();
4753        let mut unchanged = 0_u32;
4754
4755        for (module_id, configured_module) in &configured {
4756            let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
4757            else {
4758                continue;
4759            };
4760            // Compare the whole launch spec so a future launch field cannot
4761            // accidentally become a live-only policy change. Health is stored
4762            // separately and applies live without replacing the process.
4763            let launch_changed = *current_spec != configured_module.module_spec();
4764            let configuration_changed =
4765                launch_changed || *current_health != configured_module.health;
4766            let enabled_changed = *current_enabled != configured_module.enabled;
4767            if configuration_changed {
4768                configuration_changes.insert(module_id.clone());
4769            }
4770            if launch_changed {
4771                changed_pending_reload.push(module_id.clone());
4772            }
4773            if enabled_changed {
4774                enabled_changes.insert(module_id.clone());
4775            }
4776            if !configuration_changed && !enabled_changed {
4777                unchanged = unchanged.saturating_add(1);
4778            }
4779        }
4780
4781        // Everything above this point is pure computation over two snapshots.
4782        // Everything below MUTATES. The preview returns here so the boundary is a
4783        // single early return rather than a condition repeated at each mutation
4784        // site, where one missed guard would apply part of a change the caller was
4785        // told would not happen.
4786        if preview {
4787            return Ok(SupervisorRescanResult {
4788                added,
4789                removed,
4790                changed_pending_reload,
4791                enabled_changes: enabled_changes.iter().cloned().collect(),
4792                unchanged,
4793                preview: true,
4794                // Filled by the caller on both paths, so the preview reports
4795                // restart-required sections identically to an executed rescan --
4796                // the preview is where an operator is most likely to be looking.
4797                restart_required: Vec::new(),
4798                capability_warnings: Vec::new(),
4799            });
4800        }
4801
4802        for module_id in &removed {
4803            let module = &current
4804                .get(module_id)
4805                .expect("removed module came from current supervisor state")
4806                .0;
4807            module.retire().await.map_err(|err| {
4808                format!("failed to retire module_id '{module_id}' during rescan: {err}")
4809            })?;
4810            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
4811            //
4812            // `handle_route_open` resolves an absent module in three steps:
4813            // registry, then supervisor status, then tombstone. Retiring first
4814            // opens a window where ALL THREE ARE ABSENT -- the registry entry
4815            // went with the teardown above, the supervisor entry went with
4816            // `retire`, and the tombstone does not exist yet -- so a route.open
4817            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
4818            // it") for a module that was deliberately removed and whose caller
4819            // should get `module_removed` (TERMINAL, carrying a removal age).
4820            //
4821            // Writing the tombstone first closes it: during the window the
4822            // supervisor entry still answers, so the caller gets
4823            // `target_unavailable` -- retryable, and TRUE, because the module
4824            // is mid-teardown. After both statements it is `module_removed`.
4825            // No instant remains where a removed module reads as one that
4826            // never existed.
4827            //
4828            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
4829            // the absence of a test beside a fix invites deletion: these are
4830            // two sync statements with no await between them, so reaching the
4831            // window needs a second worker thread to land exactly between them
4832            // and there is no hook to force it. MEASURED: the 25 daemon_config
4833            // tests pass identically with the old order and the new one, so
4834            // the existing suite cannot see this and a green run is not
4835            // evidence either way. What the suite does hold is the
4836            // post-condition -- a removed module answers `module_removed` --
4837            // which this preserves.
4838            //
4839            // Found by an Athena panel reading the shipped tree against a
4840            // design note (2026-09-19), as the one concrete instance of that
4841            // note's class that survived contact with source. Direction is
4842            // benign: retryable where terminal was intended, never the reverse.
4843            self.supervisor.record_rescan_removal(module_id);
4844            self.supervisor.retire(module_id);
4845            self.route_outages.forget(module_id);
4846        }
4847
4848        for module_id in configured.keys() {
4849            let Some((module, _, _, _)) = current.get(module_id) else {
4850                continue;
4851            };
4852            let configured_module = configured
4853                .get(module_id)
4854                .expect("configured module id came from configured map");
4855            if configuration_changes.contains(module_id) {
4856                module
4857                    .update_configuration(
4858                        configured_module.module_spec(),
4859                        configured_module.health.clone(),
4860                        configured_module.drain_timeout_ms,
4861                    )
4862                    .await
4863                    .map_err(|err| {
4864                        format!(
4865                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
4866                        )
4867                    })?;
4868            }
4869            if enabled_changes.contains(module_id) {
4870                // A rescan that starts or stops a module applies an operator's
4871                // edit to the config, so the resulting outage was asked for.
4872                self.route_outages.mark_operator_action(module_id);
4873                module
4874                    .set_enabled(configured_module.enabled)
4875                    .await
4876                    .map_err(|err| {
4877                        self.route_outages.operator_action_ended_unrefused(module_id);
4878                        format!(
4879                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
4880                            configured_module.enabled
4881                        )
4882                    })?;
4883            }
4884        }
4885
4886        for module_id in &added {
4887            let configured_module = configured
4888                .get(module_id)
4889                .expect("added module id came from configured map");
4890            supervisor
4891                .supervise_configured_with_health(
4892                    configured_module.module_spec(),
4893                    configured_module.enabled,
4894                    configured_module.health.clone(),
4895                    configured_module.drain_timeout_ms,
4896                    configured_module.restart,
4897                )
4898                .map_err(|err| {
4899                    format!("failed to add module_id '{module_id}' during rescan: {err}")
4900                })?;
4901        }
4902
4903        Ok(SupervisorRescanResult {
4904            added,
4905            removed,
4906            changed_pending_reload,
4907            enabled_changes: enabled_changes.iter().cloned().collect(),
4908            unchanged,
4909            preview: false,
4910            // Filled by the caller, which is the only layer that can see the
4911            // previous config to diff against.
4912            restart_required: Vec::new(),
4913            capability_warnings: Vec::new(),
4914        })
4915    }
4916
4917    async fn handle_supervisor_set_enabled(
4918        &self,
4919        frame: Frame,
4920        module_id: String,
4921        enabled: bool,
4922    ) -> Result<Vec<Frame>, RouterError> {
4923        let operation_lock = self.supervisor.operation_lock();
4924        let _operation_guard = operation_lock.lock().await;
4925        let Some(module) = self.supervisor.get(&module_id) else {
4926            return Ok(vec![control_error_frame(
4927                &frame,
4928                "unknown_module",
4929                format!("module_id '{module_id}' is not supervised"),
4930            )?]);
4931        };
4932
4933        // Enabling counts as well as disabling: a module an operator starts
4934        // is refused until it registers, and that wait was asked for.
4935        self.route_outages.mark_operator_action(&module_id);
4936        let applied = match module.set_enabled(enabled).await {
4937            Ok(applied) => applied,
4938            Err(err) => {
4939                self.route_outages
4940                    .operator_action_ended_unrefused(&module_id);
4941                return Ok(vec![control_error_frame(
4942                    &frame,
4943                    "target_unavailable",
4944                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
4945                )?]);
4946            }
4947        };
4948        if !applied {
4949            // Already in the requested state: nothing was made unavailable,
4950            // so the mark must not outlive this request.
4951            self.route_outages
4952                .operator_action_ended_unrefused(&module_id);
4953        }
4954
4955        self.capability_evaluator.wake_deadline_loop();
4956        self.refresh_capability_requirements();
4957        let response = ClientControlResponse::SupervisorAck { module_id, applied };
4958        Ok(vec![control_response_body_frame(
4959            &frame,
4960            &response,
4961            "ClientControlResponse::SupervisorAck",
4962        )?])
4963    }
4964
4965    async fn handle_supervisor_health_probe(
4966        &self,
4967        frame: Frame,
4968        module_id: String,
4969    ) -> Result<Vec<Frame>, RouterError> {
4970        self.refresh_capability_requirements();
4971        let Some(registration) = self
4972            .registry
4973            .get_module(&module_id)
4974            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4975        else {
4976            return Ok(vec![control_error_frame(
4977                &frame,
4978                "unknown_module",
4979                format!("module_id '{module_id}' is not registered"),
4980            )?]);
4981        };
4982
4983        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
4984        // named for it. Making `module_registration_grants_op` return false
4985        // unconditionally reddens five tests, and every one is named for something
4986        // else -- capability relay, probe/bind demultiplexing, supervision-only
4987        // probing. They exercise a successful advertisement check on the way to their
4988        // own subject.
4989        //
4990        // Real protection, fragile in a specific way: narrowing any of those tests to
4991        // focus on its stated subject would silently remove coverage nobody knows
4992        // they are carrying. Recorded here rather than as a sixth test, because the
4993        // useful fact is WHICH tests hold the guard up -- a new test would add
4994        // coverage without telling the next person what the existing ones quietly do.
4995        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
4996        {
4997            return Ok(vec![control_error_frame(
4998                &frame,
4999                "health_not_advertised",
5000                format!("module_id '{module_id}' did not advertise health.check"),
5001            )?]);
5002        }
5003
5004        let deadline = Instant::now() + self.health_probe_timeout;
5005        let pending = match self.forwarding.begin_module_control_rpc_for(
5006            &module_id,
5007            MODULE_CONTROL_OP_HEALTH_CHECK,
5008            deadline,
5009        ) {
5010            Ok(pending) => pending,
5011            Err(err) => {
5012                return Ok(vec![control_error_frame(
5013                    &frame,
5014                    forwarding_error_code(&err),
5015                    err.to_string(),
5016                )?])
5017            }
5018        };
5019
5020        let PendingModuleControlRpc {
5021            endpoint,
5022            module_sink,
5023            negotiated_ver,
5024            corr: probe_corr,
5025            receiver,
5026        } = pending;
5027        let mut guard =
5028            ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
5029        let probe_body =
5030            serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5031                RouterError::backend(
5032                    0,
5033                    frame.header.corr,
5034                    format!("failed to encode health.check request: {err}"),
5035                )
5036            })?;
5037        let probe_frame = Frame::build_with_version(
5038            negotiated_ver,
5039            FrameType::Request,
5040            control_flags(),
5041            0,
5042            0,
5043            probe_corr,
5044            probe_body,
5045        )
5046        .map_err(RouterError::FrameBuild)?;
5047
5048        if let Err(err) = module_sink.send(probe_frame).await {
5049            return Ok(vec![control_error_frame(
5050                &frame,
5051                "target_unavailable",
5052                err.to_string(),
5053            )?]);
5054        }
5055
5056        match timeout_at(deadline, receiver).await {
5057            Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5058                guard.disarm();
5059                let Some(report) = response.health_report() else {
5060                    return Ok(vec![control_error_frame(
5061                        &frame,
5062                        "invalid_control_body",
5063                        "health.check RPC returned a non-health response",
5064                    )?]);
5065                };
5066                // Metrics go out whole here. The supervisor's cached snapshot
5067                // caps this blob (see truncate_health_metrics), and this path
5068                // exists precisely to answer without that cap -- so applying it
5069                // here would leave no way to see what the cached view drops.
5070                let HealthReport {
5071                    status,
5072                    detail,
5073                    metrics,
5074                } = report;
5075                let capability_detail = self
5076                    .capability_evaluator
5077                    .required_problem_detail(&module_id);
5078                let response = ClientControlResponse::SupervisorHealthProbe {
5079                    module_id,
5080                    status,
5081                    detail: append_capability_problem_detail(detail, capability_detail),
5082                    metrics,
5083                };
5084                Ok(vec![control_response_body_frame(
5085                    &frame,
5086                    &response,
5087                    "ClientControlResponse::SupervisorHealthProbe",
5088                )?])
5089            }
5090            Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
5091                guard.disarm();
5092                Ok(vec![control_error_body_frame(&frame, body)?])
5093            }
5094            Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5095                guard.disarm();
5096                Ok(vec![control_error_frame(
5097                    &frame,
5098                    "target_unavailable",
5099                    message,
5100                )?])
5101            }
5102            Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5103                guard.disarm();
5104                Ok(vec![control_error_frame(
5105                    &frame,
5106                    "invalid_control_body",
5107                    message,
5108                )?])
5109            }
5110            Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5111                guard.disarm();
5112                Ok(vec![control_error_frame(
5113                    &frame,
5114                    "invalid_control_body",
5115                    format!("expected module-control op '{expected}', got '{actual}'"),
5116                )?])
5117            }
5118            Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
5119                guard.disarm();
5120                Ok(vec![control_error_frame(
5121                    &frame,
5122                    "module_timeout",
5123                    format!(
5124                        "module_id '{module_id}' answered health.check after {:?}",
5125                        self.health_probe_timeout
5126                    ),
5127                )?])
5128            }
5129            Ok(Err(_)) => Ok(vec![control_error_frame(
5130                &frame,
5131                "target_unavailable",
5132                "health.check waiter was canceled before the module responded",
5133            )?]),
5134            Err(_) => Ok(vec![control_error_frame(
5135                &frame,
5136                "module_timeout",
5137                format!(
5138                    "module_id '{module_id}' did not answer health.check within {:?}",
5139                    self.health_probe_timeout
5140                ),
5141            )?]),
5142        }
5143    }
5144
5145    fn supervisor_status(
5146        &self,
5147        module_id: &str,
5148        corr: u64,
5149    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
5150        self.supervisor
5151            .get(module_id)
5152            .map(|module| {
5153                let warming = module.is_warming_for_control("status").map_err(|err| {
5154                    RouterError::backend(
5155                        0,
5156                        corr,
5157                        format!(
5158                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
5159                        ),
5160                    )
5161                })?;
5162                module.status_for_control("status").map_err(|err| {
5163                    RouterError::backend(
5164                        0,
5165                        corr,
5166                        format!(
5167                            "failed to read supervisor status for module_id '{module_id}': {err}"
5168                        ),
5169                    )
5170                }).map(|status| (status, warming))
5171            })
5172            .transpose()
5173    }
5174
5175    fn guard_module_control_op(
5176        &self,
5177        frame: &Frame,
5178        module_id: &str,
5179        op: &str,
5180    ) -> Result<Option<Frame>, RouterError> {
5181        if self.module_grants_op(module_id, op, frame.header.corr)? {
5182            return Ok(None);
5183        }
5184
5185        Ok(Some(control_error_frame(
5186            frame,
5187            "op_not_allowed",
5188            format!("module_id '{module_id}' did not grant control op '{op}'"),
5189        )?))
5190    }
5191
5192    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
5193        let Some(registration) = self
5194            .registry
5195            .get_module(module_id)
5196            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
5197        else {
5198            return Ok(false);
5199        };
5200        Ok(module_registration_grants_op(&registration.control_ops, op))
5201    }
5202
5203    fn handle_status_update(
5204        &self,
5205        endpoint: ModuleEndpointId,
5206        frame: Frame,
5207    ) -> Result<Vec<Frame>, RouterError> {
5208        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
5209            Ok(update) => update,
5210            Err(err) => {
5211                // Forward-compat: a newer module may push a channel-0 op this subc
5212                // version doesn't know. The control contract says unknown push ops
5213                // are IGNORED, never answered with an error. Only a malformed body
5214                // for an op we DO know is a real error worth surfacing.
5215                if is_known_module_push_op(&frame.body) {
5216                    return Ok(vec![control_error_frame(
5217                        &frame,
5218                        "invalid_control_body",
5219                        format!("malformed module control push body: {err}"),
5220                    )?]);
5221                }
5222                return Ok(Vec::new());
5223            }
5224        };
5225
5226        match update {
5227            ModuleControlPush::RouteStatus {
5228                route_channel,
5229                route_epoch,
5230                status,
5231            } => {
5232                self.forwarding
5233                    .cache_status(endpoint, route_channel, route_epoch, status)
5234                    .map_err(RouterError::Forwarding)?;
5235            }
5236        }
5237        Ok(Vec::new())
5238    }
5239
5240    fn handle_route_poll(
5241        &self,
5242        ctx: &RouteCtx,
5243        frame: Frame,
5244        route_channel: u16,
5245        route_epoch: u32,
5246        kind: PollKind,
5247    ) -> Result<Vec<Frame>, RouterError> {
5248        let snapshot = self
5249            .forwarding
5250            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
5251            .map_err(RouterError::Forwarding)?;
5252        let response = match (kind, snapshot) {
5253            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
5254                ClientControlResponse::RoutePoll {
5255                    route_channel,
5256                    route_epoch,
5257                    status,
5258                    live: None,
5259                }
5260            }
5261            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5262                route_channel,
5263                route_epoch,
5264                status: None,
5265                live: None,
5266            },
5267            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
5268                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
5269                // is what makes reporting `true` correct rather than a
5270                // confident guess. `process_live` returns None only when the
5271                // module id has no supervisor snapshot at all -- an
5272                // externally-started module the daemon did not spawn -- and
5273                // for those the supervisor has no opinion to offer, ever. It
5274                // is never None for a supervised module in an unknown state:
5275                // a supervised module always has a snapshot, and the answer
5276                // comes from `state == Running && process_alive`.
5277                //
5278                // The route is Bound, so the module completed a HELLO on a
5279                // live connection; "the process this route points at is
5280                // running" is therefore attested by the binding rather than
5281                // assumed. Reporting `false` for an unsupervised module would
5282                // be the actual lie -- it would tell a client its healthy
5283                // route is dead because the daemon does not manage the
5284                // process.
5285                //
5286                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
5287                // module whose liveness is genuinely unknown, e.g. a snapshot
5288                // that has not been populated yet -- THIS DEFAULT BECOMES
5289                // WRONG and must split: unsupervised stays true, unknown
5290                // becomes null so the client can tell the two apart. The
5291                // response field is already `Option<bool>`, so the wire can
5292                // carry that distinction today.
5293                let live = self
5294                    .process_liveness
5295                    .as_ref()
5296                    .and_then(|source| source.process_live(&module_id))
5297                    .unwrap_or(true);
5298                ClientControlResponse::RoutePoll {
5299                    route_channel,
5300                    route_epoch,
5301                    status: None,
5302                    live: Some(live),
5303                }
5304            }
5305            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5306                route_channel,
5307                route_epoch,
5308                status: None,
5309                live: Some(false),
5310            },
5311        };
5312
5313        Ok(vec![control_response_body_frame(
5314            &frame,
5315            &response,
5316            "ClientControlResponse::RoutePoll",
5317        )?])
5318    }
5319
5320    pub(crate) fn observe_module_control_completion(
5321        &self,
5322        completion: ModuleControlRpcCompletion,
5323    ) -> bool {
5324        match completion {
5325            ModuleControlRpcCompletion::Unknown => false,
5326            ModuleControlRpcCompletion::Settled => true,
5327            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
5328                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
5329                info!(
5330                    module_id = %module_id,
5331                    latency_ms,
5332                    "late health.check answer proves the module is alive"
5333                );
5334                match self
5335                    .supervisor
5336                    .record_late_health_answer(&module_id, latency_ms)
5337                {
5338                    Ok(true) => {}
5339                    Ok(false) => debug!(
5340                        module_id = %module_id,
5341                        latency_ms,
5342                        "late health.check answer has no active supervisor snapshot"
5343                    ),
5344                    Err(err) => warn!(
5345                        module_id = %module_id,
5346                        latency_ms,
5347                        error = %err,
5348                        "failed to record late health.check answer"
5349                    ),
5350                }
5351                true
5352            }
5353        }
5354    }
5355
5356    /// Decide whether a failure while settling a relayed `route.bind` belongs to
5357    /// the module connection whose frame is being handled, or to the client that
5358    /// relay was opened for.
5359    ///
5360    /// This runs on the MODULE connection's frame handler, where returning `Err`
5361    /// ends that connection -- and a module connection carries every client's
5362    /// routes to that module, so ending it costs the whole fleet its tools.
5363    /// `ConnectionClosing` carries the id of the connection that is closing, and
5364    /// when that id is a CLIENT's, the condition is entirely about that one
5365    /// client's route.open. A client-scoped condition has no authority over a
5366    /// shared module connection, so it is logged and the single relay is dropped:
5367    /// the client is going away, and `complete_pending_relay` already removed the
5368    /// relay before failing, so there is nothing left to settle. Anything that
5369    /// relay still reserved is released by that client's own connection teardown,
5370    /// which is already under way -- that is what "closing" means.
5371    ///
5372    /// Every other failure is a statement about THIS connection and stays fatal:
5373    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
5374    /// id in `ConnectionClosing` all mean this connection cannot keep serving
5375    /// frames correctly.
5376    fn refuse_to_end_module_connection_for_a_client(
5377        &self,
5378        module_connection_id: ConnectionId,
5379        corr: u64,
5380        err: ForwardingError,
5381    ) -> Result<(), RouterError> {
5382        if let ForwardingError::ConnectionClosing { connection_id } = err {
5383            if connection_id != module_connection_id {
5384                warn!(
5385                    module_connection_id = module_connection_id.get(),
5386                    client_connection_id = connection_id.get(),
5387                    corr,
5388                    "dropping a route.bind response for a closing client; the module connection keeps serving"
5389                );
5390                return Ok(());
5391            }
5392        }
5393        Err(RouterError::Forwarding(err))
5394    }
5395
5396    fn handle_module_relay_response(
5397        &self,
5398        connection_id: ConnectionId,
5399        frame: Frame,
5400    ) -> Result<Vec<Frame>, RouterError> {
5401        let mut secondary_error = None;
5402        let outcome = match frame.header.ty {
5403            FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
5404                Ok(probe) if probe.op == "route.bind" => {
5405                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
5406                        Ok(ModuleControlResponse::RouteBindAck {}) => {
5407                            RouteBindRelayOutcome::Accepted
5408                        }
5409                        Ok(other) => {
5410                            let message =
5411                                format!("route.bind response carried unexpected body: {other:?}");
5412                            secondary_error = Some(control_error_frame(
5413                                &frame,
5414                                "invalid_control_body",
5415                                message.clone(),
5416                            )?);
5417                            RouteBindRelayOutcome::ModuleGone(message)
5418                        }
5419                        Err(err) => {
5420                            let message = format!("malformed route.bind response body: {err}");
5421                            secondary_error = Some(control_error_frame(
5422                                &frame,
5423                                "invalid_control_body",
5424                                message.clone(),
5425                            )?);
5426                            RouteBindRelayOutcome::ModuleGone(message)
5427                        }
5428                    }
5429                }
5430                Ok(probe) => {
5431                    let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
5432                    {
5433                        Ok(response) => ModuleControlRpcOutcome::Response(response),
5434                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5435                            "malformed {} response body: {err}",
5436                            probe.op
5437                        )),
5438                    };
5439                    let completion = self
5440                        .forwarding
5441                        .complete_module_control_rpc(
5442                            connection_id,
5443                            frame.header.corr,
5444                            Some(&probe.op),
5445                            outcome,
5446                        )
5447                        .map_err(RouterError::Forwarding)?;
5448                    if !self.observe_module_control_completion(completion) {
5449                        debug!(
5450                            connection_id = connection_id.get(),
5451                            corr = frame.header.corr,
5452                            op = %probe.op,
5453                            "dropping late or unknown module-control RPC response"
5454                        );
5455                    }
5456                    return Ok(Vec::new());
5457                }
5458                Err(err) => {
5459                    if let Some(expected_op) = self
5460                        .forwarding
5461                        .pending_module_control_op(connection_id, frame.header.corr)
5462                        .map_err(RouterError::Forwarding)?
5463                    {
5464                        let completion = self
5465                            .forwarding
5466                            .complete_module_control_rpc(
5467                                connection_id,
5468                                frame.header.corr,
5469                                None,
5470                                ModuleControlRpcOutcome::MalformedResponse(format!(
5471                                    "malformed {expected_op} response body: {err}"
5472                                )),
5473                            )
5474                            .map_err(RouterError::Forwarding)?;
5475                        if !self.observe_module_control_completion(completion) {
5476                            debug!(
5477                                connection_id = connection_id.get(),
5478                                corr = frame.header.corr,
5479                                "dropping late malformed module-control RPC response"
5480                            );
5481                        }
5482                        return Ok(Vec::new());
5483                    }
5484                    let message = format!("malformed route.bind response body: {err}");
5485                    secondary_error = Some(control_error_frame(
5486                        &frame,
5487                        "invalid_control_body",
5488                        message.clone(),
5489                    )?);
5490                    RouteBindRelayOutcome::ModuleGone(message)
5491                }
5492            },
5493            FrameType::Error => {
5494                if self
5495                    .forwarding
5496                    .pending_module_control_op(connection_id, frame.header.corr)
5497                    .map_err(RouterError::Forwarding)?
5498                    .is_some()
5499                {
5500                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
5501                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
5502                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5503                            "malformed module-control ERROR body: {err}"
5504                        )),
5505                    };
5506                    let completion = self
5507                        .forwarding
5508                        .complete_module_control_rpc(
5509                            connection_id,
5510                            frame.header.corr,
5511                            None,
5512                            outcome,
5513                        )
5514                        .map_err(RouterError::Forwarding)?;
5515                    if !self.observe_module_control_completion(completion) {
5516                        debug!(
5517                            connection_id = connection_id.get(),
5518                            corr = frame.header.corr,
5519                            "dropping late or unknown module-control RPC error"
5520                        );
5521                    }
5522                    return Ok(Vec::new());
5523                }
5524                match serde_json::from_slice::<ErrorBody>(&frame.body) {
5525                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
5526                    Err(err) => {
5527                        let message = format!("malformed route.bind ERROR body: {err}");
5528                        secondary_error = Some(control_error_frame(
5529                            &frame,
5530                            "invalid_control_body",
5531                            message.clone(),
5532                        )?);
5533                        RouteBindRelayOutcome::ModuleGone(message)
5534                    }
5535                }
5536            }
5537            ty => {
5538                return Ok(vec![control_error_frame(
5539                    &frame,
5540                    "unsupported_control_frame",
5541                    format!("unsupported module channel-0 frame {ty:?}"),
5542                )?])
5543            }
5544        };
5545
5546        let settled =
5547            self.forwarding
5548                .complete_pending_relay(connection_id, frame.header.corr, outcome);
5549        let completion = match settled {
5550            Ok(completion) => completion,
5551            Err(err) => {
5552                self.refuse_to_end_module_connection_for_a_client(
5553                    connection_id,
5554                    frame.header.corr,
5555                    err,
5556                )?;
5557                return Ok(secondary_error.into_iter().collect());
5558            }
5559        };
5560        if let Some(target) = completion.abandoned.as_ref() {
5561            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
5562        }
5563        if !completion.settled {
5564            debug!(
5565                connection_id = connection_id.get(),
5566                corr = frame.header.corr,
5567                frame_type = ?frame.header.ty,
5568                "dropping late or unknown route.bind relay response"
5569            );
5570        }
5571        Ok(secondary_error.into_iter().collect())
5572    }
5573
5574    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
5575        debug!(connection_id = connection_id.get(), "handling GOODBYE");
5576        // GOODBYE ends the connection's logical session even when its socket
5577        // stays open. Use disconnect teardown so verdicts, client notices and
5578        // scope authority are released at the same lifecycle boundary.
5579        self.cleanup_connection_with_end_reason(
5580            connection_id,
5581            RegistrationEndReason::ExplicitGoodbye,
5582        )
5583        .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
5584        Ok(Vec::new())
5585    }
5586}
5587
5588impl Default for ControlHandler {
5589    fn default() -> Self {
5590        Self::new(Arc::new(Registry::default()))
5591    }
5592}
5593
5594impl crate::supervise::SwapPromotionObserver for ControlHandler {
5595    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
5596        self.apply_registration_capabilities(registration);
5597    }
5598}
5599
5600fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
5601    CapabilityRequirementStatus {
5602        consumer: status.consumer,
5603        capability: status.capability,
5604        need: match status.need {
5605            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
5606            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
5607        },
5608        verdict: status.verdict.as_str().to_string(),
5609        episode_seq: status.episode_seq,
5610        config_satisfiable: status.config_satisfiable,
5611        runtime_available: status.runtime_available,
5612        detail: status.detail,
5613    }
5614}
5615
5616fn append_capability_problem_detail(
5617    detail: Option<String>,
5618    capability_detail: Option<String>,
5619) -> Option<String> {
5620    match (detail, capability_detail) {
5621        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
5622        (Some(detail), None) => Some(detail),
5623        (None, Some(capability_detail)) => Some(capability_detail),
5624        (None, None) => None,
5625    }
5626}
5627
5628fn subc_ops() -> Vec<String> {
5629    SUBC_CONTROL_OPS
5630        .iter()
5631        .map(|op| (*op).to_string())
5632        .collect()
5633}
5634
5635fn module_subc_ops() -> Vec<String> {
5636    SUBC_CONTROL_OPS
5637        .iter()
5638        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
5639        .map(|op| (*op).to_string())
5640        .collect()
5641}
5642
5643#[cfg(test)]
5644fn module_baseline_control_ops() -> Vec<String> {
5645    MODULE_BASELINE_CONTROL_OPS
5646        .iter()
5647        .map(|op| (*op).to_string())
5648        .collect()
5649}
5650
5651fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
5652    let mut seen = HashSet::new();
5653    let mut effective = Vec::new();
5654    for op in MODULE_BASELINE_CONTROL_OPS {
5655        if seen.insert((*op).to_string()) {
5656            effective.push((*op).to_string());
5657        }
5658    }
5659    for op in declared.unwrap_or_default() {
5660        if seen.insert(op.clone()) {
5661            effective.push(op);
5662        }
5663    }
5664    effective
5665}
5666
5667fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
5668    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
5669}
5670
5671fn target_module_id(target: &RouteTarget) -> &str {
5672    match target {
5673        RouteTarget::ToolProvider { module_id }
5674        | RouteTarget::ManagementSurface { module_id }
5675        | RouteTarget::InternalService { module_id, .. } => module_id,
5676    }
5677}
5678
5679fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
5680    roles.iter().any(|role| match (target, role) {
5681        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
5682        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
5683        (
5684            RouteTarget::InternalService { service_id, .. },
5685            ProviderRole::InternalService {
5686                service_id: provided,
5687                ..
5688            },
5689        ) => service_id == provided,
5690        _ => false,
5691    })
5692}
5693
5694fn is_routable_role(role: &ProviderRole) -> bool {
5695    matches!(
5696        role,
5697        ProviderRole::ToolProvider { .. }
5698            | ProviderRole::ManagementSurface { .. }
5699            | ProviderRole::InternalService { .. }
5700    )
5701}
5702
5703#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5704enum ControlRequestBodyError {
5705    UnknownOp,
5706    InvalidBody,
5707}
5708
5709#[derive(Debug, Deserialize)]
5710struct ControlOpProbe {
5711    op: String,
5712}
5713
5714/// Channel-0 push ops this subc version understands. A push whose `op` is not in
5715/// this set is treated as a forward-compat unknown and ignored rather than errored.
5716const MODULE_PUSH_OPS: &[&str] = &["route.status"];
5717
5718fn is_known_module_push_op(body: &[u8]) -> bool {
5719    serde_json::from_slice::<ControlOpProbe>(body)
5720        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
5721        .unwrap_or(false)
5722}
5723
5724fn is_known_module_request_op(body: &[u8]) -> bool {
5725    serde_json::from_slice::<ControlOpProbe>(body)
5726        .map(|probe| is_module_to_subc_op(&probe.op))
5727        .unwrap_or(false)
5728}
5729
5730fn is_module_to_subc_op(op: &str) -> bool {
5731    MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
5732}
5733
5734fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
5735    debug!(
5736        op = %op,
5737        connection_id = connection_id.get(),
5738        corr,
5739        "control dispatch"
5740    );
5741}
5742
5743fn log_slow_control_dispatch(
5744    dispatch_started_at: Option<StdInstant>,
5745    op: &'static str,
5746    connection_id: ConnectionId,
5747    corr: u64,
5748) {
5749    let Some(dispatch_started_at) = dispatch_started_at else {
5750        return;
5751    };
5752    let elapsed = dispatch_started_at.elapsed();
5753    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
5754        warn!(
5755            op = %op,
5756            connection_id = connection_id.get(),
5757            corr,
5758            elapsed_ms = elapsed.as_millis() as u64,
5759            "slow control dispatch"
5760        );
5761    }
5762}
5763
5764fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
5765    match request {
5766        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
5767        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
5768        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
5769        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
5770        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
5771        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
5772        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
5773        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
5774        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
5775        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
5776        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
5777        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
5778        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
5779        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
5780        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
5781        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
5782        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
5783        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
5784        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
5785    }
5786}
5787
5788fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
5789    match request {
5790        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
5791        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
5792        ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
5793        ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
5794    }
5795}
5796
5797fn parse_client_control_request(
5798    body: &[u8],
5799) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
5800    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
5801        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5802            Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5803                ControlRequestBodyError::InvalidBody
5804            }
5805            Ok(_) => ControlRequestBodyError::UnknownOp,
5806            Err(_) => ControlRequestBodyError::InvalidBody,
5807        };
5808        (err, classification)
5809    })
5810}
5811
5812fn parse_module_control_request_from_module(
5813    body: &[u8],
5814) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
5815    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
5816        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5817            Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
5818            Ok(_) => ControlRequestBodyError::UnknownOp,
5819            Err(_) => ControlRequestBodyError::InvalidBody,
5820        };
5821        (err, classification)
5822    })
5823}
5824
5825#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
5826enum ProviderRoleKind {
5827    ToolProvider,
5828    PipelineStage,
5829    ManagementSurface,
5830    InternalService,
5831}
5832
5833fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
5834    match role {
5835        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
5836        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
5837        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
5838        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
5839    }
5840}
5841
5842fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
5843    roles.iter().map(provider_role_kind).collect()
5844}
5845
5846/// Most refused scope records named individually in the log per sync; the
5847/// `refused` count on the accepted line is always complete.
5848const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
5849
5850/// Per-outcome counts of one accepted `scope.sync`, for its log line.
5851#[derive(Debug, Default, PartialEq, Eq)]
5852struct ScopeOutcomeCounts {
5853    created: usize,
5854    replaced: usize,
5855    updated: usize,
5856    unchanged: usize,
5857    refused: usize,
5858}
5859
5860impl ScopeOutcomeCounts {
5861    fn of(results: &[ScopeRecordResult]) -> Self {
5862        let mut counts = Self::default();
5863        for result in results {
5864            let slot = match result.outcome {
5865                ScopeRecordOutcome::Created => &mut counts.created,
5866                ScopeRecordOutcome::Replaced => &mut counts.replaced,
5867                ScopeRecordOutcome::Updated => &mut counts.updated,
5868                ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
5869                ScopeRecordOutcome::Refused => &mut counts.refused,
5870            };
5871            *slot += 1;
5872        }
5873        counts
5874    }
5875}
5876
5877#[cfg(test)]
5878mod scope_outcome_count_tests {
5879    use super::*;
5880
5881    fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
5882        ScopeRecordResult {
5883            scope_ref: "r".to_string(),
5884            scope_epoch: 1,
5885            outcome,
5886            code: None,
5887            message: None,
5888            version: None,
5889            parent_state: None,
5890        }
5891    }
5892
5893    /// Each outcome lands in its own count, so a refused record can never be
5894    /// hidden inside the total the log already printed.
5895    #[test]
5896    fn every_outcome_is_counted_in_its_own_field() {
5897        let results = [
5898            result(ScopeRecordOutcome::Created),
5899            result(ScopeRecordOutcome::Created),
5900            result(ScopeRecordOutcome::Replaced),
5901            result(ScopeRecordOutcome::Updated),
5902            result(ScopeRecordOutcome::Unchanged),
5903            result(ScopeRecordOutcome::Refused),
5904            result(ScopeRecordOutcome::Refused),
5905            result(ScopeRecordOutcome::Refused),
5906        ];
5907        assert_eq!(
5908            ScopeOutcomeCounts::of(&results),
5909            ScopeOutcomeCounts {
5910                created: 2,
5911                replaced: 1,
5912                updated: 1,
5913                unchanged: 1,
5914                refused: 3,
5915            }
5916        );
5917    }
5918}
5919
5920/// Return whether a catalog change can create a newly violating live route.
5921/// Removing an attested claim is intentionally excluded: it makes fewer routes
5922/// forbidden and therefore must leave the existing route census untouched.
5923fn capability_census_trigger(
5924    old: Option<&CapabilityDeclarations>,
5925    new: Option<&CapabilityDeclarations>,
5926) -> bool {
5927    let old_provides = old
5928        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
5929        .unwrap_or_default();
5930    let old_denies = old
5931        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
5932        .unwrap_or_default();
5933    let new = new.cloned().unwrap_or(CapabilityDeclarations {
5934        provides: Vec::new(),
5935        requires: Vec::new(),
5936        must_never_reach: Vec::new(),
5937    });
5938
5939    new.provides
5940        .iter()
5941        .any(|capability| !old_provides.contains(capability))
5942        || new
5943            .must_never_reach
5944            .iter()
5945            .any(|capability| !old_denies.contains(capability))
5946}
5947
5948/// Find the first capability an attested opener denies that an attested target
5949/// claims. Both manifests are live registry records, never cached or client data.
5950fn denied_capability<'a>(
5951    opening_manifest: &'a ModuleManifest,
5952    target_manifest: &ModuleManifest,
5953) -> Option<&'a str> {
5954    let opening_capabilities = opening_manifest.capabilities.as_ref()?;
5955    let target_capabilities = target_manifest.capabilities.as_ref()?;
5956    opening_capabilities
5957        .must_never_reach
5958        .iter()
5959        .find(|denied| {
5960            target_capabilities
5961                .provides
5962                .iter()
5963                .any(|provided| provided == *denied)
5964        })
5965        .map(String::as_str)
5966}
5967
5968fn catalog_update_frozen_field_message(
5969    registered: &ModuleManifest,
5970    provides: &[ProviderRole],
5971) -> Option<String> {
5972    let old_has_provides = !registered.provides.is_empty();
5973    let new_has_provides = !provides.is_empty();
5974    if old_has_provides != new_has_provides {
5975        return Some(format!(
5976            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
5977            registered.module_id
5978        ));
5979    }
5980
5981    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
5982        return Some(format!(
5983            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
5984            registered.module_id
5985        ));
5986    }
5987
5988    let registered_concurrency = manifest_concurrency(registered);
5989    let mut candidate = registered.clone();
5990    candidate.provides = provides.to_vec();
5991    let candidate_concurrency = manifest_concurrency(&candidate);
5992    if candidate_concurrency != registered_concurrency {
5993        return Some(format!(
5994            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
5995            registered.module_id, registered_concurrency, candidate_concurrency
5996        ));
5997    }
5998
5999    // control_ops live beside the manifest in the HELLO body, not inside
6000    // ModuleManifest, so a provides-only catalog.update cannot change them.
6001    None
6002}
6003
6004fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
6005    manifest.provides.iter().any(is_routable_role)
6006}
6007
6008/// Returns the routable-provider concurrency subc should enforce for this manifest.
6009///
6010/// ToolProvider and ManagementSurface store their delivery concurrency directly.
6011/// InternalService has no role-specific concurrency field, so it retains the
6012/// existing ModuleManaged default for backward compatibility.
6013fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
6014    manifest
6015        .provides
6016        .iter()
6017        .find_map(|provider| match provider {
6018            ProviderRole::ToolProvider { concurrency, .. }
6019            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
6020            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
6021        })
6022        .unwrap_or(Concurrency::ModuleManaged)
6023}
6024
6025/// True when the manifest carries a ManagementSurface role whose concurrency
6026/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
6027/// bytes because the typed manifest deliberately erases that distinction: the
6028/// default exists for wire compatibility, and this probe exists so the default
6029/// stays observable. Any parse irregularity returns false -- the caller only
6030/// logs, and a malformed body already failed registration upstream.
6031fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
6032    let has_management_surface = manifest
6033        .provides
6034        .iter()
6035        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
6036    if !has_management_surface {
6037        return false;
6038    }
6039    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
6040        return false;
6041    };
6042    let Some(provides) = raw
6043        .get("manifest")
6044        .and_then(|manifest| manifest.get("provides"))
6045        .and_then(serde_json::Value::as_array)
6046    else {
6047        return false;
6048    };
6049    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
6050    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
6051    // against the management_surface_manifest_without_concurrency golden, not
6052    // recalled (the externally-tagged guess was this function's first bug).
6053    provides.iter().any(|role| {
6054        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
6055            && role.get("concurrency").is_none()
6056    })
6057}
6058
6059fn negotiate_version(peer_version: u8) -> Result<u8, String> {
6060    if peer_version != PROTOCOL_VERSION {
6061        return Err(format!(
6062            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
6063        ));
6064    }
6065    Ok(PROTOCOL_VERSION)
6066}
6067
6068fn pong(frame: &Frame) -> Result<Frame, RouterError> {
6069    Frame::build_with_version(
6070        response_version(frame),
6071        FrameType::Pong,
6072        frame.header.flags,
6073        0,
6074        0,
6075        frame.header.corr,
6076        Vec::new(),
6077    )
6078    .map_err(RouterError::FrameBuild)
6079}
6080
6081fn control_error_frame(
6082    frame: &Frame,
6083    code: &'static str,
6084    message: impl Into<String>,
6085) -> Result<Frame, RouterError> {
6086    control_error_body_frame(
6087        frame,
6088        ErrorBody {
6089            code: code.to_string(),
6090            message: message.into(),
6091            detail: None,
6092        },
6093    )
6094}
6095
6096fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
6097    let body = serde_json::to_vec(&error).map_err(|err| {
6098        RouterError::backend(
6099            0,
6100            frame.header.corr,
6101            format!("failed to encode control ERROR: {err}"),
6102        )
6103    })?;
6104
6105    Frame::build_with_version(
6106        response_version(frame),
6107        FrameType::Error,
6108        control_flags(),
6109        0,
6110        0,
6111        frame.header.corr,
6112        body,
6113    )
6114    .map_err(RouterError::FrameBuild)
6115}
6116
6117fn control_response_body_frame<T: Serialize>(
6118    frame: &Frame,
6119    reply: &T,
6120    label: &'static str,
6121) -> Result<Frame, RouterError> {
6122    let body = serde_json::to_vec(reply).map_err(|err| {
6123        RouterError::backend(
6124            0,
6125            frame.header.corr,
6126            format!("failed to encode {label}: {err}"),
6127        )
6128    })?;
6129
6130    Frame::build_with_version(
6131        response_version(frame),
6132        FrameType::Response,
6133        control_flags(),
6134        0,
6135        0,
6136        frame.header.corr,
6137        body,
6138    )
6139    .map_err(RouterError::FrameBuild)
6140}
6141
6142/// Map a forwarding failure to the wire code a client sees.
6143///
6144/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
6145/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
6146/// chosen here decides whether a caller retries or gives up.
6147///
6148/// That makes attribution the load-bearing property, not merely having a code. A
6149/// permanent fault published as a retryable one produces a fleet-wide retry storm
6150/// against something that can never recover; a transient fault published as
6151/// permanent gives up on work that would have succeeded. Both look correct in a
6152/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
6153/// the mapping per variant rather than merely asserting that some code exists.
6154///
6155/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
6156/// two codes on the same side of the boundary passes it. Measured rather than
6157/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
6158/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
6159/// a test named for something else that happens to assert the string.
6160///
6161/// That accidental coverage is deliberately left alone rather than promoted to a
6162/// named test, because it guards a property this function does not promise.
6163/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
6164/// specific code within a class, so identity is free to change and only the
6165/// partition is a contract. Splitting it out would assert a guarantee nothing
6166/// depends on — and a suite that promises more than the code does is the harder
6167/// thing to correct later, because the next reader cannot tell which assertions
6168/// are load-bearing.
6169///
6170/// Pin identity here the moment a consumer branches on a specific code.
6171fn forwarding_error_code(err: &ForwardingError) -> &'static str {
6172    match err {
6173        ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
6174        ForwardingError::NoModuleConnection => "target_unavailable",
6175        ForwardingError::ModuleReloading { .. } => "module_reloading",
6176        ForwardingError::ClientRouteChannelExhausted { .. }
6177        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
6178        ForwardingError::StaleModuleEndpoint
6179        | ForwardingError::UnknownReservation { .. }
6180        | ForwardingError::ConnectionClosing { .. }
6181        | ForwardingError::ClientEgressClosed { .. }
6182        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
6183        // Only a swap candidate's registration can produce this, and it means
6184        // exactly what a second active HELLO for a live id means.
6185        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
6186        ForwardingError::RelayCorrelationExhausted
6187        | ForwardingError::RouteOpenBuild(_)
6188        | ForwardingError::Poisoned => "forwarding_error",
6189    }
6190}
6191
6192fn response_version(frame: &Frame) -> u8 {
6193    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
6194        frame.header.ver
6195    } else {
6196        PROTOCOL_VERSION
6197    }
6198}
6199
6200fn control_flags() -> Flags {
6201    Flags::new(false, Priority::Passive, false)
6202}
6203
6204/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
6205/// channel. The target is the module (a client never saw the route), so this
6206/// takes the module path: delivered late rather than dropped when the module's
6207/// queue is momentarily full, and never closing its connection.
6208fn send_goodbye_target_best_effort(
6209    counters: &DaemonCounters,
6210    target: &GoodbyeTarget,
6211    context: &'static str,
6212) {
6213    let Ok(frame) = Frame::build_with_version(
6214        target.negotiated_ver,
6215        FrameType::Goodbye,
6216        control_flags(),
6217        target.channel,
6218        target.epoch,
6219        0,
6220        Vec::new(),
6221    ) else {
6222        return;
6223    };
6224    crate::forwarding::send_module_route_goodbye(
6225        counters,
6226        &target.sink,
6227        frame,
6228        target.module_id.as_deref(),
6229        context,
6230    );
6231}
6232
6233pub(crate) fn send_route_control_pushes(
6234    forwarding: &ForwardingTable,
6235    routes: Vec<EndpointRoute>,
6236    push: ClientControlPush,
6237) {
6238    let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
6239    for route in routes {
6240        let target = route.goodbye_target;
6241        if let Some((existing, channels)) = targets
6242            .iter_mut()
6243            .find(|(existing, _)| existing.connection_id == target.connection_id)
6244        {
6245            debug_assert_eq!(
6246                existing.negotiated_ver, target.negotiated_ver,
6247                "one connection cannot negotiate multiple frame versions"
6248            );
6249            if !channels.contains(&target.channel) {
6250                channels.push(target.channel);
6251            }
6252            continue;
6253        }
6254        let channel = target.channel;
6255        targets.push((target, vec![channel]));
6256    }
6257    for (target, mut channels) in targets {
6258        channels.sort_unstable();
6259        let mut push = push.clone();
6260        match &mut push {
6261            ClientControlPush::RouteClosing {
6262                channels: covered, ..
6263            }
6264            | ClientControlPush::RouteClosed {
6265                channels: covered, ..
6266            } => *covered = channels,
6267        }
6268        let body = match serde_json::to_vec(&push) {
6269            Ok(body) => body,
6270            Err(err) => {
6271                warn!(error = %err, "failed to serialize route lifecycle control PUSH");
6272                continue;
6273            }
6274        };
6275        let frame = match Frame::build_with_version(
6276            target.negotiated_ver,
6277            FrameType::Push,
6278            control_flags(),
6279            0,
6280            0,
6281            0,
6282            body.clone(),
6283        ) {
6284            Ok(frame) => frame,
6285            Err(err) => {
6286                warn!(
6287                    route_channel = target.channel,
6288                    error = %err,
6289                    "failed to build route lifecycle control PUSH frame"
6290                );
6291                continue;
6292            }
6293        };
6294        if let Err(err) = target.sink.try_send(frame) {
6295            if target.close_on_delivery_failure() {
6296                warn!(
6297                    target_connection_id = target.connection_id.get(),
6298                    route_channel = target.channel,
6299                    error = %err,
6300                    "route lifecycle control PUSH was not delivered to client; closing target connection"
6301                );
6302                let _ = forwarding.escalate_client_delivery_failure(
6303                    target.connection_id,
6304                    target.channel,
6305                    target.epoch,
6306                    CloseReason::new(
6307                        "route_lifecycle_push_delivery_failed",
6308                        format!(
6309                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
6310                            target.channel
6311                        ),
6312                    ),
6313                    crate::forwarding::UndeliveredFrame {
6314                        module_id: target.module_id.as_deref(),
6315                        sink: &target.sink,
6316                    },
6317                );
6318            }
6319        }
6320    }
6321}
6322
6323#[cfg(test)]
6324mod tests {
6325    #[cfg(unix)]
6326    #[tokio::test]
6327    async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
6328        let dir = subc_test_support::TestTempDir::new("rescan-live-health");
6329        let path = dir.join("subc.jsonc");
6330        std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
6331            "program":"/bin/sleep","args":["60"],"protocol":"none",
6332            "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
6333        }}}).to_string()).unwrap();
6334        let mut configured = crate::daemon_config::load(&path)
6335            .unwrap()
6336            .unwrap()
6337            .modules
6338            .pop()
6339            .unwrap();
6340        let registry = std::sync::Arc::new(crate::Registry::default());
6341        let handle = crate::SupervisorHandle::new();
6342        let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
6343            .with_handle(handle.clone());
6344        let module = supervisor
6345            .supervise_configured_with_health(
6346                configured.module_spec(),
6347                true,
6348                configured.health.clone(),
6349                None,
6350                configured.restart,
6351            )
6352            .unwrap();
6353        let handler = super::ControlHandler::new(registry).with_supervisor(handle);
6354        let before = module.status().unwrap().pid;
6355        configured.health.http = Some("http://127.0.0.1:1/healthz".into());
6356        configured.health.cadence = std::time::Duration::from_secs(3600);
6357        let health_only = handler
6358            .reconcile_supervised_modules(&supervisor, vec![configured.clone()], false)
6359            .await
6360            .unwrap();
6361        assert!(
6362            health_only.changed_pending_reload.is_empty(),
6363            "health policy is already applied live"
6364        );
6365        assert_eq!(module.status().unwrap().pid, before);
6366        assert_eq!(
6367            module.configuration().unwrap().1.http,
6368            configured.health.http
6369        );
6370        configured.args = vec!["61".into()];
6371        let launch = handler
6372            .reconcile_supervised_modules(&supervisor, vec![configured], false)
6373            .await
6374            .unwrap();
6375        assert_eq!(launch.changed_pending_reload, ["stock"]);
6376        assert_eq!(
6377            module.status().unwrap().pid,
6378            before,
6379            "a launch edit is stored until reload"
6380        );
6381        module.drain().await.unwrap();
6382    }
6383    use std::{
6384        collections::BTreeMap,
6385        fmt,
6386        path::PathBuf,
6387        sync::{Arc, Mutex},
6388        time::Duration,
6389    };
6390    use subc_test_support::TestTempDir;
6391
6392    use serde_json::{json, Value};
6393    use subc_protocol::{
6394        manifest::{
6395            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
6396            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
6397        },
6398        session::HealthStatus,
6399        FrameType,
6400    };
6401
6402    use super::*;
6403    use crate::{
6404        forwarding::{DataRoute, DataRouteState},
6405        registry::ChannelState,
6406        router::FrameSink,
6407        stderr_tail::DEFAULT_MAX_LINE_BYTES,
6408        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
6409        RouteCtx, Router,
6410    };
6411    use tokio::{
6412        sync::mpsc,
6413        time::{sleep, Instant},
6414    };
6415    use tracing::{
6416        field::{Field, Visit},
6417        Event, Subscriber,
6418    };
6419    use tracing_subscriber::{layer::Context, prelude::*, Layer};
6420
6421    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
6422    ///
6423    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
6424    /// is only populated for `tests/*.rs` integration test binaries -- this file
6425    /// compiles as part of the library target, which gets neither. This test's
6426    /// own executable path is `<target-dir>/<profile>/deps/subc_core-<hash>`,
6427    /// and the sibling binary lives two directories up at
6428    /// `<target-dir>/<profile>/fake-aft-stub`.
6429    ///
6430    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
6431    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
6432    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
6433    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
6434    /// `NotFound`, which reads as a broken test rather than an unbuilt
6435    /// dependency -- so state the cause and the remedy instead. Deliberately a
6436    /// panic and not a silent skip: a test that quietly passes when it could not
6437    /// run is worse than one that fails, because it reports health it never
6438    /// verified.
6439    fn fake_aft_stub_path() -> PathBuf {
6440        let mut path = std::env::current_exe().expect("current_exe available in tests");
6441        path.pop(); // .../deps/
6442        path.pop(); // .../<profile>/
6443        path.push(if cfg!(windows) {
6444            "fake-aft-stub.exe"
6445        } else {
6446            "fake-aft-stub"
6447        });
6448        assert!(
6449            path.exists(),
6450            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
6451             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
6452            path.display()
6453        );
6454        path
6455    }
6456
6457    /// Whether clients retry `code` in place: the predicate itself, never a copy
6458    /// of its set. A copied list breaks silently when a code is added to or
6459    /// removed from the real one, and a stale copy here would let exactly the
6460    /// failure this test exists to catch pass.
6461    fn client_retries(code: &str) -> bool {
6462        subc_protocol::error_codes::is_retryable_route_open(code)
6463    }
6464
6465    /// A code is not a label — clients branch on it, so publishing the wrong KIND
6466    /// of failure is worse than publishing none. A permanent fault dressed as
6467    /// retryable makes every client in the fleet retry forever against something
6468    /// that cannot recover; a transient fault dressed as permanent abandons work
6469    /// that would have succeeded.
6470    ///
6471    /// Asserting "a code exists" cannot catch either, because the string is free
6472    /// to say anything. This enumerates every variant and pins which side of the
6473    /// retry boundary it lands on, so a new variant must be classified here
6474    /// deliberately rather than inheriting whichever arm it was appended to.
6475    #[test]
6476    fn retryability_of_forwarding_codes_matches_the_failure() {
6477        // Transient by nature: the target is booting, reloading, or its endpoint
6478        // was swapped mid-flight. Retrying is how these resolve.
6479        let transient = [
6480            ForwardingError::NoModuleConnection,
6481            ForwardingError::ModuleReloading {
6482                module_id: "m".into(),
6483            },
6484            ForwardingError::StaleModuleEndpoint,
6485            ForwardingError::UnknownReservation {
6486                client_channel: 1,
6487                module_channel: 1,
6488            },
6489            ForwardingError::ConnectionClosing {
6490                connection_id: ConnectionId::new(1),
6491            },
6492            ForwardingError::ClientEgressClosed {
6493                connection_id: ConnectionId::new(1),
6494            },
6495            ForwardingError::ModuleEgressUnavailable {
6496                connection_id: ConnectionId::new(1),
6497            },
6498        ];
6499        for err in transient {
6500            let code = forwarding_error_code(&err);
6501            assert!(
6502                client_retries(code),
6503                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
6504            );
6505        }
6506
6507        // Not fixed by retrying. Channel and correlation exhaustion need the
6508        // caller to close routes, and a poisoned lock is a daemon that cannot
6509        // recover at all — the worst thing to advertise as retryable, since every
6510        // client would storm a daemon that will never answer.
6511        let permanent = [
6512            ForwardingError::ConnectionRoleConflict {
6513                connection_id: ConnectionId::new(1),
6514            },
6515            ForwardingError::ClientRouteChannelExhausted {
6516                connection_id: ConnectionId::new(1),
6517            },
6518            ForwardingError::ModuleRouteChannelExhausted {
6519                endpoint: ModuleEndpointId {
6520                    connection_id: ConnectionId::new(1),
6521                    generation: 1,
6522                },
6523            },
6524            ForwardingError::RelayCorrelationExhausted,
6525            ForwardingError::RouteOpenBuild("x".into()),
6526            ForwardingError::Poisoned,
6527        ];
6528        for err in permanent {
6529            let code = forwarding_error_code(&err);
6530            assert!(
6531                !client_retries(code),
6532                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
6533            );
6534        }
6535    }
6536
6537    /// The principal is the daemon's answer to "who is calling", and modules
6538    /// branch on it: aft gates bash on it, cerebellum gates browser control,
6539    /// plexus gates connector invocation. So a stamp is an authorization input in
6540    /// another process, not a label — and both possible answers SUCCEED, which is
6541    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
6542    /// first-party capability to something that never proved it; a supervised one
6543    /// stamped `Direct` silently strips a module of capability it is entitled to.
6544    ///
6545    /// Neither shows up in a test that only checks the bind succeeded. Before this
6546    /// test the only coverage was accidental —
6547    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
6548    /// stamped principal on its way past, so narrowing that wire-shape test to its
6549    /// stated subject would have deleted the last assertion on this value. It
6550    /// still asserts the stamp, which is now redundancy rather than the only
6551    /// guard: both fail under the same mutation, and this one names the reason.
6552    /// SCOPE: this handler's supervisor has spawned nothing, so
6553    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
6554    /// is unreachable here. Both assertions below are refusals, and a mutant that
6555    /// refuses everything would satisfy them.
6556    ///
6557    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
6558    /// spawns a supervised consumer, reads its live nonce, and asserts the module
6559    /// observed `principal.kind == "reserved"` carrying that module_id — verified
6560    /// at source rather than assumed, since a citation is a claim about another
6561    /// file and ages like one. Recorded because a harness that structurally
6562    /// cannot reach an arm reports "none" for that arm identically to one that
6563    /// covers it and found nothing.
6564    #[tokio::test]
6565    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
6566        let handler = ControlHandler::default();
6567        let frame =
6568            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
6569
6570        // Absent consumer_identity is the ordinary case: a human at a terminal, or
6571        // any process holding the connection file. Nothing was proved, so nothing
6572        // may be granted beyond the unattested floor.
6573        let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
6574        assert_eq!(
6575            stamped,
6576            Principal::Direct,
6577            "a caller that proved nothing must not be stamped as a supervised module"
6578        );
6579
6580        // A claimed module_id with a nonce no supervised child was given is a
6581        // forgery attempt, not a weaker caller: it must be REFUSED rather than
6582        // quietly demoted to Direct, or an impersonation attempt looks identical
6583        // to an ordinary unattested connection.
6584        let forged = handler
6585            .route_open_principal(
6586                &frame,
6587                Some(ConsumerIdentity {
6588                    module_id: "aft".to_string(),
6589                    launch_nonce: "not-a-real-nonce".to_string(),
6590                }),
6591            )
6592            .unwrap();
6593        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
6594        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
6595    }
6596
6597    /// The test above hands `route_open_principal` an identity it built itself,
6598    /// which proves the stamping rule and nothing about where the identity comes
6599    /// from. The real producer is a wire body, and the two are joined by a serde
6600    /// field name that nothing else asserts.
6601    ///
6602    /// That join fails quietly in one specific way: an unrecognised key is simply
6603    /// absent after parsing, so a renamed or misspelled `consumer_identity`
6604    /// yields `None` and every supervised module silently drops to `Direct`.
6605    /// Capability-wise that is the safe direction, but it surfaces far from its
6606    /// cause — as a module mysteriously refused bash — and it would pass every
6607    /// test that builds its own input.
6608    ///
6609    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
6610    /// would break every client the moment the daemon gains a field, trading a
6611    /// quiet demotion for a hard refusal on additive change. Asserting the join
6612    /// instead means a rename breaks a test here rather than the fleet.
6613    #[test]
6614    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
6615        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
6616        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
6617        let ClientControlRequest::RouteOpen {
6618            consumer_identity, ..
6619        } = parsed
6620        else {
6621            panic!("route.open body must parse as RouteOpen");
6622        };
6623        assert_eq!(
6624            consumer_identity,
6625            Some(ConsumerIdentity {
6626                module_id: "aft".to_string(),
6627                launch_nonce: "n".to_string(),
6628            }),
6629            "the wire field name must reach the value route_open_principal reads"
6630        );
6631    }
6632
6633    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
6634        ModuleManifest::builder(module_id, "0.1.0")
6635            .protocol_ver(protocol_ver)
6636            .provides(vec![ProviderRole::ToolProvider {
6637                tools: vec![Tool {
6638                    name: "read".to_string(),
6639                    description: None,
6640                    execution_mode: ExecutionMode::Pure,
6641                    schema: json!({"type": "object"}),
6642                }],
6643                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
6644                concurrency: Concurrency::ModuleManaged,
6645                emits_push: true,
6646                sub_supervises: true,
6647            }])
6648            .build()
6649    }
6650
6651    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
6652        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
6653    }
6654
6655    fn hello_frame_with_control_ops(
6656        module_id: &str,
6657        protocol_ver: u8,
6658        corr: u64,
6659        control_ops: Option<Vec<String>>,
6660    ) -> Frame {
6661        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
6662    }
6663
6664    fn hello_frame_with_nonce(
6665        module_id: &str,
6666        protocol_ver: u8,
6667        corr: u64,
6668        launch_nonce: Option<&str>,
6669    ) -> Frame {
6670        hello_frame_full(
6671            module_id,
6672            protocol_ver,
6673            corr,
6674            None,
6675            launch_nonce.map(ToOwned::to_owned),
6676        )
6677    }
6678
6679    fn hello_frame_full(
6680        module_id: &str,
6681        protocol_ver: u8,
6682        corr: u64,
6683        control_ops: Option<Vec<String>>,
6684        launch_nonce: Option<String>,
6685    ) -> Frame {
6686        let body = serde_json::to_vec(&ModuleHelloBody {
6687            manifest: manifest(module_id, protocol_ver),
6688            protocol_ver,
6689            control_ops,
6690            launch_nonce,
6691        })
6692        .unwrap();
6693        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6694    }
6695
6696    fn non_routable_hello_frame_with_control_ops(
6697        module_id: &str,
6698        corr: u64,
6699        control_ops: Option<Vec<String>>,
6700    ) -> Frame {
6701        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
6702        manifest.provides.clear();
6703        let body = serde_json::to_vec(&ModuleHelloBody {
6704            manifest,
6705            protocol_ver: PROTOCOL_VERSION,
6706            control_ops,
6707            launch_nonce: None,
6708        })
6709        .unwrap();
6710        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6711    }
6712
6713    fn capability_grammar_hello_frame(
6714        capabilities: Value,
6715        runtime_computed: Option<Value>,
6716        corr: u64,
6717    ) -> Frame {
6718        let mut body = serde_json::to_value(ModuleHelloBody {
6719            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
6720            protocol_ver: PROTOCOL_VERSION,
6721            control_ops: None,
6722            launch_nonce: None,
6723        })
6724        .expect("HELLO body serializes");
6725        body["manifest"]["capabilities"] = capabilities;
6726        if let Some(runtime_computed) = runtime_computed {
6727            body["runtime_computed"] = runtime_computed;
6728        }
6729        Frame::build(
6730            FrameType::Hello,
6731            control_flags(),
6732            0,
6733            0,
6734            corr,
6735            serde_json::to_vec(&body).expect("HELLO body reserializes"),
6736        )
6737        .expect("HELLO frame builds")
6738    }
6739
6740    fn channel_request(channel: u16, corr: u64) -> Frame {
6741        Frame::build(
6742            FrameType::Request,
6743            Flags::new(true, Priority::Interactive, false),
6744            channel,
6745            0,
6746            corr,
6747            b"opaque".to_vec(),
6748        )
6749        .unwrap()
6750    }
6751
6752    fn route_ctx(
6753        connection_id: ConnectionId,
6754    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
6755        let (tx, rx) = mpsc::channel(8);
6756        (
6757            RouteCtx {
6758                connection_id,
6759                egress: FrameSink::new(tx),
6760            },
6761            rx,
6762        )
6763    }
6764
6765    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
6766        serde_json::from_slice(&frame.body).unwrap()
6767    }
6768
6769    /// Register a module over a connection that has a sink and return the
6770    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
6771    /// module's own sink rather than returning it as a reply, so the ack is
6772    /// taken off `rx` here and whatever the test reads next is what followed it.
6773    async fn hello_via_sink(
6774        handler: &ControlHandler,
6775        ctx: &RouteCtx,
6776        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
6777        hello: Frame,
6778    ) -> Frame {
6779        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
6780        assert!(
6781            replies.is_empty(),
6782            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
6783        );
6784        let ack = rx
6785            .try_recv()
6786            .expect("HELLO_ACK is queued on the module sink")
6787            .frame;
6788        assert_eq!(ack.header.ty, FrameType::HelloAck);
6789        ack
6790    }
6791
6792    fn parse_error(frame: &Frame) -> Value {
6793        serde_json::from_slice(&frame.body).unwrap()
6794    }
6795
6796    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
6797        serde_json::from_slice(&frame.body).unwrap()
6798    }
6799
6800    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
6801        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
6802            route_channel,
6803            route_epoch: 0,
6804            kind,
6805        })
6806        .unwrap();
6807        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6808    }
6809
6810    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
6811        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
6812            module_id: module_id.to_string(),
6813        })
6814        .unwrap();
6815        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6816    }
6817
6818    fn route_open_frame(corr: u64, module_id: &str, project_root: TestTempDir) -> Frame {
6819        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
6820    }
6821
6822    fn route_open_frame_with_consumer_capabilities(
6823        corr: u64,
6824        module_id: &str,
6825        project_root: TestTempDir,
6826        consumer_capabilities: Option<Vec<String>>,
6827    ) -> Frame {
6828        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6829            target: RouteTarget::ToolProvider {
6830                module_id: module_id.to_string(),
6831            },
6832            identity: BindIdentity::new(
6833                project_root.path().to_path_buf(),
6834                "unit".to_string(),
6835                "session".to_string(),
6836            ),
6837            consumer_identity: None,
6838            consumer_capabilities,
6839            role_versions: None,
6840            admission_facts: None,
6841            scope: None,
6842        })
6843        .unwrap();
6844        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6845    }
6846
6847    fn route_open_frame_with_role_versions(
6848        corr: u64,
6849        module_id: &str,
6850        project_root: TestTempDir,
6851        role_versions: Option<BTreeMap<String, String>>,
6852    ) -> Frame {
6853        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6854            target: RouteTarget::ToolProvider {
6855                module_id: module_id.to_string(),
6856            },
6857            identity: BindIdentity::new(
6858                project_root.path().to_path_buf(),
6859                "unit".to_string(),
6860                format!("session-{corr}"),
6861            ),
6862            consumer_identity: None,
6863            consumer_capabilities: None,
6864            role_versions,
6865            admission_facts: None,
6866            scope: None,
6867        })
6868        .unwrap();
6869        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6870    }
6871
6872    fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
6873        entries
6874            .iter()
6875            .map(|(role, version)| (role.to_string(), version.to_string()))
6876            .collect()
6877    }
6878
6879    fn route_open_frame_with_admission_facts(
6880        corr: u64,
6881        module_id: &str,
6882        project_root: TestTempDir,
6883        consumer_identity: Option<subc_control::ConsumerIdentity>,
6884        facts: Option<Value>,
6885    ) -> Frame {
6886        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6887            target: RouteTarget::ToolProvider {
6888                module_id: module_id.to_string(),
6889            },
6890            identity: BindIdentity::new(
6891                project_root.path().to_path_buf(),
6892                "unit".to_string(),
6893                format!("session-{corr}"),
6894            ),
6895            consumer_identity,
6896            consumer_capabilities: None,
6897            role_versions: None,
6898            admission_facts: facts,
6899            scope: None,
6900        })
6901        .unwrap();
6902        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6903    }
6904
6905    #[derive(Clone, Default)]
6906    struct EventCapture {
6907        events: Arc<Mutex<Vec<CapturedEvent>>>,
6908    }
6909
6910    #[derive(Clone, Debug)]
6911    struct CapturedEvent {
6912        target: String,
6913        level: tracing::Level,
6914        fields: BTreeMap<String, String>,
6915    }
6916
6917    impl EventCapture {
6918        fn events(&self) -> Vec<CapturedEvent> {
6919            self.events.lock().unwrap().clone()
6920        }
6921    }
6922
6923    impl<S> Layer<S> for EventCapture
6924    where
6925        S: Subscriber,
6926    {
6927        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
6928            let mut visitor = EventFieldVisitor::default();
6929            event.record(&mut visitor);
6930            self.events.lock().unwrap().push(CapturedEvent {
6931                target: event.metadata().target().to_string(),
6932                level: *event.metadata().level(),
6933                fields: visitor.fields,
6934            });
6935        }
6936    }
6937
6938    #[derive(Default)]
6939    struct EventFieldVisitor {
6940        fields: BTreeMap<String, String>,
6941    }
6942
6943    impl Visit for EventFieldVisitor {
6944        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
6945            self.fields
6946                .insert(field.name().to_string(), format!("{value:?}"));
6947        }
6948    }
6949
6950    fn health_response(corr: u64, status: HealthStatus) -> Frame {
6951        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
6952            status,
6953            detail: Some("warming".to_string()),
6954            metrics: Some(json!({"queue_depth": 3})),
6955        })
6956        .unwrap();
6957        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6958    }
6959
6960    fn route_bind_ack(corr: u64) -> Frame {
6961        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
6962        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6963    }
6964
6965    fn unique_project_root(label: &str) -> TestTempDir {
6966        TestTempDir::new(label)
6967    }
6968
6969    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
6970        match parse_route_poll(frame) {
6971            ClientControlResponse::RoutePoll {
6972                status: None,
6973                live: Some(live),
6974                ..
6975            } => assert_eq!(live, expected_live),
6976            other => panic!("unexpected route.poll response: {other:?}"),
6977        }
6978    }
6979
6980    fn bind_liveness_route(
6981        registry: &Registry,
6982        forwarding: &ForwardingTable,
6983        module_id: &str,
6984    ) -> (RouteCtx, u16, u32) {
6985        let module_connection = ConnectionId::new(101);
6986        let client_connection = ConnectionId::new(202);
6987        let registration = registry
6988            .register_with_control_ops(
6989                manifest(module_id, PROTOCOL_VERSION),
6990                PROTOCOL_VERSION,
6991                module_connection,
6992                module_baseline_control_ops(),
6993            )
6994            .unwrap();
6995        let (module_tx, _module_rx) = mpsc::channel(8);
6996        let endpoint = forwarding
6997            .register_module_connection(
6998                module_connection,
6999                module_id.to_string(),
7000                PROTOCOL_VERSION,
7001                manifest_concurrency(&registration.manifest),
7002                FrameSink::new(module_tx),
7003            )
7004            .unwrap();
7005        let (client_ctx, _client_rx) = route_ctx(client_connection);
7006        let pending = forwarding
7007            .begin_route_bind_relay_for_test(
7008                client_connection,
7009                client_ctx.egress.clone(),
7010                1,
7011                module_id,
7012            )
7013            .unwrap();
7014        assert_eq!(pending.endpoint, endpoint);
7015        let route_channel = pending.client_channel;
7016        let route_epoch = pending.client_epoch;
7017        forwarding
7018            .complete_pending_relay(
7019                module_connection,
7020                pending.corr,
7021                RouteBindRelayOutcome::Accepted,
7022            )
7023            .unwrap();
7024        (client_ctx, route_channel, route_epoch)
7025    }
7026
7027    struct FakeProcessLiveness {
7028        live: Option<bool>,
7029    }
7030
7031    impl ModuleProcessLiveness for FakeProcessLiveness {
7032        fn process_live(&self, _module_id: &str) -> Option<bool> {
7033            self.live
7034        }
7035    }
7036
7037    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7038    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
7039    {
7040        let registry = Arc::new(Registry::default());
7041        let supervisor_handle = SupervisorHandle::new();
7042        let supervisor = Supervisor::new_for_test(
7043            Arc::clone(&registry),
7044            RestartPolicy::new(1, Duration::from_millis(10)),
7045        )
7046        .with_handle(supervisor_handle.clone());
7047        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
7048        let module = supervisor
7049            .spawn(ModuleSpec {
7050                module_id: "stderr-tail-wire".to_string(),
7051                program: fake_aft_stub_path(),
7052                args: Vec::new(),
7053                env: vec![
7054                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
7055                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
7056                ],
7057                reserved: false,
7058                reserved_prefixes: Vec::new(),
7059                protocol: ModuleProtocol::Subc,
7060                overlap: Default::default(),
7061            })
7062            .unwrap();
7063
7064        let deadline = Instant::now() + Duration::from_secs(5);
7065        loop {
7066            let tail = module.stderr_tail(None, None);
7067            if tail
7068                .entries
7069                .iter()
7070                .any(|entry| matches!(entry, TailEntry::ProcessStart))
7071                && tail.entries.iter().any(|entry| {
7072                    matches!(
7073                        entry,
7074                        TailEntry::Line {
7075                            truncated: true,
7076                            ..
7077                        }
7078                    )
7079                })
7080            {
7081                break;
7082            }
7083            assert!(
7084                Instant::now() < deadline,
7085                "module did not produce a truncated line and restart boundary: {tail:?}"
7086            );
7087            sleep(Duration::from_millis(10)).await;
7088        }
7089
7090        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
7091        let request = ClientControlRequest::SupervisorStderrTail {
7092            module_id: "stderr-tail-wire".to_string(),
7093            max_lines: None,
7094            max_bytes: None,
7095        };
7096        let frame = Frame::build(
7097            FrameType::Request,
7098            control_flags(),
7099            0,
7100            0,
7101            1,
7102            serde_json::to_vec(&request).unwrap(),
7103        )
7104        .unwrap();
7105        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7106        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
7107        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
7108            serde_json::from_slice(&responses[0].body).unwrap()
7109        else {
7110            panic!("expected supervisor.stderr_tail response");
7111        };
7112
7113        assert!(
7114            tail.entries
7115                .iter()
7116                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
7117            "the control response lost the restart boundary"
7118        );
7119        let Some(StderrTailEntry::Line {
7120            text,
7121            truncated,
7122            at_ms,
7123        }) = tail.entries.iter().find(|entry| {
7124            matches!(
7125                entry,
7126                StderrTailEntry::Line {
7127                    truncated: true,
7128                    ..
7129                }
7130            )
7131        })
7132        else {
7133            panic!("the control response lost the truncated line");
7134        };
7135        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
7136        assert!(*truncated);
7137        assert!(
7138            at_ms.is_some(),
7139            "the control response lost the line's capture time"
7140        );
7141    }
7142
7143    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
7144    /// read done on the worker thread would stall every other task until it
7145    /// finished; the read must run off the worker so this test's own task keeps
7146    /// running while the read is paused.
7147    #[tokio::test(flavor = "current_thread")]
7148    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
7149        let dir = TestTempDir::new("terminals-off-worker");
7150        let journal_path = dir.join("terminals.jsonl");
7151        let registry = Arc::new(Registry::default());
7152        let supervisor_handle = SupervisorHandle::new();
7153        let supervisor =
7154            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
7155                .with_handle(supervisor_handle.clone())
7156                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
7157        let module = supervisor
7158            .spawn(ModuleSpec {
7159                module_id: "terminal-off-worker".to_string(),
7160                program: fake_aft_stub_path(),
7161                args: Vec::new(),
7162                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7163                reserved: false,
7164                reserved_prefixes: Vec::new(),
7165                protocol: ModuleProtocol::Subc,
7166                overlap: Default::default(),
7167            })
7168            .unwrap();
7169        let deadline = Instant::now() + Duration::from_secs(5);
7170        while module.terminal_history().entries.len() != 2 {
7171            assert!(Instant::now() < deadline, "module did not record two exits");
7172            sleep(Duration::from_millis(10)).await;
7173        }
7174
7175        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
7176        let handler =
7177            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
7178        let frame = Frame::build(
7179            FrameType::Request,
7180            control_flags(),
7181            0,
7182            0,
7183            1,
7184            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
7185                module_id: "terminal-off-worker".to_string(),
7186            })
7187            .unwrap(),
7188        )
7189        .unwrap();
7190        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7191        let spawned_at = std::time::Instant::now();
7192        let read = tokio::spawn({
7193            let handler = Arc::clone(&handler);
7194            async move { handler.handle_control_frame(&ctx, frame).await }
7195        });
7196        // Waiting for the pause from a blocking thread keeps this task pending,
7197        // so the runtime's single worker is free to run the read task.
7198        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
7199            .await
7200            .unwrap()
7201            .expect("the history read reached its pause");
7202        let elapsed = spawned_at.elapsed();
7203        assert!(
7204            elapsed < Duration::from_secs(2) && !read.is_finished(),
7205            "this task could not run while the history read was paused \
7206             (resumed after {elapsed:?}, read finished: {})",
7207            read.is_finished()
7208        );
7209
7210        drop(release);
7211        let responses = read.await.unwrap().unwrap();
7212        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7213        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
7214            panic!("expected supervisor.terminals response");
7215        };
7216        assert_eq!(terminals.entries.len(), 2);
7217        assert_eq!(terminals.journal_skipped_lines, 0);
7218        assert_eq!(terminals.journal_read_errors, 0);
7219    }
7220
7221    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7222    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
7223        let registry = Arc::new(Registry::default());
7224        let supervisor_handle = SupervisorHandle::new();
7225        let supervisor =
7226            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
7227                .with_handle(supervisor_handle.clone());
7228        let module = supervisor
7229            .spawn(ModuleSpec {
7230                module_id: "terminal-golden".to_string(),
7231                program: fake_aft_stub_path(),
7232                args: Vec::new(),
7233                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7234                reserved: false,
7235                reserved_prefixes: Vec::new(),
7236                protocol: ModuleProtocol::Subc,
7237                overlap: Default::default(),
7238            })
7239            .unwrap();
7240
7241        let deadline = Instant::now() + Duration::from_secs(5);
7242        while module.terminal_history().entries.len() != 2 {
7243            assert!(
7244                Instant::now() < deadline,
7245                "module did not retain two terminal exits: {:?}",
7246                module.terminal_history()
7247            );
7248            sleep(Duration::from_millis(10)).await;
7249        }
7250
7251        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
7252        let request = ClientControlRequest::SupervisorTerminals {
7253            module_id: "terminal-golden".to_string(),
7254        };
7255        let frame = Frame::build(
7256            FrameType::Request,
7257            control_flags(),
7258            0,
7259            0,
7260            1,
7261            serde_json::to_vec(&request).unwrap(),
7262        )
7263        .unwrap();
7264        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7265        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
7266        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7267        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
7268            panic!("expected supervisor.terminals response");
7269        };
7270        assert_eq!(terminals.entries.len(), 2);
7271        assert_eq!(terminals.dropped, 0);
7272
7273        let mut rendered = serde_json::to_value(response).unwrap();
7274        // Wall-clock fields are the observation contract, but not stable fixture
7275        // bytes; normalize only them after the real handler has shaped the response.
7276        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
7277        for (index, entry) in rendered["entries"]
7278            .as_array_mut()
7279            .expect("terminal response entries array")
7280            .iter_mut()
7281            .enumerate()
7282        {
7283            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
7284        }
7285
7286        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7287            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
7288        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
7289        if std::env::var_os("UPDATE_GOLDEN").is_some() {
7290            std::fs::write(&golden_path, &serialized).unwrap();
7291        }
7292        let expected: Value =
7293            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
7294        assert_eq!(rendered, expected);
7295    }
7296
7297    #[test]
7298    fn hello_registers_manifest_and_returns_ack() {
7299        let registry = Arc::new(Registry::default());
7300        let handler = ControlHandler::new(Arc::clone(&registry));
7301        let conn = ConnectionId::new(1);
7302
7303        let responses = handler
7304            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
7305            .unwrap();
7306
7307        assert_eq!(responses.len(), 1);
7308        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
7309        assert_eq!(responses[0].header.channel, 0);
7310        assert_eq!(responses[0].header.corr, 7);
7311        let ack = parse_ack(&responses[0]);
7312        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
7313        assert!(ack
7314            .subc_capabilities
7315            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
7316        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
7317        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
7318        assert!(ack
7319            .subc_ops
7320            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
7321        assert!(ack
7322            .subc_ops
7323            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
7324
7325        let registration = registry.get_module("aft").unwrap().unwrap();
7326        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
7327        assert_eq!(registration.state, ChannelState::Active);
7328        assert_eq!(registration.connection_id, conn);
7329        assert_eq!(registration.control_ops, module_baseline_control_ops());
7330    }
7331
7332    #[test]
7333    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
7334        let invalid_identifiers = [
7335            ("case_change", "credentials-Provider/v1"),
7336            ("leading_zero", "credentials-provider/v01"),
7337            ("trailing_hyphen", "credentials-provider-/v1"),
7338            ("consecutive_hyphens", "credentials--provider/v1"),
7339            ("uppercase", "Credentials-provider/v1"),
7340            ("missing_v", "credentials-provider/1"),
7341            ("whitespace", "credentials provider/v1"),
7342            ("zero_version", "credentials-provider/v0"),
7343            ("out_of_range_version", "credentials-provider/v4294967296"),
7344            (
7345                "overlength_name",
7346                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
7347            ),
7348        ];
7349        let mut cases = invalid_identifiers
7350            .into_iter()
7351            .map(|(name, identifier)| {
7352                (
7353                    format!("identifier_{name}"),
7354                    "capabilities.provides[0]".to_string(),
7355                    identifier.to_string(),
7356                    json!({ "provides": [identifier] }),
7357                    None,
7358                )
7359            })
7360            .collect::<Vec<_>>();
7361        cases.extend([
7362            (
7363                "unknown_need".to_string(),
7364                "capabilities.requires[0].need".to_string(),
7365                "deferred".to_string(),
7366                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
7367                None,
7368            ),
7369            (
7370                "duplicate_provides".to_string(),
7371                "capabilities.provides[1]".to_string(),
7372                "credentials-provider/v1".to_string(),
7373                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
7374                None,
7375            ),
7376            (
7377                "duplicate_must_never_reach".to_string(),
7378                "capabilities.must_never_reach[1]".to_string(),
7379                "credentials-provider/v1".to_string(),
7380                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
7381                None,
7382            ),
7383            (
7384                "duplicate_requires_same_need".to_string(),
7385                "capabilities.requires[1]".to_string(),
7386                "credentials-provider/v1".to_string(),
7387                json!({ "requires": [
7388                    { "capability": "credentials-provider/v1", "need": "required" },
7389                    { "capability": "credentials-provider/v1", "need": "required" }
7390                ] }),
7391                None,
7392            ),
7393            (
7394                "duplicate_requires_conflicting_need".to_string(),
7395                "capabilities.requires[1]".to_string(),
7396                "credentials-provider/v1".to_string(),
7397                json!({ "requires": [
7398                    { "capability": "credentials-provider/v1", "need": "required" },
7399                    { "capability": "credentials-provider/v1", "need": "optional" }
7400                ] }),
7401                None,
7402            ),
7403            (
7404                "capabilities_root_pointer".to_string(),
7405                "runtime_computed[0]".to_string(),
7406                "/capabilities".to_string(),
7407                json!({}),
7408                Some(json!(["/capabilities"])),
7409            ),
7410            (
7411                "capabilities_descendant_pointer".to_string(),
7412                "runtime_computed[0]".to_string(),
7413                "/capabilities/provides".to_string(),
7414                json!({}),
7415                Some(json!(["/capabilities/provides"])),
7416            ),
7417            (
7418                "malformed_pointer_without_leading_slash".to_string(),
7419                "runtime_computed[0]".to_string(),
7420                "capabilities".to_string(),
7421                json!({}),
7422                Some(json!(["capabilities"])),
7423            ),
7424            (
7425                "malformed_pointer_escape".to_string(),
7426                "runtime_computed[0]".to_string(),
7427                "/roles/~2/tools".to_string(),
7428                json!({}),
7429                Some(json!(["/roles/~2/tools"])),
7430            ),
7431            (
7432                "unknown_capabilities_field".to_string(),
7433                "capabilities.future".to_string(),
7434                "<array>".to_string(),
7435                json!({ "future": [] }),
7436                None,
7437            ),
7438        ]);
7439
7440        for (index, (name, field, value, capabilities, runtime_computed)) in
7441            cases.into_iter().enumerate()
7442        {
7443            let registry = Arc::new(Registry::default());
7444            let handler = ControlHandler::new(Arc::clone(&registry));
7445            let response = handler
7446                .handle_control(
7447                    ConnectionId::new((index + 1) as u64),
7448                    capability_grammar_hello_frame(
7449                        capabilities,
7450                        runtime_computed,
7451                        index as u64 + 1,
7452                    ),
7453                )
7454                .expect("invalid HELLO returns a refusal");
7455
7456            assert_eq!(response.len(), 1, "{name} must emit one refusal");
7457            let error = parse_error(&response[0]);
7458            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
7459            let message = error["message"]
7460                .as_str()
7461                .expect("error message is a string");
7462            assert!(
7463                message.contains(&field),
7464                "{name}: field missing from {message}"
7465            );
7466            assert!(
7467                message.contains(&value),
7468                "{name}: value missing from {message}"
7469            );
7470            assert_eq!(
7471                registry
7472                    .active_registration_count()
7473                    .expect("registry reads"),
7474                0,
7475                "{name}: refused HELLO must not create a catalog entry"
7476            );
7477        }
7478    }
7479
7480    #[test]
7481    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
7482        let registry = Arc::new(Registry::default());
7483        let handler = ControlHandler::new(Arc::clone(&registry));
7484        let capabilities = json!({
7485            "provides": ["credentials-provider/v1"],
7486            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
7487            "must_never_reach": ["federation-transport/v1"]
7488        });
7489        let response = handler
7490            .handle_control(
7491                ConnectionId::new(99),
7492                capability_grammar_hello_frame(
7493                    capabilities.clone(),
7494                    Some(json!(["/roles/0/tools"])),
7495                    99,
7496                ),
7497            )
7498            .expect("valid HELLO registers");
7499        assert_eq!(response[0].header.ty, FrameType::HelloAck);
7500
7501        let request = Frame::build(
7502            FrameType::Request,
7503            control_flags(),
7504            0,
7505            0,
7506            100,
7507            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7508                .expect("catalog request serializes"),
7509        )
7510        .expect("catalog request frame builds");
7511        let response = handler
7512            .handle_catalog_list(request, None)
7513            .expect("catalog list succeeds");
7514        let ClientControlResponse::CatalogList { modules, .. } =
7515            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
7516        else {
7517            panic!("catalog request must return catalog.list");
7518        };
7519        assert_eq!(modules.len(), 1);
7520        assert_eq!(
7521            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
7522            capabilities
7523        );
7524    }
7525
7526    #[test]
7527    fn catalog_list_mirrors_management_operation_description() {
7528        let registry = Arc::new(Registry::default());
7529        let handler = ControlHandler::new(Arc::clone(&registry));
7530        let description = "List managed records and return their identifiers and metadata.";
7531        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
7532        manifest.provides = vec![ProviderRole::ManagementSurface {
7533            operations: vec![ManagementOperation {
7534                name: "records.list".to_string(),
7535                kind: ManagementOperationKind::Query,
7536                description: Some(description.to_string()),
7537            }],
7538            config_schema: json!({"type": "object"}),
7539            observability: vec![ObservabilitySurface {
7540                name: "records.stats".to_string(),
7541                kind: ObservabilityKind::Snapshot,
7542            }],
7543            identity_scope: vec![IdentityScope::Project],
7544            concurrency: Concurrency::ModuleManaged,
7545        }];
7546        registry
7547            .register_with_control_ops(
7548                manifest,
7549                PROTOCOL_VERSION,
7550                ConnectionId::new(99),
7551                Vec::new(),
7552            )
7553            .expect("described management manifest registers");
7554
7555        let request = Frame::build(
7556            FrameType::Request,
7557            control_flags(),
7558            0,
7559            0,
7560            100,
7561            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7562                .expect("catalog request serializes"),
7563        )
7564        .expect("catalog request frame builds");
7565        let response = handler
7566            .handle_catalog_list(request, None)
7567            .expect("catalog list succeeds");
7568        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7569        assert_eq!(
7570            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
7571            "catalog.list must preserve the declared operation description verbatim"
7572        );
7573    }
7574
7575    #[test]
7576    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
7577        let registry = Arc::new(Registry::default());
7578        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
7579            [("vault".to_string(), true), ("squatter".to_string(), true)],
7580            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
7581        );
7582        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
7583        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7584            provides: vec!["credentials-provider/v1".to_string()],
7585            requires: Vec::new(),
7586            must_never_reach: Vec::new(),
7587        });
7588        let frame = Frame::build(
7589            FrameType::Hello,
7590            control_flags(),
7591            0,
7592            0,
7593            77,
7594            serde_json::to_vec(&ModuleHelloBody {
7595                manifest: squatter,
7596                protocol_ver: PROTOCOL_VERSION,
7597                control_ops: None,
7598                launch_nonce: None,
7599            })
7600            .expect("HELLO serializes"),
7601        )
7602        .expect("HELLO frame builds");
7603        let response = handler
7604            .handle_control(ConnectionId::new(77), frame)
7605            .expect("reserved claim receives a typed refusal");
7606        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
7607        assert_eq!(
7608            registry
7609                .active_registration_count()
7610                .expect("registry reads"),
7611            0,
7612            "a reserved capability refusal must not leave a catalog entry"
7613        );
7614    }
7615
7616    #[test]
7617    fn stale_relay_settlement_cannot_release_the_half_open_probe() {
7618        for settlement in ["timeout", "inconclusive", "drop"] {
7619            let breakers = RouteBindBreakers::default();
7620            let RouteBindAdmission::Admitted {
7621                guard: mut old,
7622                probe: false,
7623            } = breakers.admit("prov")
7624            else {
7625                panic!("ordinary relay admitted")
7626            };
7627            let RouteBindAdmission::Admitted {
7628                guard: mut opener, ..
7629            } = breakers.admit("prov")
7630            else {
7631                panic!("second relay admitted")
7632            };
7633            assert!(
7634                !opener
7635                    .record_timeout(1, Duration::ZERO)
7636                    .unwrap()
7637                    .reopened_after_probe
7638            );
7639            let RouteBindAdmission::Admitted {
7640                guard: mut probe,
7641                probe: true,
7642            } = breakers.admit("prov")
7643            else {
7644                panic!("one cooldown probe admitted")
7645            };
7646            match settlement {
7647                "timeout" => assert!(
7648                    !old.record_timeout(1, Duration::ZERO)
7649                        .unwrap()
7650                        .reopened_after_probe
7651                ),
7652                "inconclusive" => old.record_inconclusive(),
7653                "drop" => drop(old),
7654                _ => unreachable!(),
7655            }
7656            assert!(
7657                matches!(
7658                    breakers.admit("prov"),
7659                    RouteBindAdmission::Refused {
7660                        probe_in_flight: true,
7661                        ..
7662                    }
7663                ),
7664                "{settlement} of a pre-open relay cannot release the real probe"
7665            );
7666            assert!(
7667                probe
7668                    .record_timeout(1, Duration::ZERO)
7669                    .unwrap()
7670                    .reopened_after_probe
7671            );
7672            assert!(matches!(
7673                breakers.admit("prov"),
7674                RouteBindAdmission::Admitted { probe: true, .. }
7675            ));
7676        }
7677        let breakers = RouteBindBreakers::default();
7678        let admit = || match breakers.admit("prov") {
7679            RouteBindAdmission::Admitted { guard, .. } => guard,
7680            _ => panic!("relay admitted"),
7681        };
7682        admit().record_timeout(1, Duration::ZERO);
7683        let mut old_probe = admit();
7684        breakers.reset_for_new_module_connection("prov");
7685        admit().record_timeout(1, Duration::ZERO);
7686        let _new_probe = admit();
7687        old_probe.record_inconclusive();
7688        assert!(matches!(
7689            breakers.admit("prov"),
7690            RouteBindAdmission::Refused {
7691                probe_in_flight: true,
7692                ..
7693            }
7694        ));
7695    }
7696
7697    #[tokio::test]
7698    async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
7699        for candidate in [false, true] {
7700            let registry = Arc::new(Registry::default());
7701            let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
7702                [("vault".to_string(), true), ("squatter".to_string(), true)],
7703                BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
7704            );
7705            let conn = ConnectionId::new(77);
7706            let (ctx, mut rx) = route_ctx(conn);
7707            let initial = capability_manifest("squatter", &[], &[]);
7708            if candidate {
7709                registry
7710                    .register_candidate_with_control_ops(
7711                        initial.clone(),
7712                        PROTOCOL_VERSION,
7713                        conn,
7714                        module_baseline_control_ops(),
7715                    )
7716                    .unwrap();
7717                handler
7718                    .forwarding
7719                    .register_candidate_module_connection(
7720                        conn,
7721                        "squatter".to_string(),
7722                        PROTOCOL_VERSION,
7723                        manifest_concurrency(&initial),
7724                        ctx.egress.clone(),
7725                    )
7726                    .unwrap();
7727            } else {
7728                hello_via_sink(
7729                    &handler,
7730                    &ctx,
7731                    &mut rx,
7732                    hello_frame_with_manifest(initial.clone(), 1),
7733                )
7734                .await;
7735            }
7736            let response = handler
7737                .handle_control_frame(
7738                    &ctx,
7739                    catalog_update_with_capabilities_frame(
7740                        2,
7741                        capability_manifest("squatter", &["credentials-provider/v1"], &[])
7742                            .capabilities
7743                            .unwrap(),
7744                    ),
7745                )
7746                .await
7747                .unwrap();
7748            assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
7749            assert_eq!(
7750                registry
7751                    .get_module_by_connection(conn)
7752                    .unwrap()
7753                    .unwrap()
7754                    .manifest,
7755                initial
7756            );
7757        }
7758    }
7759
7760    #[test]
7761    fn server_describe_surfaces_required_capability_verdict_fields() {
7762        let registry = Arc::new(Registry::default());
7763        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
7764            [
7765                ("consumer".to_string(), true),
7766                ("provider".to_string(), false),
7767            ],
7768            BTreeMap::new(),
7769        );
7770        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
7771        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7772            provides: Vec::new(),
7773            requires: vec![subc_protocol::manifest::CapabilityRequirement {
7774                capability: "credentials-provider/v1".to_string(),
7775                need: subc_protocol::manifest::CapabilityNeed::Required,
7776            }],
7777            must_never_reach: Vec::new(),
7778        });
7779        let hello = Frame::build(
7780            FrameType::Hello,
7781            control_flags(),
7782            0,
7783            0,
7784            78,
7785            serde_json::to_vec(&ModuleHelloBody {
7786                manifest: consumer,
7787                protocol_ver: PROTOCOL_VERSION,
7788                control_ops: None,
7789                launch_nonce: None,
7790            })
7791            .expect("HELLO serializes"),
7792        )
7793        .expect("HELLO frame builds");
7794        handler
7795            .handle_control(ConnectionId::new(78), hello)
7796            .expect("consumer registers");
7797        let describe = Frame::build(
7798            FrameType::Request,
7799            control_flags(),
7800            0,
7801            0,
7802            79,
7803            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
7804                .expect("request serializes"),
7805        )
7806        .expect("describe frame builds");
7807        let response = handler
7808            .handle_server_describe(describe)
7809            .expect("server.describe succeeds");
7810        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
7811        let requirement = &rendered["capability_requirements"][0];
7812        assert_eq!(requirement["consumer"], "consumer");
7813        assert_eq!(requirement["verdict"], "never_provided");
7814        assert_eq!(requirement["episode_seq"], 1);
7815        assert_eq!(requirement["config_satisfiable"], false);
7816        assert_eq!(requirement["runtime_available"], false);
7817        assert!(requirement["detail"]
7818            .as_str()
7819            .expect("detail string")
7820            .contains("credentials-provider/v1"));
7821    }
7822
7823    #[test]
7824    fn catalog_list_omits_capabilities_for_legacy_manifest() {
7825        let registry = Arc::new(Registry::default());
7826        let handler = ControlHandler::new(Arc::clone(&registry));
7827        let hello = handler
7828            .handle_control(
7829                ConnectionId::new(101),
7830                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
7831            )
7832            .expect("legacy HELLO registers");
7833        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
7834
7835        let request = Frame::build(
7836            FrameType::Request,
7837            control_flags(),
7838            0,
7839            0,
7840            102,
7841            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7842                .expect("catalog request serializes"),
7843        )
7844        .expect("catalog request frame builds");
7845        let response = handler
7846            .handle_catalog_list(request, None)
7847            .expect("catalog list succeeds");
7848        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7849        assert!(
7850            body["modules"][0].get("capabilities").is_none(),
7851            "legacy manifest must retain an absent capabilities field on catalog.list"
7852        );
7853    }
7854
7855    #[test]
7856    fn hello_ack_omits_storage_when_no_storage_config() {
7857        let registry = Arc::new(Registry::default());
7858        let handler = ControlHandler::new(Arc::clone(&registry));
7859        let responses = handler
7860            .handle_control(
7861                ConnectionId::new(1),
7862                hello_frame("aft", PROTOCOL_VERSION, 7),
7863            )
7864            .unwrap();
7865        let ack = parse_ack(&responses[0]);
7866        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
7867        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
7868    }
7869
7870    #[tokio::test]
7871    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
7872        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
7873        let registry = Arc::new(Registry::default());
7874        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
7875        let responses = handler
7876            .handle_control(
7877                ConnectionId::new(1),
7878                hello_frame("aft", PROTOCOL_VERSION, 7),
7879            )
7880            .unwrap();
7881        let ack = parse_ack(&responses[0]);
7882        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
7883
7884        let described = handler
7885            .handle_control_frame(
7886                &route_ctx(ConnectionId::new(2)).0,
7887                Frame::build(
7888                    FrameType::Request,
7889                    control_flags(),
7890                    0,
7891                    0,
7892                    9,
7893                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
7894                )
7895                .unwrap(),
7896            )
7897            .await
7898            .unwrap();
7899        let ClientControlResponse::ServerDescribe { machine_id, .. } =
7900            serde_json::from_slice(&described[0].body).unwrap()
7901        else {
7902            panic!("server.describe answered with another shape");
7903        };
7904        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
7905    }
7906
7907    #[test]
7908    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
7909        // With a central sqlite storage policy, each registering module gets its
7910        // own resolved descriptor in HELLO_ACK, keyed by its module id.
7911        let registry = Arc::new(Registry::default());
7912        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
7913            crate::daemon_config::StorageConfig::Sqlite {
7914                data_home: std::path::PathBuf::from("/data"),
7915            },
7916        ));
7917
7918        let responses = handler
7919            .handle_control(
7920                ConnectionId::new(1),
7921                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
7922            )
7923            .unwrap();
7924        let ack = parse_ack(&responses[0]);
7925        assert_eq!(
7926            ack.storage,
7927            Some(serde_json::json!({
7928                "module_id": "alfonso-routing",
7929                "storage_namespace": "default",
7930                "isolation": { "kind": "module" },
7931                "backend": {
7932                    "backend": "sqlite",
7933                    "path": "/data/cortexkit/alfonso-routing/store.db"
7934                }
7935            })),
7936            "the delivered descriptor is the module's own sqlite store path"
7937        );
7938    }
7939
7940    #[test]
7941    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
7942        let registry = Arc::new(Registry::default());
7943        let handler = ControlHandler::new(Arc::clone(&registry));
7944        let conn = ConnectionId::new(1);
7945        let responses = handler
7946            .handle_control(
7947                conn,
7948                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
7949            )
7950            .unwrap();
7951        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
7952        let registration = registry.get_module("aft").unwrap().unwrap();
7953        assert_eq!(registration.control_ops, module_baseline_control_ops());
7954
7955        let frame =
7956            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
7957        assert!(handler
7958            .guard_module_control_op(&frame, "aft", "route.bind")
7959            .unwrap()
7960            .is_none());
7961        let error = handler
7962            .guard_module_control_op(&frame, "aft", "test.synthetic")
7963            .unwrap()
7964            .expect("synthetic ungranted op should be rejected");
7965        assert_eq!(error.header.ty, FrameType::Error);
7966        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
7967    }
7968
7969    #[test]
7970    fn hello_control_ops_some_adds_optional_grants() {
7971        let registry = Arc::new(Registry::default());
7972        let handler = ControlHandler::new(Arc::clone(&registry));
7973        handler
7974            .handle_control(
7975                ConnectionId::new(1),
7976                hello_frame_with_control_ops(
7977                    "aft",
7978                    PROTOCOL_VERSION,
7979                    7,
7980                    Some(vec![
7981                        "future.synthetic".to_string(),
7982                        "route.bind".to_string(),
7983                    ]),
7984                ),
7985            )
7986            .unwrap();
7987        let registration = registry.get_module("aft").unwrap().unwrap();
7988        assert_eq!(
7989            registration.control_ops,
7990            vec![
7991                "route.bind".to_string(),
7992                "route.status".to_string(),
7993                "future.synthetic".to_string(),
7994            ]
7995        );
7996        let frame =
7997            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
7998        assert!(handler
7999            .guard_module_control_op(&frame, "aft", "future.synthetic")
8000            .unwrap()
8001            .is_none());
8002    }
8003
8004    #[tokio::test]
8005    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
8006        let registry = Arc::new(Registry::default());
8007        let forwarding = Arc::new(ForwardingTable::default());
8008        let handler =
8009            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
8010        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
8011        hello_via_sink(
8012            &handler,
8013            &module_ctx,
8014            &mut module_rx,
8015            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
8016        )
8017        .await;
8018
8019        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
8020        let responses = handler
8021            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
8022            .await
8023            .unwrap();
8024        assert_eq!(responses.len(), 1);
8025        assert_eq!(responses[0].header.ty, FrameType::Error);
8026        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
8027        assert!(module_rx.try_recv().is_err());
8028    }
8029
8030    #[tokio::test]
8031    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
8032        let registry = Arc::new(Registry::default());
8033        let forwarding = Arc::new(ForwardingTable::default());
8034        let handler =
8035            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
8036        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
8037        hello_via_sink(
8038            &handler,
8039            &module_ctx,
8040            &mut module_rx,
8041            hello_frame_with_control_ops(
8042                "aft",
8043                PROTOCOL_VERSION,
8044                7,
8045                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8046            ),
8047        )
8048        .await;
8049
8050        let project_root = unique_project_root("demux");
8051        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
8052        let route_handler = handler.clone();
8053        let route_task = tokio::spawn(async move {
8054            route_handler
8055                .handle_control_frame(
8056                    &route_client_ctx,
8057                    route_open_frame(100, "aft", project_root),
8058                )
8059                .await
8060                .unwrap()
8061        });
8062        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8063            .await
8064            .unwrap()
8065            .unwrap();
8066        assert!(matches!(
8067            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
8068            ModuleControlRequest::RouteBind { .. }
8069        ));
8070
8071        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
8072        let health_handler = handler.clone();
8073        let health_task = tokio::spawn(async move {
8074            health_handler
8075                .handle_control_frame(
8076                    &health_client_ctx,
8077                    supervisor_health_probe_frame(101, "aft"),
8078                )
8079                .await
8080                .unwrap()
8081        });
8082        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8083            .await
8084            .unwrap()
8085            .unwrap();
8086        assert_eq!(
8087            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
8088            ModuleControlRequest::HealthCheck {}
8089        );
8090
8091        handler
8092            .handle_control_frame(
8093                &module_ctx,
8094                health_response(health_frame.header.corr, HealthStatus::Degraded),
8095            )
8096            .await
8097            .unwrap();
8098        let health_response = health_task.await.unwrap();
8099        assert_eq!(health_response.len(), 1);
8100        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
8101            ClientControlResponse::SupervisorHealthProbe {
8102                module_id,
8103                status,
8104                detail,
8105                metrics,
8106            } => {
8107                assert_eq!(module_id, "aft");
8108                assert_eq!(status, HealthStatus::Degraded);
8109                assert_eq!(detail.as_deref(), Some("warming"));
8110                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
8111            }
8112            other => panic!("unexpected health response: {other:?}"),
8113        }
8114
8115        handler
8116            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8117            .await
8118            .unwrap();
8119        let route_response = route_task.await.unwrap();
8120        assert!(route_response.is_empty());
8121        let published = route_client_rx.recv().await.unwrap();
8122        assert!(matches!(
8123            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8124            ClientControlResponse::RouteOpen { .. }
8125        ));
8126    }
8127
8128    /// Start one `route.open` on `client_connection` and return its still-running
8129    /// handler task together with the `route.bind` the module received for it.
8130    /// The handler blocks until the module answers, so it has to run as a task
8131    /// while the test drives the module side.
8132    async fn relay_route_open(
8133        handler: &ControlHandler,
8134        client_connection: ConnectionId,
8135        client_egress: &FrameSink,
8136        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8137        corr: u64,
8138        module_id: &str,
8139        project_root_label: &str,
8140    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
8141        let ctx = RouteCtx {
8142            connection_id: client_connection,
8143            egress: client_egress.clone(),
8144        };
8145        let handler = handler.clone();
8146        let project_root = unique_project_root(project_root_label);
8147        let module_id = module_id.to_string();
8148        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
8149        let task = tokio::spawn(async move {
8150            let _guard = tracing::dispatcher::set_default(&dispatch);
8151            handler
8152                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
8153                .await
8154                .unwrap()
8155        });
8156        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
8157            .await
8158            .expect("module receives the relayed route.bind")
8159            .expect("module egress is open");
8160        (task, bind.frame)
8161    }
8162
8163    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
8164        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
8165            ModuleControlRequest::RouteBind {
8166                route_channel,
8167                epoch,
8168                ..
8169            } => (route_channel, epoch),
8170            other => panic!("expected a route.bind request, got {other:?}"),
8171        }
8172    }
8173
8174    fn published_route(frame: &Frame) -> (u16, u32) {
8175        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
8176            ClientControlResponse::RouteOpen {
8177                route_channel,
8178                route_epoch,
8179            } => (route_channel, route_epoch),
8180            other => panic!("expected a route.open response, got {other:?}"),
8181        }
8182    }
8183
8184    /// Reproduction of a production outage. A client had `route.open`s in
8185    /// flight to a module and was already marked closing -- its egress had refused a
8186    /// module frame, so the daemon asked its connection to end -- while its sink
8187    /// was still open. When the module acked those binds, the daemon refused to
8188    /// commit a route for a closing client, and that refusal was returned from
8189    /// the MODULE connection's frame handler, where a router error that has no
8190    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
8191    /// the supervisor correctly did not respawn a clean exit, and every seat lost
8192    /// its tools for hours -- one client's teardown took down a connection
8193    /// carrying ~170 other routes.
8194    ///
8195    /// The window is opened here by calling the production path that opens it
8196    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
8197    /// state that matters is "in `closing_connections`, sink still open, relay
8198    /// still pending", and it lasts only from the close request until the
8199    /// connection loop reacts to it; a socket-level test can flood a client into
8200    /// that escalation but cannot pin the module's ack inside the window. Closing
8201    /// the socket instead takes the other path entirely -- connection teardown
8202    /// removes the pending relay under the same lock, so the ack finds nothing.
8203    #[tokio::test]
8204    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
8205        let registry = Arc::new(Registry::default());
8206        let forwarding = Arc::new(ForwardingTable::default());
8207        let handler =
8208            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
8209
8210        let module_connection = ConnectionId::new(30);
8211        let (module_ctx, mut module_rx) = route_ctx(module_connection);
8212        hello_via_sink(
8213            &handler,
8214            &module_ctx,
8215            &mut module_rx,
8216            hello_frame("aft", PROTOCOL_VERSION, 7),
8217        )
8218        .await;
8219
8220        let dying_client = ConnectionId::new(31);
8221        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
8222
8223        // A published route on the dying client. The escalation below only marks
8224        // a connection closing for a route it has already published.
8225        let (first_task, first_bind) = relay_route_open(
8226            &handler,
8227            dying_client,
8228            &dying_ctx.egress,
8229            &mut module_rx,
8230            100,
8231            "aft",
8232            "closing-first",
8233        )
8234        .await;
8235        handler
8236            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
8237            .await
8238            .unwrap();
8239        assert!(first_task.await.unwrap().is_empty());
8240        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
8241
8242        // A second route.open from the same client, relayed and awaiting its ack.
8243        let (second_task, second_bind) = relay_route_open(
8244            &handler,
8245            dying_client,
8246            &dying_ctx.egress,
8247            &mut module_rx,
8248            101,
8249            "aft",
8250            "closing-second",
8251        )
8252        .await;
8253        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
8254
8255        // The window: the client is closing, its sink is still open, and its
8256        // second bind is still pending.
8257        assert!(forwarding
8258            .escalate_client_delivery_failure(
8259                dying_client,
8260                first_channel,
8261                first_epoch,
8262                CloseReason::new(
8263                    "module_to_client_delivery_failed",
8264                    "client egress refused a module frame",
8265                ),
8266                crate::forwarding::UndeliveredFrame {
8267                    module_id: None,
8268                    sink: &dying_ctx.egress,
8269                },
8270            )
8271            .unwrap());
8272        assert!(!dying_ctx.egress.is_closed());
8273
8274        // The frame that used to end the module connection.
8275        let ack = handler
8276            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
8277            .await;
8278        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
8279        if module_loop_error.is_some() {
8280            // What the server's connection loop does with a router error that has
8281            // no ERROR-frame translation: end the connection, which releases the
8282            // module's registration and every route on it.
8283            handler.cleanup_connection(module_connection).unwrap();
8284        }
8285        // Read the module's next frame before opening the co-tenant's route, so
8286        // the GOODBYE assertion below is about THIS ack and not about later
8287        // traffic. `None` means the module was told nothing.
8288        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8289            .await
8290            .ok()
8291            .flatten();
8292
8293        // 1. The module connection is still registered.
8294        assert!(
8295            registry
8296                .get_module_by_connection(module_connection)
8297                .unwrap()
8298                .is_some(),
8299            "one client's closing connection ended the shared module connection: \
8300             {module_loop_error:?}"
8301        );
8302        // ...and still serving: another client can open and use a route on it.
8303        let cotenant = ConnectionId::new(32);
8304        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
8305        let (cotenant_task, cotenant_bind) = relay_route_open(
8306            &handler,
8307            cotenant,
8308            &cotenant_ctx.egress,
8309            &mut module_rx,
8310            102,
8311            "aft",
8312            "closing-cotenant",
8313        )
8314        .await;
8315        handler
8316            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
8317            .await
8318            .unwrap();
8319        assert!(cotenant_task.await.unwrap().is_empty());
8320        let (cotenant_channel, cotenant_epoch) =
8321            published_route(&cotenant_rx.recv().await.unwrap());
8322        assert!(matches!(
8323            forwarding
8324                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
8325                .unwrap(),
8326            DataRoute::Client(DataRouteState::Bound(_))
8327        ));
8328
8329        // 2. The module was told to drop the binding it created for the route
8330        //    that will never be published.
8331        let goodbye = post_ack_module_frame
8332            .expect("module receives a GOODBYE for the abandoned route channel");
8333        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
8334        assert_eq!(goodbye.header.channel, abandoned_channel);
8335        assert_eq!(goodbye.header.epoch, abandoned_epoch);
8336
8337        // 3. The dying client received nothing: no route was ever published to
8338        //    it. Its route.open is answered as unavailable, which the connection
8339        //    loop would write to a socket that is already going away.
8340        assert!(dying_rx.try_recv().is_err());
8341        let second_response = second_task.await.unwrap();
8342        assert_eq!(second_response.len(), 1);
8343        assert_eq!(
8344            parse_error(&second_response[0])["code"],
8345            "target_unavailable"
8346        );
8347    }
8348
8349    /// The fence at the module-loop boundary, stated as its own contract: which
8350    /// forwarding failures are allowed to end the module connection that is being
8351    /// served. A `ConnectionClosing` naming some client is about that client, and
8352    /// a module connection is shared; the same error naming the module's own
8353    /// connection is about this connection and must stay fatal, as must failures
8354    /// that are about the forwarding table itself.
8355    #[test]
8356    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
8357        let handler = ControlHandler::default();
8358        let module_connection = ConnectionId::new(30);
8359        let client_connection = ConnectionId::new(31);
8360
8361        handler
8362            .refuse_to_end_module_connection_for_a_client(
8363                module_connection,
8364                77,
8365                ForwardingError::ConnectionClosing {
8366                    connection_id: client_connection,
8367                },
8368            )
8369            .expect("a closing client must never end the module connection");
8370
8371        assert!(matches!(
8372            handler.refuse_to_end_module_connection_for_a_client(
8373                module_connection,
8374                78,
8375                ForwardingError::ConnectionClosing {
8376                    connection_id: module_connection,
8377                },
8378            ),
8379            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
8380                connection_id
8381            })) if connection_id == module_connection
8382        ));
8383        assert!(matches!(
8384            handler.refuse_to_end_module_connection_for_a_client(
8385                module_connection,
8386                79,
8387                ForwardingError::Poisoned,
8388            ),
8389            Err(RouterError::Forwarding(ForwardingError::Poisoned))
8390        ));
8391        assert!(matches!(
8392            handler.refuse_to_end_module_connection_for_a_client(
8393                module_connection,
8394                80,
8395                ForwardingError::StaleModuleEndpoint,
8396            ),
8397            Err(RouterError::Forwarding(
8398                ForwardingError::StaleModuleEndpoint
8399            ))
8400        ));
8401    }
8402
8403    /// The spawn-attestation guard is what stops a connected module from claiming
8404    /// another module's identity and being stamped `Reserved` for it. Every other
8405    /// test that supplies a consumer_identity supplies a CORRECT one, because a
8406    /// correct one is what the rest of the flow needs -- so the guard's rejection
8407    /// branch was never the subject of an assertion, only its acceptance branch.
8408    ///
8409    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
8410    /// whole subc-core library suite green; only the forwarding integration tests
8411    /// notice, and they notice for unrelated reasons. This test exists so the
8412    /// refusal itself is asserted where the guard lives: it fails if the identity
8413    /// check stops refusing, which is the direction that matters, since a guard
8414    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
8415    #[tokio::test]
8416    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
8417        let registry = Arc::new(Registry::default());
8418        let forwarding = Arc::new(ForwardingTable::default());
8419        let supervisor = SupervisorHandle::new();
8420        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8421        let handler =
8422            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8423                .with_supervisor(supervisor);
8424
8425        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
8426        hello_via_sink(
8427            &handler,
8428            &target_ctx,
8429            &mut target_rx,
8430            hello_frame("target", PROTOCOL_VERSION, 1),
8431        )
8432        .await;
8433
8434        // A real supervised module id presenting the wrong nonce. This is the
8435        // impersonation case: the attacker knows a privileged module_id, which is
8436        // public, and guesses at the nonce, which is not.
8437        let wrong_nonce = handler
8438            .handle_control_frame(
8439                &route_ctx(ConnectionId::new(91)).0,
8440                route_open_frame_with_admission_facts(
8441                    20,
8442                    "target",
8443                    unique_project_root("admission-facts"),
8444                    Some(subc_control::ConsumerIdentity {
8445                        module_id: "fed".to_string(),
8446                        launch_nonce: "not-the-real-nonce".to_string(),
8447                    }),
8448                    None,
8449                ),
8450            )
8451            .await
8452            .unwrap();
8453        assert_eq!(
8454            parse_error(&wrong_nonce[0])["code"],
8455            "bad_consumer_identity",
8456            "a mismatched launch nonce must be refused, not stamped Reserved"
8457        );
8458
8459        // A module id the supervisor never spawned at all, so no nonce exists to
8460        // compare against. An implementation that treats "no record" as "nothing
8461        // to check" fails open here while passing the case above.
8462        let never_spawned = handler
8463            .handle_control_frame(
8464                &route_ctx(ConnectionId::new(92)).0,
8465                route_open_frame_with_admission_facts(
8466                    21,
8467                    "target",
8468                    unique_project_root("admission-facts"),
8469                    Some(subc_control::ConsumerIdentity {
8470                        module_id: "never-spawned".to_string(),
8471                        launch_nonce: "any-nonce".to_string(),
8472                    }),
8473                    None,
8474                ),
8475            )
8476            .await
8477            .unwrap();
8478        assert_eq!(
8479            parse_error(&never_spawned[0])["code"],
8480            "bad_consumer_identity",
8481            "an unspawned module_id must be refused rather than accepted for lack of a record"
8482        );
8483    }
8484
8485    /// The refusal test above proves the guard says NO. Nothing proved it can say
8486    /// YES, and the difference is not academic: replacing the whole authorization
8487    /// with `false` -- admitting no consumer identity at all, revoking Reserved
8488    /// standing for every supervised module in the fleet -- leaves 110 of the 111
8489    /// library tests GREEN. The one that notices does so by HANGING, because it
8490    /// waits for a bind that can no longer happen.
8491    ///
8492    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
8493    /// or flaky test, invites a RETRY rather than an investigation, and the retry
8494    /// hangs too and gets blamed on the runner. So a total revocation of the
8495    /// daemon's trust grant would have shipped behind a symptom nobody attributes
8496    /// to code.
8497    ///
8498    /// The bias is structural rather than accidental. A REFUSAL looks like a
8499    /// failure someone writes a test for; a GRANT looks like the happy path. Every
8500    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
8501    /// suite for that reason, and this one is the purest case in the daemon.
8502    ///
8503    /// This test asserts the EFFECT rather than the absence of an error: the module
8504    /// receives a RouteBind and it carries `Reserved` naming the attested module.
8505    /// A guard that admitted nobody would produce no bind at all; one that admitted
8506    /// everybody would stamp the wrong principal, which the refusal test catches.
8507    #[tokio::test]
8508    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
8509        let registry = Arc::new(Registry::default());
8510        let forwarding = Arc::new(ForwardingTable::default());
8511        let supervisor = SupervisorHandle::new();
8512        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8513        let handler =
8514            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8515                .with_supervisor(supervisor);
8516
8517        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
8518        hello_via_sink(
8519            &handler,
8520            &target_ctx,
8521            &mut target_rx,
8522            hello_frame("target", PROTOCOL_VERSION, 1),
8523        )
8524        .await;
8525
8526        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
8527        let route_handler = handler.clone();
8528        let route_task = tokio::spawn(async move {
8529            route_handler
8530                .handle_control_frame(
8531                    &client_ctx,
8532                    route_open_frame_with_admission_facts(
8533                        30,
8534                        "target",
8535                        unique_project_root("admission-facts"),
8536                        Some(subc_control::ConsumerIdentity {
8537                            module_id: "fed".to_string(),
8538                            launch_nonce: "fed-nonce".to_string(),
8539                        }),
8540                        None,
8541                    ),
8542                )
8543                .await
8544                .unwrap()
8545        });
8546
8547        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
8548        // the very mutation it exists to catch -- a guard that admits nobody -- no
8549        // bind is ever sent, so it HUNG rather than failing. That reproduces the
8550        // exact defect being fixed: a total revocation detected only as a stalled
8551        // suite, which reads as flakiness and invites a retry. An acceptance test
8552        // that waits for an effect must bound the wait, or a red becomes a hang.
8553        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8554            .await
8555            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
8556            .expect("module control channel closed before route.bind");
8557        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8558        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
8559            panic!("expected route.bind")
8560        };
8561        assert_eq!(
8562            principal,
8563            Some(Principal::Reserved {
8564                module_id: "fed".to_string()
8565            }),
8566            "a correctly attested consumer must be stamped Reserved for its own id"
8567        );
8568
8569        handler
8570            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8571            .await
8572            .unwrap();
8573        assert!(route_task.await.unwrap().is_empty());
8574        assert!(
8575            matches!(
8576                serde_json::from_slice::<ClientControlResponse>(
8577                    &client_rx.recv().await.unwrap().body
8578                )
8579                .unwrap(),
8580                ClientControlResponse::RouteOpen { .. }
8581            ),
8582            "the route must actually open, not merely avoid an error"
8583        );
8584    }
8585
8586    #[tokio::test(start_paused = true)]
8587    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
8588        let registry = Arc::new(Registry::default());
8589        let forwarding = Arc::new(ForwardingTable::default());
8590        let supervisor = SupervisorHandle::new();
8591        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8592        let handler =
8593            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8594                .with_supervisor(supervisor);
8595
8596        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
8597        hello_via_sink(
8598            &handler,
8599            &target_ctx,
8600            &mut target_rx,
8601            hello_frame("target", PROTOCOL_VERSION, 1),
8602        )
8603        .await;
8604
8605        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
8606        let direct_handler = handler.clone();
8607        let direct_open = tokio::spawn(async move {
8608            direct_handler
8609                .handle_control_frame(
8610                    &direct_ctx,
8611                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
8612                )
8613                .await
8614                .unwrap()
8615        });
8616        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8617            .await
8618            .expect("no direct route.bind within 5s")
8619            .expect("target control channel closed before direct route.bind");
8620        handler
8621            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
8622            .await
8623            .unwrap();
8624        assert!(direct_open.await.unwrap().is_empty());
8625        let _ = direct_rx.recv().await.unwrap();
8626
8627        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
8628        let reserved_handler = handler.clone();
8629        let reserved_open = tokio::spawn(async move {
8630            reserved_handler
8631                .handle_control_frame(
8632                    &reserved_ctx,
8633                    route_open_frame_with_admission_facts(
8634                        3,
8635                        "target",
8636                        unique_project_root("admission-facts"),
8637                        Some(ConsumerIdentity {
8638                            module_id: "fed".to_string(),
8639                            launch_nonce: "fed-nonce".to_string(),
8640                        }),
8641                        None,
8642                    ),
8643                )
8644                .await
8645                .unwrap()
8646        });
8647        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8648            .await
8649            .expect("no reserved route.bind within 5s")
8650            .expect("target control channel closed before reserved route.bind");
8651        handler
8652            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
8653            .await
8654            .unwrap();
8655        assert!(reserved_open.await.unwrap().is_empty());
8656        let _ = reserved_rx.recv().await.unwrap();
8657
8658        forwarding
8659            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
8660            .unwrap();
8661        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
8662        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
8663            module_id: Some("target".to_string()),
8664        })
8665        .unwrap();
8666        let census_frame =
8667            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
8668        let response = handler
8669            .handle_control_frame(&census_ctx, census_frame)
8670            .await
8671            .unwrap()
8672            .pop()
8673            .unwrap();
8674        let actual: Value = serde_json::from_slice(&response.body).unwrap();
8675        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
8676        assert!(matches!(
8677            decoded,
8678            ClientControlResponse::SupervisorRoutes { .. }
8679        ));
8680        let routes = actual["modules"][0]["routes"].as_array().unwrap();
8681        assert_eq!(routes.len(), 2);
8682        assert!(routes.iter().all(|route| route["draining"] == true));
8683        // The census carries WHY: the reason the drain was begun with, in the
8684        // route.closing vocabulary, on every draining route this drain marked.
8685        assert!(
8686            routes.iter().all(|route| route["drain_reason"] == "reload"),
8687            "draining routes must name the drain's reason: {routes:?}"
8688        );
8689        assert!(routes.iter().any(|route| {
8690            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
8691        }));
8692        assert!(routes.iter().any(|route| {
8693            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
8694        }));
8695
8696        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8697            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
8698        if std::env::var_os("UPDATE_GOLDEN").is_some() {
8699            std::fs::write(
8700                &golden_path,
8701                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
8702            )
8703            .unwrap();
8704        }
8705        let expected: Value =
8706            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
8707        assert_eq!(actual, expected);
8708    }
8709
8710    async fn query_live_roots(
8711        handler: &ControlHandler,
8712        module_ctx: &RouteCtx,
8713    ) -> ModuleControlResponseToModule {
8714        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
8715        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
8716        let response = handler
8717            .handle_control_frame(module_ctx, frame)
8718            .await
8719            .unwrap()
8720            .pop()
8721            .unwrap();
8722        serde_json::from_slice(&response.body).unwrap()
8723    }
8724
8725    #[tokio::test(start_paused = true)]
8726    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
8727        let registry = Arc::new(Registry::default());
8728        let forwarding = Arc::new(ForwardingTable::default());
8729        let handler = ControlHandler::with_forwarding(registry, forwarding);
8730        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
8731        hello_via_sink(
8732            &handler,
8733            &target_ctx,
8734            &mut target_rx,
8735            hello_frame("target", PROTOCOL_VERSION, 1),
8736        )
8737        .await;
8738        let root = unique_project_root("live-roots-known");
8739        let path = ProjectRootId::from_path_allowing_missing(root.path())
8740            .unwrap()
8741            .as_path()
8742            .to_path_buf();
8743        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
8744        let open_handler = handler.clone();
8745        let opened = tokio::spawn(async move {
8746            open_handler
8747                .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
8748                .await
8749                .unwrap()
8750        });
8751        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8752            .await
8753            .unwrap()
8754            .unwrap();
8755        handler
8756            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
8757            .await
8758            .unwrap();
8759        assert!(opened.await.unwrap().is_empty());
8760        let _ = client_rx.recv().await.unwrap();
8761
8762        let root = unique_project_root("live-roots-pending");
8763        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
8764            .unwrap()
8765            .as_path()
8766            .to_path_buf();
8767        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
8768        let open_handler = handler.clone();
8769        let pending = tokio::spawn(async move {
8770            open_handler
8771                .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
8772                .await
8773                .unwrap()
8774        });
8775        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8776            .await
8777            .unwrap()
8778            .unwrap();
8779        let actual = query_live_roots(&handler, &target_ctx).await;
8780        let ModuleControlResponseToModule::LiveRoots {
8781            roots,
8782            unknown_root_bindings,
8783            total_bindings,
8784        } = actual
8785        else {
8786            panic!("expected live roots")
8787        };
8788        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
8789        assert_eq!(unknown_root_bindings, 0);
8790        assert_eq!(
8791            roots.len(),
8792            2,
8793            "root-known arm must retain each canonical root"
8794        );
8795        assert_eq!(
8796            total_bindings,
8797            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8798        );
8799        let counts = roots
8800            .iter()
8801            .map(|root| (root.project_root.clone(), root.bound, root.pending))
8802            .collect::<Vec<_>>();
8803        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
8804        expected.sort_by(|a, b| a.0.cmp(&b.0));
8805        assert_eq!(
8806            counts, expected,
8807            "roots must sort by path and count pending separately"
8808        );
8809        handler
8810            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
8811            .await
8812            .unwrap();
8813        assert!(pending.await.unwrap().is_empty());
8814    }
8815
8816    #[tokio::test(start_paused = true)]
8817    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
8818        let forwarding = Arc::new(ForwardingTable::default());
8819        let handler =
8820            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
8821        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
8822        hello_via_sink(
8823            &handler,
8824            &target_ctx,
8825            &mut target_rx,
8826            hello_frame("target", PROTOCOL_VERSION, 1),
8827        )
8828        .await;
8829        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
8830        let pending = forwarding
8831            .begin_route_bind_relay_for_test(
8832                client_ctx.connection_id,
8833                client_ctx.egress.clone(),
8834                2,
8835                "target",
8836            )
8837            .unwrap();
8838        forwarding
8839            .complete_pending_relay(
8840                target_ctx.connection_id,
8841                pending.corr,
8842                RouteBindRelayOutcome::Accepted,
8843            )
8844            .unwrap();
8845        let actual = query_live_roots(&handler, &target_ctx).await;
8846        let ModuleControlResponseToModule::LiveRoots {
8847            roots,
8848            unknown_root_bindings,
8849            total_bindings,
8850        } = actual
8851        else {
8852            panic!("expected live roots")
8853        };
8854        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
8855        assert_eq!(
8856            unknown_root_bindings, 1,
8857            "unknown-root arm must not read as no bindings"
8858        );
8859        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
8860        assert_eq!(
8861            total_bindings,
8862            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8863        );
8864    }
8865
8866    /// A module reads HELLO_ACK as its first frame and exits on anything else,
8867    /// so the ack has to be on its outbound queue before the module is
8868    /// routable. The connection loop writes a handler's replies only after the
8869    /// handler returns; this test stops in exactly that gap, runs a real
8870    /// route.open from another connection, and only then writes whatever the
8871    /// HELLO handler returned, the way the loop would. If the ack were still a
8872    /// reply, the route.bind request would reach the module first.
8873    #[tokio::test(start_paused = true)]
8874    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
8875        let forwarding = Arc::new(ForwardingTable::default());
8876        let handler =
8877            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
8878        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
8879        let replies = handler
8880            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
8881            .await
8882            .unwrap();
8883        let queued_by_hello = module_rx.len();
8884
8885        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
8886        let open_handler = handler.clone();
8887        let open = tokio::spawn(async move {
8888            open_handler
8889                .handle_control_frame(
8890                    &client_ctx,
8891                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
8892                )
8893                .await
8894                .unwrap()
8895        });
8896        // Let the route.open run until its route.bind is on the module's queue.
8897        let mut spins = 0;
8898        while module_rx.len() == queued_by_hello {
8899            spins += 1;
8900            assert!(spins < 10_000, "route.open never queued a route.bind");
8901            tokio::task::yield_now().await;
8902        }
8903
8904        // Now the connection loop's half: write the HELLO handler's replies.
8905        for reply in replies {
8906            module_ctx.egress.send(reply).await.unwrap();
8907        }
8908
8909        let first = module_rx.recv().await.unwrap().frame;
8910        assert_eq!(
8911            first.header.ty,
8912            FrameType::HelloAck,
8913            "the first frame a registering module reads must be its HELLO_ACK"
8914        );
8915        assert_eq!(first.header.corr, 7);
8916        let second = module_rx.recv().await.unwrap().frame;
8917        assert_eq!(second.header.ty, FrameType::Request);
8918        assert!(
8919            matches!(
8920                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
8921                ModuleControlRequest::RouteBind { .. }
8922            ),
8923            "the route.bind follows the ack"
8924        );
8925        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
8926
8927        handler
8928            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
8929            .await
8930            .unwrap();
8931        assert!(open.await.unwrap().is_empty());
8932        let _ = client_rx.recv().await.unwrap();
8933    }
8934
8935    #[tokio::test(start_paused = true)]
8936    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
8937        let handler = ControlHandler::with_forwarding(
8938            Arc::new(Registry::default()),
8939            Arc::new(ForwardingTable::default()),
8940        );
8941        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
8942        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
8943        hello_via_sink(
8944            &handler,
8945            &first_ctx,
8946            &mut first_rx,
8947            hello_frame("first", PROTOCOL_VERSION, 1),
8948        )
8949        .await;
8950        hello_via_sink(
8951            &handler,
8952            &second_ctx,
8953            &mut second_rx,
8954            hello_frame("second", PROTOCOL_VERSION, 2),
8955        )
8956        .await;
8957        let root = unique_project_root("second-only");
8958        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
8959        let cloned = handler.clone();
8960        let open = tokio::spawn(async move {
8961            cloned
8962                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
8963                .await
8964                .unwrap()
8965        });
8966        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
8967            .await
8968            .unwrap()
8969            .unwrap();
8970        let first = query_live_roots(&handler, &first_ctx).await;
8971        let second = query_live_roots(&handler, &second_ctx).await;
8972        assert!(
8973            matches!(
8974                first,
8975                ModuleControlResponseToModule::LiveRoots {
8976                    total_bindings: 0,
8977                    ..
8978                }
8979            ),
8980            "cross-module scope must not expose another module's roots"
8981        );
8982        assert!(
8983            matches!(
8984                second,
8985                ModuleControlResponseToModule::LiveRoots {
8986                    total_bindings: 1,
8987                    ..
8988                }
8989            ),
8990            "second module must see its pending route"
8991        );
8992        handler
8993            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
8994            .await
8995            .unwrap();
8996        assert!(open.await.unwrap().is_empty());
8997    }
8998
8999    #[tokio::test(start_paused = true)]
9000    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
9001        let handler = ControlHandler::with_forwarding(
9002            Arc::new(Registry::default()),
9003            Arc::new(ForwardingTable::default()),
9004        );
9005        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
9006        hello_via_sink(
9007            &handler,
9008            &target_ctx,
9009            &mut target_rx,
9010            hello_frame("target", PROTOCOL_VERSION, 1),
9011        )
9012        .await;
9013        let actual = query_live_roots(&handler, &target_ctx).await;
9014        let ModuleControlResponseToModule::LiveRoots {
9015            roots,
9016            unknown_root_bindings,
9017            total_bindings,
9018        } = actual
9019        else {
9020            panic!("expected live roots")
9021        };
9022        assert!(roots.is_empty());
9023        assert_eq!(unknown_root_bindings, 0);
9024        assert_eq!(total_bindings, 0);
9025        assert_eq!(
9026            total_bindings,
9027            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
9028        );
9029    }
9030
9031    /// Read the vendored fed corpus rather than hand-building a package.
9032    ///
9033    /// A hand-built object encodes what the test author believed the carrier
9034    /// emits. These vectors are what it actually emits, and one of them exists
9035    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
9036    /// ignores additive unknown fields at the traversal emit terminus."
9037    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
9038        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9039            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
9040        let text = std::fs::read_to_string(&path)
9041            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
9042        let vectors: Vec<(String, Value)> = text
9043            .lines()
9044            .filter(|line| !line.trim().is_empty())
9045            .map(|line| {
9046                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
9047                let id = entry["corpus_id"]
9048                    .as_str()
9049                    .expect("every vector carries a corpus_id")
9050                    .to_string();
9051                (id, entry["package"].clone())
9052            })
9053            .collect();
9054        // Pin the count: a corpus that silently shrinks would take its coverage
9055        // with it, and a suite reading N-1 vectors reports the same clean pass
9056        // as one reading N.
9057        assert_eq!(
9058            vectors.len(),
9059            3,
9060            "vendored fed corpus changed size; re-sync from subc-federation"
9061        );
9062
9063        // Pin what makes the corpus DISCRIMINATING, not just present.
9064        //
9065        // The relay test below takes its expected value from the corpus, so the
9066        // corpus supplies the test's power to detect a lossy relay rather than
9067        // its correctness. A relay that dropped unrecognised fields would still
9068        // be caught -- but only by a package carrying fields it does not know.
9069        // Shrink every package to the handful of keys any implementation would
9070        // recognise and the test keeps passing over an input that can no longer
9071        // fail, which is the same clean green as a corpus that shrank away.
9072        //
9073        // So assert the precondition rather than duplicating the packages here:
9074        // at least one vector must carry a field beyond the small common set.
9075        // That is one claim to maintain instead of nine, and it fails loudly if
9076        // a re-sync ever flattens the corpus.
9077        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
9078        let richest = vectors
9079            .iter()
9080            .filter_map(|(_, package)| package.as_object())
9081            .map(|object| {
9082                object
9083                    .keys()
9084                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
9085                    .count()
9086            })
9087            .max()
9088            .unwrap_or(0);
9089        assert!(
9090            richest >= 2,
9091            "vendored corpus no longer carries a package with unmodelled fields, \
9092             so the relay test can no longer distinguish a verbatim relay from a lossy one"
9093        );
9094
9095        vectors
9096    }
9097
9098    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
9099    ///
9100    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
9101    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
9102    /// three-key object, so a relay that quietly dropped fields it did not
9103    /// recognise would satisfy it. These vectors carry nine keys including ones
9104    /// this crate has no type for, so a typed relay fails here and only here.
9105    #[tokio::test]
9106    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
9107        for (corpus_id, package) in fed_admission_facts_vectors() {
9108            let registry = Arc::new(Registry::default());
9109            let forwarding = Arc::new(ForwardingTable::default());
9110            let supervisor = SupervisorHandle::new();
9111            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9112            let handler =
9113                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9114                    .with_supervisor(supervisor)
9115                    .with_admission_facts_config(
9116                        Some("fed".to_string()),
9117                        Some(vec!["target".to_string()]),
9118                    );
9119
9120            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
9121            hello_via_sink(
9122                &handler,
9123                &target_ctx,
9124                &mut target_rx,
9125                hello_frame("target", PROTOCOL_VERSION, 1),
9126            )
9127            .await;
9128
9129            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
9130            let route_handler = handler.clone();
9131            let expected = package.clone();
9132            let route_task = tokio::spawn(async move {
9133                route_handler
9134                    .handle_control_frame(
9135                        &client_ctx,
9136                        route_open_frame_with_admission_facts(
9137                            20,
9138                            "target",
9139                            unique_project_root("admission-facts"),
9140                            Some(subc_control::ConsumerIdentity {
9141                                module_id: "fed".to_string(),
9142                                launch_nonce: "fed-nonce".to_string(),
9143                            }),
9144                            Some(package),
9145                        ),
9146                    )
9147                    .await
9148                    .unwrap()
9149            });
9150
9151            let bind_frame = target_rx.recv().await.unwrap();
9152            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9153            let ModuleControlRequest::RouteBind {
9154                admission_facts, ..
9155            } = bind
9156            else {
9157                panic!("{corpus_id}: expected route.bind")
9158            };
9159            assert_eq!(
9160                admission_facts,
9161                Some(expected),
9162                "{corpus_id}: relay must not add, drop or reshape any field"
9163            );
9164
9165            handler
9166                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9167                .await
9168                .unwrap();
9169            route_task.await.unwrap();
9170        }
9171    }
9172
9173    #[tokio::test]
9174    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
9175        let registry = Arc::new(Registry::default());
9176        let forwarding = Arc::new(ForwardingTable::default());
9177        let supervisor = SupervisorHandle::new();
9178        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9179        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
9180        let handler =
9181            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9182                .with_supervisor(supervisor)
9183                .with_admission_facts_config(
9184                    Some("fed".to_string()),
9185                    Some(vec!["target".to_string()]),
9186                );
9187
9188        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
9189        hello_via_sink(
9190            &handler,
9191            &target_ctx,
9192            &mut target_rx,
9193            hello_frame("target", PROTOCOL_VERSION, 1),
9194        )
9195        .await;
9196        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
9197        hello_via_sink(
9198            &handler,
9199            &other_ctx,
9200            &mut other_rx,
9201            hello_frame("other", PROTOCOL_VERSION, 2),
9202        )
9203        .await;
9204
9205        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
9206        let expected_facts = facts.clone();
9207        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
9208        let route_handler = handler.clone();
9209        let route_task = tokio::spawn(async move {
9210            route_handler
9211                .handle_control_frame(
9212                    &client_ctx,
9213                    route_open_frame_with_admission_facts(
9214                        10,
9215                        "target",
9216                        unique_project_root("admission-facts"),
9217                        Some(subc_control::ConsumerIdentity {
9218                            module_id: "fed".to_string(),
9219                            launch_nonce: "fed-nonce".to_string(),
9220                        }),
9221                        Some(facts.clone()),
9222                    ),
9223                )
9224                .await
9225                .unwrap()
9226        });
9227        let bind_frame = target_rx.recv().await.unwrap();
9228        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9229        let ModuleControlRequest::RouteBind {
9230            admission_facts, ..
9231        } = bind
9232        else {
9233            panic!("expected route.bind")
9234        };
9235        assert_eq!(admission_facts, Some(expected_facts));
9236        handler
9237            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9238            .await
9239            .unwrap();
9240        assert!(route_task.await.unwrap().is_empty());
9241        assert!(matches!(
9242            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
9243                .unwrap(),
9244            ClientControlResponse::RouteOpen { .. }
9245        ));
9246
9247        let direct = handler
9248            .handle_control_frame(
9249                &route_ctx(ConnectionId::new(73)).0,
9250                route_open_frame_with_admission_facts(
9251                    11,
9252                    "target",
9253                    unique_project_root("admission-facts"),
9254                    None,
9255                    Some(json!({"x": 1})),
9256                ),
9257            )
9258            .await
9259            .unwrap();
9260        assert_eq!(
9261            parse_error(&direct[0])["code"],
9262            "admission_facts_not_permitted"
9263        );
9264
9265        let different_reserved = handler
9266            .handle_control_frame(
9267                &route_ctx(ConnectionId::new(77)).0,
9268                route_open_frame_with_admission_facts(
9269                    15,
9270                    "target",
9271                    unique_project_root("admission-facts"),
9272                    Some(subc_control::ConsumerIdentity {
9273                        module_id: "other".to_string(),
9274                        launch_nonce: "other-nonce".to_string(),
9275                    }),
9276                    Some(json!({"x": 1})),
9277                ),
9278            )
9279            .await
9280            .unwrap();
9281        assert_eq!(
9282            parse_error(&different_reserved[0])["code"],
9283            "admission_facts_not_permitted"
9284        );
9285
9286        let other_target = handler
9287            .handle_control_frame(
9288                &route_ctx(ConnectionId::new(74)).0,
9289                route_open_frame_with_admission_facts(
9290                    12,
9291                    "other",
9292                    unique_project_root("admission-facts"),
9293                    Some(subc_control::ConsumerIdentity {
9294                        module_id: "fed".to_string(),
9295                        launch_nonce: "fed-nonce".to_string(),
9296                    }),
9297                    Some(json!({"x": 1})),
9298                ),
9299            )
9300            .await
9301            .unwrap();
9302        assert_eq!(
9303            parse_error(&other_target[0])["code"],
9304            "admission_facts_target_not_allowed"
9305        );
9306
9307        let nonexistent = handler
9308            .handle_control_frame(
9309                &route_ctx(ConnectionId::new(75)).0,
9310                route_open_frame_with_admission_facts(
9311                    13,
9312                    "missing",
9313                    unique_project_root("admission-facts"),
9314                    None,
9315                    Some(json!({"x": 1})),
9316                ),
9317            )
9318            .await
9319            .unwrap();
9320        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
9321
9322        let described = handler
9323            .handle_control_frame(
9324                &route_ctx(ConnectionId::new(76)).0,
9325                Frame::build(
9326                    FrameType::Request,
9327                    control_flags(),
9328                    0,
9329                    0,
9330                    14,
9331                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
9332                )
9333                .unwrap(),
9334            )
9335            .await
9336            .unwrap();
9337        let ClientControlResponse::ServerDescribe { capabilities, .. } =
9338            serde_json::from_slice(&described[0].body).unwrap()
9339        else {
9340            panic!("expected server.describe response")
9341        };
9342        assert!(capabilities
9343            .iter()
9344            .any(|cap| cap == "admission_facts_relay_v1"));
9345    }
9346
9347    #[tokio::test]
9348    async fn admission_facts_without_configured_carrier_are_rejected() {
9349        let registry = Arc::new(Registry::default());
9350        let forwarding = Arc::new(ForwardingTable::default());
9351        let handler = ControlHandler::with_forwarding(registry, forwarding);
9352        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
9353        hello_via_sink(
9354            &handler,
9355            &target_ctx,
9356            &mut target_rx,
9357            hello_frame("target", PROTOCOL_VERSION, 1),
9358        )
9359        .await;
9360
9361        let responses = handler
9362            .handle_control_frame(
9363                &route_ctx(ConnectionId::new(79)).0,
9364                route_open_frame_with_admission_facts(
9365                    16,
9366                    "target",
9367                    unique_project_root("admission-facts"),
9368                    None,
9369                    Some(json!({"x": 1})),
9370                ),
9371            )
9372            .await
9373            .unwrap();
9374        assert_eq!(
9375            parse_error(&responses[0])["code"],
9376            "admission_facts_not_permitted"
9377        );
9378    }
9379
9380    #[tokio::test]
9381    async fn route_open_relays_consumer_capabilities_verbatim() {
9382        let registry = Arc::new(Registry::default());
9383        let forwarding = Arc::new(ForwardingTable::default());
9384        let handler =
9385            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9386        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
9387        hello_via_sink(
9388            &handler,
9389            &module_ctx,
9390            &mut module_rx,
9391            hello_frame("aft", PROTOCOL_VERSION, 7),
9392        )
9393        .await;
9394
9395        let expected = vec!["elicitation".to_string(), "roots".to_string()];
9396        let expected_for_request = expected.clone();
9397        let project_root = unique_project_root("consumer-capabilities-present");
9398        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
9399        let route_handler = handler.clone();
9400        let route_task = tokio::spawn(async move {
9401            route_handler
9402                .handle_control_frame(
9403                    &client_ctx,
9404                    route_open_frame_with_consumer_capabilities(
9405                        401,
9406                        "aft",
9407                        project_root,
9408                        Some(expected_for_request),
9409                    ),
9410                )
9411                .await
9412                .unwrap()
9413        });
9414        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9415            .await
9416            .unwrap()
9417            .unwrap();
9418        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9419        let ModuleControlRequest::RouteBind {
9420            consumer_capabilities,
9421            ..
9422        } = bind
9423        else {
9424            panic!("expected route.bind request, got {bind:?}");
9425        };
9426        assert_eq!(consumer_capabilities, Some(expected.clone()));
9427
9428        handler
9429            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9430            .await
9431            .unwrap();
9432        let route_response = route_task.await.unwrap();
9433        assert!(route_response.is_empty());
9434        let published = client_rx.recv().await.unwrap();
9435        assert!(matches!(
9436            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9437            ClientControlResponse::RouteOpen { .. }
9438        ));
9439    }
9440
9441    #[tokio::test]
9442    async fn route_open_without_consumer_capabilities_relays_none() {
9443        let registry = Arc::new(Registry::default());
9444        let forwarding = Arc::new(ForwardingTable::default());
9445        let handler =
9446            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9447        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
9448        hello_via_sink(
9449            &handler,
9450            &module_ctx,
9451            &mut module_rx,
9452            hello_frame("aft", PROTOCOL_VERSION, 7),
9453        )
9454        .await;
9455
9456        let project_root = unique_project_root("consumer-capabilities-absent");
9457        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
9458        let route_handler = handler.clone();
9459        let route_task = tokio::spawn(async move {
9460            route_handler
9461                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
9462                .await
9463                .unwrap()
9464        });
9465        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9466            .await
9467            .unwrap()
9468            .unwrap();
9469        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9470        let ModuleControlRequest::RouteBind {
9471            consumer_capabilities,
9472            ..
9473        } = bind
9474        else {
9475            panic!("expected route.bind request, got {bind:?}");
9476        };
9477        assert_eq!(consumer_capabilities, None);
9478
9479        handler
9480            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9481            .await
9482            .unwrap();
9483        let route_response = route_task.await.unwrap();
9484        assert!(route_response.is_empty());
9485        let published = client_rx.recv().await.unwrap();
9486        assert!(matches!(
9487            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9488            ClientControlResponse::RouteOpen { .. }
9489        ));
9490    }
9491
9492    /// Opens a route to a freshly registered `aft` with `sent` as the
9493    /// route.open's role_versions, acks the bind, and returns the role_versions
9494    /// the module's bind carried.
9495    async fn bind_role_versions_for(
9496        sent: Option<BTreeMap<String, String>>,
9497        connection: u64,
9498    ) -> Option<BTreeMap<String, String>> {
9499        let registry = Arc::new(Registry::default());
9500        let forwarding = Arc::new(ForwardingTable::default());
9501        let handler =
9502            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9503        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
9504        hello_via_sink(
9505            &handler,
9506            &module_ctx,
9507            &mut module_rx,
9508            hello_frame("aft", PROTOCOL_VERSION, 7),
9509        )
9510        .await;
9511        let project_root = unique_project_root("role-versions");
9512        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
9513        let route_handler = handler.clone();
9514        let route_task = tokio::spawn(async move {
9515            route_handler
9516                .handle_control_frame(
9517                    &client_ctx,
9518                    route_open_frame_with_role_versions(403, "aft", project_root, sent),
9519                )
9520                .await
9521                .unwrap()
9522        });
9523        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9524            .await
9525            .expect("a well-formed route.open reaches the module as a bind")
9526            .unwrap();
9527        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9528        let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
9529            panic!("expected route.bind request, got {bind:?}");
9530        };
9531        handler
9532            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9533            .await
9534            .unwrap();
9535        assert!(route_task.await.unwrap().is_empty());
9536        let published = client_rx.recv().await.unwrap();
9537        assert!(matches!(
9538            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9539            ClientControlResponse::RouteOpen { .. }
9540        ));
9541        role_versions
9542    }
9543
9544    #[tokio::test]
9545    async fn route_open_relays_role_versions_verbatim() {
9546        let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
9547        assert_eq!(
9548            bind_role_versions_for(Some(sent.clone()), 141).await,
9549            Some(sent)
9550        );
9551    }
9552
9553    /// An empty map declares nothing, so the provider sees no field rather
9554    /// than an empty object it would have to treat as a second "none".
9555    #[tokio::test]
9556    async fn route_open_with_empty_or_absent_role_versions_relays_none() {
9557        assert_eq!(bind_role_versions_for(None, 143).await, None);
9558        assert_eq!(
9559            bind_role_versions_for(Some(BTreeMap::new()), 145).await,
9560            None
9561        );
9562    }
9563
9564    #[tokio::test]
9565    async fn route_open_refuses_malformed_role_versions_before_any_bind() {
9566        let registry = Arc::new(Registry::default());
9567        let forwarding = Arc::new(ForwardingTable::default());
9568        let handler =
9569            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9570        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
9571        hello_via_sink(
9572            &handler,
9573            &module_ctx,
9574            &mut module_rx,
9575            hello_frame("aft", PROTOCOL_VERSION, 7),
9576        )
9577        .await;
9578
9579        let nine: BTreeMap<String, String> = (0..9)
9580            .map(|index| (format!("role-{index}"), "v1".to_string()))
9581            .collect();
9582        for (label, malformed) in [
9583            (
9584                "invalid role name",
9585                role_versions(&[("Tool_Provider", "v1")]),
9586            ),
9587            ("invalid version", role_versions(&[("tool-provider", "v0")])),
9588            ("nine entries", nine),
9589        ] {
9590            // A refused open answers at once; one that reached the module
9591            // would wait for its bind ack and trip this timeout.
9592            let responses = tokio::time::timeout(
9593                Duration::from_secs(1),
9594                handler.handle_control_frame(
9595                    &route_ctx(ConnectionId::new(148)).0,
9596                    route_open_frame_with_role_versions(
9597                        404,
9598                        "aft",
9599                        unique_project_root("role-versions-malformed"),
9600                        Some(malformed),
9601                    ),
9602                ),
9603            )
9604            .await
9605            .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
9606            .unwrap();
9607            assert_eq!(responses.len(), 1, "{label}");
9608            assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
9609            let error = parse_error(&responses[0]);
9610            assert_eq!(error["code"], "invalid_request", "{label}: {error}");
9611            assert_eq!(
9612                error["detail"]["field"], "role_versions",
9613                "{label}: {error}"
9614            );
9615            assert!(
9616                !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
9617                "{label}: a malformed declaration is terminal"
9618            );
9619            assert!(
9620                module_rx.try_recv().is_err(),
9621                "{label}: the module must never see a bind"
9622            );
9623        }
9624    }
9625
9626    /// `route-role-versions/v1` is in HELLO_ACK and `server.describe`, so a
9627    /// consumer can tell this daemon forwards the field from one that would
9628    /// drop it.
9629    #[tokio::test]
9630    async fn route_role_versions_capability_is_advertised() {
9631        let handler = ControlHandler::new(Arc::new(Registry::default()));
9632        let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
9633        let ack = hello_via_sink(
9634            &handler,
9635            &ctx,
9636            &mut rx,
9637            hello_frame("m", PROTOCOL_VERSION, 1),
9638        )
9639        .await;
9640        let ack = parse_ack(&ack);
9641        assert!(
9642            ack.subc_capabilities
9643                .iter()
9644                .any(|c| c == "route-role-versions/v1"),
9645            "{:?}",
9646            ack.subc_capabilities
9647        );
9648
9649        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
9650        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
9651        let reply = handler
9652            .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
9653            .await
9654            .unwrap()
9655            .pop()
9656            .unwrap();
9657        let ClientControlResponse::ServerDescribe { capabilities, .. } =
9658            serde_json::from_slice(&reply.body).unwrap()
9659        else {
9660            panic!("not a server.describe reply");
9661        };
9662        assert!(
9663            capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
9664            "{capabilities:?}"
9665        );
9666    }
9667
9668    #[tokio::test]
9669    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
9670        let registry = Arc::new(Registry::default());
9671        let forwarding = Arc::new(ForwardingTable::default());
9672        let handler =
9673            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9674                .with_health_probe_timeout(Duration::from_secs(5));
9675        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
9676        hello_via_sink(
9677            &handler,
9678            &module_ctx,
9679            &mut module_rx,
9680            non_routable_hello_frame_with_control_ops(
9681                "mcp",
9682                300,
9683                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9684            ),
9685        )
9686        .await;
9687        assert!(registry
9688            .get_module("mcp")
9689            .unwrap()
9690            .unwrap()
9691            .manifest
9692            .provides
9693            .is_empty());
9694
9695        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
9696        let route_response = handler
9697            .handle_control_frame(
9698                &route_client_ctx,
9699                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
9700            )
9701            .await
9702            .unwrap();
9703        assert_eq!(route_response[0].header.ty, FrameType::Error);
9704        assert_eq!(
9705            parse_error(&route_response[0])["code"],
9706            "target_unavailable"
9707        );
9708        assert!(parse_error(&route_response[0])["message"]
9709            .as_str()
9710            .unwrap()
9711            .contains("does not provide the requested target"));
9712        assert!(module_rx.try_recv().is_err());
9713
9714        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
9715        let health_handler = handler.clone();
9716        let health_task = tokio::spawn(async move {
9717            health_handler
9718                .handle_control_frame(
9719                    &health_client_ctx,
9720                    supervisor_health_probe_frame(302, "mcp"),
9721                )
9722                .await
9723                .unwrap()
9724        });
9725        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9726            .await
9727            .unwrap()
9728            .unwrap();
9729        assert_eq!(
9730            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
9731            ModuleControlRequest::HealthCheck {}
9732        );
9733        handler
9734            .handle_control_frame(
9735                &module_ctx,
9736                health_response(health_frame.header.corr, HealthStatus::Ok),
9737            )
9738            .await
9739            .unwrap();
9740        let health_response = health_task.await.unwrap();
9741        assert_eq!(health_response[0].header.ty, FrameType::Response);
9742        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
9743            ClientControlResponse::SupervisorHealthProbe {
9744                module_id, status, ..
9745            } => {
9746                assert_eq!(module_id, "mcp");
9747                assert_eq!(status, HealthStatus::Ok);
9748            }
9749            other => panic!("unexpected health response: {other:?}"),
9750        }
9751
9752        // Exercise the forwarding cleanup path directly while leaving the registry
9753        // advertisement in place. If cleanup leaves a stale control sink behind,
9754        // the next probe will enqueue onto it and wait for the long probe timeout
9755        // instead of returning an immediate no-connection error.
9756        forwarding
9757            .cleanup_connection(module_ctx.connection_id)
9758            .unwrap();
9759        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
9760        let cleanup_response = tokio::time::timeout(
9761            Duration::from_millis(200),
9762            handler.handle_control_frame(
9763                &cleanup_probe_ctx,
9764                supervisor_health_probe_frame(303, "mcp"),
9765            ),
9766        )
9767        .await
9768        .expect("probe should fail immediately when the control lane is gone")
9769        .unwrap();
9770        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
9771        assert_eq!(
9772            parse_error(&cleanup_response[0])["code"],
9773            "target_unavailable"
9774        );
9775        assert!(parse_error(&cleanup_response[0])["message"]
9776            .as_str()
9777            .unwrap()
9778            .contains("no module connection"));
9779
9780        handler
9781            .cleanup_connection(module_ctx.connection_id)
9782            .unwrap();
9783    }
9784
9785    #[tokio::test]
9786    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
9787        let registry = Arc::new(Registry::default());
9788        let supervisor_handle = SupervisorHandle::new();
9789        let supervisor =
9790            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
9791                .with_handle(supervisor_handle.clone())
9792                .with_connection_file_path(
9793                    std::env::temp_dir()
9794                        .join(format!("subc-route-open-warming-{}", std::process::id())),
9795                );
9796        let module = supervisor
9797            .supervise_configured(
9798                ModuleSpec {
9799                    module_id: "warming".to_string(),
9800                    program: fake_aft_stub_path(),
9801                    args: Vec::new(),
9802                    env: Vec::new(),
9803                    reserved: false,
9804                    reserved_prefixes: Vec::new(),
9805                    protocol: ModuleProtocol::Subc,
9806                    overlap: Default::default(),
9807                },
9808                true,
9809            )
9810            .unwrap();
9811        assert_eq!(module.state().unwrap(), ModuleState::Running);
9812
9813        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9814        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
9815        let response = handler
9816            .handle_control_frame(
9817                &ctx,
9818                route_open_frame(304, "warming", unique_project_root("warming")),
9819            )
9820            .await
9821            .unwrap();
9822        module.stop().await.unwrap();
9823
9824        assert_eq!(response[0].header.ty, FrameType::Error);
9825        let error = parse_error(&response[0]);
9826        assert_eq!(error["code"], "module_warming");
9827        assert!(error["message"]
9828            .as_str()
9829            .unwrap()
9830            .contains("state=running, enabled=true, live=false"));
9831    }
9832
9833    #[test]
9834    fn route_open_connection_cap_logs_admission_reason_and_capacity() {
9835        let handler = ControlHandler::new(Arc::new(Registry::default()));
9836        let capture = EventCapture::default();
9837        let _subscriber =
9838            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9839        let (ctx, _rx) = route_ctx(ConnectionId::new(96));
9840        let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
9841        let pending = (0..limit).collect::<Vec<_>>();
9842        let response = handler
9843            .route_open_capacity_refusal(
9844                &ctx,
9845                &route_open_frame(396, "busy", unique_project_root("connection-cap")),
9846                "busy",
9847                pending.len(),
9848                limit,
9849            )
9850            .unwrap();
9851        assert_eq!(parse_error(&response)["code"], "target_unavailable");
9852        let event = capture
9853            .events()
9854            .into_iter()
9855            .find(|event| {
9856                event.target == "control"
9857                    && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
9858            })
9859            .expect("connection admission refusal event");
9860        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
9861        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
9862    }
9863
9864    #[test]
9865    fn route_open_target_cap_logs_admission_reason_and_capacity() {
9866        let handler = ControlHandler::new(Arc::new(Registry::default()));
9867        let capture = EventCapture::default();
9868        let _subscriber =
9869            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9870        let (ctx, _rx) = route_ctx(ConnectionId::new(97));
9871        let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
9872        let guards = (0..limit)
9873            .map(|_| {
9874                handler
9875                    .route_bind_concurrency
9876                    .try_admit("busy", limit)
9877                    .unwrap()
9878            })
9879            .collect::<Vec<_>>();
9880        let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
9881            Err(in_flight) => in_flight,
9882            Ok(_) => panic!("target cap must refuse after {limit} admissions"),
9883        };
9884        let response = handler
9885            .route_open_target_capacity_refusal(
9886                &ctx,
9887                &route_open_frame(397, "busy", unique_project_root("target-cap")),
9888                "busy",
9889                in_flight,
9890            )
9891            .unwrap();
9892        assert_eq!(parse_error(&response)["code"], "target_unavailable");
9893        let event = capture
9894            .events()
9895            .into_iter()
9896            .find(|event| {
9897                event.target == "control"
9898                    && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
9899            })
9900            .expect("target admission refusal event");
9901        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
9902        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
9903        drop(guards);
9904    }
9905
9906    /// One wire code has several senders, so the refusal line names the check
9907    /// that refused. This drives the shared refusal path for ordinary refusals
9908    /// with an unregistered
9909    /// target and requires the branch label on the event.
9910    #[tokio::test]
9911    async fn route_open_refusal_names_the_check_that_refused() {
9912        let handler = ControlHandler::new(Arc::new(Registry::default()));
9913        let capture = EventCapture::default();
9914        let _subscriber =
9915            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9916        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
9917        let response = handler
9918            .handle_control_frame(
9919                &ctx,
9920                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
9921            )
9922            .await
9923            .unwrap();
9924
9925        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9926        let event = capture
9927            .events()
9928            .into_iter()
9929            .find(|event| {
9930                event.target == "control"
9931                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
9932            })
9933            .expect("route.open refusal event");
9934        assert_eq!(
9935            event.fields.get("reason"),
9936            Some(&"\"not_registered\"".to_string())
9937        );
9938    }
9939
9940    #[tokio::test]
9941    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
9942        let registry = Arc::new(Registry::default());
9943        let supervisor_handle = SupervisorHandle::new();
9944        let supervisor =
9945            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
9946                .with_handle(supervisor_handle.clone())
9947                .with_connection_file_path(std::env::temp_dir().join(format!(
9948                    "subc-route-open-refusal-info-{}",
9949                    std::process::id()
9950                )));
9951        let module = supervisor
9952            .supervise_configured(
9953                ModuleSpec {
9954                    module_id: "warming".to_string(),
9955                    program: fake_aft_stub_path(),
9956                    args: Vec::new(),
9957                    env: Vec::new(),
9958                    reserved: false,
9959                    reserved_prefixes: Vec::new(),
9960                    protocol: ModuleProtocol::Subc,
9961                    overlap: Default::default(),
9962                },
9963                true,
9964            )
9965            .unwrap();
9966        assert_eq!(module.state().unwrap(), ModuleState::Running);
9967
9968        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9969        assert!(handler
9970            .counters()
9971            .snapshot()
9972            .get("route_open_refused_by_code")
9973            .is_none());
9974        let capture = EventCapture::default();
9975        let _subscriber =
9976            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9977        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
9978        let response = handler
9979            .handle_control_frame(
9980                &ctx,
9981                route_open_frame(394, "warming", unique_project_root("refusal-info")),
9982            )
9983            .await
9984            .unwrap();
9985        module.stop().await.unwrap();
9986
9987        assert_eq!(parse_error(&response[0])["code"], "module_warming");
9988        let event = capture
9989            .events()
9990            .into_iter()
9991            .find(|event| {
9992                event.target == "control"
9993                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
9994            })
9995            .expect("route.open refusal event");
9996        assert_eq!(
9997            event.fields.get("module_id"),
9998            Some(&"\"warming\"".to_string())
9999        );
10000        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
10001        assert_eq!(
10002            event.fields.get("reason"),
10003            Some(&"\"supervised_not_registered\"".to_string())
10004        );
10005        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
10006        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
10007        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
10008        assert_eq!(
10009            handler.counters().snapshot()["route_open_refused_by_code"],
10010            json!({ "module_warming": 1 })
10011        );
10012    }
10013
10014    const OUTAGE_START: &str = "route.open refusing module: not serving";
10015    const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
10016
10017    fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
10018        capture
10019            .events()
10020            .into_iter()
10021            .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
10022            .collect()
10023    }
10024
10025    fn supervise_stub(
10026        registry: &Arc<Registry>,
10027        module_id: &str,
10028        enabled: bool,
10029    ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
10030        let supervisor_handle = SupervisorHandle::new();
10031        let supervisor =
10032            Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
10033                .with_handle(supervisor_handle.clone())
10034                .with_connection_file_path(std::env::temp_dir().join(format!(
10035                    "subc-route-outage-{module_id}-{}",
10036                    std::process::id()
10037                )));
10038        let module = supervisor
10039            .supervise_configured(
10040                ModuleSpec {
10041                    module_id: module_id.to_string(),
10042                    program: fake_aft_stub_path(),
10043                    args: Vec::new(),
10044                    env: Vec::new(),
10045                    reserved: false,
10046                    reserved_prefixes: Vec::new(),
10047                    protocol: ModuleProtocol::Subc,
10048                    overlap: Default::default(),
10049                },
10050                enabled,
10051            )
10052            .unwrap();
10053        (supervisor_handle, module)
10054    }
10055
10056    fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
10057        let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
10058            module_id: module_id.to_string(),
10059            drain_timeout_ms: Some(50),
10060        })
10061        .unwrap();
10062        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
10063    }
10064
10065    /// Two handlers built over one forwarding table must share one outage
10066    /// tracker; separate trackers would each log their own opening line for
10067    /// the same outage.
10068    #[test]
10069    fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
10070        let registry = Arc::new(Registry::default());
10071        let forwarding = Arc::new(ForwardingTable::default());
10072        let first = ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10073        let second = ControlHandler::with_forwarding(registry, forwarding);
10074        assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
10075    }
10076
10077    /// A client can name any module id it likes. Refusing an unknown one,
10078    /// however often, must not create outage state or outage lines, or the
10079    /// tracker would be a memory sink any client could fill.
10080    #[tokio::test(flavor = "current_thread")]
10081    async fn route_open_unknown_module_refusals_add_no_outage_state() {
10082        let handler = ControlHandler::new(Arc::new(Registry::default()));
10083        let capture = EventCapture::default();
10084        let _subscriber =
10085            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10086        let (ctx, _rx) = route_ctx(ConnectionId::new(90));
10087        for corr in 0..8 {
10088            let response = handler
10089                .handle_control_frame(
10090                    &ctx,
10091                    route_open_frame(
10092                        380 + corr,
10093                        &format!("nobody-{corr}"),
10094                        unique_project_root("outage-unknown"),
10095                    ),
10096                )
10097                .await
10098                .unwrap();
10099            assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10100        }
10101
10102        assert_eq!(handler.route_outages.tracked_module_count(), 0);
10103        assert!(outage_lines(&capture, OUTAGE_START).is_empty());
10104        assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
10105    }
10106
10107    /// Drives the refusal path end to end: a supervised module that served
10108    /// before and stopped being registered with no instruction to stop is a
10109    /// WARN, and the same module refused after an operator `supervisor.restart`
10110    /// is an INFO.
10111    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10112    async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
10113        let registry = Arc::new(Registry::default());
10114        let (supervisor_handle, module) = supervise_stub(&registry, "outage-restart", true);
10115        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
10116        let capture = EventCapture::default();
10117        let _subscriber =
10118            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10119        let (ctx, _rx) = route_ctx(ConnectionId::new(91));
10120        // The stub never registers, so pretend it served once: otherwise every
10121        // refusal would fall in its startup window.
10122        handler.route_outages.record_accepted("outage-restart");
10123
10124        let response = handler
10125            .handle_control_frame(
10126                &ctx,
10127                route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
10128            )
10129            .await
10130            .unwrap();
10131        assert_eq!(response[0].header.ty, FrameType::Error);
10132        let starts = outage_lines(&capture, OUTAGE_START);
10133        assert_eq!(starts.len(), 1, "{starts:?}");
10134        assert_eq!(starts[0].level, tracing::Level::WARN);
10135        assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
10136        assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
10137        assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
10138        handler.route_outages.record_accepted("outage-restart");
10139        assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
10140
10141        let restart = handler
10142            .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
10143            .await
10144            .unwrap();
10145        assert_eq!(
10146            restart[0].header.ty,
10147            FrameType::Response,
10148            "{:?}",
10149            parse_error(&restart[0])
10150        );
10151        handler
10152            .handle_control_frame(
10153                &ctx,
10154                route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
10155            )
10156            .await
10157            .unwrap();
10158        module.stop().await.unwrap();
10159
10160        let starts = outage_lines(&capture, OUTAGE_START);
10161        assert_eq!(starts.len(), 2, "{starts:?}");
10162        assert_eq!(starts[1].level, tracing::Level::INFO);
10163        assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
10164    }
10165
10166    /// A restart refused before it touched the module (here: the module is
10167    /// disabled) must clear its operator mark, so the next real outage is
10168    /// still reported as a warning.
10169    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10170    async fn failed_operator_restart_leaves_no_operator_mark() {
10171        let registry = Arc::new(Registry::default());
10172        let (supervisor_handle, _module) = supervise_stub(&registry, "outage-disabled", false);
10173        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
10174        let capture = EventCapture::default();
10175        let _subscriber =
10176            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10177        let (ctx, _rx) = route_ctx(ConnectionId::new(92));
10178        handler.route_outages.record_accepted("outage-disabled");
10179
10180        let restart = handler
10181            .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
10182            .await
10183            .unwrap();
10184        assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
10185        assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
10186
10187        handler
10188            .handle_control_frame(
10189                &ctx,
10190                route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
10191            )
10192            .await
10193            .unwrap();
10194        let starts = outage_lines(&capture, OUTAGE_START);
10195        assert_eq!(starts.len(), 1, "{starts:?}");
10196        assert_eq!(starts[0].level, tracing::Level::WARN);
10197    }
10198
10199    #[tokio::test(flavor = "current_thread")]
10200    async fn route_open_unknown_module_escapes_target_module_id() {
10201        let handler = ControlHandler::new(Arc::new(Registry::default()));
10202        let capture = EventCapture::default();
10203        let _subscriber =
10204            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10205        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
10206        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
10207        let response = handler
10208            .handle_control_frame(
10209                &ctx,
10210                route_open_frame(
10211                    395,
10212                    hostile_module_id,
10213                    unique_project_root("hostile-target-module-id"),
10214                ),
10215            )
10216            .await
10217            .unwrap();
10218
10219        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10220        let event = capture
10221            .events()
10222            .into_iter()
10223            .find(|event| {
10224                event.target == "control"
10225                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
10226            })
10227            .expect("route.open unknown-module refusal event");
10228        let logged = event.fields.get("module_id").expect("module_id field");
10229        assert!(!logged.bytes().any(|byte| byte < 0x20));
10230        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
10231    }
10232
10233    #[tokio::test(flavor = "current_thread")]
10234    async fn route_open_module_rejection_uses_daemon_counter_key() {
10235        let registry = Arc::new(Registry::default());
10236        let forwarding = Arc::new(ForwardingTable::default());
10237        let handler =
10238            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10239        let module_connection = ConnectionId::new(95);
10240        let (module_ctx, mut module_rx) = route_ctx(module_connection);
10241        hello_via_sink(
10242            &handler,
10243            &module_ctx,
10244            &mut module_rx,
10245            hello_frame("aft", PROTOCOL_VERSION, 395),
10246        )
10247        .await;
10248
10249        let client_connection = ConnectionId::new(96);
10250        let (client_ctx, _client_rx) = route_ctx(client_connection);
10251        let capture = EventCapture::default();
10252        let _subscriber =
10253            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10254        let (route_task, bind) = relay_route_open(
10255            &handler,
10256            client_connection,
10257            &client_ctx.egress,
10258            &mut module_rx,
10259            396,
10260            "aft",
10261            "hostile-module-code",
10262        )
10263        .await;
10264        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
10265        let rejection = Frame::build(
10266            FrameType::Error,
10267            control_flags(),
10268            0,
10269            0,
10270            bind.header.corr,
10271            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
10272        )
10273        .unwrap();
10274        handler
10275            .handle_control_frame(&module_ctx, rejection)
10276            .await
10277            .unwrap();
10278
10279        let response = route_task.await.unwrap();
10280        assert_eq!(parse_error(&response[0])["code"], hostile_code);
10281        let counters = handler.counters().snapshot();
10282        assert_eq!(
10283            counters["route_open_refused_by_code"],
10284            json!({ "module_rejected": 1 })
10285        );
10286        assert!(counters["route_open_refused_by_code"]
10287            .get(hostile_code)
10288            .is_none());
10289
10290        let event = capture
10291            .events()
10292            .into_iter()
10293            .find(|event| {
10294                event.target == "control"
10295                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
10296            })
10297            .expect("route.open module-rejection refusal event");
10298        let logged = event.fields.get("module_code").expect("module_code field");
10299        assert!(!logged.bytes().any(|byte| byte < 0x20));
10300        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
10301    }
10302
10303    #[tokio::test]
10304    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
10305        let registry = Arc::new(Registry::default());
10306        let supervisor_handle = SupervisorHandle::new();
10307        let missing_program = std::env::temp_dir().join(format!(
10308            "subc-route-open-missing-program-{}",
10309            std::process::id()
10310        ));
10311        let supervisor =
10312            Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
10313                .with_handle(supervisor_handle.clone());
10314        let module = supervisor
10315            .supervise_configured(
10316                ModuleSpec {
10317                    module_id: "failed".to_string(),
10318                    program: missing_program,
10319                    args: Vec::new(),
10320                    env: Vec::new(),
10321                    reserved: false,
10322                    reserved_prefixes: Vec::new(),
10323                    protocol: ModuleProtocol::Subc,
10324                    overlap: Default::default(),
10325                },
10326                true,
10327            )
10328            .unwrap();
10329        assert_eq!(module.state().unwrap(), ModuleState::Failed);
10330
10331        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
10332        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
10333        let response = handler
10334            .handle_control_frame(
10335                &ctx,
10336                route_open_frame(305, "failed", unique_project_root("failed")),
10337            )
10338            .await
10339            .unwrap();
10340
10341        assert_eq!(response[0].header.ty, FrameType::Error);
10342        let error = parse_error(&response[0]);
10343        assert_eq!(error["code"], "target_unavailable");
10344        assert!(error["message"]
10345            .as_str()
10346            .unwrap()
10347            .contains("state=failed, enabled=true, live=false"));
10348    }
10349
10350    #[tokio::test]
10351    async fn route_open_role_mismatch_remains_target_unavailable() {
10352        let registry = Arc::new(Registry::default());
10353        let handler = ControlHandler::new(Arc::clone(&registry));
10354        handler
10355            .handle_control(
10356                ConnectionId::new(41),
10357                non_routable_hello_frame_with_control_ops("health-only", 306, None),
10358            )
10359            .unwrap();
10360
10361        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
10362        let response = handler
10363            .handle_control_frame(
10364                &ctx,
10365                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
10366            )
10367            .await
10368            .unwrap();
10369
10370        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10371        assert!(parse_error(&response[0])["message"]
10372            .as_str()
10373            .unwrap()
10374            .contains("does not provide the requested target"));
10375    }
10376
10377    #[tokio::test]
10378    async fn route_open_inactive_registration_remains_target_unavailable() {
10379        let registry = Arc::new(Registry::default());
10380        let handler = ControlHandler::new(Arc::clone(&registry));
10381        handler
10382            .handle_control(
10383                ConnectionId::new(43),
10384                hello_frame("inactive", PROTOCOL_VERSION, 308),
10385            )
10386            .unwrap();
10387        assert!(registry
10388            .set_module_state_for_test("inactive", ChannelState::Closed)
10389            .unwrap());
10390
10391        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
10392        let response = handler
10393            .handle_control_frame(
10394                &ctx,
10395                route_open_frame(309, "inactive", unique_project_root("inactive")),
10396            )
10397            .await
10398            .unwrap();
10399
10400        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10401        assert!(parse_error(&response[0])["message"]
10402            .as_str()
10403            .unwrap()
10404            .contains("is not active"));
10405    }
10406
10407    #[tokio::test]
10408    async fn late_health_reply_is_recorded_through_the_module_response_path() {
10409        let registry = Arc::new(Registry::default());
10410        let forwarding = Arc::new(ForwardingTable::default());
10411        let supervisor_handle = SupervisorHandle::new();
10412        let supervisor =
10413            Supervisor::new_for_test(Arc::clone(&registry), crate::RestartPolicy::default())
10414                .with_forwarding(Arc::clone(&forwarding))
10415                .with_handle(supervisor_handle.clone());
10416        let module = supervisor
10417            .supervise_configured(
10418                crate::ModuleSpec {
10419                    module_id: "late-health-response".to_string(),
10420                    program: PathBuf::from("disabled-module"),
10421                    args: Vec::new(),
10422                    env: Vec::new(),
10423                    reserved: false,
10424                    reserved_prefixes: Vec::new(),
10425                    protocol: ModuleProtocol::Subc,
10426                    overlap: Default::default(),
10427                },
10428                false,
10429            )
10430            .unwrap();
10431        let handler =
10432            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10433                .with_supervisor(supervisor_handle);
10434        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
10435        handler
10436            .handle_control_frame(
10437                &module_ctx,
10438                hello_frame_with_control_ops(
10439                    "late-health-response",
10440                    PROTOCOL_VERSION,
10441                    7,
10442                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10443                ),
10444            )
10445            .await
10446            .unwrap();
10447        let probe_started_at = Instant::now() - Duration::from_millis(80);
10448        let pending = forwarding
10449            .begin_health_probe_rpc_for(
10450                "late-health-response",
10451                MODULE_CONTROL_OP_HEALTH_CHECK,
10452                probe_started_at,
10453                Instant::now() - Duration::from_millis(1),
10454            )
10455            .unwrap();
10456        assert!(forwarding
10457            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
10458            .unwrap());
10459
10460        let responses = handler
10461            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
10462            .await
10463            .unwrap();
10464
10465        assert!(responses.is_empty());
10466        let health = module.status().unwrap().health;
10467        assert_eq!(health.late_answer_count, 1);
10468        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
10469    }
10470
10471    #[tokio::test]
10472    async fn health_probe_timeout_and_module_death_are_typed() {
10473        let registry = Arc::new(Registry::default());
10474        let forwarding = Arc::new(ForwardingTable::default());
10475        let handler =
10476            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10477                .with_health_probe_timeout(Duration::from_millis(50));
10478        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
10479        hello_via_sink(
10480            &handler,
10481            &module_ctx,
10482            &mut module_rx,
10483            hello_frame_with_control_ops(
10484                "aft",
10485                PROTOCOL_VERSION,
10486                7,
10487                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10488            ),
10489        )
10490        .await;
10491
10492        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10493        let responses = handler
10494            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
10495            .await
10496            .unwrap();
10497        assert_eq!(responses[0].header.ty, FrameType::Error);
10498        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
10499        let _ = module_rx.try_recv();
10500
10501        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
10502        let health_handler = handler.clone();
10503        let death_task = tokio::spawn(async move {
10504            health_handler
10505                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
10506                .await
10507                .unwrap()
10508        });
10509        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10510            .await
10511            .unwrap()
10512            .unwrap();
10513        handler
10514            .cleanup_connection(module_ctx.connection_id)
10515            .unwrap();
10516        let responses = death_task.await.unwrap();
10517        assert_eq!(responses[0].header.ty, FrameType::Error);
10518        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
10519    }
10520
10521    #[test]
10522    fn hello_requires_exact_protocol_version() {
10523        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
10524            let registry = Arc::new(Registry::default());
10525            let handler = ControlHandler::new(Arc::clone(&registry));
10526            let responses = handler
10527                .handle_control(
10528                    ConnectionId::new(connection),
10529                    hello_frame("aft", offered, 9),
10530                )
10531                .unwrap();
10532
10533            assert_eq!(responses.len(), 1);
10534            assert_eq!(responses[0].header.ty, FrameType::Error);
10535            let error = parse_error(&responses[0]);
10536            assert_eq!(error["code"], "version_unsupported");
10537            assert!(registry.get_module("aft").unwrap().is_none());
10538            assert_eq!(registry.active_registration_count().unwrap(), 0);
10539        }
10540    }
10541
10542    #[test]
10543    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
10544        let registry = Arc::new(Registry::default());
10545        let forwarding = Arc::new(ForwardingTable::default());
10546        let handler =
10547            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10548        let module_connection = ConnectionId::new(301);
10549        let registration = registry
10550            .register_with_control_ops(
10551                manifest("aft-push", PROTOCOL_VERSION),
10552                PROTOCOL_VERSION,
10553                module_connection,
10554                module_baseline_control_ops(),
10555            )
10556            .unwrap();
10557        let (module_tx, _module_rx) = mpsc::channel(8);
10558        let endpoint = forwarding
10559            .register_module_connection(
10560                module_connection,
10561                "aft-push".to_string(),
10562                PROTOCOL_VERSION,
10563                manifest_concurrency(&registration.manifest),
10564                FrameSink::new(module_tx),
10565            )
10566            .unwrap();
10567
10568        // A push op this version does not know is ignored (forward-compat), not errored.
10569        let unknown = Frame::build(
10570            FrameType::Push,
10571            control_flags(),
10572            0,
10573            0,
10574            5,
10575            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
10576        )
10577        .unwrap();
10578        let out = handler.handle_status_update(endpoint, unknown).unwrap();
10579        assert!(
10580            out.is_empty(),
10581            "unknown push op must be ignored, got {out:?}"
10582        );
10583
10584        // A malformed body for a KNOWN op is a real error worth surfacing.
10585        let malformed = Frame::build(
10586            FrameType::Push,
10587            control_flags(),
10588            0,
10589            0,
10590            6,
10591            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
10592        )
10593        .unwrap();
10594        let out = handler.handle_status_update(endpoint, malformed).unwrap();
10595        assert_eq!(out.len(), 1);
10596        assert_eq!(out[0].header.ty, FrameType::Error);
10597        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
10598    }
10599
10600    #[test]
10601    fn hello_rejected_when_connection_already_owns_client_routes() {
10602        let registry = Arc::new(Registry::default());
10603        let forwarding = Arc::new(ForwardingTable::default());
10604        let handler =
10605            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10606        // Commits a client route on connection 202 (bound to a module on conn 101).
10607        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
10608        let client_connection = ConnectionId::new(202);
10609
10610        // That same connection now tries to register as a module: rejected, so one
10611        // connection never holds both client-route and module-endpoint state.
10612        let responses = handler
10613            .handle_control(
10614                client_connection,
10615                hello_frame("aft-second", PROTOCOL_VERSION, 9),
10616            )
10617            .unwrap();
10618        assert_eq!(responses[0].header.ty, FrameType::Error);
10619        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
10620        assert!(registry.get_module("aft-second").unwrap().is_none());
10621    }
10622
10623    #[tokio::test]
10624    async fn second_hello_preserves_registration_routes_and_launch_nonce() {
10625        let registry = Arc::new(Registry::default());
10626        let forwarding = Arc::new(ForwardingTable::default());
10627        let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
10628        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
10629        hello_via_sink(
10630            &handler,
10631            &module_ctx,
10632            &mut module_rx,
10633            hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
10634        )
10635        .await;
10636        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
10637        let pending = forwarding
10638            .begin_route_bind_relay_for_test(
10639                client_ctx.connection_id,
10640                client_ctx.egress.clone(),
10641                2,
10642                "alpha",
10643            )
10644            .unwrap();
10645        forwarding
10646            .complete_pending_relay(
10647                module_ctx.connection_id,
10648                pending.corr,
10649                RouteBindRelayOutcome::Accepted,
10650            )
10651            .unwrap();
10652        client_rx.try_recv().unwrap();
10653        for module_id in ["beta", "alpha"] {
10654            let replies = handler
10655                .handle_control_frame(
10656                    &module_ctx,
10657                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
10658                )
10659                .await
10660                .unwrap();
10661            assert_eq!(replies.len(), 1, "second HELLO must be refused");
10662            assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
10663        }
10664        assert_eq!(registry.list_modules().unwrap().1.len(), 1);
10665        assert!(registry.get_module("beta").unwrap().is_none());
10666        assert!(matches!(
10667            forwarding
10668                .lookup_data_route(
10669                    client_ctx.connection_id,
10670                    pending.client_channel,
10671                    pending.client_epoch,
10672                )
10673                .unwrap(),
10674            DataRoute::Client(DataRouteState::Bound(_))
10675        ));
10676        assert!(handler
10677            .hello_launch_nonces
10678            .lock()
10679            .unwrap()
10680            .presented(module_ctx.connection_id, Some("alpha-nonce")));
10681        assert!(module_rx.try_recv().is_err());
10682    }
10683
10684    #[test]
10685    fn reserved_module_hello_requires_matching_launch_nonce() {
10686        let registry = Arc::new(Registry::default());
10687        let supervisor = SupervisorHandle::new();
10688        // The supervisor recorded the nonce it injected when it spawned the reserved
10689        // module; the HELLO verifier checks against the same shared handle.
10690        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
10691        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
10692
10693        // A HELLO with NO nonce is rejected.
10694        let no_nonce = handler
10695            .handle_control(
10696                ConnectionId::new(1),
10697                hello_frame("vault", PROTOCOL_VERSION, 1),
10698            )
10699            .unwrap();
10700        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
10701        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
10702        assert!(registry.get_module("vault").unwrap().is_none());
10703
10704        // A HELLO with the WRONG nonce is rejected.
10705        let wrong = handler
10706            .handle_control(
10707                ConnectionId::new(2),
10708                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
10709            )
10710            .unwrap();
10711        assert_eq!(wrong[0].header.ty, FrameType::Error);
10712        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
10713        assert!(registry.get_module("vault").unwrap().is_none());
10714
10715        // A HELLO with the CORRECT nonce registers.
10716        let ok = handler
10717            .handle_control(
10718                ConnectionId::new(3),
10719                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
10720            )
10721            .unwrap();
10722        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
10723        assert!(registry.get_module("vault").unwrap().is_some());
10724    }
10725
10726    #[test]
10727    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
10728        let registry = Arc::new(Registry::default());
10729        let supervisor = SupervisorHandle::new();
10730        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
10731        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
10732        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
10733
10734        let squat = handler
10735            .handle_control(
10736                ConnectionId::new(1),
10737                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
10738            )
10739            .unwrap();
10740        assert_eq!(squat[0].header.ty, FrameType::Error);
10741        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
10742        assert!(parse_error(&squat[0])["message"]
10743            .as_str()
10744            .unwrap()
10745            .contains("fed:"));
10746
10747        let accepted_peer = handler
10748            .handle_control(
10749                ConnectionId::new(2),
10750                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
10751            )
10752            .unwrap();
10753        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
10754
10755        let accepted_short = handler
10756            .handle_control(
10757                ConnectionId::new(3),
10758                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
10759            )
10760            .unwrap();
10761        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
10762
10763        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
10764            let response = handler
10765                .handle_control(
10766                    ConnectionId::new(conn),
10767                    hello_frame(module_id, PROTOCOL_VERSION, conn),
10768                )
10769                .unwrap();
10770            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
10771        }
10772    }
10773
10774    #[test]
10775    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
10776        let registry = Arc::new(Registry::default());
10777        let supervisor = SupervisorHandle::new();
10778        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
10779        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
10780        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
10781        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
10782
10783        let owner_nonce = handler
10784            .handle_control(
10785                ConnectionId::new(1),
10786                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
10787            )
10788            .unwrap();
10789        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
10790        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
10791        assert!(registry.get_module("fed:special").unwrap().is_none());
10792
10793        let exact_nonce = handler
10794            .handle_control(
10795                ConnectionId::new(2),
10796                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
10797            )
10798            .unwrap();
10799        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
10800        assert!(registry.get_module("fed:special").unwrap().is_some());
10801    }
10802
10803    #[test]
10804    fn non_reserved_module_ignores_launch_nonce() {
10805        let registry = Arc::new(Registry::default());
10806        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
10807        // registration succeeds whether a spawned process echoes a nonce or not.
10808        let handler = ControlHandler::new(Arc::clone(&registry));
10809        let no_nonce = handler
10810            .handle_control(
10811                ConnectionId::new(1),
10812                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
10813            )
10814            .unwrap();
10815        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
10816        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
10817
10818        let echoed_nonce = handler
10819            .handle_control(
10820                ConnectionId::new(2),
10821                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
10822            )
10823            .unwrap();
10824        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
10825        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
10826    }
10827
10828    #[test]
10829    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
10830        let handler = ControlHandler::default();
10831        let conn = ConnectionId::new(1);
10832        let malformed = Frame::build(
10833            FrameType::Hello,
10834            control_flags(),
10835            0,
10836            0,
10837            3,
10838            b"{not json".to_vec(),
10839        )
10840        .unwrap();
10841
10842        let error = handler.handle_control(conn, malformed).unwrap();
10843        assert_eq!(error[0].header.ty, FrameType::Error);
10844        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
10845
10846        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
10847        let pong = handler.handle_control(conn, ping).unwrap();
10848        assert_eq!(pong[0].header.ty, FrameType::Pong);
10849        assert_eq!(pong[0].header.corr, 4);
10850    }
10851
10852    #[test]
10853    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
10854        let registry = Arc::new(Registry::default());
10855        let handler = ControlHandler::new(Arc::clone(&registry));
10856
10857        handler
10858            .handle_control(
10859                ConnectionId::new(1),
10860                hello_frame("aft", PROTOCOL_VERSION, 1),
10861            )
10862            .unwrap();
10863        let duplicate = handler
10864            .handle_control(
10865                ConnectionId::new(2),
10866                hello_frame("aft", PROTOCOL_VERSION, 2),
10867            )
10868            .unwrap();
10869
10870        assert_eq!(duplicate[0].header.ty, FrameType::Error);
10871        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10872        let registration = registry.get_module("aft").unwrap().unwrap();
10873        assert_eq!(registration.connection_id, ConnectionId::new(1));
10874    }
10875
10876    #[test]
10877    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
10878        let registry = Arc::new(Registry::default());
10879        let forwarding = Arc::new(ForwardingTable::default());
10880        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
10881        let handler =
10882            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10883                .with_process_liveness(process_liveness);
10884        let (ctx, route_channel, route_epoch) =
10885            bind_liveness_route(&registry, &forwarding, "aft-dead");
10886        let responses = handler
10887            .handle_route_poll(
10888                &ctx,
10889                route_poll_frame(41, PollKind::Liveness, route_channel),
10890                route_channel,
10891                route_epoch,
10892                PollKind::Liveness,
10893            )
10894            .unwrap();
10895
10896        assert_eq!(responses.len(), 1);
10897        assert_eq!(responses[0].header.ty, FrameType::Response);
10898        assert_route_poll_liveness(&responses[0], false);
10899    }
10900
10901    #[test]
10902    fn liveness_poll_without_process_source_uses_bound_route() {
10903        let registry = Arc::new(Registry::default());
10904        let forwarding = Arc::new(ForwardingTable::default());
10905        let handler =
10906            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10907        let (ctx, route_channel, route_epoch) =
10908            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
10909        let responses = handler
10910            .handle_route_poll(
10911                &ctx,
10912                route_poll_frame(42, PollKind::Liveness, route_channel),
10913                route_channel,
10914                route_epoch,
10915                PollKind::Liveness,
10916            )
10917            .unwrap();
10918
10919        assert_route_poll_liveness(&responses[0], true);
10920    }
10921
10922    #[test]
10923    fn liveness_poll_untracked_process_source_uses_bound_route() {
10924        let registry = Arc::new(Registry::default());
10925        let forwarding = Arc::new(ForwardingTable::default());
10926        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
10927        let handler =
10928            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10929                .with_process_liveness(process_liveness);
10930        let (ctx, route_channel, route_epoch) =
10931            bind_liveness_route(&registry, &forwarding, "aft-untracked");
10932        let responses = handler
10933            .handle_route_poll(
10934                &ctx,
10935                route_poll_frame(43, PollKind::Liveness, route_channel),
10936                route_channel,
10937                route_epoch,
10938                PollKind::Liveness,
10939            )
10940            .unwrap();
10941
10942        assert_route_poll_liveness(&responses[0], true);
10943    }
10944
10945    #[tokio::test]
10946    async fn unknown_op_returns_unknown_control_op() {
10947        let handler = ControlHandler::default();
10948        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
10949        let request = Frame::build(
10950            FrameType::Request,
10951            control_flags(),
10952            0,
10953            0,
10954            55,
10955            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
10956        )
10957        .unwrap();
10958
10959        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10960
10961        assert_eq!(response.len(), 1);
10962        assert_eq!(response[0].header.ty, FrameType::Error);
10963        assert_eq!(response[0].header.corr, 55);
10964        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
10965    }
10966
10967    #[tokio::test]
10968    async fn supervisor_provenance_rejects_unknown_exact_module() {
10969        let handler = ControlHandler::default();
10970        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
10971        let request = Frame::build(
10972            FrameType::Request,
10973            control_flags(),
10974            0,
10975            0,
10976            57,
10977            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
10978        )
10979        .unwrap();
10980
10981        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10982
10983        assert_eq!(response.len(), 1);
10984        assert_eq!(response[0].header.ty, FrameType::Error);
10985        assert_eq!(response[0].header.corr, 57);
10986        let error = parse_error(&response[0]);
10987        assert_eq!(error["code"], "unknown_module");
10988        assert_eq!(error["message"], "module_id 'missing' is not supervised");
10989    }
10990
10991    #[test]
10992    fn provenance_probe_override_keeps_handler_tests_deterministic() {
10993        let expected = subc_control::RunningImageAgreement::Unavailable {
10994            reason: subc_control::RunningImageUnavailableReason::HashFailed,
10995        };
10996        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
10997        assert_eq!(handler.provenance_probe_override, Some(expected));
10998    }
10999
11000    #[test]
11001    fn reload_verdict_detects_configured_program_different_from_spawned_path() {
11002        let verdict = reload_verdict(
11003            std::path::Path::new("/bin/new"),
11004            Some(std::path::Path::new("/bin/old")),
11005            subc_control::RunningImageAgreement::Unavailable {
11006                reason: subc_control::RunningImageUnavailableReason::HashFailed,
11007            },
11008        );
11009        assert!(matches!(
11010            verdict.path,
11011            subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
11012                if configured == std::path::Path::new("/bin/new")
11013                    && spawned_from == std::path::Path::new("/bin/old")
11014        ));
11015    }
11016
11017    #[test]
11018    fn reload_verdict_detects_replaced_image_at_same_path() {
11019        let image = subc_control::RunningImageAgreement::Mismatch {
11020            running: subc_control::RunningImageEvidence::LinuxProcSha256 {
11021                digest: "old".into(),
11022            },
11023            disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
11024                digest: "new".into(),
11025            },
11026        };
11027        let verdict = reload_verdict(
11028            std::path::Path::new("/bin/same"),
11029            Some(std::path::Path::new("/bin/same")),
11030            image.clone(),
11031        );
11032        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11033        assert_eq!(verdict.image, image);
11034    }
11035
11036    #[test]
11037    fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
11038        let image = subc_control::RunningImageAgreement::Unavailable {
11039            reason: subc_control::RunningImageUnavailableReason::NotRunning,
11040        };
11041        let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
11042        assert_eq!(
11043            verdict.path,
11044            subc_control::ReloadPathAgreement::Unavailable {
11045                reason: subc_control::ReloadPathUnavailableReason::NotRunning,
11046            }
11047        );
11048        assert_eq!(verdict.image, image);
11049
11050        let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
11051            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
11052        };
11053        let verdict = reload_verdict(
11054            std::path::Path::new("/bin/same"),
11055            Some(std::path::Path::new("/bin/same")),
11056            unconfirmed.clone(),
11057        );
11058        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11059        assert_eq!(verdict.image, unconfirmed);
11060    }
11061
11062    #[test]
11063    fn reload_verdict_preserves_each_image_unavailability_reason() {
11064        use subc_control::RunningImageUnavailableReason as Reason;
11065
11066        for reason in [
11067            Reason::NotRunning,
11068            Reason::UnsupportedPlatform,
11069            Reason::RunningExecutableUnreadable,
11070            Reason::SpawnedPathUnreadable,
11071            Reason::HashFailed,
11072            Reason::ProcessIdentityUnconfirmed,
11073            Reason::Unknown("future_probe_reason".to_string()),
11074        ] {
11075            let image = subc_control::RunningImageAgreement::Unavailable {
11076                reason: reason.clone(),
11077            };
11078            let verdict = reload_verdict(
11079                std::path::Path::new("/bin/same"),
11080                Some(std::path::Path::new("/bin/same")),
11081                image.clone(),
11082            );
11083            assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11084            assert_eq!(verdict.image, image, "{reason:?}");
11085        }
11086    }
11087
11088    #[tokio::test]
11089    async fn malformed_control_bodies_return_invalid_control_body() {
11090        let handler = ControlHandler::default();
11091        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
11092
11093        for (corr, body) in [
11094            (56, br#"{"route_channel":1}"#.as_slice()),
11095            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
11096            (
11097                58,
11098                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
11099            ),
11100        ] {
11101            let request = Frame::build(
11102                FrameType::Request,
11103                control_flags(),
11104                0,
11105                0,
11106                corr,
11107                body.to_vec(),
11108            )
11109            .unwrap();
11110            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
11111
11112            assert_eq!(response.len(), 1);
11113            assert_eq!(response[0].header.ty, FrameType::Error);
11114            assert_eq!(response[0].header.corr, corr);
11115            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
11116        }
11117    }
11118
11119    #[tokio::test]
11120    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
11121        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
11122        let registry = Arc::new(Registry::default());
11123        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
11124        let router = Router::with_control_handler(Arc::clone(&control));
11125        let connection = router.begin_connection();
11126        let (ctx, mut rx) = route_ctx(connection.id());
11127
11128        router
11129            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
11130            .await
11131            .unwrap();
11132        let response = rx.recv().await.unwrap();
11133        let ack = parse_ack(&response);
11134        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11135        let channel = 1;
11136
11137        let goodbye =
11138            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
11139        router.route_for_connection(&ctx, goodbye).await.unwrap();
11140        assert!(rx.try_recv().is_err());
11141        assert!(registry.get_module("aft").unwrap().is_none());
11142
11143        router
11144            .route_for_connection(&ctx, channel_request(channel, 13))
11145            .await
11146            .unwrap();
11147        let error_frame = rx.recv().await.unwrap();
11148        assert_eq!(error_frame.header.ty, FrameType::Error);
11149        assert_eq!(error_frame.header.channel, channel);
11150        let captured = crate::router::test_log::captured_logs(&logs);
11151        assert_eq!(
11152            captured
11153                .lines()
11154                .filter(|line| {
11155                    line.contains("module_id=aft")
11156                        && line.contains("reason=explicit_goodbye")
11157                        && line.contains("module registration ended")
11158                })
11159                .count(),
11160            1,
11161            "unexpected GOODBYE registry log: {captured}"
11162        );
11163    }
11164
11165    #[tokio::test]
11166    async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
11167        let registry = Arc::new(Registry::default());
11168        let handler = ControlHandler::new(registry).with_capability_config(
11169            [("prov".to_string(), true), ("cons".to_string(), true)],
11170            BTreeMap::new(),
11171        );
11172        let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
11173        register_capability_manifest(
11174            &handler,
11175            &provider_ctx,
11176            &mut provider_rx,
11177            capability_manifest("prov", &["thing/v1"], &[]),
11178            1,
11179        )
11180        .await;
11181        let mut consumer = capability_manifest("cons", &[], &[]);
11182        consumer.capabilities.as_mut().unwrap().requires.push(
11183            subc_protocol::manifest::CapabilityRequirement {
11184                capability: "thing/v1".to_string(),
11185                need: subc_protocol::manifest::CapabilityNeed::Required,
11186            },
11187        );
11188        let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
11189        register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
11190        assert_eq!(
11191            handler.capability_evaluator.verdict("cons", "thing/v1"),
11192            Some(CapabilityVerdict::Provided)
11193        );
11194        let (mut client_rx, _) = open_route_for_capability_test(
11195            &handler,
11196            &provider_ctx,
11197            &mut provider_rx,
11198            703,
11199            3,
11200            "prov",
11201            None,
11202        )
11203        .await;
11204        let goodbye =
11205            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
11206        handler
11207            .handle_control_frame(&provider_ctx, goodbye)
11208            .await
11209            .unwrap();
11210        assert_eq!(
11211            handler.capability_evaluator.verdict("cons", "thing/v1"),
11212            Some(CapabilityVerdict::NeverProvided)
11213        );
11214        let closed = client_rx
11215            .try_recv()
11216            .expect("GOODBYE pushes route.closed before route GOODBYE");
11217        assert!(
11218            matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
11219            ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
11220        );
11221        assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
11222        assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11223    }
11224
11225    #[tokio::test]
11226    async fn dropping_router_connection_releases_registration() {
11227        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
11228        let registry = Arc::new(Registry::default());
11229        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
11230        let router = Router::with_control_handler(Arc::clone(&control));
11231        let connection = router.begin_connection();
11232        let connection_id = connection.id();
11233        let (ctx, mut rx) = route_ctx(connection_id);
11234
11235        router
11236            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
11237            .await
11238            .unwrap();
11239        let response = rx.recv().await.unwrap();
11240        let ack = parse_ack(&response);
11241        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11242        assert!(registry.get_module("aft").unwrap().is_some());
11243
11244        drop(connection);
11245
11246        assert!(registry.get_module("aft").unwrap().is_none());
11247        assert_eq!(registry.active_registration_count().unwrap(), 0);
11248
11249        control
11250            .cleanup_connection(ConnectionId::new(u64::MAX))
11251            .unwrap();
11252        let captured = crate::router::test_log::captured_logs(&logs);
11253        println!("captured registry lifecycle logs:\n{captured}");
11254        let events: Vec<_> = captured
11255            .lines()
11256            .filter(|line| {
11257                line.contains("module registered module_id=aft ")
11258                    || line.contains("module registration ended")
11259            })
11260            .collect();
11261        assert_eq!(
11262            events.len(),
11263            2,
11264            "unexpected registry lifecycle logs: {captured}"
11265        );
11266        assert!(events[0].contains("module registered module_id=aft "));
11267        assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
11268        assert!(events[1].contains(&format!(
11269            "module_id=aft connection_id={}",
11270            connection_id.get()
11271        )));
11272        assert!(events[1].contains("reason=connection_closed"));
11273        assert!(events[1].contains("module registration ended"));
11274        assert_eq!(
11275            captured
11276                .lines()
11277                .filter(|line| line.contains("module registered module_id=aft "))
11278                .count(),
11279            1,
11280            "legacy registration admission line must appear once: {captured}"
11281        );
11282    }
11283
11284    #[tokio::test]
11285    async fn hello_registration_keeps_legacy_module_registered_line_once() {
11286        let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
11287        let registry = Arc::new(Registry::default());
11288        let control = ControlHandler::new(Arc::clone(&registry));
11289        let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
11290        hello_via_sink(
11291            &control,
11292            &ctx,
11293            &mut rx,
11294            hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
11295        )
11296        .await;
11297
11298        let captured = crate::router::test_log::captured_logs(&logs);
11299        let admissions: Vec<_> = captured
11300            .lines()
11301            .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
11302            .collect();
11303        assert_eq!(
11304            admissions.len(),
11305            1,
11306            "legacy admission line must remain exactly once: {captured}"
11307        );
11308        assert!(admissions[0].contains("routable_provider=true"));
11309        assert!(admissions[0].contains("connection_id=777"));
11310    }
11311
11312    fn capability_manifest(
11313        module_id: &str,
11314        provides: &[&str],
11315        must_never_reach: &[&str],
11316    ) -> ModuleManifest {
11317        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
11318        manifest.capabilities = Some(CapabilityDeclarations {
11319            provides: provides
11320                .iter()
11321                .map(|capability| (*capability).to_string())
11322                .collect(),
11323            requires: Vec::new(),
11324            must_never_reach: must_never_reach
11325                .iter()
11326                .map(|capability| (*capability).to_string())
11327                .collect(),
11328        });
11329        manifest
11330    }
11331
11332    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
11333        Frame::build(
11334            FrameType::Hello,
11335            control_flags(),
11336            0,
11337            0,
11338            corr,
11339            serde_json::to_vec(&ModuleHelloBody {
11340                protocol_ver: manifest.protocol_ver,
11341                manifest,
11342                control_ops: None,
11343                launch_nonce: None,
11344            })
11345            .expect("capability test HELLO serializes"),
11346        )
11347        .expect("capability test HELLO frame builds")
11348    }
11349
11350    fn catalog_update_with_capabilities_frame(
11351        corr: u64,
11352        capabilities: CapabilityDeclarations,
11353    ) -> Frame {
11354        Frame::build(
11355            FrameType::Request,
11356            control_flags(),
11357            0,
11358            0,
11359            corr,
11360            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
11361                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
11362                capabilities: Some(capabilities),
11363                ready: None,
11364            })
11365            .expect("capability catalog.update serializes"),
11366        )
11367        .expect("capability catalog.update frame builds")
11368    }
11369
11370    async fn register_capability_manifest(
11371        handler: &ControlHandler,
11372        ctx: &RouteCtx,
11373        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11374        manifest: ModuleManifest,
11375        corr: u64,
11376    ) {
11377        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
11378    }
11379
11380    async fn open_route_for_capability_test(
11381        handler: &ControlHandler,
11382        target_ctx: &RouteCtx,
11383        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11384        client_connection_id: u64,
11385        corr: u64,
11386        target_module_id: &str,
11387        consumer_identity: Option<ConsumerIdentity>,
11388    ) -> (
11389        mpsc::Receiver<crate::router::OutboundFrame>,
11390        ModuleControlRequest,
11391    ) {
11392        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
11393        let route_handler = handler.clone();
11394        let target_module_id = target_module_id.to_string();
11395        let route_task = tokio::spawn(async move {
11396            route_handler
11397                .handle_control_frame(
11398                    &client_ctx,
11399                    route_open_frame_with_admission_facts(
11400                        corr,
11401                        &target_module_id,
11402                        unique_project_root("admission-facts"),
11403                        consumer_identity,
11404                        None,
11405                    ),
11406                )
11407                .await
11408                .expect("capability test route.open succeeds")
11409        });
11410        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
11411            .await
11412            .expect("capability test route.open must reach route.bind")
11413            .expect("target control receiver stays open");
11414        let bind_request: ModuleControlRequest =
11415            serde_json::from_slice(&bind.body).expect("route.bind decodes");
11416        handler
11417            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
11418            .await
11419            .expect("capability test route.bind ACK succeeds");
11420        assert!(route_task.await.expect("route.open task joins").is_empty());
11421        let opened = client_rx
11422            .recv()
11423            .await
11424            .expect("successful route.open publishes a response");
11425        assert!(matches!(
11426            serde_json::from_slice::<ClientControlResponse>(&opened.body),
11427            Ok(ClientControlResponse::RouteOpen { .. })
11428        ));
11429        (client_rx, bind_request)
11430    }
11431
11432    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
11433        assert_eq!(frame.header.ty, FrameType::Push);
11434        assert_eq!(frame.header.channel, 0);
11435        let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
11436            .expect("route.closed control push decodes");
11437        let ClientControlPush::RouteClosed { channels, .. } = &push else {
11438            panic!("expected route.closed");
11439        };
11440        assert_eq!(channels.len(), 1, "exactly one violating route closed");
11441        let channels = channels.clone();
11442        assert_eq!(
11443            push,
11444            ClientControlPush::RouteClosed {
11445                module_id: target_module_id.to_string(),
11446                channels,
11447                reason: RouteCloseReason::CapabilityDenied,
11448                drained: false,
11449                abandoned: 0,
11450                excluded_subscriptions: 0,
11451                terminal: Some(false),
11452            }
11453        );
11454    }
11455
11456    #[tokio::test]
11457    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
11458        let registry = Arc::new(Registry::default());
11459        let forwarding = Arc::new(ForwardingTable::default());
11460        let supervisor = SupervisorHandle::new();
11461        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11462        let handler =
11463            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11464                .with_supervisor(supervisor);
11465        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
11466        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
11467        register_capability_manifest(
11468            &handler,
11469            &target_ctx,
11470            &mut target_rx,
11471            capability_manifest("target", &["credentials-provider/v1"], &[]),
11472            1,
11473        )
11474        .await;
11475        register_capability_manifest(
11476            &handler,
11477            &opener_ctx,
11478            &mut opener_rx,
11479            capability_manifest("opener", &[], &["credentials-provider/v1"]),
11480            2,
11481        )
11482        .await;
11483
11484        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
11485        let replies = handler
11486            .handle_control_frame(
11487                &client_ctx,
11488                route_open_frame_with_admission_facts(
11489                    3,
11490                    "target",
11491                    unique_project_root("admission-facts"),
11492                    Some(ConsumerIdentity {
11493                        module_id: "opener".to_string(),
11494                        launch_nonce: "opener-nonce".to_string(),
11495                    }),
11496                    None,
11497                ),
11498            )
11499            .await
11500            .expect("denied route.open returns a typed frame");
11501        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
11502        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11503        assert!(
11504            target_rx.try_recv().is_err(),
11505            "forbidden route.open must not relay route.bind"
11506        );
11507    }
11508
11509    #[tokio::test]
11510    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
11511        let registry = Arc::new(Registry::default());
11512        let forwarding = Arc::new(ForwardingTable::default());
11513        let supervisor = SupervisorHandle::new();
11514        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11515        let handler =
11516            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11517                .with_supervisor(supervisor);
11518        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
11519        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
11520        register_capability_manifest(
11521            &handler,
11522            &target_ctx,
11523            &mut target_rx,
11524            capability_manifest("target", &["credentials-provider/v1"], &[]),
11525            1,
11526        )
11527        .await;
11528        register_capability_manifest(
11529            &handler,
11530            &old_opener_ctx,
11531            &mut old_opener_rx,
11532            capability_manifest("opener", &[], &[]),
11533            2,
11534        )
11535        .await;
11536        let (mut client_rx, _) = open_route_for_capability_test(
11537            &handler,
11538            &target_ctx,
11539            &mut target_rx,
11540            712,
11541            3,
11542            "target",
11543            Some(ConsumerIdentity {
11544                module_id: "opener".to_string(),
11545                launch_nonce: "opener-nonce".to_string(),
11546            }),
11547        )
11548        .await;
11549        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11550
11551        handler
11552            .cleanup_connection(old_opener_ctx.connection_id)
11553            .expect("old opener registration cleans up");
11554        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
11555        register_capability_manifest(
11556            &handler,
11557            &new_opener_ctx,
11558            &mut new_opener_rx,
11559            capability_manifest("opener", &[], &["credentials-provider/v1"]),
11560            4,
11561        )
11562        .await;
11563
11564        assert_capability_denied_push(
11565            client_rx
11566                .try_recv()
11567                .expect("HELLO deny addition must emit route.closed")
11568                .frame,
11569            "target",
11570        );
11571        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11572        assert!(matches!(
11573            target_rx.try_recv(),
11574            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
11575        ));
11576    }
11577
11578    #[tokio::test]
11579    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
11580        let registry = Arc::new(Registry::default());
11581        let forwarding = Arc::new(ForwardingTable::default());
11582        let supervisor = SupervisorHandle::new();
11583        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11584        let handler =
11585            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11586                .with_supervisor(supervisor);
11587        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
11588        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
11589        register_capability_manifest(
11590            &handler,
11591            &target_ctx,
11592            &mut target_rx,
11593            capability_manifest("target", &[], &[]),
11594            1,
11595        )
11596        .await;
11597        register_capability_manifest(
11598            &handler,
11599            &opener_ctx,
11600            &mut opener_rx,
11601            capability_manifest("opener", &[], &["credentials-provider/v1"]),
11602            2,
11603        )
11604        .await;
11605        let (mut client_rx, _) = open_route_for_capability_test(
11606            &handler,
11607            &target_ctx,
11608            &mut target_rx,
11609            722,
11610            3,
11611            "target",
11612            Some(ConsumerIdentity {
11613                module_id: "opener".to_string(),
11614                launch_nonce: "opener-nonce".to_string(),
11615            }),
11616        )
11617        .await;
11618        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11619
11620        let replies = handler
11621            .handle_control_frame(
11622                &target_ctx,
11623                catalog_update_with_capabilities_frame(
11624                    4,
11625                    CapabilityDeclarations {
11626                        provides: vec!["credentials-provider/v1".to_string()],
11627                        requires: Vec::new(),
11628                        must_never_reach: Vec::new(),
11629                    },
11630                ),
11631            )
11632            .await
11633            .expect("claim catalog.update succeeds");
11634        assert!(matches!(
11635            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
11636            Ok(ModuleControlResponseToModule::CatalogUpdate {})
11637        ));
11638        assert_capability_denied_push(
11639            client_rx
11640                .try_recv()
11641                .expect("claim addition must emit route.closed")
11642                .frame,
11643            "target",
11644        );
11645        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11646        assert!(matches!(
11647            target_rx.try_recv(),
11648            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
11649        ));
11650    }
11651
11652    #[tokio::test]
11653    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
11654        let registry = Arc::new(Registry::default());
11655        let forwarding = Arc::new(ForwardingTable::default());
11656        let supervisor = SupervisorHandle::new();
11657        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11658        let handler =
11659            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11660                .with_supervisor(supervisor);
11661        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
11662        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
11663        register_capability_manifest(
11664            &handler,
11665            &target_ctx,
11666            &mut target_rx,
11667            capability_manifest("target", &["credentials-provider/v1"], &[]),
11668            1,
11669        )
11670        .await;
11671        register_capability_manifest(
11672            &handler,
11673            &opener_ctx,
11674            &mut opener_rx,
11675            capability_manifest("opener", &[], &[]),
11676            2,
11677        )
11678        .await;
11679        let (mut client_rx, _) = open_route_for_capability_test(
11680            &handler,
11681            &target_ctx,
11682            &mut target_rx,
11683            732,
11684            3,
11685            "target",
11686            Some(ConsumerIdentity {
11687                module_id: "opener".to_string(),
11688                launch_nonce: "opener-nonce".to_string(),
11689            }),
11690        )
11691        .await;
11692        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11693
11694        handler
11695            .handle_control_frame(
11696                &target_ctx,
11697                catalog_update_with_capabilities_frame(
11698                    4,
11699                    CapabilityDeclarations {
11700                        provides: Vec::new(),
11701                        requires: Vec::new(),
11702                        must_never_reach: Vec::new(),
11703                    },
11704                ),
11705            )
11706            .await
11707            .expect("claim removal catalog.update succeeds");
11708        assert_eq!(
11709            forwarding.active_binding_count().unwrap(),
11710            1,
11711            "removing an attested target claim must leave the route census unchanged"
11712        );
11713        assert!(
11714            client_rx.try_recv().is_err(),
11715            "claim removal must not emit route.closed capability_denied"
11716        );
11717        assert!(
11718            target_rx.try_recv().is_err(),
11719            "claim removal must not send the target a route GOODBYE"
11720        );
11721    }
11722
11723    /// A direct client may open a route to a denied capability provider; this
11724    /// policy applies only to attested supervised module origins, not to direct clients.
11725    #[tokio::test]
11726    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
11727        let registry = Arc::new(Registry::default());
11728        let forwarding = Arc::new(ForwardingTable::default());
11729        let supervisor = SupervisorHandle::new();
11730        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11731        let handler =
11732            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11733                .with_supervisor(supervisor);
11734        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
11735        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
11736        register_capability_manifest(
11737            &handler,
11738            &target_ctx,
11739            &mut target_rx,
11740            capability_manifest("target", &["credentials-provider/v1"], &[]),
11741            1,
11742        )
11743        .await;
11744        register_capability_manifest(
11745            &handler,
11746            &opener_ctx,
11747            &mut opener_rx,
11748            capability_manifest("opener", &[], &["credentials-provider/v1"]),
11749            2,
11750        )
11751        .await;
11752
11753        let (_client_rx, bind) = open_route_for_capability_test(
11754            &handler,
11755            &target_ctx,
11756            &mut target_rx,
11757            742,
11758            3,
11759            "target",
11760            None,
11761        )
11762        .await;
11763        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11764            panic!("direct scope-honesty route must bind");
11765        };
11766        assert_eq!(principal, Some(Principal::Direct));
11767        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11768    }
11769
11770    /// A module that denies a capability receives no self-route exemption when it
11771    /// also attestedly provides that capability.
11772    #[tokio::test]
11773    async fn must_never_reach_self_route_is_capability_forbidden() {
11774        let registry = Arc::new(Registry::default());
11775        let forwarding = Arc::new(ForwardingTable::default());
11776        let supervisor = SupervisorHandle::new();
11777        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
11778        let handler =
11779            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11780                .with_supervisor(supervisor);
11781        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
11782        register_capability_manifest(
11783            &handler,
11784            &self_ctx,
11785            &mut self_rx,
11786            capability_manifest(
11787                "self-provider",
11788                &["credentials-provider/v1"],
11789                &["credentials-provider/v1"],
11790            ),
11791            1,
11792        )
11793        .await;
11794
11795        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
11796        let replies = handler
11797            .handle_control_frame(
11798                &client_ctx,
11799                route_open_frame_with_admission_facts(
11800                    2,
11801                    "self-provider",
11802                    unique_project_root("admission-facts"),
11803                    Some(ConsumerIdentity {
11804                        module_id: "self-provider".to_string(),
11805                        launch_nonce: "self-nonce".to_string(),
11806                    }),
11807                    None,
11808                ),
11809            )
11810            .await
11811            .expect("self-route refusal returns a typed frame");
11812        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
11813        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11814        assert!(
11815            self_rx.try_recv().is_err(),
11816            "self denial must not relay route.bind"
11817        );
11818    }
11819
11820    #[test]
11821    fn unsupported_channel_zero_frame_returns_error() {
11822        let handler = ControlHandler::default();
11823        let request = Frame::build(
11824            FrameType::Request,
11825            control_flags(),
11826            0,
11827            0,
11828            21,
11829            b"opaque".to_vec(),
11830        )
11831        .unwrap();
11832
11833        let response = handler
11834            .handle_control(ConnectionId::new(1), request)
11835            .unwrap();
11836
11837        assert_eq!(response[0].header.ty, FrameType::Error);
11838        assert_eq!(
11839            parse_error(&response[0])["code"],
11840            "unsupported_control_frame"
11841        );
11842    }
11843
11844    /// Blue/green swap at the control-plane boundary. The supervisor that opens
11845    /// a swap is not wired yet, so the candidate is registered here directly
11846    /// into the registry and forwarding candidate slots, the way the swap's
11847    /// HELLO admission will.
11848    mod swap {
11849        use super::*;
11850
11851        const INCUMBENT: ConnectionId = ConnectionId::new(30);
11852        const CANDIDATE: ConnectionId = ConnectionId::new(40);
11853
11854        struct Swap {
11855            registry: Arc<Registry>,
11856            forwarding: Arc<ForwardingTable>,
11857            handler: ControlHandler,
11858            incumbent_ctx: RouteCtx,
11859            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
11860            candidate_ctx: RouteCtx,
11861            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
11862        }
11863
11864        async fn swap_with_incumbent() -> Swap {
11865            let registry = Arc::new(Registry::default());
11866            let forwarding = Arc::new(ForwardingTable::default());
11867            let handler =
11868                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11869            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
11870            hello_via_sink(
11871                &handler,
11872                &incumbent_ctx,
11873                &mut incumbent_rx,
11874                hello_frame("aft", PROTOCOL_VERSION, 7),
11875            )
11876            .await;
11877            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
11878            Swap {
11879                registry,
11880                forwarding,
11881                handler,
11882                incumbent_ctx,
11883                incumbent_rx,
11884                candidate_ctx,
11885                candidate_rx,
11886            }
11887        }
11888
11889        fn register_candidate(swap: &Swap, ready: Option<bool>) {
11890            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
11891            candidate_manifest.ready = ready;
11892            let registration = swap
11893                .registry
11894                .register_candidate_with_control_ops(
11895                    candidate_manifest,
11896                    PROTOCOL_VERSION,
11897                    CANDIDATE,
11898                    module_baseline_control_ops(),
11899                )
11900                .unwrap();
11901            swap.forwarding
11902                .register_candidate_module_connection(
11903                    CANDIDATE,
11904                    "aft".to_string(),
11905                    PROTOCOL_VERSION,
11906                    manifest_concurrency(&registration.manifest),
11907                    swap.candidate_ctx.egress.clone(),
11908                )
11909                .unwrap();
11910        }
11911
11912        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
11913            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
11914            swap.registry.promote_candidate("aft").unwrap().unwrap();
11915            cutover.incumbent.unwrap()
11916        }
11917
11918        fn keyed_total(counters: &Value, key: &str) -> u64 {
11919            counters[key]
11920                .as_object()
11921                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
11922                .unwrap_or(0)
11923        }
11924
11925        #[tokio::test]
11926        async fn replacement_logs_old_end_and_new_admission_once() {
11927            let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
11928            let mut swap = swap_with_incumbent().await;
11929            swap.handler
11930                .supervisor
11931                .open_swap("aft", "candidate-nonce".to_string());
11932            hello_via_sink(
11933                &swap.handler,
11934                &swap.candidate_ctx,
11935                &mut swap.candidate_rx,
11936                hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
11937            )
11938            .await;
11939            cutover(&swap);
11940            swap.handler.cleanup_connection(INCUMBENT).unwrap();
11941
11942            let captured = crate::router::test_log::captured_logs(&logs);
11943            println!("captured replacement registry logs:\n{captured}");
11944            let new_admissions: Vec<_> = captured
11945                .lines()
11946                .filter(|line| {
11947                    line.contains("connection_id=40")
11948                        && line.contains("swap candidate registered; not routable until cutover")
11949                })
11950                .collect();
11951            assert_eq!(
11952                new_admissions.len(),
11953                1,
11954                "unexpected admission logs: {captured}"
11955            );
11956            assert!(new_admissions[0].contains("module_id=aft"));
11957            assert!(new_admissions[0].contains("connection_id=40"));
11958            assert!(
11959                new_admissions[0].contains("swap candidate registered; not routable until cutover")
11960            );
11961            assert!(new_admissions[0].contains("ready=true"));
11962
11963            let old_admissions: Vec<_> = captured
11964                .lines()
11965                .filter(|line| {
11966                    line.contains("module registered module_id=aft ")
11967                        && line.contains("connection_id=30")
11968                })
11969                .collect();
11970            assert_eq!(
11971                old_admissions.len(),
11972                1,
11973                "unexpected incumbent admission: {captured}"
11974            );
11975
11976            let old_ends: Vec<_> = captured
11977                .lines()
11978                .filter(|line| {
11979                    line.contains("connection_id=30") && line.contains("module registration ended")
11980                })
11981                .collect();
11982            assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
11983            assert!(old_ends[0].contains("module_id=aft"));
11984            assert!(old_ends[0].contains("connection_id=30"));
11985            assert!(old_ends[0].contains("reason=replaced"));
11986            assert!(old_ends[0].contains("replaced_by_connection_id=40"));
11987            assert_eq!(
11988                captured
11989                    .lines()
11990                    .filter(|line| line.contains("module registration promoted"))
11991                    .count(),
11992                1,
11993                "unexpected promotion logs: {captured}"
11994            );
11995        }
11996
11997        /// An ack from the incumbent for a bind it was sent before cutover,
11998        /// arriving before the incumbent is drained. The incumbent is the live
11999        /// connection carrying every other client's routes, so the ack must
12000        /// not end it: the waiting client is told to retry, the reservation is
12001        /// given back, and the incumbent is told to drop just that binding.
12002        #[tokio::test]
12003        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
12004            let mut swap = swap_with_incumbent().await;
12005            let handler = swap.handler.clone();
12006
12007            // A co-tenant route, bound on the incumbent before the swap.
12008            let cotenant = ConnectionId::new(31);
12009            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
12010            let (cotenant_task, cotenant_bind) = relay_route_open(
12011                &handler,
12012                cotenant,
12013                &cotenant_ctx.egress,
12014                &mut swap.incumbent_rx,
12015                100,
12016                "aft",
12017                "swap-cotenant",
12018            )
12019            .await;
12020            handler
12021                .handle_control_frame(
12022                    &swap.incumbent_ctx,
12023                    route_bind_ack(cotenant_bind.header.corr),
12024                )
12025                .await
12026                .unwrap();
12027            assert!(cotenant_task.await.unwrap().is_empty());
12028            let (cotenant_channel, cotenant_epoch) =
12029                published_route(&cotenant_rx.recv().await.unwrap());
12030
12031            // A second route.open, relayed to the incumbent and not yet acked.
12032            let caller = ConnectionId::new(32);
12033            let (caller_ctx, mut caller_rx) = route_ctx(caller);
12034            let (caller_task, caller_bind) = relay_route_open(
12035                &handler,
12036                caller,
12037                &caller_ctx.egress,
12038                &mut swap.incumbent_rx,
12039                101,
12040                "aft",
12041                "swap-caller",
12042            )
12043            .await;
12044            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
12045
12046            register_candidate(&swap, None);
12047            cutover(&swap);
12048
12049            // The incumbent acks after promotion and before any drain.
12050            let ack = handler
12051                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
12052                .await;
12053            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
12054            if module_loop_error.is_some() {
12055                // What the connection loop does with an untranslated router
12056                // error: end the connection, releasing every route on it.
12057                handler.cleanup_connection(INCUMBENT).unwrap();
12058            }
12059
12060            // 1. The incumbent's other routes survive.
12061            assert!(
12062                cotenant_rx.try_recv().is_err(),
12063                "the co-tenant route on the incumbent was torn down by one late ack: \
12064                 {module_loop_error:?}"
12065            );
12066            assert!(matches!(
12067                swap.forwarding
12068                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
12069                    .unwrap(),
12070                DataRoute::Client(DataRouteState::Bound(_))
12071            ));
12072            assert_eq!(module_loop_error, None);
12073            assert!(swap
12074                .registry
12075                .get_module_by_connection(INCUMBENT)
12076                .unwrap()
12077                .is_some());
12078
12079            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
12080            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
12081                .await
12082                .expect("the incumbent is told to drop the abandoned binding")
12083                .unwrap()
12084                .frame;
12085            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12086            assert_eq!(goodbye.header.channel, abandoned_channel);
12087            assert_eq!(goodbye.header.epoch, abandoned_epoch);
12088            assert!(swap.incumbent_rx.try_recv().is_err());
12089
12090            // 3. The waiting client gets a retryable refusal and no route.
12091            let response = caller_task.await.unwrap();
12092            assert_eq!(response.len(), 1);
12093            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
12094            assert!(caller_rx.try_recv().is_err());
12095
12096            // 4. The reservation pair is given back, and the pending bind
12097            //    settled exactly once: one accepted open (the co-tenant) and one
12098            //    refused open (the caller), nothing counted twice.
12099            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
12100            let counters = handler.counters().snapshot();
12101            assert_eq!(
12102                keyed_total(&counters, "route_open_accepted_by_principal"),
12103                1
12104            );
12105            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
12106            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
12107        }
12108
12109        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
12110        /// id would resolve to the promoted candidate and every new route.open
12111        /// would be refused as reloading, leaving neither process routable.
12112        #[tokio::test]
12113        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
12114            let mut swap = swap_with_incumbent().await;
12115            register_candidate(&swap, None);
12116            let incumbent = cutover(&swap);
12117            swap.forwarding
12118                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
12119                .unwrap()
12120                .expect("the incumbent is still registered");
12121
12122            let client = ConnectionId::new(33);
12123            let (client_ctx, mut client_rx) = route_ctx(client);
12124            let route_handler = swap.handler.clone();
12125            let open_ctx = RouteCtx {
12126                connection_id: client,
12127                egress: client_ctx.egress.clone(),
12128            };
12129            let mut route_task = tokio::spawn(async move {
12130                route_handler
12131                    .handle_control_frame(
12132                        &open_ctx,
12133                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
12134                    )
12135                    .await
12136                    .unwrap()
12137            });
12138            let bind = tokio::select! {
12139                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
12140                response = &mut route_task => {
12141                    let response = response.unwrap();
12142                    panic!(
12143                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
12144                        parse_error(&response[0])["code"]
12145                    );
12146                }
12147            };
12148            swap.handler
12149                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
12150                .await
12151                .unwrap();
12152            assert!(route_task.await.unwrap().is_empty());
12153            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
12154            match swap
12155                .forwarding
12156                .lookup_data_route(client, channel, epoch)
12157                .unwrap()
12158            {
12159                DataRoute::Client(DataRouteState::Bound(route)) => {
12160                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
12161                }
12162                other => panic!("expected a bound route on the candidate, got {other:?}"),
12163            }
12164            assert!(swap.incumbent_rx.try_recv().is_err());
12165        }
12166
12167        /// A candidate declares itself ready with `catalog.update` on its own
12168        /// connection. If the connection-keyed registry lookups searched only the
12169        /// active slot, this would answer `not_registered` and the candidate
12170        /// would never become ready.
12171        #[tokio::test]
12172        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
12173            let swap = swap_with_incumbent().await;
12174            register_candidate(&swap, Some(false));
12175            let update = Frame::build(
12176                FrameType::Request,
12177                control_flags(),
12178                0,
12179                0,
12180                55,
12181                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
12182                    provides: manifest("aft", PROTOCOL_VERSION).provides,
12183                    capabilities: None,
12184                    ready: Some(true),
12185                })
12186                .unwrap(),
12187            )
12188            .unwrap();
12189
12190            let replies = swap
12191                .handler
12192                .handle_control_frame(&swap.candidate_ctx, update)
12193                .await
12194                .unwrap();
12195
12196            assert_eq!(replies.len(), 1);
12197            assert_eq!(
12198                replies[0].header.ty,
12199                FrameType::Response,
12200                "candidate catalog.update was refused: {:?}",
12201                serde_json::from_slice::<Value>(&replies[0].body).ok()
12202            );
12203            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
12204            assert_eq!(
12205                swap.registry
12206                    .get_module("aft")
12207                    .unwrap()
12208                    .unwrap()
12209                    .connection_id,
12210                INCUMBENT
12211            );
12212        }
12213    }
12214
12215    /// The HELLO gate while the supervisor has a swap open: only the nonce it
12216    /// minted for the candidate admits a second process, into the candidate
12217    /// slot, and that check runs ahead of the reserved-module gate.
12218    mod swap_admission {
12219        use super::*;
12220
12221        const INCUMBENT_NONCE: &str = "incumbent-nonce";
12222        const CANDIDATE_NONCE: &str = "candidate-nonce";
12223
12224        fn handler_with_incumbent(
12225            module_id: &str,
12226            reserved: bool,
12227        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
12228            let registry = Arc::new(Registry::default());
12229            let supervisor = SupervisorHandle::new();
12230            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
12231            if reserved {
12232                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
12233            }
12234            let handler =
12235                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
12236            let incumbent = handler
12237                .handle_control(
12238                    ConnectionId::new(1),
12239                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
12240                )
12241                .unwrap();
12242            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
12243            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
12244            (registry, supervisor, handler)
12245        }
12246
12247        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
12248        /// admits every nonce, so while a swap is open the swap gate is the only
12249        /// thing between a key-holder and the candidate slot. A nonce the
12250        /// supervisor did not mint, or none at all, is refused, and neither the
12251        /// incumbent's registration nor the candidate slot moves.
12252        #[test]
12253        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
12254            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
12255
12256            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
12257                let replies = handler
12258                    .handle_control(
12259                        ConnectionId::new(connection),
12260                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
12261                    )
12262                    .unwrap();
12263                assert_eq!(replies[0].header.ty, FrameType::Error);
12264                assert_eq!(
12265                    parse_error(&replies[0])["code"],
12266                    "swap_token_invalid",
12267                    "nonce {nonce:?}"
12268                );
12269            }
12270            assert!(registry.get_candidate("aft").unwrap().is_none());
12271            assert_eq!(
12272                registry.get_module("aft").unwrap().unwrap().connection_id,
12273                ConnectionId::new(1)
12274            );
12275
12276            // Control: the minted token is admitted, into the candidate slot,
12277            // and only once.
12278            let admitted = handler
12279                .handle_control(
12280                    ConnectionId::new(4),
12281                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
12282                )
12283                .unwrap();
12284            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
12285            assert_eq!(
12286                registry
12287                    .get_candidate("aft")
12288                    .unwrap()
12289                    .unwrap()
12290                    .connection_id,
12291                ConnectionId::new(4)
12292            );
12293            assert_eq!(
12294                registry.get_module("aft").unwrap().unwrap().connection_id,
12295                ConnectionId::new(1),
12296                "the candidate must not take the active slot"
12297            );
12298            let replayed = handler
12299                .handle_control(
12300                    ConnectionId::new(5),
12301                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
12302                )
12303                .unwrap();
12304            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
12305
12306            // The case only this gate covers: the incumbent has died mid-swap,
12307            // so its duplicate refusal is gone too, and without the gate a
12308            // key-holder would take the id's ACTIVE slot.
12309            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
12310            let squatter = handler
12311                .handle_control(
12312                    ConnectionId::new(6),
12313                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
12314                )
12315                .unwrap();
12316            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
12317            assert!(
12318                registry.get_module("aft").unwrap().is_none(),
12319                "a squatter took the active slot of an id being swapped"
12320            );
12321        }
12322
12323        /// Design mutation arm (iii). A reserved module's candidate presents a
12324        /// nonce the reserved gate has never seen (that gate holds the
12325        /// incumbent's), so the swap gate must run first or the candidate is
12326        /// refused `reserved_module` and a reserved module can never be swapped.
12327        #[test]
12328        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
12329            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
12330
12331            let replies = handler
12332                .handle_control(
12333                    ConnectionId::new(2),
12334                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
12335                )
12336                .unwrap();
12337
12338            assert_eq!(
12339                replies[0].header.ty,
12340                FrameType::HelloAck,
12341                "reserved candidate refused: {:?}",
12342                serde_json::from_slice::<Value>(&replies[0].body).ok()
12343            );
12344            assert_eq!(
12345                registry
12346                    .get_candidate("vault")
12347                    .unwrap()
12348                    .unwrap()
12349                    .connection_id,
12350                ConnectionId::new(2)
12351            );
12352        }
12353
12354        /// With no swap open the gate is inert: the incumbent's reserved gate
12355        /// and duplicate refusal behave exactly as before.
12356        #[test]
12357        fn without_an_open_swap_the_ordinary_gates_decide() {
12358            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
12359            supervisor.close_swap("vault");
12360
12361            let candidate = handler
12362                .handle_control(
12363                    ConnectionId::new(2),
12364                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
12365                )
12366                .unwrap();
12367            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
12368            let duplicate = handler
12369                .handle_control(
12370                    ConnectionId::new(3),
12371                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
12372                )
12373                .unwrap();
12374            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
12375            assert!(registry.get_candidate("vault").unwrap().is_none());
12376        }
12377    }
12378
12379    /// `scope.sync` and `scope.describe` through the real control handler: who
12380    /// may sync is decided by the registration and launch nonce of the module
12381    /// connection, never by the request body.
12382    mod scopes {
12383        use subc_protocol::scope::{
12384            ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
12385            ScopeStatus,
12386        };
12387
12388        use super::*;
12389
12390        const OWNER: &str = "prefrontal-core";
12391
12392        fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
12393            ScopeRecord {
12394                scope_ref: scope_ref.to_string(),
12395                scope_epoch,
12396                kind: ScopeKind::Head,
12397                parent: None,
12398                child_owners: Vec::new(),
12399                carriers: Vec::new(),
12400                attributes: Default::default(),
12401            }
12402        }
12403
12404        async fn call(
12405            handler: &ControlHandler,
12406            ctx: &RouteCtx,
12407            request: &ModuleControlRequestFromModule,
12408        ) -> Frame {
12409            let body = serde_json::to_vec(request).unwrap();
12410            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
12411            let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
12412            assert_eq!(replies.len(), 1, "{replies:?}");
12413            replies.pop().unwrap()
12414        }
12415
12416        async fn sync(
12417            handler: &ControlHandler,
12418            ctx: &RouteCtx,
12419            generation: u64,
12420            scopes: Vec<ScopeRecord>,
12421        ) -> Result<ModuleControlResponseToModule, String> {
12422            let reply = call(
12423                handler,
12424                ctx,
12425                &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
12426            )
12427            .await;
12428            match reply.header.ty {
12429                FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
12430                _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
12431            }
12432        }
12433
12434        async fn describe(
12435            handler: &ControlHandler,
12436            ctx: &RouteCtx,
12437            owner: &str,
12438            scope_ref: &str,
12439        ) -> ModuleControlResponseToModule {
12440            let reply = call(
12441                handler,
12442                ctx,
12443                &ModuleControlRequestFromModule::ScopeDescribe {
12444                    owner: Principal::Reserved {
12445                        module_id: owner.to_string(),
12446                    },
12447                    scope_ref: scope_ref.to_string(),
12448                },
12449            )
12450            .await;
12451            assert_eq!(
12452                reply.header.ty,
12453                FrameType::Response,
12454                "{:?}",
12455                parse_error(&reply)
12456            );
12457            serde_json::from_slice(&reply.body).unwrap()
12458        }
12459
12460        /// Register `module_id` on `connection` with `nonce`, returning its ctx.
12461        async fn module(
12462            handler: &ControlHandler,
12463            connection: u64,
12464            module_id: &str,
12465            nonce: Option<&str>,
12466        ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
12467            let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
12468            hello_via_sink(
12469                handler,
12470                &ctx,
12471                &mut rx,
12472                hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
12473            )
12474            .await;
12475            (ctx, rx)
12476        }
12477
12478        /// `direct` and every other client connection has no registration, so
12479        /// it can neither sync nor own a scope.
12480        #[tokio::test]
12481        async fn a_client_connection_cannot_sync_or_describe() {
12482            let handler = ControlHandler::new(Arc::new(Registry::default()));
12483            let (ctx, _rx) = route_ctx(ConnectionId::new(9));
12484            for request in [
12485                ModuleControlRequestFromModule::ScopeSync {
12486                    generation: 1,
12487                    scopes: vec![head("s", 1)],
12488                },
12489                ModuleControlRequestFromModule::ScopeDescribe {
12490                    owner: Principal::Direct,
12491                    scope_ref: "s".to_string(),
12492                },
12493            ] {
12494                let reply = call(&handler, &ctx, &request).await;
12495                assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
12496            }
12497            assert!(
12498                !handler
12499                    .scopes
12500                    .read()
12501                    .unwrap()
12502                    .describe(
12503                        &Principal::Reserved {
12504                            module_id: OWNER.to_string()
12505                        },
12506                        "s"
12507                    )
12508                    .owner_synced
12509            );
12510        }
12511
12512        /// A module the supervisor did not spawn registers without a launch
12513        /// nonce, so it is never an owner's current launch.
12514        #[tokio::test]
12515        async fn a_module_without_a_supervised_launch_cannot_sync() {
12516            let handler = ControlHandler::new(Arc::new(Registry::default()));
12517            let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
12518            assert_eq!(
12519                sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
12520                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12521            );
12522        }
12523
12524        #[tokio::test]
12525        async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
12526            let supervisor = SupervisorHandle::new();
12527            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
12528            let handler = ControlHandler::new(Arc::new(Registry::default()))
12529                .with_supervisor(supervisor.clone());
12530            let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
12531            sync(&handler, &incumbent, 1, vec![head("s", 1)])
12532                .await
12533                .expect("the current launch syncs");
12534
12535            // A swap candidate registers with the swap token and is refused
12536            // while the incumbent keeps syncing.
12537            supervisor.open_swap(OWNER, "n2".to_string());
12538            let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
12539            assert_eq!(
12540                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
12541                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12542            );
12543            sync(&handler, &incumbent, 2, vec![head("s", 1)])
12544                .await
12545                .expect("the serving owner syncs during the swap");
12546
12547            // The swap fails and is rolled back. The candidate never held sync
12548            // authority, and still cannot sync.
12549            supervisor.close_swap(OWNER);
12550            assert_eq!(
12551                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
12552                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12553            );
12554            sync(&handler, &incumbent, 3, vec![head("s", 1)])
12555                .await
12556                .expect("the serving owner syncs after the rollback");
12557            handler.cleanup_connection(candidate.connection_id).unwrap();
12558
12559            // A swap that cuts over. Promotion records the candidate's nonce as
12560            // the module's spawn nonce, which is what `set_spawn_nonce` does
12561            // here; the promoted connection then takes authority at any
12562            // generation and the superseded incumbent is refused.
12563            supervisor.open_swap(OWNER, "n3".to_string());
12564            let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
12565            supervisor.set_spawn_nonce(OWNER, "n3".to_string());
12566            let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
12567                .await
12568                .expect("the promoted launch takes authority");
12569            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
12570                panic!("unexpected reply {reply:?}");
12571            };
12572            assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
12573            assert_eq!(
12574                sync(&handler, &incumbent, 4, Vec::new()).await,
12575                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12576            );
12577        }
12578
12579        /// Authority dies with its connection: the cleanup path releases it,
12580        /// so the owner's next connection takes it at any generation.
12581        #[tokio::test]
12582        async fn closing_the_authority_connection_frees_sync_authority() {
12583            let supervisor = SupervisorHandle::new();
12584            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
12585            let handler = ControlHandler::new(Arc::new(Registry::default()))
12586                .with_supervisor(supervisor.clone());
12587            let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
12588            sync(&handler, &first, 10, vec![head("s", 1)])
12589                .await
12590                .unwrap();
12591            handler.cleanup_connection(first.connection_id).unwrap();
12592
12593            let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
12594            sync(&handler, &second, 1, vec![head("s", 1)])
12595                .await
12596                .expect("the next connection takes the released authority");
12597        }
12598
12599        #[tokio::test]
12600        async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
12601            let supervisor = SupervisorHandle::new();
12602            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
12603            let handler =
12604                ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
12605            let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
12606            sync(&handler, &first, 10, vec![head("s", 1)])
12607                .await
12608                .unwrap();
12609            handler
12610                .handle_control_frame(
12611                    &first,
12612                    Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
12613                )
12614                .await
12615                .unwrap();
12616            let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
12617            sync(&handler, &second, 1, vec![head("s", 1)])
12618                .await
12619                .expect("GOODBYE releases authority even if the old socket remains open");
12620        }
12621
12622        #[tokio::test]
12623        async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
12624            let registry = Arc::new(Registry::default());
12625            let supervisor_handle = SupervisorHandle::new();
12626            let supervisor =
12627                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
12628                    .with_handle(supervisor_handle.clone())
12629                    .with_daemon_incarnation("incarnation-7".to_string());
12630            // Configured with enabled: false, so the supervisor lists the
12631            // module without spawning a process for it.
12632            supervisor
12633                .supervise_configured(
12634                    ModuleSpec {
12635                        module_id: OWNER.to_string(),
12636                        program: PathBuf::from("/nonexistent/prefrontal-core"),
12637                        args: Vec::new(),
12638                        env: Vec::new(),
12639                        reserved: false,
12640                        reserved_prefixes: Vec::new(),
12641                        protocol: ModuleProtocol::Subc,
12642                        overlap: Default::default(),
12643                    },
12644                    false,
12645                )
12646                .unwrap();
12647            supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
12648            let handler =
12649                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
12650            let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
12651
12652            // Configured but not yet synced: a reader waits for the owner.
12653            let ModuleControlResponseToModule::ScopeDescribe {
12654                status,
12655                daemon_incarnation,
12656                owner_synced,
12657                owner_configured,
12658                scope,
12659                ..
12660            } = describe(&handler, &reader, OWNER, "s").await
12661            else {
12662                panic!("not a describe reply");
12663            };
12664            assert_eq!(status, ScopeStatus::NotLive);
12665            assert_eq!(daemon_incarnation, "incarnation-7");
12666            assert!(!owner_synced);
12667            assert!(owner_configured);
12668            assert!(scope.is_none());
12669
12670            // Not a supervised module: the owner will never sync, and a reader
12671            // refuses rather than waits.
12672            let ModuleControlResponseToModule::ScopeDescribe {
12673                status,
12674                owner_configured,
12675                ..
12676            } = describe(&handler, &reader, "ghost", "s").await
12677            else {
12678                panic!("not a describe reply");
12679            };
12680            assert_eq!(status, ScopeStatus::NotLive);
12681            assert!(!owner_configured);
12682
12683            // Live, with the stamp fields and the computed owner_authorized.
12684            let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
12685            sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
12686            let ModuleControlResponseToModule::ScopeDescribe {
12687                status,
12688                scope_epoch,
12689                owner_synced,
12690                scope,
12691                ..
12692            } = describe(&handler, &reader, OWNER, "s").await
12693            else {
12694                panic!("not a describe reply");
12695            };
12696            assert_eq!(status, ScopeStatus::Live);
12697            assert_eq!(scope_epoch, Some(4));
12698            assert!(owner_synced);
12699            let stamp = scope.expect("a live scope carries its stamp");
12700            assert!(
12701                stamp.owner_authorized,
12702                "prefrontal-core is the default authority"
12703            );
12704            assert_eq!(stamp.kind, ScopeKind::Head);
12705        }
12706
12707        #[tokio::test]
12708        async fn scope_authority_owners_decides_owner_authorized() {
12709            let supervisor = SupervisorHandle::new();
12710            supervisor.set_spawn_nonce("broca", "b1".to_string());
12711            let handler = ControlHandler::new(Arc::new(Registry::default()))
12712                .with_supervisor(supervisor)
12713                .with_scope_authority_owners(vec!["broca".to_string()]);
12714            let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
12715            let mut gated = head("s", 1);
12716            gated.attributes.agent_id = Some("agent".to_string());
12717            sync(&handler, &broca, 1, vec![gated]).await.unwrap();
12718            let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
12719                describe(&handler, &broca, "broca", "s").await
12720            else {
12721                panic!("not a describe reply");
12722            };
12723            assert!(scope.unwrap().owner_authorized);
12724        }
12725
12726        /// With route admission, the stamp, the commit re-check and drains in
12727        /// place, the feature is advertised: the module ops in HELLO_ACK, and
12728        /// `scopes/v1` in HELLO_ACK and `server.describe`.
12729        #[tokio::test]
12730        async fn scope_ops_and_the_scopes_capability_are_advertised() {
12731            let handler = ControlHandler::new(Arc::new(Registry::default()));
12732            let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
12733            let ack = hello_via_sink(
12734                &handler,
12735                &ctx,
12736                &mut rx,
12737                hello_frame("m", PROTOCOL_VERSION, 1),
12738            )
12739            .await;
12740            let ack = parse_ack(&ack);
12741            for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
12742                assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
12743            }
12744            assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
12745
12746            let (client, _client_rx) = route_ctx(ConnectionId::new(2));
12747            let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12748            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12749            let reply = handler
12750                .handle_control_frame(&client, frame)
12751                .await
12752                .unwrap()
12753                .pop()
12754                .unwrap();
12755            let ClientControlResponse::ServerDescribe { capabilities, .. } =
12756                serde_json::from_slice(&reply.body).unwrap()
12757            else {
12758                panic!("not a server.describe reply");
12759            };
12760            assert!(
12761                capabilities.iter().any(|c| c == CAP_SCOPES_V1),
12762                "{capabilities:?}"
12763            );
12764        }
12765
12766        // ---- route admission, stamps, commit re-check and drains ----------
12767
12768        const PLEXUS: &str = "plexus";
12769        const OTHER: &str = "other";
12770        const AFT: &str = "aft";
12771        const BROCA: &str = "broca";
12772        const MAGIC: &str = "magic-context";
12773
12774        fn nonce(module_id: &str) -> String {
12775            format!("nonce-{module_id}")
12776        }
12777
12778        fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
12779            let (tx, rx) = mpsc::channel(64);
12780            (
12781                RouteCtx {
12782                    connection_id: ConnectionId::new(connection),
12783                    egress: FrameSink::new(tx),
12784                },
12785                rx,
12786            )
12787        }
12788
12789        /// A daemon with a configured owner (prefrontal-core) registered on its
12790        /// own module connection, two routable targets (plexus, other), and
12791        /// launch nonces minted for the modules that open routes as carriers.
12792        struct Rig {
12793            handler: ControlHandler,
12794            forwarding: Arc<ForwardingTable>,
12795            owner: RouteCtx,
12796            _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12797            modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
12798            generation: u64,
12799            next_connection: u64,
12800            _supervisor: Supervisor,
12801        }
12802
12803        async fn rig() -> Rig {
12804            rig_with_flow_support(true).await
12805        }
12806
12807        async fn rig_with_flow_support(flow_support: bool) -> Rig {
12808            let registry = Arc::new(Registry::default());
12809            let forwarding = Arc::new(ForwardingTable::default());
12810            let supervisor_handle = SupervisorHandle::new();
12811            let supervisor =
12812                Supervisor::new_for_test(Arc::clone(&registry), RestartPolicy::default())
12813                    .with_handle(supervisor_handle.clone());
12814            supervisor
12815                .supervise_configured(
12816                    ModuleSpec {
12817                        module_id: OWNER.to_string(),
12818                        program: PathBuf::from("/nonexistent/prefrontal-core"),
12819                        args: Vec::new(),
12820                        env: Vec::new(),
12821                        reserved: false,
12822                        reserved_prefixes: Vec::new(),
12823                        protocol: ModuleProtocol::Subc,
12824                        overlap: Default::default(),
12825                    },
12826                    false,
12827                )
12828                .unwrap();
12829            for module_id in [OWNER, AFT, BROCA, MAGIC] {
12830                supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
12831            }
12832            let handler =
12833                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12834                    .with_supervisor(supervisor_handle);
12835            let (owner, mut owner_rx) = wide_ctx(1);
12836            hello_via_sink(
12837                &handler,
12838                &owner,
12839                &mut owner_rx,
12840                hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
12841            )
12842            .await;
12843            let mut modules = BTreeMap::new();
12844            for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
12845                let (ctx, mut rx) = wide_ctx(connection);
12846                let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
12847                let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
12848                // A decoder version alone must not admit flow routes. Every
12849                // target here declares wire crate version 0.29.0; only one that
12850                // declares `flow-scopes/v1` promises flow behaviour.
12851                body["manifest"]["provenance"] =
12852                    serde_json::json!({"wire_crate_version": "0.29.0"});
12853                if flow_support {
12854                    body["manifest"]["capabilities"] =
12855                        serde_json::json!({"provides": ["flow-scopes/v1"]});
12856                }
12857                let hello = Frame::build(
12858                    FrameType::Hello,
12859                    control_flags(),
12860                    0,
12861                    0,
12862                    connection,
12863                    serde_json::to_vec(&body).unwrap(),
12864                )
12865                .unwrap();
12866                hello_via_sink(&handler, &ctx, &mut rx, hello).await;
12867                modules.insert(module_id.to_string(), (ctx, rx));
12868            }
12869            Rig {
12870                handler,
12871                forwarding,
12872                owner,
12873                _owner_rx: owner_rx,
12874                modules,
12875                generation: 0,
12876                next_connection: 100,
12877                _supervisor: supervisor,
12878            }
12879        }
12880
12881        fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
12882            ScopeCarrier {
12883                principal: Principal::Reserved {
12884                    module_id: module_id.to_string(),
12885                },
12886                targets: targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()),
12887            }
12888        }
12889
12890        /// The scope most tests open under: aft carries to any module, broca
12891        /// only to plexus and other, and the owner delegates as agent-1.
12892        fn session(scope_epoch: u64) -> ScopeRecord {
12893            let mut record = head("s", scope_epoch);
12894            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
12895            record.attributes.agent_id = Some("agent-1".to_string());
12896            record.attributes.delegates = true;
12897            record
12898        }
12899
12900        impl Rig {
12901            async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
12902                self.generation += 1;
12903                sync(&self.handler, &self.owner, self.generation, scopes)
12904                    .await
12905                    .expect("the owner's sync is accepted");
12906            }
12907
12908            fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
12909                ScopeSelector {
12910                    owner: Principal::Reserved {
12911                        module_id: OWNER.to_string(),
12912                    },
12913                    scope_ref: scope_ref.to_string(),
12914                    scope_epoch,
12915                }
12916            }
12917
12918            fn open_frame(
12919                &mut self,
12920                opener: Option<&str>,
12921                target: &str,
12922                scope: Option<ScopeSelector>,
12923            ) -> (
12924                RouteCtx,
12925                mpsc::Receiver<crate::router::OutboundFrame>,
12926                Frame,
12927            ) {
12928                self.next_connection += 1;
12929                let (ctx, rx) = wide_ctx(self.next_connection);
12930                let root = unique_project_root("scoped-open");
12931                let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
12932                    target: RouteTarget::ToolProvider {
12933                        module_id: target.to_string(),
12934                    },
12935                    identity: BindIdentity::new(
12936                        root.path().to_path_buf(),
12937                        "unit".to_string(),
12938                        "session".to_string(),
12939                    ),
12940                    consumer_identity: opener.map(|module_id| ConsumerIdentity {
12941                        module_id: module_id.to_string(),
12942                        launch_nonce: nonce(module_id),
12943                    }),
12944                    consumer_capabilities: None,
12945                    role_versions: None,
12946                    admission_facts: None,
12947                    scope,
12948                })
12949                .unwrap();
12950                let frame = Frame::build(
12951                    FrameType::Request,
12952                    control_flags(),
12953                    0,
12954                    0,
12955                    self.next_connection,
12956                    body,
12957                )
12958                .unwrap();
12959                (ctx, rx, frame)
12960            }
12961
12962            /// Open and expect a refusal before anything is relayed.
12963            async fn refused(
12964                &mut self,
12965                opener: Option<&str>,
12966                target: &str,
12967                scope: Option<ScopeSelector>,
12968            ) -> String {
12969                self.refusal_body(opener, target, scope).await["code"]
12970                    .as_str()
12971                    .unwrap()
12972                    .to_string()
12973            }
12974
12975            async fn refusal_body(
12976                &mut self,
12977                opener: Option<&str>,
12978                target: &str,
12979                scope: Option<ScopeSelector>,
12980            ) -> Value {
12981                let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
12982                let replies = tokio::time::timeout(
12983                    Duration::from_secs(2),
12984                    self.handler.handle_control_frame(&ctx, frame),
12985                )
12986                .await
12987                .expect("the open must be refused before waiting for a bind ack")
12988                .unwrap();
12989                assert_eq!(replies.len(), 1, "{replies:?}");
12990                assert_eq!(replies[0].header.ty, FrameType::Error);
12991                let (_, module_rx) = self.modules.get_mut(target).unwrap();
12992                assert!(
12993                    module_rx.try_recv().is_err(),
12994                    "a refused open relays nothing"
12995                );
12996                assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
12997                parse_error(&replies[0])
12998            }
12999
13000            /// Start an open and return its task and the bind the target got.
13001            async fn relayed(
13002                &mut self,
13003                opener: Option<&str>,
13004                target: &str,
13005                scope: Option<ScopeSelector>,
13006            ) -> Relayed {
13007                let (ctx, rx, frame) = self.open_frame(opener, target, scope);
13008                let handler = self.handler.clone();
13009                let task_ctx = ctx.clone();
13010                let task = tokio::spawn(async move {
13011                    handler
13012                        .handle_control_frame(&task_ctx, frame)
13013                        .await
13014                        .unwrap()
13015                });
13016                let (_, module_rx) = self.modules.get_mut(target).unwrap();
13017                let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
13018                    .await
13019                    .expect("the target receives the relayed route.bind")
13020                    .unwrap()
13021                    .frame;
13022                Relayed {
13023                    target: target.to_string(),
13024                    client: ctx,
13025                    client_rx: rx,
13026                    task,
13027                    bind,
13028                }
13029            }
13030
13031            async fn ack(&self, relayed: &Relayed) {
13032                let (module, _) = &self.modules[&relayed.target];
13033                self.handler
13034                    .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
13035                    .await
13036                    .unwrap();
13037            }
13038
13039            /// Open, ack and return the bound route.
13040            async fn bound(
13041                &mut self,
13042                opener: Option<&str>,
13043                target: &str,
13044                scope: Option<ScopeSelector>,
13045            ) -> Bound {
13046                let relayed = self.relayed(opener, target, scope).await;
13047                self.ack(&relayed).await;
13048                let Relayed {
13049                    target,
13050                    client,
13051                    mut client_rx,
13052                    task,
13053                    bind,
13054                } = relayed;
13055                assert!(
13056                    task.await.unwrap().is_empty(),
13057                    "the open is answered by commit"
13058                );
13059                let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
13060                Bound {
13061                    target,
13062                    client,
13063                    client_rx,
13064                    channel,
13065                    epoch,
13066                    bind,
13067                }
13068            }
13069
13070            fn live(&self, route: &Bound) -> bool {
13071                matches!(
13072                    self.forwarding
13073                        .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
13074                        .unwrap(),
13075                    DataRoute::Client(DataRouteState::Bound(_))
13076                )
13077            }
13078        }
13079
13080        struct Relayed {
13081            target: String,
13082            client: RouteCtx,
13083            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13084            task: tokio::task::JoinHandle<Vec<Frame>>,
13085            bind: Frame,
13086        }
13087
13088        struct Bound {
13089            target: String,
13090            client: RouteCtx,
13091            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13092            channel: u16,
13093            epoch: u32,
13094            bind: Frame,
13095        }
13096
13097        impl Bound {
13098            /// The reason of the `route.closed` this client was sent, after
13099            /// checking it also got a GOODBYE on exactly this route.
13100            fn closed_reason(&mut self) -> RouteCloseReason {
13101                let mut reason = None;
13102                let mut goodbye = false;
13103                while let Ok(outbound) = self.client_rx.try_recv() {
13104                    let frame = outbound.frame;
13105                    match frame.header.ty {
13106                        FrameType::Goodbye => {
13107                            assert_eq!(
13108                                (frame.header.channel, frame.header.epoch),
13109                                (self.channel, self.epoch)
13110                            );
13111                            goodbye = true;
13112                        }
13113                        FrameType::Push => {
13114                            let ClientControlPush::RouteClosed {
13115                                reason: r,
13116                                module_id,
13117                                ..
13118                            } = serde_json::from_slice(&frame.body).unwrap()
13119                            else {
13120                                panic!("unexpected push");
13121                            };
13122                            assert_eq!(module_id, self.target);
13123                            reason = Some(r);
13124                        }
13125                        other => panic!("unexpected frame {other:?}"),
13126                    }
13127                }
13128                assert!(goodbye, "the client is sent a GOODBYE for the closed route");
13129                reason.expect("the client is told why the route closed")
13130            }
13131
13132            fn untouched(&mut self) -> bool {
13133                self.client_rx.try_recv().is_err()
13134            }
13135
13136            fn stamp(&self) -> Option<ScopeStamp> {
13137                match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
13138                    ModuleControlRequest::RouteBind { scope, .. } => scope,
13139                    other => panic!("expected a route.bind, got {other:?}"),
13140                }
13141            }
13142        }
13143
13144        #[tokio::test]
13145        async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
13146        ) {
13147            let mut rig = rig().await;
13148            let mut record = session(1);
13149            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
13150            record.child_owners = vec![Principal::Reserved {
13151                module_id: MAGIC.to_string(),
13152            }];
13153            rig.sync(vec![record]).await;
13154            let scope = || Some(rig_selector("s", Some(1)));
13155
13156            // Admitted: the owner, a bare carrier to any module, a targeted
13157            // carrier to its listed module.
13158            rig.bound(Some(OWNER), PLEXUS, scope()).await;
13159            rig.bound(Some(AFT), OTHER, scope()).await;
13160            rig.bound(Some(BROCA), PLEXUS, scope()).await;
13161
13162            // Refused scope_not_carrier: a targeted carrier to an unlisted
13163            // module, a module that is not listed at all (a child owner is not
13164            // a carrier), and a direct key-holder.
13165            for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
13166                assert_eq!(
13167                    rig.refused(opener, target, scope()).await,
13168                    error_codes::SCOPE_NOT_CARRIER,
13169                    "{opener:?} -> {target}"
13170                );
13171            }
13172        }
13173
13174        fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
13175            ScopeSelector {
13176                owner: Principal::Reserved {
13177                    module_id: OWNER.to_string(),
13178                },
13179                scope_ref: scope_ref.to_string(),
13180                scope_epoch,
13181            }
13182        }
13183
13184        #[tokio::test]
13185        async fn an_open_without_an_epoch_is_refused_the_owners_included() {
13186            let mut rig = rig().await;
13187            rig.sync(vec![session(1)]).await;
13188            for opener in [OWNER, AFT] {
13189                assert_eq!(
13190                    rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
13191                        .await,
13192                    error_codes::SCOPE_EPOCH_REQUIRED,
13193                    "{opener}"
13194                );
13195            }
13196        }
13197
13198        #[tokio::test]
13199        async fn admission_separates_not_synced_not_live_and_ended() {
13200            let mut rig = rig().await;
13201            // Before the configured owner's first sync: retryable.
13202            let code = rig
13203                .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13204                .await;
13205            assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
13206            assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
13207
13208            // An owner that is not configured will never sync: terminal.
13209            let ghost = ScopeSelector {
13210                owner: Principal::Reserved {
13211                    module_id: "ghost".to_string(),
13212                },
13213                scope_ref: "s".to_string(),
13214                scope_epoch: Some(1),
13215            };
13216            assert_eq!(
13217                rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
13218                error_codes::SCOPE_NOT_LIVE
13219            );
13220
13221            rig.sync(vec![session(2)]).await;
13222            assert_eq!(
13223                rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
13224                    .await,
13225                error_codes::SCOPE_NOT_LIVE
13226            );
13227            for epoch in [1, 3] {
13228                assert_eq!(
13229                    rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
13230                        .await,
13231                    error_codes::SCOPE_ENDED,
13232                    "epoch {epoch}"
13233                );
13234            }
13235            // Control: the live epoch is admitted.
13236            rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
13237                .await;
13238        }
13239
13240        #[tokio::test]
13241        async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
13242            let mut rig = rig().await;
13243            rig.sync(vec![session(1)]).await;
13244            let route = rig
13245                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13246                .await;
13247            let stamp = route.stamp().expect("a scoped bind carries the stamp");
13248            assert_eq!(stamp.scope_ref, "s");
13249            assert_eq!(stamp.scope_epoch, 1);
13250            assert_eq!(stamp.kind, ScopeKind::Head);
13251            assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
13252            assert!(stamp.attributes.delegates);
13253            assert!(stamp.owner_authorized);
13254            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
13255            assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
13256
13257            // broca owns a scope of its own on its own module connection; it is
13258            // not in scope_authority_owners, so its stamp is not authorized.
13259            let (broca, mut broca_rx) = wide_ctx(50);
13260            hello_via_sink(
13261                &rig.handler,
13262                &broca,
13263                &mut broca_rx,
13264                hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
13265            )
13266            .await;
13267            sync(&rig.handler, &broca, 1, vec![head("b", 1)])
13268                .await
13269                .unwrap();
13270            let own = ScopeSelector {
13271                owner: Principal::Reserved {
13272                    module_id: BROCA.to_string(),
13273                },
13274                scope_ref: "b".to_string(),
13275                scope_epoch: Some(1),
13276            };
13277            let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
13278            assert!(!route.stamp().unwrap().owner_authorized);
13279        }
13280
13281        #[tokio::test]
13282        async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
13283            let mut rig = rig().await;
13284            let mut record = head("s", 1);
13285            record.carriers = vec![carrier(AFT, None)];
13286            let flow_id = "Flow:run-7/step_2!~";
13287            record.attributes.flow_id = Some(flow_id.to_string());
13288            let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
13289                .await
13290                .unwrap();
13291            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
13292                panic!("not a sync reply");
13293            };
13294            assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
13295            let route = rig
13296                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13297                .await;
13298            assert!(rig.live(&route), "the stamped bind committed");
13299            let stamp = route.stamp().expect("a flow scope carries a stamp");
13300            assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
13301            assert_eq!(stamp.attributes.agent_id, None);
13302            assert!(!stamp.attributes.delegates);
13303            assert!(stamp.owner_authorized);
13304            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
13305            assert_eq!(unscoped.stamp(), None);
13306        }
13307
13308        #[tokio::test]
13309        async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
13310            let mut rig = rig_with_flow_support(false).await;
13311            let mut record = session(1);
13312            record.attributes.flow_id = Some("flow:7".to_string());
13313            rig.sync(vec![record]).await;
13314            for opener in [OWNER, AFT] {
13315                let body = rig
13316                    .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
13317                    .await;
13318                assert_eq!(body["code"], "target_flow_unsupported");
13319                let message = body["message"].as_str().unwrap();
13320                for required in [PLEXUS, "flow-scopes/v1"] {
13321                    assert!(message.contains(required), "{message}");
13322                }
13323            }
13324        }
13325
13326        #[tokio::test]
13327        async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
13328            let mut rig = rig_with_flow_support(true).await;
13329            let mut record = session(1);
13330            record.attributes.flow_id = Some("flow:7".to_string());
13331            rig.sync(vec![record]).await;
13332            for opener in [OWNER, AFT] {
13333                let route = rig
13334                    .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
13335                    .await;
13336                assert!(rig.live(&route));
13337                assert_eq!(
13338                    route.stamp().unwrap().attributes.flow_id.as_deref(),
13339                    Some("flow:7")
13340                );
13341            }
13342        }
13343
13344        #[tokio::test]
13345        async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
13346            use std::future::Future;
13347
13348            let mut rig = rig_with_flow_support(true).await;
13349            let mut record = session(1);
13350            record.attributes.flow_id = Some("flow:7".to_string());
13351            rig.sync(vec![record]).await;
13352            let (client, mut client_rx, frame) =
13353                rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
13354            // Hold the route-open response permit so admission sees the first
13355            // target but relay reservation cannot capture an endpoint yet.
13356            for _ in 0..64 {
13357                client.egress.try_send(route_bind_ack(1)).unwrap();
13358            }
13359            let handler = rig.handler.clone();
13360            let mut open = Box::pin(handler.handle_control_frame(&client, frame));
13361            std::future::poll_fn(|cx| {
13362                assert!(open.as_mut().poll(cx).is_pending());
13363                std::task::Poll::Ready(())
13364            })
13365            .await;
13366            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
13367
13368            let old_connection = rig.modules[PLEXUS].0.connection_id;
13369            rig.handler.cleanup_connection(old_connection).unwrap();
13370            let (replacement, mut replacement_rx) = wide_ctx(200);
13371            let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
13372            let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
13373            body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
13374            let hello = Frame::build(
13375                FrameType::Hello,
13376                control_flags(),
13377                0,
13378                0,
13379                200,
13380                serde_json::to_vec(&body).unwrap(),
13381            )
13382            .unwrap();
13383            hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
13384
13385            client_rx.try_recv().unwrap();
13386            let replies = tokio::time::timeout(Duration::from_secs(2), open)
13387                .await
13388                .expect("the replacement is refused without waiting for a bind ack")
13389                .unwrap();
13390            assert_eq!(replies.len(), 1);
13391            let body = parse_error(&replies[0]);
13392            assert_eq!(body["code"], "target_flow_unsupported");
13393            for required in [PLEXUS, "flow-scopes/v1"] {
13394                assert!(body["message"].as_str().unwrap().contains(required));
13395            }
13396            assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
13397            assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
13398            assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
13399            assert!(rig
13400                .handler
13401                .registry
13402                .get_module_by_connection(replacement.connection_id)
13403                .unwrap()
13404                .is_some());
13405        }
13406
13407        #[tokio::test]
13408        async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
13409        {
13410            let mut rig = rig_with_flow_support(false).await;
13411            rig.sync(vec![session(1)]).await;
13412            let route = rig
13413                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13414                .await;
13415            assert!(rig.live(&route));
13416            assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
13417            let mut record = session(1);
13418            record.attributes.flow_id = Some("flow:7".to_string());
13419            rig.sync(vec![record]).await;
13420            let unscoped = rig.bound(Some(AFT), OTHER, None).await;
13421            assert!(rig.live(&unscoped));
13422            assert_eq!(unscoped.stamp(), None);
13423        }
13424
13425        #[tokio::test]
13426        async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
13427            let mut rig = rig().await;
13428            let mut record = session(1);
13429            record.attributes.flow_id = Some("flow:7".to_string());
13430            rig.sync(vec![record.clone()]).await;
13431            let before = rig.forwarding.published_scope_tag(OWNER, "s");
13432            let mut owner_route = rig
13433                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
13434                .await;
13435            let mut carrier_route = rig
13436                .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
13437                .await;
13438            let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
13439            rig.sync(vec![record.clone()]).await;
13440            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
13441            assert!(rig.live(&owner_route) && owner_route.untouched());
13442            assert!(rig.live(&carrier_route) && carrier_route.untouched());
13443
13444            record.attributes.flow_id = Some("flow:8".to_string());
13445            rig.sync(vec![record]).await;
13446            let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
13447            let before = before.unwrap();
13448            assert_eq!(after.scope_epoch, before.scope_epoch);
13449            assert!(after.version > before.version);
13450            for route in [&mut owner_route, &mut carrier_route] {
13451                assert!(!rig.live(route));
13452                assert_eq!(
13453                    route.closed_reason(),
13454                    RouteCloseReason::ScopeDelegationChanged
13455                );
13456            }
13457            assert!(rig.live(&unscoped) && unscoped.untouched());
13458            // Each provider also receives a GOODBYE for its drained route;
13459            // consume it before expecting the next route.bind on that sink.
13460            for target in [PLEXUS, OTHER] {
13461                let (_, module_rx) = rig.modules.get_mut(target).unwrap();
13462                let goodbye = module_rx
13463                    .try_recv()
13464                    .expect("the provider sees the drain")
13465                    .frame;
13466                assert_eq!(goodbye.header.ty, FrameType::Goodbye);
13467                assert!(module_rx.try_recv().is_err());
13468            }
13469            let rebound = rig
13470                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13471                .await;
13472            assert_eq!(
13473                rebound.stamp().unwrap().attributes.flow_id.as_deref(),
13474                Some("flow:8")
13475            );
13476        }
13477
13478        /// The owner's sync lands between admission and the module's ack. The
13479        /// open is refused by name, the module's other routes stay up, and the
13480        /// reserved pair is released. Changed content is retryable; an ended
13481        /// scope is not.
13482        #[tokio::test]
13483        async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
13484            let mut rig = rig().await;
13485            rig.sync(vec![session(1)]).await;
13486            let mut cotenant = rig
13487                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
13488                .await;
13489
13490            let mut changed = session(1);
13491            changed.child_owners.push(Principal::Reserved {
13492                module_id: MAGIC.to_string(),
13493            });
13494            let mut ended = None;
13495            for (code, next) in [
13496                (error_codes::SCOPE_CHANGED, vec![changed]),
13497                (error_codes::SCOPE_ENDED, Vec::new()),
13498            ] {
13499                let relayed = rig
13500                    .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13501                    .await;
13502                let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
13503                ended = Some(next.is_empty());
13504                rig.sync(next).await;
13505                rig.ack(&relayed).await;
13506                let replies = relayed.task.await.unwrap();
13507                assert_eq!(replies.len(), 1, "{replies:?}");
13508                assert_eq!(parse_error(&replies[0])["code"], code);
13509                assert_eq!(
13510                    subc_protocol::error_codes::is_retryable_route_open(code),
13511                    code == error_codes::SCOPE_CHANGED
13512                );
13513                assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
13514                // The module is told to drop just the binding it created.
13515                let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
13516                // Collected, because ending the scope also closes the co-tenant
13517                // route, whose GOODBYE comes first.
13518                let mut goodbyes = Vec::new();
13519                while let Ok(outbound) = plexus_rx.try_recv() {
13520                    assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
13521                    goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
13522                }
13523                assert!(
13524                    goodbyes.contains(&(bind_channel, bind_epoch)),
13525                    "{goodbyes:?}"
13526                );
13527                assert!(rig
13528                    .handler
13529                    .registry
13530                    .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
13531                    .unwrap()
13532                    .is_some());
13533            }
13534            assert_eq!(ended, Some(true));
13535            // The co-tenant stayed up through the change, and closed only when
13536            // the scope ended, by the drain rule rather than by the commit.
13537            assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
13538        }
13539
13540        /// Each row of the drain table on one set of routes: the owner's, a
13541        /// bare carrier's, and a targeted carrier's to each of its targets.
13542        #[tokio::test]
13543        async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
13544            struct Case {
13545                name: &'static str,
13546                change: fn(&mut ScopeRecord),
13547                /// Closed routes by index: owner->plexus, aft->plexus,
13548                /// broca->plexus, broca->other.
13549                closed: [Option<RouteCloseReason>; 4],
13550            }
13551            use RouteCloseReason::*;
13552            let cases = [
13553                Case {
13554                    name: "a carrier entry removed",
13555                    change: |r| {
13556                        r.carriers.retain(|c| {
13557                            c.principal
13558                                != Principal::Reserved {
13559                                    module_id: AFT.to_string(),
13560                                }
13561                        })
13562                    },
13563                    closed: [None, Some(ScopeCarrierRemoved), None, None],
13564                },
13565                Case {
13566                    name: "a target removed from a carrier",
13567                    change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
13568                    closed: [None, None, None, Some(ScopeCarrierRemoved)],
13569                },
13570                Case {
13571                    name: "a bare carrier narrowed to targets",
13572                    change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
13573                    closed: [None, Some(ScopeCarrierRemoved), None, None],
13574                },
13575                Case {
13576                    name: "delegates turned off",
13577                    change: |r| r.attributes.delegates = false,
13578                    closed: [Some(ScopeDelegationChanged); 4],
13579                },
13580                Case {
13581                    name: "agent_id changed",
13582                    change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
13583                    closed: [Some(ScopeDelegationChanged); 4],
13584                },
13585                Case {
13586                    name: "a carrier added, child owners changed, the record re-sent",
13587                    change: |r| {
13588                        r.carriers.push(carrier(MAGIC, None));
13589                        r.child_owners.push(Principal::Reserved {
13590                            module_id: MAGIC.to_string(),
13591                        });
13592                    },
13593                    closed: [None; 4],
13594                },
13595                Case {
13596                    name: "a target added",
13597                    change: |r| {
13598                        r.carriers[1]
13599                            .targets
13600                            .as_mut()
13601                            .unwrap()
13602                            .push("third".to_string())
13603                    },
13604                    closed: [None; 4],
13605                },
13606                Case {
13607                    name: "delegates turned on",
13608                    change: |r| r.attributes.delegates = true,
13609                    closed: [None; 4],
13610                },
13611            ];
13612            for case in cases {
13613                let mut rig = rig().await;
13614                rig.sync(vec![session(1)]).await;
13615                let scope = || Some(rig_selector("s", Some(1)));
13616                let mut routes = [
13617                    rig.bound(Some(OWNER), PLEXUS, scope()).await,
13618                    rig.bound(Some(AFT), PLEXUS, scope()).await,
13619                    rig.bound(Some(BROCA), PLEXUS, scope()).await,
13620                    rig.bound(Some(BROCA), OTHER, scope()).await,
13621                ];
13622                let mut record = session(1);
13623                (case.change)(&mut record);
13624                rig.sync(vec![record]).await;
13625                for (index, expected) in case.closed.iter().enumerate() {
13626                    let route = &mut routes[index];
13627                    match expected {
13628                        Some(reason) => {
13629                            assert!(!rig.live(route), "{}: route {index} still live", case.name);
13630                            assert_eq!(
13631                                route.closed_reason(),
13632                                *reason,
13633                                "{}: route {index}",
13634                                case.name
13635                            );
13636                        }
13637                        None => {
13638                            assert!(rig.live(route), "{}: route {index} closed", case.name);
13639                            assert!(
13640                                route.untouched(),
13641                                "{}: route {index} was told something",
13642                                case.name
13643                            );
13644                        }
13645                    }
13646                }
13647            }
13648        }
13649
13650        #[tokio::test]
13651        async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
13652            // Removed, and replaced by a higher epoch.
13653            for next in [Vec::new(), vec![session(2)]] {
13654                let mut rig = rig().await;
13655                rig.sync(vec![session(1)]).await;
13656                let mut route = rig
13657                    .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13658                    .await;
13659                rig.sync(next).await;
13660                assert!(!rig.live(&route));
13661                assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
13662            }
13663
13664            // A child whose parent ends: its routes close as parent-ended, the
13665            // child stays live, and routes under the parent close as ended.
13666            let mut rig = rig().await;
13667            let mut child = session(1);
13668            child.scope_ref = "child".to_string();
13669            child.kind = ScopeKind::Worker;
13670            child.parent = Some(ScopeParent {
13671                owner: Principal::Reserved {
13672                    module_id: OWNER.to_string(),
13673                },
13674                scope_ref: "s".to_string(),
13675                scope_epoch: 1,
13676            });
13677            rig.sync(vec![session(1), child.clone()]).await;
13678            let mut child_route = rig
13679                .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
13680                .await;
13681            assert_eq!(
13682                child_route.stamp().unwrap().parent_state,
13683                Some(ParentState::Linked)
13684            );
13685            rig.sync(vec![child]).await;
13686            assert!(!rig.live(&child_route));
13687            assert_eq!(
13688                child_route.closed_reason(),
13689                RouteCloseReason::ScopeParentEnded
13690            );
13691        }
13692
13693        #[tokio::test]
13694        async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
13695        ) {
13696            let mut rig = rig().await;
13697            rig.sync(vec![session(1)]).await;
13698            let mut route = rig
13699                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13700                .await;
13701            let before = rig.forwarding.published_scope_tag(OWNER, "s");
13702            rig.sync(vec![session(1)]).await;
13703            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
13704            assert!(rig.live(&route) && route.untouched());
13705
13706            // A call in flight on the route when another carrier is added. A
13707            // forwarded REQUEST holds one credit on the route's flow until the
13708            // module answers; the router takes it exactly like this.
13709            let DataRoute::Client(DataRouteState::Bound(binding)) = rig
13710                .forwarding
13711                .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
13712                .unwrap()
13713            else {
13714                panic!("the route is bound");
13715            };
13716            binding.flow.acquire_tagged(9, false).await.unwrap();
13717            let mut widened = session(1);
13718            widened.carriers.push(carrier(MAGIC, None));
13719            rig.sync(vec![widened]).await;
13720            assert!(rig.live(&route) && route.untouched());
13721            let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
13722            assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
13723            // The call's credit is still held on an open flow, so its answer
13724            // will be delivered: closing the route would have closed the flow.
13725            assert_eq!(binding.flow.in_flight(), 1);
13726            binding
13727                .flow
13728                .acquire_tagged(10, false)
13729                .await
13730                .expect("the flow is still open");
13731        }
13732
13733        /// A swap's superseded endpoint keeps its routes until drained; ending
13734        /// the scope closes them there too.
13735        #[tokio::test]
13736        async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
13737            let mut rig = rig().await;
13738            rig.sync(vec![session(1)]).await;
13739            let mut on_incumbent = rig
13740                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13741                .await;
13742
13743            // Swap plexus: register a candidate and cut over, leaving the
13744            // incumbent superseded with the route still on it.
13745            let (candidate, _candidate_rx) = wide_ctx(9);
13746            let registration = rig
13747                .handler
13748                .registry
13749                .register_candidate_with_control_ops(
13750                    manifest(PLEXUS, PROTOCOL_VERSION),
13751                    PROTOCOL_VERSION,
13752                    candidate.connection_id,
13753                    module_baseline_control_ops(),
13754                )
13755                .unwrap();
13756            rig.forwarding
13757                .register_candidate_module_connection(
13758                    candidate.connection_id,
13759                    PLEXUS.to_string(),
13760                    PROTOCOL_VERSION,
13761                    manifest_concurrency(&registration.manifest),
13762                    candidate.egress.clone(),
13763                )
13764                .unwrap();
13765            rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
13766            rig.handler
13767                .registry
13768                .promote_candidate(PLEXUS)
13769                .unwrap()
13770                .unwrap();
13771            assert!(rig.live(&on_incumbent), "cutover alone does not drain");
13772
13773            rig.sync(Vec::new()).await;
13774            assert!(!rig.live(&on_incumbent));
13775            assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
13776            let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
13777            let goodbye = incumbent_rx
13778                .try_recv()
13779                .expect("the superseded endpoint is told")
13780                .frame;
13781            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
13782        }
13783    }
13784}
13785
13786#[cfg(test)]
13787mod concurrency_default_exposure_tests {
13788    use super::*;
13789
13790    fn hello_body(role_json: &str) -> Vec<u8> {
13791        format!(
13792            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
13793        )
13794        .into_bytes()
13795    }
13796
13797    fn manifest_from(body: &[u8]) -> ModuleManifest {
13798        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
13799        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
13800            .expect("manifest parses")
13801    }
13802
13803    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
13804
13805    #[test]
13806    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
13807        let body = hello_body(&format!(
13808            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
13809        ));
13810        let manifest = manifest_from(&body);
13811        // Precondition: serde really resolved it to the default, so the typed
13812        // manifest alone cannot answer the question this probe exists for.
13813        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
13814        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
13815    }
13816
13817    #[test]
13818    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
13819        let body = hello_body(&format!(
13820            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
13821        ));
13822        let manifest = manifest_from(&body);
13823        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
13824        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
13825    }
13826
13827    #[test]
13828    fn non_management_roles_are_never_reported() {
13829        let body = hello_body(
13830            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
13831        );
13832        let manifest = manifest_from(&body);
13833        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
13834    }
13835}