Skip to main content

subc_daemon/
control.rs

1use std::{
2    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3    fmt,
4    path::{Path, PathBuf},
5    sync::{Arc, Mutex, RwLock},
6    time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14    ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20    error_codes,
21    manifest::{
22        validate_hello_capability_grammar, validate_hello_self_signal_declarations,
23        CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
24        ProviderRole,
25    },
26    scope::{
27        ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
28        CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_DESCRIBE_OP, SCOPE_SYNC_OP,
29    },
30    session::{
31        validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
32        ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
33        MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE, ROLE_VERSIONS_FIELD,
34    },
35    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
36    Priority, RouteTarget, PROTOCOL_VERSION,
37};
38use tokio::time::{timeout_at, Instant};
39use tracing::{debug, info, warn};
40
41use crate::{
42    capability_requirements::{
43        log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
44        CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
45        RuntimeModule,
46    },
47    daemon_config::RestartRequiredSection,
48    forwarding::{
49        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
50        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
51        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
52    },
53    observability::{
54        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
55    },
56    provenance::{
57        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
58    },
59    registry::{ChannelState, ConnectionId, Registry, RegistryError},
60    router::{RouteCtx, RouterError},
61    scopes::{BoundScope, HelloLaunchNonces, ScopeTable},
62    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
63    stderr_tail::{CaptureState, TailEntry},
64    supervise::{
65        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
66        SupervisorHandle, SwapHelloAdmission,
67    },
68    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
69};
70
71/// Lowest envelope version this subc build will negotiate.
72///
73/// Module HELLO negotiation is exact: peers must use the daemon's locked
74/// protocol version. Older and newer peers receive `version_unsupported` and
75/// are not registered.
76pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
77
78const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
79const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
80const CAP_PING_PONG: &str = "ping_pong_v1";
81const CAP_SESSION_ATTACH: &str = "session_attach_v1";
82const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
83
84const SUBC_CONTROL_OPS: &[&str] = &[
85    ops::SERVER_DESCRIBE,
86    ops::CATALOG_LIST,
87    ops::ROUTE_OPEN,
88    ops::ROUTE_POLL,
89    ops::ROUTE_CLOSING,
90    ops::ROUTE_CLOSED,
91    ops::SUPERVISOR_LIST,
92    ops::SUPERVISOR_RESTART,
93    ops::SUPERVISOR_SWAP,
94    ops::SUPERVISOR_RELOAD,
95    ops::SUPERVISOR_RESCAN,
96    ops::SUPERVISOR_RELEASE_RESERVED,
97    ops::SUPERVISOR_SET_ENABLED,
98    ops::SUPERVISOR_HEALTH_PROBE,
99    ops::SUPERVISOR_HEALTH,
100    ops::SUPERVISOR_STDERR_TAIL,
101    ops::SUPERVISOR_TERMINALS,
102    ops::SUPERVISOR_ROUTES,
103    ops::SUPERVISOR_PROVENANCE,
104    ops::SUPERVISOR_SPAWN_SNAPSHOT,
105    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
106];
107
108const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
109    MODULE_TO_SUBC_OP_CATALOG_UPDATE,
110    "supervisor.live_roots",
111    SCOPE_SYNC_OP,
112    SCOPE_DESCRIBE_OP,
113];
114
115/// Module-originated ops the daemon answers but does not advertise in
116/// `HELLO_ACK`. Empty today; an op is served from here while the feature it
117/// belongs to is incomplete, so no module is told it works before it does.
118const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
119
120const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
121
122/// How long subc waits for a module to ack a relayed route.bind before returning
123/// `module_timeout`. The ack waits on the module's own configure, which for AFT
124/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
125/// and DB-open work — on a cold page cache or a large repo that legitimately
126/// exceeds a couple of seconds. The default is generous because rejecting a VALID
127/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
128/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
129pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
130
131/// How many CONSECUTIVE full-budget relay timeouts against one target module
132/// open that module's bind-relay breaker.
133///
134/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
135/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
136/// so three consecutive full-budget timeouts take ~36s to observe: every client
137/// whose open contributed to opening the breaker had already given up on its
138/// own. That is what makes opening the breaker unable to turn a call that would
139/// have succeeded into a refusal — it can only make an already-failing module
140/// fail faster.
141///
142/// Two would be reachable inside one default deadline. One would convict a
143/// module on a single cold-cache bind, which is exactly the valid-but-slow case
144/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
145pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
146
147/// How long a module's bind-relay breaker stays open before exactly one
148/// `route.open` is let through as a probe.
149///
150/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
151/// re-pays a full-budget stall almost continuously, and the breaker stops being
152/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
153/// deadline: a client that starts retrying after the module recovers has to get
154/// a probe opportunity inside its own deadline, or the breaker converts a
155/// recovered module into a failed call — the failure it exists to prevent,
156/// pointed the other way.
157///
158/// 20s sits between those with room on both sides, and it caps what a wedged
159/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
160/// rather than one per `route.open` per connection. The stall that motivated
161/// this, with its measurements, is written up in
162/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
163/// waited the whole budget out.
164pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
165
166const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
167const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
168
169fn reload_verdict(
170    configured: &Path,
171    spawned_from: Option<&Path>,
172    image: subc_control::RunningImageAgreement,
173) -> PendingReloadVerdict {
174    let path = match spawned_from {
175        Some(spawned_from) if configured == spawned_from => ReloadPathAgreement::Match,
176        Some(spawned_from) => ReloadPathAgreement::Mismatch {
177            configured: configured.to_path_buf(),
178            spawned_from: spawned_from.to_path_buf(),
179        },
180        None => ReloadPathAgreement::Unavailable {
181            reason: if matches!(
182                image,
183                subc_control::RunningImageAgreement::Unavailable {
184                    reason: subc_control::RunningImageUnavailableReason::NotRunning
185                }
186            ) {
187                ReloadPathUnavailableReason::NotRunning
188            } else {
189                ReloadPathUnavailableReason::SpawnedPathUnavailable
190            },
191        },
192    };
193    PendingReloadVerdict { path, image }
194}
195
196#[derive(Clone)]
197struct DaemonProvenanceFacts {
198    build: DaemonBuildProvenance,
199    pid: Option<u32>,
200    started_at_ms: Option<u64>,
201    start_clock: Option<crate::clock::StartClock>,
202    executable_path: Option<PathBuf>,
203    executable_identity: Option<SpawnedFileIdentity>,
204    process_start_time: Option<u64>,
205    probe: ExecutableIdentityProbe,
206}
207
208impl Default for DaemonProvenanceFacts {
209    fn default() -> Self {
210        Self {
211            build: DaemonBuildProvenance {
212                build_git_sha: None,
213                build_lock_digest: None,
214            },
215            pid: None,
216            started_at_ms: None,
217            start_clock: None,
218            executable_path: None,
219            executable_identity: None,
220            process_start_time: None,
221            probe: ExecutableIdentityProbe::default(),
222        }
223    }
224}
225
226#[derive(Debug, Clone)]
227struct SupervisorRescanContext {
228    supervisor: Supervisor,
229    config_path: PathBuf,
230    configured_port: Option<u16>,
231    storage_config: Option<crate::daemon_config::StorageConfig>,
232    admission_facts_carrier_module_id: Option<String>,
233    admission_facts_targets: Option<Vec<String>>,
234    scope_authority_owners: Vec<String>,
235}
236
237/// Refusal labels passed to `observe_route_open_refusal` that mean the target
238/// module is not serving right now, and so open or extend an outage in the
239/// route outage tracker. Every one of them is only reachable after the target
240/// was found in the registry, which is what keeps an arbitrary client-chosen
241/// id from ever creating tracker state.
242///
243/// Deliberately absent: `not_registered` and `removed` (the id may be
244/// anything a client sent, and a removed module is gone on purpose),
245/// `protocol_none` (such a module never serves routes, so nothing is out),
246/// `role_not_provided`, `op_not_allowed`, `bad_consumer_identity`, the
247/// capability and admission-facts refusals (they refuse the caller, not a
248/// module outage), and `relay_reservation_failed` (its code ranges over
249/// capacity limits as well as a vanished connection). Capacity, breaker,
250/// relay-timeout and module-rejection refusals do not pass through that
251/// function at all; the breaker logs its own transitions.
252///
253/// The two not-serving refusals that bypass that function record themselves
254/// at their own sites: `supervised_not_registered` and `declared_not_ready`.
255/// `required_capability_unprovided` is not tracked: the module itself is up,
256/// and the outage belongs to the missing provider.
257const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
258    "reloading",
259    "supervisor_not_live",
260    "registration_not_active",
261    "no_forwarding_connection",
262    "relay_send_failed",
263];
264
265/// Real channel-0 control handler for subc itself.
266#[derive(Clone)]
267pub struct ControlHandler {
268    registry: Arc<Registry>,
269    forwarding: Arc<ForwardingTable>,
270    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
271    supervisor: SupervisorHandle,
272    subc_capabilities: Arc<[String]>,
273    /// Daemon-wide route.bind relay budget. Used as the fallback when the
274    /// target module has no per-module override in
275    /// `route_bind_relay_timeouts`.
276    route_bind_relay_timeout: Duration,
277    /// Per-module route.bind relay budget overrides, keyed by module id. When
278    /// `handle_route_open` resolves the deadline for a target module, a
279    /// per-module entry wins over the daemon-wide value above.
280    route_bind_relay_timeouts: BTreeMap<String, Duration>,
281    /// Per-target-module bind-relay breaker state. Shared with the forwarding
282    /// table, which is where a new module connection resets it.
283    route_bind_breakers: RouteBindBreakers,
284    /// Live relay admissions keyed by target module. Shared through the
285    /// forwarding table so cloned or separately built handlers enforce one cap.
286    route_bind_concurrency: RouteBindConcurrency,
287    /// Start and end of each module's not-serving period as seen by
288    /// `route.open`, so an outage gets one line at each edge instead of only
289    /// the per-refusal INFO lines. Taken from the forwarding table, so every
290    /// handler built over one table shares it.
291    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
292    /// Consecutive relay timeouts that open a module's breaker.
293    route_bind_breaker_threshold: u32,
294    /// How long a breaker stays open before one probe is admitted.
295    route_bind_breaker_cooldown: Duration,
296    health_probe_timeout: Duration,
297    /// Central storage policy. When set, each registering module receives its
298    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
299    storage_config: Option<crate::daemon_config::StorageConfig>,
300    /// The machine id established at boot, served on every HELLO_ACK and on
301    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
302    /// changes the file, never this value. `None` serves no id.
303    machine_id: Option<crate::machine_id::MachineId>,
304    admission_facts_carrier_module_id: Option<String>,
305    admission_facts_targets: Option<Vec<String>>,
306    /// Scope records with their sync authorities and tombstones; see
307    /// `crate::scopes`. Shared by clones of this handler, so every connection
308    /// reads and writes one table.
309    scopes: Arc<RwLock<ScopeTable>>,
310    /// The configured `scope_authority_owners`, kept so a rescan can report a
311    /// changed value as needing a daemon restart; rescan never applies it.
312    scope_authority_owners: Vec<String>,
313    /// The launch nonce each module connection presented at HELLO, which is how
314    /// a `scope.sync` is matched to the owner's current launch.
315    hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
316    rescan: Option<SupervisorRescanContext>,
317    connected_clients: ConnectedClients,
318    counters: DaemonCounters,
319    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
320    daemon_provenance: DaemonProvenanceFacts,
321    #[cfg(test)]
322    control_dispatch_delay: Option<Duration>,
323    #[cfg(test)]
324    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
325}
326
327impl fmt::Debug for ControlHandler {
328    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
329        f.debug_struct("ControlHandler")
330            .field("registry", &self.registry)
331            .field("forwarding", &self.forwarding)
332            .field("process_liveness", &self.process_liveness.is_some())
333            .field("supervisor", &self.supervisor)
334            .field("subc_capabilities", &self.subc_capabilities)
335            .finish()
336    }
337}
338
339struct RouteOpenRequest {
340    target: RouteTarget,
341    identity: BindIdentity,
342    consumer_identity: Option<ConsumerIdentity>,
343    consumer_capabilities: Option<Vec<String>>,
344    role_versions: Option<BTreeMap<String, String>>,
345    admission_facts: Option<serde_json::Value>,
346    scope: Option<ScopeSelector>,
347}
348
349struct RouteBindReservationGuard {
350    forwarding: Arc<ForwardingTable>,
351    endpoint: ModuleEndpointId,
352    relay_corr: u64,
353    armed: bool,
354}
355
356struct ModuleControlRpcGuard {
357    forwarding: Arc<ForwardingTable>,
358    endpoint: ModuleEndpointId,
359    corr: u64,
360    armed: bool,
361}
362
363impl ModuleControlRpcGuard {
364    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
365        Self {
366            forwarding,
367            endpoint,
368            corr,
369            armed: true,
370        }
371    }
372
373    fn disarm(&mut self) {
374        self.armed = false;
375    }
376}
377
378impl Drop for ModuleControlRpcGuard {
379    fn drop(&mut self) {
380        if self.armed {
381            let _ = self
382                .forwarding
383                .cancel_module_control_rpc(self.endpoint, self.corr);
384        }
385    }
386}
387
388impl RouteBindReservationGuard {
389    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
390        Self {
391            forwarding,
392            endpoint,
393            relay_corr,
394            armed: true,
395        }
396    }
397
398    fn release_and_disarm(&mut self) {
399        if !self.armed {
400            return;
401        }
402        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
403            self.endpoint,
404            self.relay_corr,
405            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
406        ) {
407            send_goodbye_target_best_effort(
408                &self.forwarding.counters(),
409                &target,
410                "canceled route.bind",
411            );
412        }
413        self.armed = false;
414    }
415
416    fn disarm(&mut self) {
417        self.armed = false;
418    }
419}
420
421impl Drop for RouteBindReservationGuard {
422    fn drop(&mut self) {
423        self.release_and_disarm();
424    }
425}
426
427/// Per-target-module circuit breaker around the `route.bind` relay.
428///
429/// The connection reader is serial per connection, so a module whose `on_bind`
430/// sits on the ack blocks every LATER frame on the connections that call it,
431/// including calls to unrelated modules. This does not make any module's bind
432/// fast; it stops the daemon paying the full budget again and again for a
433/// condition it has already observed.
434///
435/// State is keyed by TARGET MODULE and shared by every connection: a wedged
436/// module wedges everyone, so what one connection learned should protect the
437/// rest.
438///
439/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
440/// relay to that module has actually timed out, and is removed again when a
441/// relay is accepted or the module reconnects, so it cannot grow with traffic
442/// or with modules that behave.
443///
444/// # Why a `std` mutex here is not the head-of-line defect again
445///
446/// Acquisition never awaits. The critical section is a hash lookup plus a few
447/// integer updates, with no I/O and no `.await` inside it, so a reader task
448/// cannot be descheduled behind it the way it can behind
449/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
450/// primitive, held for the same kind of work, as the refusal counter this very
451/// path already increments.
452///
453/// It is also NOT on the data-plane splice path: only `route.open` and module
454/// registration touch it, so bound-route frames gain no state check and no
455/// contention.
456#[derive(Debug, Clone, Default)]
457pub(crate) struct RouteBindBreakers {
458    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
459}
460
461#[derive(Debug, Clone, Default)]
462pub(crate) struct RouteBindConcurrency {
463    modules: Arc<Mutex<HashMap<String, usize>>>,
464}
465
466struct RouteBindConcurrencyGuard {
467    concurrency: RouteBindConcurrency,
468    module_id: String,
469}
470
471impl RouteBindConcurrency {
472    /// Admit without waiting. Waiting here would move the bind stall from the
473    /// module reply to a semaphore and restore reader head-of-line blocking.
474    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
475        let mut modules = self
476            .modules
477            .lock()
478            .expect("route.bind concurrency mutex poisoned");
479        let in_flight = modules.entry(module_id.to_string()).or_default();
480        if *in_flight >= limit {
481            return Err(*in_flight);
482        }
483        *in_flight += 1;
484        Ok(RouteBindConcurrencyGuard {
485            concurrency: self.clone(),
486            module_id: module_id.to_string(),
487        })
488    }
489}
490
491impl Drop for RouteBindConcurrencyGuard {
492    fn drop(&mut self) {
493        let mut modules = self
494            .concurrency
495            .modules
496            .lock()
497            .expect("route.bind concurrency mutex poisoned");
498        let remove = {
499            let in_flight = modules
500                .get_mut(&self.module_id)
501                .expect("admitted route.bind has a concurrency entry");
502            *in_flight -= 1;
503            *in_flight == 0
504        };
505        if remove {
506            modules.remove(&self.module_id);
507        }
508    }
509}
510
511#[derive(Debug, Default)]
512struct ModuleBreakerState {
513    /// Relay timeouts observed with no accepted relay in between.
514    consecutive_timeouts: u32,
515    /// `Some` while the breaker is open: the instant the cooldown expires and
516    /// the next arrival may probe. `None` means closed.
517    cooldown_until: Option<Instant>,
518    /// A half-open probe has been admitted and has not settled yet. This is
519    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
520    /// that read the cooldown, so concurrent opens arriving at the moment the
521    /// cooldown expires cannot all decide that they are the probe.
522    probe_in_flight: bool,
523}
524
525/// What the breaker decided for one `route.open`, before any relay work.
526enum RouteBindAdmission<'a> {
527    Admitted {
528        guard: RouteBindBreakerGuard<'a>,
529        /// This open is the single half-open probe, so the transition is worth
530        /// one log line.
531        probe: bool,
532    },
533    Refused {
534        consecutive_timeouts: u32,
535        /// What is left of the cooldown. Zero when the refusal is because the
536        /// one probe is already in flight rather than because the cooldown has
537        /// not elapsed.
538        retry_in: Duration,
539        probe_in_flight: bool,
540    },
541}
542
543/// An outstanding admission, which must be told how its relay settled.
544///
545/// `Drop` settles it as inconclusive, so an early return between admission and
546/// the relay -- or the whole handler being cancelled when the client
547/// disconnects -- releases a half-open probe slot instead of leaving the
548/// breaker wedged half-open with no further probes.
549struct RouteBindBreakerGuard<'a> {
550    breakers: RouteBindBreakers,
551    module_id: &'a str,
552    settled: bool,
553}
554
555impl RouteBindBreakerGuard<'_> {
556    /// The module answered within the budget and took the bind. THE ONLY
557    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
558    /// breaker, which is a transition worth logging.
559    fn record_accepted(&mut self) -> bool {
560        self.settled = true;
561        self.breakers.record_accepted(self.module_id)
562    }
563
564    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
565    /// COUNTS TOWARD OPENING.
566    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
567        self.settled = true;
568        self.breakers
569            .record_timeout(self.module_id, threshold, cooldown)
570    }
571
572    /// Everything else: the module REJECTED the bind, its connection went away
573    /// mid-relay, or the waiter was cancelled.
574    ///
575    /// None of these is evidence that a module is slow, and each already has
576    /// its own refusal with its own code. A module that rejects a bind in
577    /// microseconds is healthy and must never be convicted for it; a module
578    /// that died has said nothing about the module that replaces it. So these
579    /// neither increment nor reset the count -- they only release a probe slot.
580    fn record_inconclusive(&mut self) {
581        self.settled = true;
582        self.breakers.record_inconclusive(self.module_id);
583    }
584}
585
586impl Drop for RouteBindBreakerGuard<'_> {
587    fn drop(&mut self) {
588        if !self.settled {
589            self.breakers.record_inconclusive(self.module_id);
590        }
591    }
592}
593
594/// The breaker moved to open, reported so the caller can log it outside the
595/// lock. Opening is rare and load-bearing; the refusals that follow are
596/// frequent and are counted rather than logged.
597struct BreakerOpened {
598    consecutive_timeouts: u32,
599    /// True when a failed probe re-opened an already-open breaker, which reads
600    /// very differently in a log from a first opening.
601    reopened_after_probe: bool,
602}
603
604impl RouteBindBreakers {
605    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
606        self.modules
607            .lock()
608            .expect("route.bind breaker mutex poisoned")
609    }
610
611    /// Decide whether this `route.open` may attempt its relay. Takes the map
612    /// lock and nothing else, and never awaits.
613    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
614        let admitted = |probe| RouteBindAdmission::Admitted {
615            guard: RouteBindBreakerGuard {
616                breakers: self.clone(),
617                module_id,
618                settled: false,
619            },
620            probe,
621        };
622
623        let mut modules = self.lock();
624        let Some(state) = modules.get_mut(module_id) else {
625            return admitted(false);
626        };
627        let Some(cooldown_until) = state.cooldown_until else {
628            return admitted(false);
629        };
630        if state.probe_in_flight {
631            return RouteBindAdmission::Refused {
632                consecutive_timeouts: state.consecutive_timeouts,
633                retry_in: Duration::ZERO,
634                probe_in_flight: true,
635            };
636        }
637        let now = Instant::now();
638        if now < cooldown_until {
639            return RouteBindAdmission::Refused {
640                consecutive_timeouts: state.consecutive_timeouts,
641                retry_in: cooldown_until - now,
642                probe_in_flight: false,
643            };
644        }
645        state.probe_in_flight = true;
646        admitted(true)
647    }
648
649    fn record_accepted(&self, module_id: &str) -> bool {
650        self.lock()
651            .remove(module_id)
652            .is_some_and(|state| state.cooldown_until.is_some())
653    }
654
655    fn record_timeout(
656        &self,
657        module_id: &str,
658        threshold: u32,
659        cooldown: Duration,
660    ) -> Option<BreakerOpened> {
661        let mut modules = self.lock();
662        let state = modules.entry(module_id.to_string()).or_default();
663        let was_open = state.cooldown_until.is_some();
664        let was_probe = state.probe_in_flight;
665        state.probe_in_flight = false;
666        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
667        if state.consecutive_timeouts < threshold {
668            return None;
669        }
670        state.cooldown_until = Some(Instant::now() + cooldown);
671        Some(BreakerOpened {
672            consecutive_timeouts: state.consecutive_timeouts,
673            reopened_after_probe: was_open && was_probe,
674        })
675    }
676
677    fn record_inconclusive(&self, module_id: &str) {
678        if let Some(state) = self.lock().get_mut(module_id) {
679            state.probe_in_flight = false;
680        }
681    }
682
683    /// Discard what was learned about a module, because the process it was
684    /// learned about is gone. Returns the discarded count when it was non-zero.
685    ///
686    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
687    /// `module_id` is a configuration identity that outlives any particular
688    /// child; what the breaker observed was the process behind the module
689    /// connection of the moment. When a new connection registers under that id
690    /// the verdict's subject no longer exists, so the verdict is stale by
691    /// construction rather than merely likely to be wrong. Keeping it would
692    /// apply a dead process's record to a live one, which is the same defect
693    /// class this breaker exists to stop the daemon committing.
694    ///
695    /// A half-open probe in flight is discarded with the rest: it was a
696    /// question about the old process.
697    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
698        self.lock()
699            .remove(module_id)
700            .map(|state| state.consecutive_timeouts)
701            .filter(|discarded| *discarded > 0)
702    }
703
704    /// Open breakers, for the `server.describe` counters object. `None` when
705    /// none is open, so the key stays absent rather than present-and-empty.
706    ///
707    /// This is the operator's answer to "is this module refusing instantly or
708    /// is it fine?", which look identical from a client that retries and then
709    /// succeeds.
710    fn open_snapshot(&self) -> Option<serde_json::Value> {
711        let now = Instant::now();
712        let modules = self.lock();
713        let open = modules
714            .iter()
715            .filter_map(|(module_id, state)| {
716                let cooldown_until = state.cooldown_until?;
717                Some((
718                    module_id.clone(),
719                    serde_json::json!({
720                        "consecutive_timeouts": state.consecutive_timeouts,
721                        "cooldown_remaining_ms":
722                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
723                        "probe_in_flight": state.probe_in_flight,
724                    }),
725                ))
726            })
727            .collect::<serde_json::Map<String, serde_json::Value>>();
728        (!open.is_empty()).then_some(serde_json::Value::Object(open))
729    }
730}
731
732impl ControlHandler {
733    pub fn new(registry: Arc<Registry>) -> Self {
734        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
735    }
736
737    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
738        let counters = forwarding.counters();
739        // Taken from the forwarding table rather than created here, so that the
740        // breaker a `route.open` consults is the same one a module's
741        // registration resets, however many handlers are built over one table.
742        let route_bind_breakers = forwarding.route_bind_breakers();
743        let route_bind_concurrency = forwarding.route_bind_concurrency();
744        let route_outages = forwarding.route_outages();
745        Self {
746            registry,
747            forwarding,
748            process_liveness: None,
749            supervisor: SupervisorHandle::new(),
750            subc_capabilities: Arc::from([
751                CAP_MANIFEST_REGISTRATION.to_string(),
752                CAP_CHANNEL_LIFECYCLE.to_string(),
753                CAP_PING_PONG.to_string(),
754                CAP_SESSION_ATTACH.to_string(),
755                CAP_ADMISSION_FACTS_RELAY.to_string(),
756                CAP_SCOPES_V1.to_string(),
757                CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
758            ]),
759            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
760            route_bind_relay_timeouts: BTreeMap::new(),
761            route_bind_breakers,
762            route_bind_concurrency,
763            route_outages,
764            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
765            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
766            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
767            storage_config: None,
768            machine_id: None,
769            admission_facts_carrier_module_id: None,
770            admission_facts_targets: None,
771            scopes: Arc::new(RwLock::new(ScopeTable::new(
772                crate::daemon_config::default_scope_authority_owners(),
773            ))),
774            scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
775            hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
776            rescan: None,
777            connected_clients: ConnectedClients::new(),
778            counters,
779            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
780            daemon_provenance: DaemonProvenanceFacts::default(),
781            #[cfg(test)]
782            control_dispatch_delay: None,
783            #[cfg(test)]
784            provenance_probe_override: None,
785        }
786    }
787
788    /// Set the central storage policy: registering modules then receive their
789    /// resolved storage descriptor in HELLO_ACK.
790    pub fn with_storage_config(
791        mut self,
792        storage_config: Option<crate::daemon_config::StorageConfig>,
793    ) -> Self {
794        self.storage_config = storage_config;
795        self
796    }
797
798    /// Set the machine id served to every registering module (HELLO_ACK) and on
799    /// `server.describe`.
800    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
801        self.machine_id = machine_id;
802        self
803    }
804
805    /// Configure the exact reserved module and target ids permitted to relay
806    /// opaque admission facts. Config-file loading validates this authority;
807    /// this builder keeps the same policy available to embedded test daemons.
808    pub fn with_admission_facts_config(
809        mut self,
810        carrier_module_id: Option<String>,
811        targets: Option<Vec<String>>,
812    ) -> Self {
813        self.admission_facts_carrier_module_id = carrier_module_id;
814        self.admission_facts_targets = targets;
815        self
816    }
817
818    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
819    /// Replaces the scope table with an empty one under the new list, so call it
820    /// while building the handler, before any module can sync.
821    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
822        self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
823        self.scope_authority_owners = owners;
824        self
825    }
826
827    /// Override the route.bind relay timeout. Used by tests that assert the
828    /// timeout path so they don't block on the production-safe default.
829    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
830        self.route_bind_relay_timeout = timeout;
831        self
832    }
833
834    /// Install per-module route.bind relay budget overrides. A module id
835    /// listed here wins over the daemon-wide default set via
836    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
837    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
838    /// the same `Duration` the bind path will use.
839    pub fn with_route_bind_relay_timeouts(
840        mut self,
841        timeouts: impl IntoIterator<Item = (String, Duration)>,
842    ) -> Self {
843        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
844        self
845    }
846
847    /// Resolve the route.bind relay budget for a specific target module id.
848    /// Per-module overrides win; the daemon-wide value (set via
849    /// `with_route_bind_relay_timeout` or the built-in default) is the
850    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
851    /// the same resolution `handle_route_open` will use.
852    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
853        self.route_bind_relay_timeouts
854            .get(module_id)
855            .copied()
856            .unwrap_or(self.route_bind_relay_timeout)
857    }
858
859    /// Override the per-module bind-relay breaker policy.
860    ///
861    /// Used by tests, which cannot spend three production budgets opening a
862    /// breaker or twenty seconds waiting for its cooldown. The production
863    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
864    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
865    /// reasoning for the numbers.
866    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
867        self.route_bind_breaker_threshold = threshold.max(1);
868        self.route_bind_breaker_cooldown = cooldown;
869        self
870    }
871
872    #[cfg(test)]
873    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
874        self.health_probe_timeout = timeout;
875        self
876    }
877
878    #[cfg(test)]
879    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
880        self.control_dispatch_delay = Some(delay);
881        self
882    }
883
884    pub fn with_process_liveness(
885        mut self,
886        process_liveness: Arc<dyn ModuleProcessLiveness>,
887    ) -> Self {
888        self.process_liveness = Some(process_liveness);
889        self
890    }
891
892    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
893        self.supervisor = supervisor;
894        self
895    }
896
897    pub fn with_daemon_provenance(
898        mut self,
899        pid: u32,
900        started_at_ms: u64,
901        executable_path: Option<PathBuf>,
902        build_git_sha: Option<String>,
903        build_lock_digest: Option<String>,
904    ) -> Self {
905        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
906        let process_start_time = process_start_time(pid);
907        self.daemon_provenance = DaemonProvenanceFacts {
908            build: DaemonBuildProvenance {
909                build_git_sha,
910                build_lock_digest,
911            },
912            pid: Some(pid),
913            started_at_ms: Some(started_at_ms),
914            start_clock: None,
915            executable_path,
916            executable_identity,
917            process_start_time,
918            probe: ExecutableIdentityProbe::default(),
919        };
920        self
921    }
922
923    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
924        self.daemon_provenance.start_clock = Some(clock);
925        self
926    }
927
928    #[cfg(test)]
929    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
930        self.provenance_probe_override = Some(result);
931        self
932    }
933
934    /// Install the configured module set and its reserved capability bindings.
935    /// Bindings are configuration-scoped and may point at a provider that has not
936    /// been installed yet, so this does not require the bound module to exist.
937    pub fn with_capability_config(
938        self,
939        modules: impl IntoIterator<Item = (String, bool)>,
940        reserved_capabilities: BTreeMap<String, String>,
941    ) -> Self {
942        self.capability_evaluator
943            .configure(modules, reserved_capabilities);
944        self
945    }
946
947    pub fn with_supervisor_rescan(
948        mut self,
949        supervisor: Supervisor,
950        config_path: impl Into<PathBuf>,
951        configured_port: Option<u16>,
952    ) -> Self {
953        self.rescan = Some(SupervisorRescanContext {
954            supervisor,
955            config_path: config_path.into(),
956            configured_port,
957            storage_config: self.storage_config.clone(),
958            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
959            admission_facts_targets: self.admission_facts_targets.clone(),
960            scope_authority_owners: self.scope_authority_owners.clone(),
961        });
962        self
963    }
964
965    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
966        self.connected_clients = connected_clients;
967        self
968    }
969
970    pub fn forwarding(&self) -> Arc<ForwardingTable> {
971        Arc::clone(&self.forwarding)
972    }
973
974    pub(crate) fn counters(&self) -> DaemonCounters {
975        self.counters.clone()
976    }
977
978    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
979    /// requirement event without depending on an operator polling a status command.
980    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
981        tokio::spawn(async move {
982            loop {
983                self.capability_evaluator
984                    .wait_for_change_or_deadline()
985                    .await;
986                self.refresh_capability_requirements();
987            }
988        });
989    }
990
991    fn runtime_capability_snapshot(
992        &self,
993    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
994        let runtime = self
995            .supervisor
996            .list()
997            .into_iter()
998            .map(|module| {
999                let status = module.status().map_err(|err| {
1000                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1001                })?;
1002                Ok(RuntimeModule {
1003                    module_id: status.module_id,
1004                    state: status.state,
1005                    enabled: status.enabled,
1006                })
1007            })
1008            .collect::<Result<Vec<_>, RouterError>>()?;
1009        let (_, registrations) = self.registry.list_modules().map_err(|err| {
1010            RouterError::backend(
1011                0,
1012                0,
1013                format!("failed to list capability registrations: {err}"),
1014            )
1015        })?;
1016        let registrations = registrations
1017            .into_iter()
1018            .map(|registration| RegisteredModule {
1019                module_id: registration.manifest.module_id,
1020                module_version: registration.manifest.module_version,
1021                capabilities: registration.manifest.capabilities,
1022            })
1023            .collect();
1024        Ok((runtime, registrations))
1025    }
1026
1027    /// The capability side effects of a module becoming the active registration
1028    /// for its id: cache its manifest (warning if its claims drifted), run the
1029    /// deny census when its declarations call for one, and recompute the
1030    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
1031    /// candidate's does not, and the supervisor does it at promotion instead,
1032    /// through [`crate::supervise::SwapPromotionObserver`].
1033    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1034        let cached_registration = RegisteredModule {
1035            module_id: registration.manifest.module_id.clone(),
1036            module_version: registration.manifest.module_version.clone(),
1037            capabilities: registration.manifest.capabilities.clone(),
1038        };
1039        if self.capability_evaluator.record_hello(&cached_registration) {
1040            warn!(
1041                module_id = %cached_registration.module_id,
1042                "capability claims drifted from the cached manifest"
1043            );
1044        }
1045        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1046            self.enforce_capability_denies();
1047        }
1048        self.refresh_capability_requirements();
1049    }
1050
1051    /// Point the shared supervisor handle at this handler for swap promotions.
1052    /// Called wherever a handler is put behind the `Arc` the router serves, so
1053    /// it can be held weakly.
1054    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1055        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1056            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1057        self.supervisor.set_swap_promotion_observer(observer);
1058    }
1059
1060    pub fn refresh_capability_requirements(&self) {
1061        match self.runtime_capability_snapshot() {
1062            Ok((runtime, registrations)) => {
1063                log_requirement_events(
1064                    self.capability_evaluator
1065                        .evaluate_now(&runtime, &registrations),
1066                );
1067            }
1068            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1069        }
1070    }
1071
1072    /// Reconcile only live, attested route bindings after a capability deny edge
1073    /// or target claim was added. This is deliberately a control-plane census:
1074    /// the opaque forwarding hot path must not grow a per-frame capability check.
1075    fn enforce_capability_denies(&self) {
1076        let (_, registrations) = match self.registry.list_modules() {
1077            Ok(snapshot) => snapshot,
1078            Err(err) => {
1079                warn!(error = %err, "failed to read registrations for capability deny census");
1080                return;
1081            }
1082        };
1083        let manifests = registrations
1084            .into_iter()
1085            .map(|registration| {
1086                (
1087                    registration.manifest.module_id.clone(),
1088                    registration.manifest,
1089                )
1090            })
1091            .collect::<BTreeMap<_, _>>();
1092        let census = match self.forwarding.route_census(None) {
1093            Ok(census) => census,
1094            Err(err) => {
1095                warn!(error = %err, "failed to read route census for capability deny enforcement");
1096                return;
1097            }
1098        };
1099
1100        for (target_module_id, routes) in census {
1101            let Some(target_manifest) = manifests.get(&target_module_id) else {
1102                continue;
1103            };
1104            let mut closed_routes = Vec::new();
1105            let mut module_goodbyes = Vec::new();
1106            for route in routes {
1107                let Principal::Reserved {
1108                    module_id: opening_module_id,
1109                } = &route.principal
1110                else {
1111                    continue;
1112                };
1113                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1114                    continue;
1115                };
1116                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1117                    continue;
1118                };
1119
1120                match self.forwarding.release_client_route(
1121                    route.goodbye_target.connection_id,
1122                    route.goodbye_target.channel,
1123                    route.goodbye_target.epoch,
1124                ) {
1125                    Ok(RouteRelease::Removed(module_goodbye)) => {
1126                        warn!(
1127                            opening_module_id,
1128                            target_module_id,
1129                            capability,
1130                            "force-closing route because an attested capability deny edge now matches"
1131                        );
1132                        closed_routes.push(route);
1133                        module_goodbyes.push(module_goodbye);
1134                    }
1135                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1136                    Err(err) => warn!(
1137                        opening_module_id,
1138                        target_module_id,
1139                        capability,
1140                        error = %err,
1141                        "failed to force-close capability-denied route"
1142                    ),
1143                }
1144            }
1145
1146            if closed_routes.is_empty() {
1147                continue;
1148            }
1149            send_route_control_pushes(
1150                &self.forwarding,
1151                closed_routes,
1152                ClientControlPush::RouteClosed {
1153                    module_id: target_module_id,
1154                    reason: RouteCloseReason::CapabilityDenied,
1155                    drained: false,
1156                    abandoned: 0,
1157                    excluded_subscriptions: 0,
1158                    terminal: Some(false),
1159                },
1160            );
1161            self.emit_route_goodbyes(module_goodbyes);
1162        }
1163    }
1164
1165    /// Why a registered module is not accepting new route binds, or `None` when
1166    /// it is. This is the module's effective readiness: its declared readiness
1167    /// first, then every `need: required` capability it declares evaluating to
1168    /// `provided`. `route.open` and `catalog.list` both read it here so the
1169    /// catalog never reports a module routable that `route.open` would refuse.
1170    fn not_ready_reason(
1171        &self,
1172        registration: &crate::registry::ModuleRegistration,
1173    ) -> Option<NotReadyReason> {
1174        if !registration.ready {
1175            return Some(NotReadyReason {
1176                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1177                capability: None,
1178            });
1179        }
1180        self.first_unprovided_required_capability(registration)
1181            .map(|capability| NotReadyReason {
1182                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1183                capability: Some(capability),
1184            })
1185    }
1186
1187    /// The lexicographically first capability this registration declares
1188    /// `need: required` whose evaluator verdict is not `provided`.
1189    ///
1190    /// The verdicts are the capability evaluator's own; nothing here decides
1191    /// what "provided" means. The evaluator counts a capability provided as
1192    /// soon as a module claiming it has REGISTERED, not once that module is
1193    /// ready. That distinction is what keeps two modules that require each
1194    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1195    /// is ready", each would wait for the other to become ready first and
1196    /// neither ever would. Do not tighten it to readiness.
1197    ///
1198    /// A required capability with no verdict at all means this registration's
1199    /// HELLO or catalog.update landed after the last recompute; recompute once
1200    /// rather than let a missing verdict read as either answer. If it is still
1201    /// missing (the recompute itself failed) the capability counts as
1202    /// unprovided: the refusal is retryable, and routing a module whose
1203    /// required provider is unknown is the outcome this check exists to stop.
1204    fn first_unprovided_required_capability(
1205        &self,
1206        registration: &crate::registry::ModuleRegistration,
1207    ) -> Option<String> {
1208        let required = registration
1209            .manifest
1210            .capabilities
1211            .iter()
1212            .flat_map(|declarations| declarations.requires.iter())
1213            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1214            .map(|requirement| requirement.capability.as_str())
1215            .collect::<BTreeSet<_>>();
1216        if required.is_empty() {
1217            return None;
1218        }
1219        let module_id = registration.manifest.module_id.as_str();
1220        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1221        if required
1222            .iter()
1223            .any(|capability| verdict(capability).is_none())
1224        {
1225            self.refresh_capability_requirements();
1226        }
1227        required
1228            .into_iter()
1229            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1230            .map(str::to_string)
1231    }
1232
1233    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1234        self.capability_evaluator
1235            .statuses()
1236            .into_iter()
1237            .map(capability_requirement_status)
1238            .collect()
1239    }
1240
1241    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1242    /// registration-release watch. The signal is what the supervisor waits on
1243    /// before spawning a replacement, so it must only fire once forwarding
1244    /// teardown is also done (see [`Self::cleanup_connection`] /
1245    /// [`Self::handle_goodbye`]). Used directly only where there is no forwarding
1246    /// state to tear down (a HELLO that failed before module registration).
1247    fn deregister_connection(
1248        &self,
1249        connection_id: ConnectionId,
1250    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1251        self.registry.deregister_connection(connection_id)
1252    }
1253
1254    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1255        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1256            return None;
1257        }
1258        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1259            parse_client_control_request(&frame.body)
1260        else {
1261            return None;
1262        };
1263        Some(target_module_id(&target).to_string())
1264    }
1265
1266    pub(crate) fn route_open_capacity_refusal(
1267        &self,
1268        ctx: &RouteCtx,
1269        frame: &Frame,
1270        target_module_id: &str,
1271        in_flight: usize,
1272        limit: usize,
1273    ) -> Result<Frame, RouterError> {
1274        self.route_open_admission_refusal_frame(
1275            ctx,
1276            frame,
1277            target_module_id,
1278            "open_admission_full",
1279            (in_flight, limit),
1280            format!(
1281                "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1282            ),
1283        )
1284    }
1285
1286    fn route_open_target_capacity_refusal(
1287        &self,
1288        ctx: &RouteCtx,
1289        frame: &Frame,
1290        target_module_id: &str,
1291        in_flight: usize,
1292    ) -> Result<Frame, RouterError> {
1293        self.route_open_admission_refusal_frame(
1294            ctx,
1295            frame,
1296            target_module_id,
1297            "target_binds_full",
1298            (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1299            format!(
1300                "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1301            ),
1302        )
1303    }
1304
1305    /// Admission pressure clears as existing binds settle, so its refusal must
1306    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
1307    /// `module_reloading`, `module_warming`, `target_unavailable`, or
1308    /// `module_timeout`. `target_unavailable` is honest for an attempt that
1309    /// cannot currently reach its target; `module_timeout` would falsely claim
1310    /// that a wait expired. A new, cleaner code would be terminal to deployed
1311    /// clients, so it requires a client-tolerance rollout before daemon emission.
1312    fn route_open_admission_refusal_frame(
1313        &self,
1314        ctx: &RouteCtx,
1315        frame: &Frame,
1316        target_module_id: &str,
1317        reason: &'static str,
1318        (in_flight, limit): (usize, usize),
1319        message: impl Into<String>,
1320    ) -> Result<Frame, RouterError> {
1321        let code = error_codes::TARGET_UNAVAILABLE;
1322        self.counters.increment_route_open_refused(code);
1323        info!(
1324            target: "control",
1325            code,
1326            reason,
1327            module_id = ?target_module_id,
1328            connection_id = ctx.connection_id.get(),
1329            in_flight,
1330            limit,
1331            "route.open refused"
1332        );
1333        control_error_frame(frame, code, message.into())
1334    }
1335
1336    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
1337    ///
1338    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
1339    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
1340    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
1341    #[cfg(test)]
1342    pub fn handle_control(
1343        &self,
1344        connection_id: ConnectionId,
1345        frame: Frame,
1346    ) -> Result<Vec<Frame>, RouterError> {
1347        match frame.header.ty {
1348            FrameType::Ping => Ok(vec![pong(&frame)?]),
1349            FrameType::Hello => self.handle_hello(connection_id, None, frame),
1350            FrameType::Goodbye => self.handle_goodbye(connection_id),
1351            ty => Ok(vec![control_error_frame(
1352                &frame,
1353                "unsupported_control_frame",
1354                format!("unsupported channel-0 frame {ty:?}"),
1355            )?]),
1356        }
1357    }
1358
1359    pub async fn handle_control_frame(
1360        &self,
1361        ctx: &RouteCtx,
1362        frame: Frame,
1363    ) -> Result<Vec<Frame>, RouterError> {
1364        self.handle_control_frame_timed(ctx, frame, None).await
1365    }
1366
1367    pub(crate) async fn handle_control_frame_timed(
1368        &self,
1369        ctx: &RouteCtx,
1370        frame: Frame,
1371        dispatch_started_at: Option<StdInstant>,
1372    ) -> Result<Vec<Frame>, RouterError> {
1373        match frame.header.ty {
1374            FrameType::Ping => Ok(vec![pong(&frame)?]),
1375            FrameType::Hello => {
1376                self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1377            }
1378            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1379            FrameType::Cancel => {
1380                if self
1381                    .supervisor
1382                    .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1383                {
1384                    Ok(Vec::new())
1385                } else {
1386                    Ok(vec![control_error_frame(
1387                        &frame,
1388                        "unknown_subscription",
1389                        "no supervisor spawn subscription has this correlation id",
1390                    )?])
1391                }
1392            }
1393            FrameType::Request => {
1394                if self
1395                    .forwarding
1396                    .module_endpoint_for_connection(ctx.connection_id)
1397                    .map_err(RouterError::Forwarding)?
1398                    .is_some()
1399                {
1400                    if !is_known_module_request_op(&frame.body) {
1401                        return Ok(vec![control_error_frame(
1402                            &frame,
1403                            "unsupported_control_frame",
1404                            "module-originated channel-0 REQUEST is not supported",
1405                        )?]);
1406                    }
1407                    let request = match parse_module_control_request_from_module(&frame.body) {
1408                        Ok(request) => request,
1409                        Err((err, ControlRequestBodyError::UnknownOp)) => {
1410                            return Ok(vec![control_error_frame(
1411                                &frame,
1412                                "unsupported_control_frame",
1413                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
1414                            )?])
1415                        }
1416                        Err((err, ControlRequestBodyError::InvalidBody)) => {
1417                            return Ok(vec![control_error_frame(
1418                                &frame,
1419                                "invalid_control_body",
1420                                format!("malformed module control body: {err}"),
1421                            )?])
1422                        }
1423                    };
1424                    let op = module_control_request_op(&request);
1425                    let corr = frame.header.corr;
1426                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
1427                    let result =
1428                        self.handle_module_control_request(ctx.connection_id, frame, request);
1429                    log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1430                    return result;
1431                }
1432
1433                if is_known_module_request_op(&frame.body) {
1434                    return Ok(vec![control_error_frame(
1435                        &frame,
1436                        "not_registered",
1437                        "catalog.update requires an active module registration owned by this connection",
1438                    )?]);
1439                }
1440
1441                let request = match parse_client_control_request(&frame.body) {
1442                    Ok(request) => request,
1443                    Err((err, ControlRequestBodyError::UnknownOp)) => {
1444                        return Ok(vec![control_error_frame(
1445                            &frame,
1446                            "unknown_control_op",
1447                            format!("unknown client control op: {err}"),
1448                        )?])
1449                    }
1450                    Err((err, ControlRequestBodyError::InvalidBody)) => {
1451                        return Ok(vec![control_error_frame(
1452                            &frame,
1453                            "invalid_control_body",
1454                            format!("malformed client control body: {err}"),
1455                        )?])
1456                    }
1457                };
1458                let op = client_control_request_op(&request);
1459                let corr = frame.header.corr;
1460                log_control_dispatch_arrival(op, ctx.connection_id, corr);
1461                #[cfg(test)]
1462                if let Some(delay) = self.control_dispatch_delay {
1463                    tokio::time::sleep(delay).await;
1464                }
1465                let result = self
1466                    .handle_client_control_request(ctx, frame, request)
1467                    .await;
1468                log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1469                result
1470            }
1471            FrameType::Push => {
1472                let Some(endpoint) = self
1473                    .forwarding
1474                    .module_endpoint_for_connection(ctx.connection_id)
1475                    .map_err(RouterError::Forwarding)?
1476                else {
1477                    return Ok(vec![control_error_frame(
1478                        &frame,
1479                        "unsupported_control_frame",
1480                        "client-originated channel-0 PUSH is not supported",
1481                    )?]);
1482                };
1483                self.handle_status_update(endpoint, frame)
1484            }
1485            FrameType::Response | FrameType::Error
1486                if self
1487                    .forwarding
1488                    .module_endpoint_for_connection(ctx.connection_id)
1489                    .map_err(RouterError::Forwarding)?
1490                    .is_some() =>
1491            {
1492                self.handle_module_relay_response(ctx.connection_id, frame)
1493            }
1494            ty => Ok(vec![control_error_frame(
1495                &frame,
1496                "unsupported_control_frame",
1497                format!("unsupported channel-0 frame {ty:?}"),
1498            )?]),
1499        }
1500    }
1501
1502    pub fn cleanup_connection(
1503        &self,
1504        connection_id: ConnectionId,
1505    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1506        let crash_closed = self
1507            .registry
1508            .get_module_by_connection(connection_id)?
1509            .and_then(|registration| {
1510                self.forwarding
1511                    .module_endpoint_for_connection(connection_id)
1512                    .ok()
1513                    .flatten()
1514                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1515                    .map(|routes| (registration.manifest.module_id, routes))
1516            });
1517        let crash_closed = crash_closed.map(|(module_id, routes)| {
1518            let terminal = match self.supervisor.get(&module_id) {
1519                None => false,
1520                Some(module) => match module.will_recover_after_connection_loss() {
1521                    Ok(will_recover) => !will_recover,
1522                    Err(err) => {
1523                        warn!(
1524                            %module_id,
1525                            error = %err,
1526                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
1527                        );
1528                        false
1529                    }
1530                },
1531            };
1532            // The forwarding table gates all providers at the start of daemon
1533            // shutdown, before their connections are closed. An ordinary
1534            // module disconnect still reports crash if that gate is not set.
1535            let reason = match self.forwarding.is_daemon_draining() {
1536                Ok(true) => RouteCloseReason::Restart,
1537                Ok(false) => RouteCloseReason::Crash,
1538                Err(err) => {
1539                    warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1540                    RouteCloseReason::Crash
1541                }
1542            };
1543            (module_id, routes, reason, terminal)
1544        });
1545        let registrations = self.deregister_connection(connection_id);
1546        let cleanup = self.forwarding.cleanup_connection_counted(connection_id);
1547        // The route.closed push waits for forwarding teardown because only
1548        // teardown knows how many pending route.bind relays it aborted. It still
1549        // goes out before the GOODBYEs for the released routes, and its targets
1550        // were captured above, before teardown removed those routes.
1551        if let Some((module_id, routes, reason, terminal)) = crash_closed {
1552            let abandoned = cleanup
1553                .as_ref()
1554                .map_or(0, |cleanup| cleanup.abandoned_relays);
1555            send_route_control_pushes(
1556                &self.forwarding,
1557                routes,
1558                ClientControlPush::RouteClosed {
1559                    module_id,
1560                    reason,
1561                    drained: false,
1562                    abandoned,
1563                    excluded_subscriptions: 0,
1564                    terminal: Some(terminal),
1565                },
1566            );
1567        }
1568        if let Ok(cleanup) = cleanup {
1569            self.emit_route_goodbyes(cleanup.released);
1570        }
1571        // Signal the registration-release watch only now that BOTH registry and
1572        // forwarding teardown are done, so a supervisor waiting to spawn a
1573        // replacement never observes release while old routes still exist.
1574        if matches!(&registrations, Ok(r) if !r.is_empty()) {
1575            crate::supervise::notify_registration_release();
1576            self.capability_evaluator.wake_deadline_loop();
1577            self.refresh_capability_requirements();
1578        }
1579        self.supervisor.remove_spawn_subscribers(connection_id);
1580        // Sync authority dies with its connection, so the owner's next
1581        // connection can take it; the owner's scopes stay as they are.
1582        self.hello_launch_nonces
1583            .lock()
1584            .unwrap_or_else(|poisoned| poisoned.into_inner())
1585            .forget(connection_id);
1586        self.scopes
1587            .write()
1588            .unwrap_or_else(|poisoned| poisoned.into_inner())
1589            .release_connection(connection_id);
1590        registrations
1591    }
1592
1593    pub(crate) fn handle_route_goodbye(
1594        &self,
1595        connection_id: ConnectionId,
1596        route_channel: u16,
1597        route_epoch: u32,
1598    ) -> Result<bool, RouterError> {
1599        debug!(
1600            connection_id = connection_id.get(),
1601            route_channel, route_epoch, "handling route GOODBYE"
1602        );
1603        let RouteRelease::Removed(released_route) = self
1604            .forwarding
1605            .release_client_route(connection_id, route_channel, route_epoch)
1606            .map_err(RouterError::Forwarding)?
1607        else {
1608            return Ok(false);
1609        };
1610        self.emit_route_goodbyes(vec![released_route]);
1611        Ok(true)
1612    }
1613
1614    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1615        for released in released_routes {
1616            let frame = match Frame::build_with_version(
1617                released.negotiated_ver,
1618                FrameType::Goodbye,
1619                control_flags(),
1620                released.channel,
1621                released.epoch,
1622                0,
1623                Vec::new(),
1624            ) {
1625                Ok(frame) => frame,
1626                Err(err) => {
1627                    warn!(
1628                        route_channel = released.channel,
1629                        error = %err,
1630                        "failed to build route GOODBYE frame"
1631                    );
1632                    continue;
1633                }
1634            };
1635            if !released.close_on_delivery_failure() {
1636                crate::forwarding::send_module_route_goodbye(
1637                    &self.counters,
1638                    &released.sink,
1639                    frame,
1640                    released.module_id.as_deref(),
1641                    "client route released",
1642                );
1643                continue;
1644            }
1645            if let Err(err) = released.sink.try_send(frame) {
1646                warn!(
1647                    target_connection_id = released.connection_id.get(),
1648                    route_channel = released.channel,
1649                    error = %err,
1650                    "route GOODBYE was not delivered to client; closing target connection"
1651                );
1652                if self
1653                    .forwarding
1654                    .escalate_client_delivery_failure(
1655                        released.connection_id,
1656                        released.channel,
1657                        released.epoch,
1658                        CloseReason::new(
1659                            "route_goodbye_delivery_failed",
1660                            format!(
1661                                "failed to enqueue route GOODBYE for channel {}: {err}",
1662                                released.channel
1663                            ),
1664                        ),
1665                        crate::forwarding::UndeliveredFrame {
1666                            module_id: released.module_id.as_deref(),
1667                            sink: &released.sink,
1668                        },
1669                    )
1670                    .unwrap_or(false)
1671                {
1672                    self.counters.increment_goodbye_relay_client_failed();
1673                }
1674            }
1675        }
1676    }
1677
1678    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
1679    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
1680    /// own commit failed after the module had already accepted). Without this, a
1681    /// module that accepts late keeps a binding subc has torn down, so a later
1682    /// frame on that module channel could misdeliver if the channel is reused.
1683    ///
1684    /// Never closes the shared module connection on failure: a dropped notification
1685    /// only wastes a bounded amount of warm module-side state, which the module's
1686    /// own idle reaper reclaims. Only call this once the route.bind relay was
1687    /// actually enqueued to the module — if the relay send itself failed, the
1688    /// module never created a binding and there is nothing to tear down.
1689    fn send_abandoned_route_bind_goodbye(
1690        &self,
1691        module_sink: &crate::FrameSink,
1692        negotiated_ver: u8,
1693        module_channel: u16,
1694        module_epoch: u32,
1695    ) {
1696        let frame = match Frame::build_with_version(
1697            negotiated_ver,
1698            FrameType::Goodbye,
1699            control_flags(),
1700            module_channel,
1701            module_epoch,
1702            0,
1703            Vec::new(),
1704        ) {
1705            Ok(frame) => frame,
1706            Err(err) => {
1707                warn!(
1708                    route_channel = module_channel,
1709                    error = %err,
1710                    "failed to build GOODBYE for abandoned route.bind"
1711                );
1712                return;
1713            }
1714        };
1715        crate::forwarding::send_module_route_goodbye(
1716            &self.counters,
1717            module_sink,
1718            frame,
1719            None,
1720            "abandoned route.bind",
1721        );
1722    }
1723
1724    fn handle_hello(
1725        &self,
1726        connection_id: ConnectionId,
1727        sink: Option<crate::FrameSink>,
1728        frame: Frame,
1729    ) -> Result<Vec<Frame>, RouterError> {
1730        debug!(
1731            connection_id = connection_id.get(),
1732            corr = frame.header.corr,
1733            "handling HELLO"
1734        );
1735        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1736            Ok(value) => value,
1737            Err(err) => {
1738                return Ok(vec![control_error_frame(
1739                    &frame,
1740                    "invalid_hello",
1741                    format!("malformed HELLO body: {err}"),
1742                )?])
1743            }
1744        };
1745        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1746            return Ok(vec![control_error_frame(
1747                &frame,
1748                "invalid_capability_grammar",
1749                err.to_string(),
1750            )?]);
1751        }
1752        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
1753            return Ok(vec![control_error_frame(
1754                &frame,
1755                "invalid_manifest",
1756                err.to_string(),
1757            )?]);
1758        }
1759        if let Some(provenance) = hello_value
1760            .get("manifest")
1761            .and_then(|manifest| manifest.get("provenance"))
1762        {
1763            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
1764                return Ok(vec![control_error_frame(
1765                    &frame,
1766                    "invalid_manifest",
1767                    format!("malformed manifest provenance: {err}"),
1768                )?]);
1769            }
1770        }
1771        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
1772            Ok(hello) => hello,
1773            Err(err) => {
1774                return Ok(vec![control_error_frame(
1775                    &frame,
1776                    "invalid_hello",
1777                    format!("malformed HELLO body: {err}"),
1778                )?])
1779            }
1780        };
1781
1782        if hello.protocol_ver != hello.manifest.protocol_ver {
1783            return Ok(vec![control_error_frame(
1784                &frame,
1785                "invalid_manifest",
1786                format!(
1787                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
1788                    hello.protocol_ver, hello.manifest.protocol_ver
1789                ),
1790            )?]);
1791        }
1792
1793        if hello.manifest.module_id.trim().is_empty() {
1794            return Ok(vec![control_error_frame(
1795                &frame,
1796                "invalid_manifest",
1797                "manifest module_id must not be empty",
1798            )?]);
1799        }
1800
1801        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
1802            Ok(negotiated_ver) => negotiated_ver,
1803            Err(message) => {
1804                return Ok(vec![control_error_frame(
1805                    &frame,
1806                    "version_unsupported",
1807                    message,
1808                )?])
1809            }
1810        };
1811
1812        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
1813        // swap is open for this id, the only HELLO admitted as a second process
1814        // is the one carrying the candidate's launch nonce (the swap token), and
1815        // it registers into the candidate slot rather than being refused as a
1816        // duplicate. Run after the reserved gate, a reserved module's candidate
1817        // would be refused `reserved_module` for presenting a nonce that gate
1818        // does not know. See `SupervisorHandle::swap_hello_admission`.
1819        let swap_admission = self
1820            .supervisor
1821            .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
1822        if swap_admission == SwapHelloAdmission::Refused {
1823            warn!(
1824                module_id = %hello.manifest.module_id,
1825                connection_id = connection_id.get(),
1826                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
1827            );
1828            return Ok(vec![control_error_frame(
1829                &frame,
1830                "swap_token_invalid",
1831                format!(
1832                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
1833                    hello.manifest.module_id
1834                ),
1835            )?]);
1836        }
1837        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
1838
1839        // Reserved-module identity gate: a module_id configured `reserved` may be
1840        // registered ONLY by the process subc spawned for it, proven by echoing the
1841        // one-time launch nonce subc injected. A non-reserved id has no recorded
1842        // nonce and always passes. This blocks a key-holder from impersonating a
1843        // security-boundary module (e.g. the credential vault) while the real one is
1844        // down/restarting and its registration slot is momentarily free. A swap
1845        // candidate has already proven the same thing with its own nonce above.
1846        if let Some(rejection) = (!swap_candidate)
1847            .then(|| {
1848                self.supervisor.reserved_hello_rejection(
1849                    &hello.manifest.module_id,
1850                    hello.launch_nonce.as_deref(),
1851                )
1852            })
1853            .flatten()
1854        {
1855            let message = match rejection {
1856                ReservedHelloRejection::Exact { module_id } => format!(
1857                    "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
1858                ),
1859                ReservedHelloRejection::Prefix {
1860                    prefix,
1861                    owner_module_id,
1862                } => format!(
1863                    "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
1864                    hello.manifest.module_id
1865                ),
1866            };
1867            return Ok(vec![control_error_frame(
1868                &frame,
1869                "reserved_module",
1870                message,
1871            )?]);
1872        }
1873
1874        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
1875            &hello.manifest.module_id,
1876            hello.manifest.capabilities.as_ref(),
1877        );
1878        if let Some(refusal) = reserved_capability_refusals.first() {
1879            let capability = refusal.capability.clone();
1880            let bound_module = refusal.claimants[0].clone();
1881            log_duplicate_claim_events(reserved_capability_refusals);
1882            return Ok(vec![control_error_frame(
1883                &frame,
1884                "reserved_capability",
1885                format!(
1886                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
1887                    capability, bound_module, hello.manifest.module_id
1888                ),
1889            )?]);
1890        }
1891
1892        // A connection that already opened client routes must not also register as
1893        // a module: cleanup would then release only one side and leak the other.
1894        if self
1895            .forwarding
1896            .connection_has_client_routes(connection_id)
1897            .map_err(RouterError::Forwarding)?
1898        {
1899            return Ok(vec![control_error_frame(
1900                &frame,
1901                "invalid_hello",
1902                "connection has open client routes and cannot also register as a module",
1903            )?]);
1904        }
1905
1906        // Kept for scope sync authority, which goes only to the connection that
1907        // presented the module's current launch nonce. Recorded before the
1908        // registration is attempted: a connection whose registration then fails
1909        // has no registration, so it cannot sync anyway, and cleanup forgets it.
1910        self.hello_launch_nonces
1911            .lock()
1912            .unwrap_or_else(|poisoned| poisoned.into_inner())
1913            .record(connection_id, hello.launch_nonce.as_deref());
1914        let control_ops = effective_module_control_ops(hello.control_ops);
1915        // Built before anything is registered so an encoding failure leaves no
1916        // registry or forwarding state behind.
1917        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
1918        if swap_candidate {
1919            return self.register_swap_candidate(
1920                connection_id,
1921                sink,
1922                &frame,
1923                hello.manifest,
1924                negotiated_ver,
1925                control_ops,
1926                hello_ack,
1927            );
1928        }
1929        let registration = match self.registry.register_with_control_ops(
1930            hello.manifest,
1931            negotiated_ver,
1932            connection_id,
1933            control_ops,
1934        ) {
1935            Ok(registration) => registration,
1936            Err(RegistryError::DuplicateModuleId { module_id }) => {
1937                return Ok(vec![control_error_frame(
1938                    &frame,
1939                    "duplicate_module_id",
1940                    format!(
1941                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
1942                    ),
1943                )?])
1944            }
1945            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
1946                return Ok(vec![control_error_frame(
1947                    &frame,
1948                    "invalid_module_id",
1949                    err.to_string(),
1950                )?])
1951            }
1952            Err(err) => {
1953                return Ok(vec![control_error_frame(
1954                    &frame,
1955                    "registry_error",
1956                    err.to_string(),
1957                )?])
1958            }
1959        };
1960
1961        let reply = if let Some(sink) = sink {
1962            // The forwarding table's module store is also the daemon-to-module
1963            // control-RPC lane, so every HELLO gets a live endpoint even when the
1964            // manifest has no routable provider role. Non-routable modules still
1965            // cannot receive route.bind in production: `handle_route_open` checks
1966            // the registry manifest with `target_has_required_role` before the
1967            // only production call to `begin_route_bind_relay_for` below that
1968            // route.open path. The remaining direct relay callers are unit tests
1969            // and benchmark harnesses that construct forwarding state explicitly.
1970            //
1971            // The HELLO_ACK is queued by the forwarding table itself, before the
1972            // endpoint becomes visible, and is NOT returned as a reply. A module
1973            // reads HELLO_ACK first and exits on anything else; a reply is only
1974            // written after this handler returns, by which time a route.open on
1975            // another connection could already have queued a route.bind request
1976            // for this module ahead of it.
1977            let concurrency = manifest_concurrency(&registration.manifest);
1978            if let Err(err) = self.forwarding.register_module_connection_acked(
1979                connection_id,
1980                registration.manifest.module_id.clone(),
1981                negotiated_ver,
1982                concurrency,
1983                sink,
1984                hello_ack,
1985            ) {
1986                // Forwarding registration failed, so there is no forwarding
1987                // state to tear down. Remove the registry entry and signal the
1988                // release watch directly.
1989                if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
1990                    crate::supervise::notify_registration_release();
1991                }
1992                return Ok(vec![control_error_frame(
1993                    &frame,
1994                    forwarding_error_code(&err),
1995                    err.to_string(),
1996                )?]);
1997            }
1998            Vec::new()
1999        } else {
2000            // No sink means no forwarding endpoint, so nothing can be routed
2001            // ahead of the ack; it goes out as the reply.
2002            vec![hello_ack]
2003        };
2004
2005        // Exposure over assumption: Concurrency's serde default is pinned to the
2006        // pre-field behavior (ModuleManaged), so a management surface that is
2007        // genuinely Serial and just never declared it inherits concurrent
2008        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
2009        // turns "no module has been bitten yet" into the checkable claim "no
2010        // module is exposed" -- one read of the boot log instead of a fleet
2011        // audit. Detected from the raw HELLO bytes because the serde default
2012        // deliberately erases the absent/declared distinction from the type.
2013        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
2014            info!(
2015                module_id = %registration.manifest.module_id,
2016                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2017            );
2018        }
2019
2020        self.apply_registration_capabilities(&registration);
2021
2022        info!(
2023            module_id = %registration.manifest.module_id,
2024            module_version = %registration.manifest.module_version,
2025            negotiated_ver,
2026            routable_provider = manifest_provides_routable_role(&registration.manifest),
2027            connection_id = connection_id.get(),
2028            "module registered"
2029        );
2030
2031        Ok(reply)
2032    }
2033
2034    /// Register a HELLO the swap gate admitted into the candidate slot of the
2035    /// registry and of forwarding, where it is reachable over its own
2036    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
2037    /// nothing routes to it until the supervisor cuts over.
2038    ///
2039    /// Registry first, then forwarding, the same order as an ordinary HELLO;
2040    /// a forwarding failure removes the registry entry again. The capability
2041    /// census is not run: it describes routable modules, and this one is not
2042    /// routable until promotion.
2043    #[allow(clippy::too_many_arguments)]
2044    fn register_swap_candidate(
2045        &self,
2046        connection_id: ConnectionId,
2047        sink: Option<crate::FrameSink>,
2048        frame: &Frame,
2049        manifest: ModuleManifest,
2050        negotiated_ver: u8,
2051        control_ops: Vec<String>,
2052        hello_ack: Frame,
2053    ) -> Result<Vec<Frame>, RouterError> {
2054        let module_id = manifest.module_id.clone();
2055        let registration = match self.registry.register_candidate_with_control_ops(
2056            manifest,
2057            negotiated_ver,
2058            connection_id,
2059            control_ops,
2060        ) {
2061            Ok(registration) => registration,
2062            Err(RegistryError::DuplicateModuleId { module_id }) => {
2063                return Ok(vec![control_error_frame(
2064                    frame,
2065                    "duplicate_module_id",
2066                    format!(
2067                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2068                    ),
2069                )?])
2070            }
2071            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2072                return Ok(vec![control_error_frame(
2073                    frame,
2074                    "invalid_module_id",
2075                    err.to_string(),
2076                )?])
2077            }
2078            Err(err) => {
2079                return Ok(vec![control_error_frame(
2080                    frame,
2081                    "registry_error",
2082                    err.to_string(),
2083                )?])
2084            }
2085        };
2086        let reply = if let Some(sink) = sink {
2087            // Same ordering as an ordinary HELLO: the forwarding table queues
2088            // the HELLO_ACK before the candidate endpoint is inserted, because
2089            // a module exits if its first frame after HELLO is anything else.
2090            let concurrency = manifest_concurrency(&registration.manifest);
2091            if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2092                connection_id,
2093                module_id.clone(),
2094                negotiated_ver,
2095                concurrency,
2096                sink,
2097                hello_ack,
2098            ) {
2099                if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
2100                    crate::supervise::notify_registration_release();
2101                }
2102                return Ok(vec![control_error_frame(
2103                    frame,
2104                    forwarding_error_code(&err),
2105                    err.to_string(),
2106                )?]);
2107            }
2108            Vec::new()
2109        } else {
2110            vec![hello_ack]
2111        };
2112        self.supervisor.mark_swap_candidate_admitted(&module_id);
2113        info!(
2114            module_id = %module_id,
2115            module_version = %registration.manifest.module_version,
2116            negotiated_ver,
2117            ready = registration.ready,
2118            connection_id = connection_id.get(),
2119            "swap candidate registered; not routable until cutover"
2120        );
2121        Ok(reply)
2122    }
2123
2124    fn build_hello_ack(
2125        &self,
2126        frame: &Frame,
2127        negotiated_ver: u8,
2128        module_id: &str,
2129    ) -> Result<Frame, RouterError> {
2130        let ack = ModuleHelloAckBody {
2131            negotiated_ver,
2132            subc_ops: module_subc_ops(),
2133            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2134            storage: self
2135                .storage_config
2136                .as_ref()
2137                .map(|cfg| cfg.descriptor_for(module_id)),
2138            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2139        };
2140        let body = serde_json::to_vec(&ack).map_err(|err| {
2141            RouterError::backend(
2142                0,
2143                frame.header.corr,
2144                format!("failed to encode HELLO_ACK: {err}"),
2145            )
2146        })?;
2147
2148        Frame::build_with_version(
2149            negotiated_ver,
2150            FrameType::HelloAck,
2151            control_flags(),
2152            0,
2153            0,
2154            frame.header.corr,
2155            body,
2156        )
2157        .map_err(RouterError::FrameBuild)
2158    }
2159
2160    async fn handle_client_control_request(
2161        &self,
2162        ctx: &RouteCtx,
2163        frame: Frame,
2164        request: ClientControlRequest,
2165    ) -> Result<Vec<Frame>, RouterError> {
2166        match request {
2167            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2168            ClientControlRequest::CatalogList { module_id } => {
2169                self.handle_catalog_list(frame, module_id)
2170            }
2171            ClientControlRequest::RouteOpen {
2172                target,
2173                identity,
2174                consumer_identity,
2175                consumer_capabilities,
2176                role_versions,
2177                admission_facts,
2178                scope,
2179            } => {
2180                self.handle_route_open(
2181                    ctx,
2182                    frame,
2183                    RouteOpenRequest {
2184                        target,
2185                        identity,
2186                        consumer_identity,
2187                        consumer_capabilities,
2188                        role_versions,
2189                        admission_facts,
2190                        scope,
2191                    },
2192                )
2193                .await
2194            }
2195            ClientControlRequest::RoutePoll {
2196                route_channel,
2197                route_epoch,
2198                kind,
2199            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2200            ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2201            ClientControlRequest::SupervisorSpawnSnapshot {} => {
2202                self.handle_supervisor_spawn_snapshot(frame)
2203            }
2204            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2205                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2206            }
2207            ClientControlRequest::SupervisorRestart {
2208                module_id,
2209                drain_timeout_ms,
2210            } => {
2211                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2212                    .await
2213            }
2214            ClientControlRequest::SupervisorSwap {
2215                module_id,
2216                ready_timeout_ms,
2217            } => {
2218                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2219                    .await
2220            }
2221            ClientControlRequest::SupervisorReload { module_id } => {
2222                self.handle_supervisor_reload(frame, module_id).await
2223            }
2224            ClientControlRequest::SupervisorRescan { preview } => {
2225                self.handle_supervisor_rescan(frame, preview).await
2226            }
2227            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2228                self.handle_supervisor_release_reserved(frame, module_id)
2229                    .await
2230            }
2231            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2232                self.handle_supervisor_set_enabled(frame, module_id, enabled)
2233                    .await
2234            }
2235            ClientControlRequest::SupervisorHealthProbe { module_id } => {
2236                self.handle_supervisor_health_probe(frame, module_id).await
2237            }
2238            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2239            ClientControlRequest::SupervisorRoutes { module_id } => {
2240                self.handle_supervisor_routes(frame, module_id)
2241            }
2242            ClientControlRequest::SupervisorProvenance { module_id } => {
2243                self.handle_supervisor_provenance(frame, module_id).await
2244            }
2245            ClientControlRequest::SupervisorStderrTail {
2246                module_id,
2247                max_lines,
2248                max_bytes,
2249            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2250            ClientControlRequest::SupervisorTerminals { module_id } => {
2251                self.handle_supervisor_terminals(frame, module_id).await
2252            }
2253        }
2254    }
2255
2256    fn handle_module_control_request(
2257        &self,
2258        connection_id: ConnectionId,
2259        frame: Frame,
2260        request: ModuleControlRequestFromModule,
2261    ) -> Result<Vec<Frame>, RouterError> {
2262        match request {
2263            ModuleControlRequestFromModule::CatalogUpdate {
2264                provides,
2265                capabilities,
2266                ready,
2267            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2268            ModuleControlRequestFromModule::LiveRoots {} => {
2269                let registered = self
2270                    .registry
2271                    .get_module_by_connection(connection_id)
2272                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2273                let Some(registration) = registered else {
2274                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2275                };
2276                let response = self
2277                    .forwarding
2278                    .live_roots(&registration.manifest.module_id)
2279                    .map_err(RouterError::Forwarding)?;
2280                Ok(vec![control_response_body_frame(
2281                    &frame,
2282                    &response,
2283                    "ModuleControlResponseToModule::LiveRoots",
2284                )?])
2285            }
2286            ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2287                self.handle_scope_sync(connection_id, frame, generation, scopes)
2288            }
2289            ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2290                self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2291            }
2292        }
2293    }
2294
2295    /// `scope.sync`: the owner is the module registered on this connection.
2296    /// A connection with no registration (every client connection, `direct`
2297    /// included) is refused `not_registered` before the table is consulted.
2298    fn handle_scope_sync(
2299        &self,
2300        connection_id: ConnectionId,
2301        frame: Frame,
2302        generation: u64,
2303        scopes: Vec<ScopeRecord>,
2304    ) -> Result<Vec<Frame>, RouterError> {
2305        let Some(registration) = self
2306            .registry
2307            .get_module_by_connection(connection_id)
2308            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2309        else {
2310            return Ok(vec![control_error_frame(
2311                &frame,
2312                "not_registered",
2313                "scope.sync requires an active module registration owned by this connection",
2314            )?]);
2315        };
2316        let owner = registration.manifest.module_id;
2317        let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
2318        let is_current_launch = |connection: ConnectionId| {
2319            self.hello_launch_nonces
2320                .lock()
2321                .unwrap_or_else(|poisoned| poisoned.into_inner())
2322                .presented(connection, current_nonce.as_deref())
2323        };
2324        // Lock order is the scope table, then the forwarding table: the new
2325        // tags are published, and the routes the change closes are selected,
2326        // while the scope table is still write-locked, so no admission can read
2327        // a record whose tag is not yet published.
2328        let mut table = self
2329            .scopes
2330            .write()
2331            .unwrap_or_else(|poisoned| poisoned.into_inner());
2332        let outcome = table.sync(&owner, connection_id, is_current_launch, generation, scopes);
2333        let drained = match &outcome {
2334            Ok(applied) => self
2335                .forwarding
2336                .publish_scope_changes(&applied.tag_changes)
2337                .map_err(RouterError::Forwarding)?,
2338            Err(_) => Vec::new(),
2339        };
2340        drop(table);
2341        match outcome {
2342            Ok(applied) => {
2343                let counts = ScopeOutcomeCounts::of(&applied.results);
2344                info!(
2345                    owner = %owner,
2346                    generation,
2347                    records = applied.results.len(),
2348                    created = counts.created,
2349                    replaced = counts.replaced,
2350                    updated = counts.updated,
2351                    unchanged = counts.unchanged,
2352                    refused = counts.refused,
2353                    ended = applied.ended.len(),
2354                    tag_changes = applied.tag_changes.len(),
2355                    routes_closed = drained.len(),
2356                    "scope sync accepted"
2357                );
2358                // An accepted sync can still refuse individual records, and the
2359                // owner is the only party that sees the reply. Name them here so
2360                // an operator can tell a refused session from a missing one
2361                // without the owner's logs. Capped so a sync that refuses
2362                // thousands cannot flood the log; the count above is complete.
2363                for refused in applied
2364                    .results
2365                    .iter()
2366                    .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
2367                    .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
2368                {
2369                    warn!(
2370                        owner = %owner,
2371                        generation,
2372                        scope_ref = %refused.scope_ref,
2373                        scope_epoch = refused.scope_epoch,
2374                        code = refused.code.as_deref().unwrap_or(""),
2375                        "scope record refused"
2376                    );
2377                }
2378                self.close_scope_drained_routes(drained);
2379                let response = ModuleControlResponseToModule::ScopeSync {
2380                    generation,
2381                    results: applied.results,
2382                    ended: applied.ended,
2383                };
2384                Ok(vec![control_response_body_frame(
2385                    &frame,
2386                    &response,
2387                    "ModuleControlResponseToModule::ScopeSync",
2388                )?])
2389            }
2390            Err(refusal) => {
2391                info!(
2392                    owner = %owner,
2393                    generation,
2394                    code = refusal.code,
2395                    "scope sync refused"
2396                );
2397                Ok(vec![control_error_frame(
2398                    &frame,
2399                    refusal.code,
2400                    refusal.message,
2401                )?])
2402            }
2403        }
2404    }
2405
2406    /// Tell both ends of each route a scope change closed. The module gets a
2407    /// channel-scoped GOODBYE and so does the client: the GOODBYE is what ends
2408    /// the client's route handle. The client also gets `route.closed` with the
2409    /// scope reason, one push per module and reason, so it can tell a revoked
2410    /// route from an ordinary close and not reopen it.
2411    fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
2412        if drained.is_empty() {
2413            return;
2414        }
2415        let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
2416            BTreeMap::new();
2417        let mut goodbyes = Vec::with_capacity(drained.len() * 2);
2418        for route in drained {
2419            warn!(
2420                module_id = %route.module_id,
2421                reason = ?route.reason,
2422                client_connection_id = route.client.connection_id.get(),
2423                route_channel = route.client.channel,
2424                "closing route because its scope changed"
2425            );
2426            pushes
2427                .entry((route.module_id.clone(), format!("{:?}", route.reason)))
2428                .or_insert_with(|| (route.reason, Vec::new()))
2429                .1
2430                .push(EndpointRoute {
2431                    goodbye_target: route.client.clone(),
2432                    principal: Principal::Unverified,
2433                    bound_at: Instant::now(),
2434                    draining: false,
2435                    drain_reason: None,
2436                });
2437            goodbyes.push(route.module);
2438            goodbyes.push(route.client);
2439        }
2440        for ((module_id, _), (reason, routes)) in pushes {
2441            send_route_control_pushes(
2442                &self.forwarding,
2443                routes,
2444                ClientControlPush::RouteClosed {
2445                    module_id,
2446                    reason,
2447                    drained: false,
2448                    abandoned: 0,
2449                    excluded_subscriptions: 0,
2450                    terminal: Some(false),
2451                },
2452            );
2453        }
2454        self.emit_route_goodbyes(goodbyes);
2455    }
2456
2457    /// `scope.describe`: any registered module may read any scope, because a
2458    /// provider must read the scope a route it serves is stamped with.
2459    fn handle_scope_describe(
2460        &self,
2461        connection_id: ConnectionId,
2462        frame: Frame,
2463        owner: Principal,
2464        scope_ref: String,
2465    ) -> Result<Vec<Frame>, RouterError> {
2466        let registered = self
2467            .registry
2468            .get_module_by_connection(connection_id)
2469            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2470        if registered.is_none() {
2471            return Ok(vec![control_error_frame(
2472                &frame,
2473                "not_registered",
2474                "scope.describe requires an active module registration owned by this connection",
2475            )?]);
2476        }
2477        let description = self
2478            .scopes
2479            .read()
2480            .unwrap_or_else(|poisoned| poisoned.into_inner())
2481            .describe(&owner, &scope_ref);
2482        let owner_configured = match &owner {
2483            Principal::Reserved { module_id } => self.supervisor.get(module_id).is_some(),
2484            _ => false,
2485        };
2486        let response = ModuleControlResponseToModule::ScopeDescribe {
2487            status: description.status,
2488            scope_epoch: description.scope_epoch,
2489            daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
2490            owner_synced: description.owner_synced,
2491            owner_configured,
2492            scope: description.stamp,
2493        };
2494        Ok(vec![control_response_body_frame(
2495            &frame,
2496            &response,
2497            "ModuleControlResponseToModule::ScopeDescribe",
2498        )?])
2499    }
2500
2501    fn handle_catalog_update(
2502        &self,
2503        connection_id: ConnectionId,
2504        frame: Frame,
2505        provides: Vec<ProviderRole>,
2506        capabilities: Option<CapabilityDeclarations>,
2507        ready: Option<bool>,
2508    ) -> Result<Vec<Frame>, RouterError> {
2509        self.refresh_capability_requirements();
2510        let Some(registration) = self
2511            .registry
2512            .get_module_by_connection(connection_id)
2513            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2514        else {
2515            return Ok(vec![control_error_frame(
2516                &frame,
2517                "not_registered",
2518                "catalog.update requires an active module registration owned by this connection",
2519            )?]);
2520        };
2521
2522        if let Some(message) =
2523            catalog_update_frozen_field_message(&registration.manifest, &provides)
2524        {
2525            return Ok(vec![control_error_frame(
2526                &frame,
2527                "catalog_update_frozen_field",
2528                message,
2529            )?]);
2530        }
2531
2532        let mut candidate = registration.manifest.clone();
2533        candidate.provides = provides.clone();
2534        candidate.capabilities = capabilities
2535            .clone()
2536            .or_else(|| registration.manifest.capabilities.clone());
2537        if let Err(err) = candidate.validate_capability_grammar() {
2538            return Ok(vec![control_error_frame(
2539                &frame,
2540                "invalid_capability_grammar",
2541                err.to_string(),
2542            )?]);
2543        }
2544
2545        let updated = self
2546            .registry
2547            .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
2548            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2549        if updated.is_none() {
2550            return Ok(vec![control_error_frame(
2551                &frame,
2552                "not_registered",
2553                "catalog.update requires an active module registration owned by this connection",
2554            )?]);
2555        }
2556        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
2557            log_duplicate_claim_events(
2558                self.capability_evaluator
2559                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
2560            );
2561        }
2562        if capability_census_trigger(
2563            registration.manifest.capabilities.as_ref(),
2564            updated
2565                .as_ref()
2566                .and_then(|entry| entry.manifest.capabilities.as_ref()),
2567        ) {
2568            self.enforce_capability_denies();
2569        }
2570        self.refresh_capability_requirements();
2571
2572        let response = ModuleControlResponseToModule::CatalogUpdate {};
2573        control_response_body_frame(
2574            &frame,
2575            &response,
2576            "ModuleControlResponseToModule::CatalogUpdate",
2577        )
2578        .map(|frame| vec![frame])
2579    }
2580
2581    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2582        self.refresh_capability_requirements();
2583        // A bare connection count is ambiguous between many clients holding a
2584        // route each and one client accumulating hundreds, so publish the
2585        // concentration alongside it. Route state is best-effort here: a
2586        // diagnostic endpoint must still answer if the forwarding lock is
2587        // contended.
2588        let mut counters = self.counters.snapshot();
2589        if let (Ok((connections_with_routes, max)), Some(obj)) = (
2590            self.forwarding.client_route_concentration(),
2591            counters.as_object_mut(),
2592        ) {
2593            obj.insert(
2594                "client_connections_with_routes".into(),
2595                connections_with_routes.into(),
2596            );
2597            obj.insert("max_routes_on_one_connection".into(), max.into());
2598        }
2599        // A module that is being fast-refused and a module that is fine look
2600        // identical from a client that retries and succeeds, so name the open
2601        // breakers here. This rides the existing free-form counters object
2602        // rather than a new wire field, so no sibling that deserializes
2603        // `ServerDescribe` has to be rebuilt to keep reading it.
2604        if let (Some(open_breakers), Some(obj)) = (
2605            self.route_bind_breakers.open_snapshot(),
2606            counters.as_object_mut(),
2607        ) {
2608            obj.insert("route_bind_breakers_open".into(), open_breakers);
2609        }
2610        let response = ClientControlResponse::ServerDescribe {
2611            protocol_ver: PROTOCOL_VERSION,
2612            subc_ops: subc_ops(),
2613            capabilities: self.subc_capabilities.as_ref().to_vec(),
2614            connected_clients: self.connected_clients.count(),
2615            counters: Some(counters),
2616            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
2617            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
2618            capability_requirements: self.capability_requirement_statuses(),
2619            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2620        };
2621        Ok(vec![control_response_body_frame(
2622            &frame,
2623            &response,
2624            "ClientControlResponse::ServerDescribe",
2625        )?])
2626    }
2627
2628    fn handle_catalog_list(
2629        &self,
2630        frame: Frame,
2631        module_id: Option<String>,
2632    ) -> Result<Vec<Frame>, RouterError> {
2633        let (generation, modules) = self.registry.list_modules().map_err(|err| {
2634            RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
2635        })?;
2636        let entries = modules
2637            .into_iter()
2638            .filter(|registration| {
2639                module_id
2640                    .as_deref()
2641                    .map(|wanted| registration.manifest.module_id == wanted)
2642                    .unwrap_or(true)
2643            })
2644            .map(|registration| {
2645                let not_ready = self.not_ready_reason(&registration);
2646                let roles = registration.manifest.provides;
2647                CatalogEntry {
2648                    module_id: registration.manifest.module_id,
2649                    ready: not_ready.is_none(),
2650                    not_ready,
2651                    module_version: Some(registration.manifest.module_version),
2652                    roles,
2653                    control_ops: registration.control_ops,
2654                    capabilities: registration.manifest.capabilities,
2655                    self_signals: registration.manifest.self_signals,
2656                }
2657            })
2658            .collect();
2659        let response = ClientControlResponse::CatalogList {
2660            generation,
2661            modules: entries,
2662            subc_ops: subc_ops(),
2663        };
2664        Ok(vec![control_response_body_frame(
2665            &frame,
2666            &response,
2667            "ClientControlResponse::CatalogList",
2668        )?])
2669    }
2670
2671    fn route_open_principal(
2672        &self,
2673        frame: &Frame,
2674        consumer_identity: Option<ConsumerIdentity>,
2675    ) -> Result<Result<Principal, Frame>, RouterError> {
2676        let Some(consumer_identity) = consumer_identity else {
2677            return Ok(Ok(Principal::Direct));
2678        };
2679
2680        if self.supervisor.spawned_consumer_authorized(
2681            &consumer_identity.module_id,
2682            &consumer_identity.launch_nonce,
2683        ) {
2684            return Ok(Ok(Principal::Reserved {
2685                module_id: consumer_identity.module_id,
2686            }));
2687        }
2688
2689        Ok(Err(control_error_frame(
2690            frame,
2691            "bad_consumer_identity",
2692            format!(
2693                "consumer_identity for module_id '{}' did not match a supervised launch nonce",
2694                consumer_identity.module_id
2695            ),
2696        )?))
2697    }
2698
2699    /// Ordinary `route.open` refusals go through here; admission and breaker
2700    /// refusals log separately with their capacity or breaker state. The daemon can
2701    /// attest which code it sent: without the event, a client's "the daemon
2702    /// refused me" and the daemon's own view could only be reconciled by
2703    /// argument. Malformed input (`invalid_project_root`) does not come here;
2704    /// rejecting a request that was never a valid open is not a refusal of one.
2705    fn route_open_refusal_frame(
2706        &self,
2707        ctx: &RouteCtx,
2708        frame: &Frame,
2709        module_id: &str,
2710        reason: &'static str,
2711        code: &'static str,
2712        message: impl Into<String>,
2713    ) -> Result<Frame, RouterError> {
2714        self.observe_route_open_refusal(ctx, module_id, reason, code);
2715        control_error_frame(frame, code, message.into())
2716    }
2717
2718    /// Refuse a `route.open` because the target module's bind-relay breaker is
2719    /// open, without attempting the relay.
2720    ///
2721    /// The wire code is `module_timeout`, which is the truth (the module has
2722    /// not been answering binds) and which both SDKs already classify as
2723    /// retryable with capped backoff. Reusing it is what keeps this change out
2724    /// of both SDKs; the daemon-side distinction lives in the counter key
2725    /// instead.
2726    ///
2727    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
2728    /// While a breaker is open this fires on every open to that module, and the
2729    /// stall written up in `docs/designs/route-open-head-of-line.md` already
2730    /// produced 261 lines about a single module inside 3000 lines of daemon
2731    /// log. The rare transitions are logged at warn/info instead and the volume
2732    /// is carried by the counter, so the evidence survives without the flood.
2733    /// The debug line keeps a per-refusal record reachable for whoever turns
2734    /// the level up.
2735    fn route_open_breaker_refusal_frame(
2736        &self,
2737        ctx: &RouteCtx,
2738        frame: &Frame,
2739        module_id: &str,
2740        consecutive_timeouts: u32,
2741        retry_in: Duration,
2742        probe_in_flight: bool,
2743    ) -> Result<Frame, RouterError> {
2744        self.counters
2745            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
2746        debug!(
2747            target: "control",
2748            code = "module_timeout",
2749            module_id = ?module_id,
2750            connection_id = ctx.connection_id.get(),
2751            consecutive_timeouts,
2752            retry_in_ms = retry_in.as_millis() as u64,
2753            probe_in_flight,
2754            "route.open refused by open bind-relay breaker"
2755        );
2756        let detail = if probe_in_flight {
2757            "one probe bind is already in flight; retry once it settles".to_string()
2758        } else {
2759            format!("not relaying for another {retry_in:?}")
2760        };
2761        control_error_frame(
2762            frame,
2763            "module_timeout",
2764            format!(
2765                "module_id '{module_id}' failed {consecutive_timeouts} consecutive route.bind \
2766                 relays; {detail}"
2767            ),
2768        )
2769    }
2770
2771    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
2772    /// requester's bytes (an unknown target is whatever the client sent) and
2773    /// is Debug-formatted so control characters land in the log escaped
2774    /// rather than as terminal sequences for whoever tails it.
2775    ///
2776    /// `reason` names the check that refused, because one wire code has
2777    /// several senders: after a module registers, `target_unavailable` can
2778    /// come from a missing role, an inactive registration, a supervisor that
2779    /// has not marked the process live, a missing forwarding connection, or a
2780    /// failed relay, and a log that records only the code cannot say which of
2781    /// them fired. It is a static, daemon-chosen label per branch, so it is
2782    /// safe to print plainly and stays a closed set.
2783    fn observe_route_open_refusal(
2784        &self,
2785        ctx: &RouteCtx,
2786        module_id: &str,
2787        reason: &'static str,
2788        code: &'static str,
2789    ) {
2790        self.counters.increment_route_open_refused(code);
2791        info!(
2792            target: "control",
2793            code,
2794            reason,
2795            module_id = ?module_id,
2796            connection_id = ctx.connection_id.get(),
2797            "route.open refused"
2798        );
2799        if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
2800            self.route_outages.record_not_serving(module_id, reason);
2801        }
2802    }
2803
2804    /// Record an ACCEPTED route.open.
2805    ///
2806    /// Refusals have been logged and counted since the attestation work; accepts
2807    /// were invisible, so the daemon knew every principal it stamped and wrote
2808    /// none of them down. The party that attests the identity was the only party
2809    /// not recording it, which left a credential vault unable to name the sender
2810    /// of a call that reached it (claustrum #43) and left the launch-nonce
2811    /// concurrency question unanswerable from the outside.
2812    ///
2813    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
2814    /// `code`/`module_id`/`connection_id` returns both directions of the same
2815    /// decision rather than two shapes a reader has to join by hand.
2816    ///
2817    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
2818    /// and the difference carries information rather than being an
2819    /// inconsistency. This line is only reachable after a successful bind to a
2820    /// REGISTERED module, so the value has already passed HELLO validation
2821    /// including the path-hazard refusal and cannot contain control bytes. A
2822    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
2823    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
2824    ///
2825    /// Bare is also what every other daemon line already emits (`module
2826    /// registered`, `configured module supervised`). Shipping `?module_id` here
2827    /// made this instrument the only one in the file whose ids did not answer
2828    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
2829    /// line whose whole purpose is being grepped beside its sibling.
2830    ///
2831    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
2832    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
2833    /// forwards every field type through it and a bare `&str` and a `?`-escaped
2834    /// one are recorded identically. A test written against that harness passes
2835    /// either way -- I wrote one, measured it, and deleted it rather than ship a
2836    /// green assertion that cannot fail. The same limit applies to the escaping
2837    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
2838    /// it reads as a guard on the Debug escaping and cannot detect its removal.
2839    /// Fencing either needs the real formatter, not the capture layer.
2840    ///
2841    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
2842    /// unix-socket options and subc is loopback TCP, so there is no peer identity
2843    /// to record. The ephemeral port would decay within minutes and answer only a
2844    /// live question. The identity question is instead answered by counting
2845    /// distinct live connections presenting one module's `consumer_identity` --
2846    /// "is anyone else holding this secret" rather than "is this the right
2847    /// process".
2848    fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
2849        self.route_outages.record_accepted(module_id);
2850        self.counters.increment_route_open_accepted(principal);
2851        info!(
2852            target: "control",
2853            principal,
2854            module_id,
2855            connection_id = ctx.connection_id.get(),
2856            "route.open accepted"
2857        );
2858    }
2859
2860    fn supervised_absent_route_open_refusal_frame(
2861        &self,
2862        ctx: &RouteCtx,
2863        frame: &Frame,
2864        module_id: &str,
2865        code: &'static str,
2866        status: &crate::supervise::ModuleStatus,
2867    ) -> Result<Frame, RouterError> {
2868        self.counters.increment_route_open_refused(code);
2869        info!(
2870            target: "control",
2871            code,
2872            reason = "supervised_not_registered",
2873            module_id = ?module_id,
2874            connection_id = ctx.connection_id.get(),
2875            state = %status.state,
2876            enabled = status.enabled,
2877            live = status.live,
2878            "route.open refused"
2879        );
2880        // A supervised module whose process has not registered is not
2881        // serving, whatever the reason; the supervisor knows this id, so it is
2882        // safe to track.
2883        self.route_outages
2884            .record_not_serving(module_id, "supervised_not_registered");
2885        control_error_frame(
2886            frame,
2887            code,
2888            format!(
2889                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
2890                status.state, status.enabled, status.live
2891            ),
2892        )
2893    }
2894
2895    async fn handle_route_open(
2896        &self,
2897        ctx: &RouteCtx,
2898        frame: Frame,
2899        request: RouteOpenRequest,
2900    ) -> Result<Vec<Frame>, RouterError> {
2901        let RouteOpenRequest {
2902            target,
2903            mut identity,
2904            consumer_identity,
2905            consumer_capabilities,
2906            role_versions,
2907            admission_facts,
2908            scope,
2909        } = request;
2910        let target_module_id = target_module_id(&target).to_string();
2911        debug!(
2912            connection_id = ctx.connection_id.get(),
2913            corr = frame.header.corr,
2914            module_id = %target_module_id,
2915            "handling route.open"
2916        );
2917
2918        // A malformed declaration is refused first, before anything about the
2919        // target is looked up: the same body would be refused against any
2920        // module, so the caller learns nothing by retrying or waiting. An empty
2921        // map declares nothing and travels as no field at all, so a provider
2922        // only ever sees a missing field or a non-empty one.
2923        let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
2924        if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
2925            self.observe_route_open_refusal(
2926                ctx,
2927                &target_module_id,
2928                "invalid_role_versions",
2929                error_codes::INVALID_REQUEST,
2930            );
2931            return Ok(vec![control_error_body_frame(
2932                &frame,
2933                ErrorBody {
2934                    code: error_codes::INVALID_REQUEST.to_string(),
2935                    message: error.to_string(),
2936                    detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
2937                },
2938            )?]);
2939        }
2940
2941        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
2942        // opposite. Below, a caller learns whether a module is unregistered,
2943        // supervised-but-down (with state/enabled/live), or registered without the
2944        // requested role. Elsewhere that is an enumeration leak: a probe learning
2945        // the shape of a fleet it cannot otherwise see.
2946        //
2947        // It is not one here, and the reason is the ACCESS MODEL rather than
2948        // anything about these errors. Reaching route.open requires the
2949        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
2950        // connection file, so any caller who completes it already runs as this
2951        // user -- and can read subc.jsonc for the module list and `ck module
2952        // status` for live state. The reply discloses nothing the caller cannot
2953        // read more easily from disk, while the precision is load-bearing:
2954        // `unknown_module` is retryable and a missing role is not.
2955        //
2956        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
2957        // remote transport, a sandboxed caller, a shared-host mode -- THAT
2958        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
2959        let Some(registration) = self
2960            .registry
2961            .get_module(&target_module_id)
2962            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2963        else {
2964            if let Some((status, warming)) =
2965                self.supervisor_status(&target_module_id, frame.header.corr)?
2966            {
2967                // BEFORE the two availability codes below, because for a module
2968                // that speaks no subc wire both of them are false comfort: they
2969                // say "not right now" and are retried, and this module will
2970                // never register no matter how long the caller waits. The
2971                // absence here is the declaration being honoured, not a module
2972                // that is late.
2973                if status.protocol == ModuleProtocol::None {
2974                    return Ok(vec![self.route_open_refusal_frame(
2975                        ctx,
2976                        &frame,
2977                        &target_module_id,
2978                        "protocol_none",
2979                        error_codes::MODULE_NO_PROTOCOL,
2980                        format!(
2981                            "module_id '{target_module_id}' is declared protocol: none; \
2982                             it speaks no subc wire and serves no routes"
2983                        ),
2984                    )?]);
2985                }
2986                let code = if warming {
2987                    "module_warming"
2988                } else {
2989                    "target_unavailable"
2990                };
2991                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
2992                    ctx,
2993                    &frame,
2994                    &target_module_id,
2995                    code,
2996                    &status,
2997                )?]);
2998            }
2999            if let Some(removed_ago_ms) =
3000                self.supervisor.removal_tombstone_age_ms(&target_module_id)
3001            {
3002                return Ok(vec![self.route_open_refusal_frame(
3003                    ctx,
3004                    &frame,
3005                    &target_module_id,
3006                    "removed",
3007                    error_codes::MODULE_REMOVED,
3008                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
3009                )?]);
3010            }
3011            return Ok(vec![self.route_open_refusal_frame(
3012                ctx,
3013                &frame,
3014                &target_module_id,
3015                "not_registered",
3016                error_codes::UNKNOWN_MODULE,
3017                format!("module_id '{target_module_id}' is not registered"),
3018            )?]);
3019        };
3020
3021        // Best-effort only: registry readiness and forwarding reservation use
3022        // different locks, so a module can flip readiness between this read and
3023        // the relay. Modules must still tolerate an `on_bind` while not ready.
3024        if !registration.ready {
3025            self.counters
3026                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
3027            info!(
3028                target: "control",
3029                code = error_codes::MODULE_WARMING,
3030                module_id = ?target_module_id,
3031                connection_id = ctx.connection_id.get(),
3032                reason = "declared_not_ready",
3033                "route.open refused"
3034            );
3035            // The module is registered but says it cannot take work, which is
3036            // an outage from the caller's side even though its process is up.
3037            self.route_outages
3038                .record_not_serving(&target_module_id, "declared_not_ready");
3039            return Ok(vec![control_error_body_frame(
3040                &frame,
3041                ErrorBody {
3042                    code: error_codes::MODULE_WARMING.to_string(),
3043                    message: format!(
3044                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
3045                    ),
3046                    detail: Some(serde_json::json!({
3047                        "reason": "declared_not_ready"
3048                    })),
3049                },
3050            )?]);
3051        }
3052
3053        // Effective readiness, second half: a module that declares a capability
3054        // `need: required` is not routable while that capability has no
3055        // registered provider. It is enforced HERE, as a retryable routing
3056        // refusal, and deliberately not as spawn ordering or a boot block. The
3057        // module is still started and registered and can make its own calls;
3058        // spawn ordering is a promise that cannot be kept once a provider
3059        // crashes at runtime, and refusing to boot would stop the whole
3060        // machine, including the tools needed to fix its configuration.
3061        //
3062        // "Provided" is the evaluator's verdict, which counts a provider as
3063        // soon as it has REGISTERED, not once it is ready. Two modules that
3064        // require each other's capabilities are therefore both routable once
3065        // both register; counting readiness instead would deadlock them.
3066        //
3067        // Only new opens are refused. Routes already bound when a provider
3068        // goes away stay bound: nothing here tears them down, and the module
3069        // answers them as it can. Like the readiness read above this is
3070        // best-effort against a provider registering or leaving concurrently.
3071        if let Some(capability) = self.first_unprovided_required_capability(&registration) {
3072            self.counters
3073                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
3074            info!(
3075                target: "control",
3076                code = error_codes::MODULE_WARMING,
3077                module_id = ?target_module_id,
3078                connection_id = ctx.connection_id.get(),
3079                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3080                capability = %capability,
3081                "route.open refused"
3082            );
3083            return Ok(vec![control_error_body_frame(
3084                &frame,
3085                ErrorBody {
3086                    code: error_codes::MODULE_WARMING.to_string(),
3087                    message: format!(
3088                        "module_id '{target_module_id}' requires capability '{capability}', \
3089                         which no registered module provides; retry"
3090                    ),
3091                    detail: Some(serde_json::json!({
3092                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3093                        "capability": capability,
3094                    })),
3095                },
3096            )?]);
3097        }
3098
3099        if !target_has_required_role(&target, &registration.manifest.provides) {
3100            return Ok(vec![self.route_open_refusal_frame(
3101                ctx,
3102                &frame,
3103                &target_module_id,
3104                "role_not_provided",
3105                "target_unavailable",
3106                format!("module_id '{target_module_id}' does not provide the requested target"),
3107            )?]);
3108        }
3109
3110        if registration.state != ChannelState::Active {
3111            return Ok(vec![self.route_open_refusal_frame(
3112                ctx,
3113                &frame,
3114                &target_module_id,
3115                "registration_not_active",
3116                "target_unavailable",
3117                format!("module_id '{target_module_id}' is not active"),
3118            )?]);
3119        }
3120
3121        if self
3122            .forwarding
3123            .module_is_draining(&target_module_id)
3124            .map_err(RouterError::Forwarding)?
3125        {
3126            return Ok(vec![self.route_open_refusal_frame(
3127                ctx,
3128                &frame,
3129                &target_module_id,
3130                "reloading",
3131                "module_reloading",
3132                format!("module_id '{target_module_id}' is reloading"),
3133            )?]);
3134        }
3135
3136        if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
3137            process_liveness.process_live(&target_module_id) == Some(false)
3138        }) {
3139            // A module the supervisor is restarting or reloading can still hold
3140            // a registration: the old process before its connection closes, or
3141            // a new one that registered while the supervisor was draining. The
3142            // forwarding table does not see that as draining, but the consumer
3143            // should still be told to retry soon, exactly as for the drain
3144            // above, rather than that the target is unavailable.
3145            if process_liveness.process_replacing(&target_module_id) {
3146                return Ok(vec![self.route_open_refusal_frame(
3147                    ctx,
3148                    &frame,
3149                    &target_module_id,
3150                    "reloading",
3151                    "module_reloading",
3152                    format!("module_id '{target_module_id}' is reloading"),
3153                )?]);
3154            }
3155            return Ok(vec![self.route_open_refusal_frame(
3156                ctx,
3157                &frame,
3158                &target_module_id,
3159                "supervisor_not_live",
3160                "target_unavailable",
3161                format!("module_id '{target_module_id}' is not live"),
3162            )?]);
3163        }
3164
3165        if !self
3166            .forwarding
3167            .has_live_module_connection(&target_module_id)
3168            .map_err(RouterError::Forwarding)?
3169        {
3170            return Ok(vec![self.route_open_refusal_frame(
3171                ctx,
3172                &frame,
3173                &target_module_id,
3174                "no_forwarding_connection",
3175                "target_unavailable",
3176                format!("module_id '{target_module_id}' has no live forwarding connection"),
3177            )?]);
3178        }
3179
3180        if let Some(error) =
3181            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
3182        {
3183            self.observe_route_open_refusal(
3184                ctx,
3185                &target_module_id,
3186                "op_not_allowed",
3187                "op_not_allowed",
3188            );
3189            return Ok(vec![error]);
3190        }
3191
3192        let principal = match self.route_open_principal(&frame, consumer_identity)? {
3193            Ok(principal) => principal,
3194            Err(error) => {
3195                self.observe_route_open_refusal(
3196                    ctx,
3197                    &target_module_id,
3198                    "bad_consumer_identity",
3199                    "bad_consumer_identity",
3200                );
3201                return Ok(vec![error]);
3202            }
3203        };
3204
3205        // This is attested, control-plane policy for supervised module origins.
3206        // Keep it before route reservation and out of the opaque forwarding hot
3207        // path: data frames must never acquire a per-frame capability check.
3208        if let Principal::Reserved {
3209            module_id: opening_module_id,
3210        } = &principal
3211        {
3212            if let Some(opening_registration) = self
3213                .registry
3214                .get_module(opening_module_id)
3215                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3216            {
3217                if let Some(capability) =
3218                    denied_capability(&opening_registration.manifest, &registration.manifest)
3219                {
3220                    warn!(
3221                        opening_module_id,
3222                        target_module_id,
3223                        capability,
3224                        "refusing route.open because an attested capability deny edge matches"
3225                    );
3226                    return Ok(vec![self.route_open_refusal_frame(
3227                        ctx,
3228                        &frame,
3229                        &target_module_id,
3230                        "capability_deny_edge",
3231                        "capability_forbidden",
3232                        format!(
3233                            "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
3234                        ),
3235                    )?]);
3236                }
3237            }
3238        }
3239
3240        if admission_facts.is_some() {
3241            let carrier_matches = matches!(
3242                &principal,
3243                Principal::Reserved { module_id }
3244                    if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
3245            );
3246            if !carrier_matches {
3247                return Ok(vec![self.route_open_refusal_frame(
3248                    ctx,
3249                    &frame,
3250                    &target_module_id,
3251                    "admission_facts_carrier_not_permitted",
3252                    "admission_facts_not_permitted",
3253                    "admission facts may only be carried by the configured reserved module",
3254                )?]);
3255            }
3256
3257            let target_allowed = self
3258                .admission_facts_targets
3259                .as_ref()
3260                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
3261            if !target_allowed {
3262                return Ok(vec![self.route_open_refusal_frame(
3263                    ctx,
3264                    &frame,
3265                    &target_module_id,
3266                    "admission_facts_target_not_listed",
3267                    "admission_facts_target_not_allowed",
3268                    format!(
3269                        "admission facts are not permitted for target module_id '{target_module_id}'"
3270                    ),
3271                )?]);
3272            }
3273
3274            // Keep the value opaque to subc. The downstream admission validator owns
3275            // schema and semantic checks; this daemon only enforces carrier authority
3276            // and the configured destination allowlist.
3277        }
3278
3279        // Scope admission, on the attested principal above and never on the
3280        // request body. The tag read here travels with the pending bind and is
3281        // compared with the published one at commit, so a sync between here
3282        // and the module's ack refuses the open instead of binding a stamp
3283        // that is no longer true.
3284        let (bound_scope, scope_stamp) = match scope {
3285            None => (None, None),
3286            Some(selector) => {
3287                let owner_configured = match &selector.owner {
3288                    Principal::Reserved { module_id } => self.supervisor.get(module_id).is_some(),
3289                    _ => false,
3290                };
3291                let admitted = self
3292                    .scopes
3293                    .read()
3294                    .unwrap_or_else(|poisoned| poisoned.into_inner())
3295                    .admit(&principal, &target_module_id, &selector, owner_configured);
3296                match admitted {
3297                    Ok(admission) => (
3298                        Some(BoundScope {
3299                            owner: admission.owner,
3300                            scope_ref: admission.stamp.scope_ref.clone(),
3301                            tag: admission.tag,
3302                        }),
3303                        Some(admission.stamp),
3304                    ),
3305                    Err(refusal) => {
3306                        return Ok(vec![self.route_open_refusal_frame(
3307                            ctx,
3308                            &frame,
3309                            &target_module_id,
3310                            refusal.code,
3311                            refusal.code,
3312                            refusal.message,
3313                        )?]);
3314                    }
3315                }
3316            }
3317        };
3318
3319        // Bind admits a root that no longer exists on disk, because refusing here
3320        // closes the only exit from a paused run: cancel needs a bound route, and a
3321        // renamed or reclaimed directory makes that route unopenable forever. The
3322        // run itself is intact and still addressable by its recorded identity.
3323        //
3324        // This does NOT relax the rule the strict constructor protects. That rule is
3325        // that no root is ever aliased into NEW durable state -- a missing component
3326        // can reappear as a symlink elsewhere, which would move the identity and
3327        // split a session's history across two of them. The engine now refuses the
3328        // two operations that create such state (send and import) at admission,
3329        // which is a narrower way to hold the same invariant: reads and terminations
3330        // are admitted, writes are not. That refusal had to ship before this line
3331        // changed, or there is an interval where a send commits under a provisional
3332        // identity -- the exact failure the original policy existed to prevent.
3333        //
3334        // Resolution follows realpath rather than lexical cleanup: the longest
3335        // existing ancestor is canonicalized and the missing tail re-appended, so a
3336        // live root is unchanged and a vanished leaf keeps the identity it was
3337        // admitted under. Lexical cleanup would mint a DIFFERENT identity for the
3338        // same caller the moment the directory vanished, which strands the run more
3339        // quietly than refusing it.
3340        let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
3341            Ok(project_root) => project_root,
3342            Err(err) => {
3343                return Ok(vec![control_error_frame(
3344                    &frame,
3345                    "invalid_project_root",
3346                    err.to_string(),
3347                )?])
3348            }
3349        };
3350        identity.project_root = project_root.as_path().to_path_buf();
3351
3352        // Last gate before any relay work, and deliberately after the cheap
3353        // registry and availability checks above: those name a more precise
3354        // condition (unknown, removed, reloading) and a caller is better served
3355        // by the precise code than by this one.
3356        //
3357        // Everything below this point costs an egress permit, a reserved handle
3358        // pair and, if the module does not answer, the whole relay budget. The
3359        // reader no longer waits for that budget, so cap each target explicitly;
3360        // serial dispatch used to provide the accidental cap of one relay per
3361        // connection. Admission is a mutex-protected count and never waits.
3362        let _concurrency_guard = match self
3363            .route_bind_concurrency
3364            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
3365        {
3366            Ok(guard) => guard,
3367            Err(in_flight) => {
3368                return Ok(vec![self.route_open_target_capacity_refusal(
3369                    ctx,
3370                    &frame,
3371                    &target_module_id,
3372                    in_flight,
3373                )?]);
3374            }
3375        };
3376
3377        // A module that has already burned the whole budget `threshold` times
3378        // in a row does not get to charge it again until a probe says it recovered.
3379        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
3380            RouteBindAdmission::Admitted { guard, probe } => {
3381                if probe {
3382                    info!(
3383                        module_id = %target_module_id,
3384                        connection_id = ctx.connection_id.get(),
3385                        "route.bind breaker half-open: admitting one probe"
3386                    );
3387                }
3388                guard
3389            }
3390            RouteBindAdmission::Refused {
3391                consecutive_timeouts,
3392                retry_in,
3393                probe_in_flight,
3394            } => {
3395                return Ok(vec![self.route_open_breaker_refusal_frame(
3396                    ctx,
3397                    &frame,
3398                    &target_module_id,
3399                    consecutive_timeouts,
3400                    retry_in,
3401                    probe_in_flight,
3402                )?]);
3403            }
3404        };
3405
3406        // Resolve the per-module budget here so the wait matches the operator's
3407        // intent for this specific target. A per-module override in
3408        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
3409        // daemons) wins over the daemon-wide default.
3410        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
3411        let relay_deadline = Instant::now() + route_bind_relay_timeout;
3412        let pending = match self
3413            .forwarding
3414            .begin_route_bind_relay_for(
3415                ctx.connection_id,
3416                ctx.egress.clone(),
3417                response_version(&frame),
3418                frame.header.corr,
3419                &target_module_id,
3420                principal.clone(),
3421                bound_scope,
3422                Some(project_root),
3423                relay_deadline,
3424            )
3425            .await
3426        {
3427            Ok(pending) => pending,
3428            Err(err) => {
3429                return Ok(vec![self.route_open_refusal_frame(
3430                    ctx,
3431                    &frame,
3432                    &target_module_id,
3433                    "relay_reservation_failed",
3434                    forwarding_error_code(&err),
3435                    err.to_string(),
3436                )?])
3437            }
3438        };
3439        let crate::forwarding::PendingRouteBindRelay {
3440            endpoint,
3441            module_sink,
3442            negotiated_ver,
3443            client_channel,
3444            client_epoch,
3445            module_channel,
3446            module_epoch,
3447            corr: relay_corr,
3448            receiver,
3449        } = pending;
3450        let mut reservation =
3451            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
3452
3453        debug!(
3454            connection_id = ctx.connection_id.get(),
3455            client_channel,
3456            client_epoch,
3457            module_channel,
3458            module_epoch,
3459            "reserved route handle pair"
3460        );
3461        // Rendered BEFORE the move into the relay, because the accept arm below
3462        // is where it is logged and the principal is gone by then.
3463        let principal_label = match &principal {
3464            Principal::Reserved { module_id } => format!("reserved:{module_id}"),
3465            Principal::Direct => "direct".to_string(),
3466            other => format!("{other:?}"),
3467        };
3468        let relay = ModuleControlRequest::RouteBind {
3469            route_channel: module_channel,
3470            epoch: module_epoch,
3471            target,
3472            identity,
3473            principal: Some(principal),
3474            consumer_capabilities,
3475            role_versions,
3476            admission_facts,
3477            scope: scope_stamp,
3478        };
3479        let relay_body = serde_json::to_vec(&relay).map_err(|err| {
3480            RouterError::backend(
3481                0,
3482                frame.header.corr,
3483                format!("failed to encode route.bind request: {err}"),
3484            )
3485        })?;
3486        let relay_frame = Frame::build_with_version(
3487            negotiated_ver,
3488            FrameType::Request,
3489            control_flags(),
3490            0,
3491            0,
3492            relay_corr,
3493            relay_body,
3494        )
3495        .map_err(RouterError::FrameBuild)?;
3496
3497        if let Err(err) = module_sink.send(relay_frame).await {
3498            reservation.release_and_disarm();
3499            return Ok(vec![self.route_open_refusal_frame(
3500                ctx,
3501                &frame,
3502                &target_module_id,
3503                "relay_send_failed",
3504                "target_unavailable",
3505                err.to_string(),
3506            )?]);
3507        }
3508
3509        if !self
3510            .forwarding
3511            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
3512            .map_err(RouterError::Forwarding)?
3513        {
3514            self.send_abandoned_route_bind_goodbye(
3515                &module_sink,
3516                negotiated_ver,
3517                module_channel,
3518                module_epoch,
3519            );
3520        }
3521
3522        match timeout_at(relay_deadline, receiver).await {
3523            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
3524                reservation.disarm();
3525                if breaker.record_accepted() {
3526                    info!(
3527                        module_id = %target_module_id,
3528                        "route.bind breaker closed: the probe was accepted"
3529                    );
3530                }
3531                self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
3532                Ok(Vec::new())
3533            }
3534            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
3535                reservation.release_and_disarm();
3536                // A module that says no in microseconds is healthy. Rejection
3537                // is a different condition with its own refusal and must not
3538                // move the breaker.
3539                breaker.record_inconclusive();
3540                // The daemon's own commit re-check refused the bind because the
3541                // scope ended or changed after admission. The module accepted;
3542                // counting it as a module rejection would blame the module.
3543                let scope_code = match body.code.as_str() {
3544                    error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
3545                    error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
3546                    _ => None,
3547                };
3548                if let Some(code) = scope_code {
3549                    self.observe_route_open_refusal(
3550                        ctx,
3551                        &target_module_id,
3552                        "scope_changed_before_commit",
3553                        code,
3554                    );
3555                    return Ok(vec![control_error_body_frame(&frame, body)?]);
3556                }
3557                self.counters
3558                    .increment_route_open_refused("module_rejected");
3559                info!(
3560                    target: "control",
3561                    code = "module_rejected",
3562                    module_code = ?body.code,
3563                    module_id = ?target_module_id,
3564                    connection_id = ctx.connection_id.get(),
3565                    "route.open refused"
3566                );
3567                Ok(vec![control_error_body_frame(&frame, body)?])
3568            }
3569            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
3570                reservation.release_and_disarm();
3571                breaker.record_inconclusive();
3572                // Fires when the module's connection closes while a relayed
3573                // bind is pending -- typically a caller racing a module restart
3574                // whose bind was relayed BEFORE the drain mark went up. Logged
3575                // because the caller sees only its own error and the fleet has
3576                // already spent one diagnosis round unable to tell this arm
3577                // from a relay timeout without daemon-side evidence.
3578                tracing::warn!(
3579                    module_id = %target_module_id,
3580                    "route.bind relay abandoned: {message}"
3581                );
3582                Ok(vec![self.route_open_refusal_frame(
3583                    ctx,
3584                    &frame,
3585                    &target_module_id,
3586                    "relay_abandoned",
3587                    "target_unavailable",
3588                    message,
3589                )?])
3590            }
3591            Ok(Err(_)) => {
3592                reservation.release_and_disarm();
3593                breaker.record_inconclusive();
3594                Ok(vec![self.route_open_refusal_frame(
3595                    ctx,
3596                    &frame,
3597                    &target_module_id,
3598                    "relay_waiter_canceled",
3599                    "target_unavailable",
3600                    "route.bind relay waiter was canceled before the module responded",
3601                )?])
3602            }
3603            Err(_) => {
3604                reservation.release_and_disarm();
3605                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
3606                // answer at all is the one condition a fast refusal can
3607                // usefully stand in for; every other arm already answered.
3608                if let Some(opened) = breaker.record_timeout(
3609                    self.route_bind_breaker_threshold,
3610                    self.route_bind_breaker_cooldown,
3611                ) {
3612                    warn!(
3613                        module_id = %target_module_id,
3614                        consecutive_timeouts = opened.consecutive_timeouts,
3615                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
3616                        reopened_after_probe = opened.reopened_after_probe,
3617                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
3618                    );
3619                }
3620                // The generous budget just burned to no answer: the module is
3621                // registered and its connection is up, but its bind handler sat
3622                // on the ack for the full budget (warm-on-bind, cold configure,
3623                // or a wedged handler). Every earlier unavailability shape
3624                // fast-refuses BEFORE the relay, so this arm firing means the
3625                // slowness is module-side -- log it so the per-module timeline
3626                // is reconstructable without client audit rows.
3627                tracing::warn!(
3628                    module_id = %target_module_id,
3629                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
3630                    "route.bind relay timed out: module did not ack within budget"
3631                );
3632                Ok(vec![self.route_open_refusal_frame(
3633                    ctx,
3634                    &frame,
3635                    &target_module_id,
3636                    "relay_timed_out",
3637                    "module_timeout",
3638                    format!(
3639                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
3640                        route_bind_relay_timeout
3641                    ),
3642                )?])
3643            }
3644        }
3645    }
3646
3647    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3648        let response = ClientControlResponse::SupervisorSpawnSnapshot {
3649            snapshot: self.supervisor.spawn_snapshot(),
3650        };
3651        Ok(vec![control_response_body_frame(
3652            &frame,
3653            &response,
3654            "ClientControlResponse::SupervisorSpawnSnapshot",
3655        )?])
3656    }
3657
3658    fn handle_supervisor_spawn_subscribe(
3659        &self,
3660        ctx: &RouteCtx,
3661        frame: Frame,
3662        since: Option<SpawnCursor>,
3663    ) -> Result<Vec<Frame>, RouterError> {
3664        match self.supervisor.subscribe_spawns(
3665            ctx.connection_id,
3666            frame.header.corr,
3667            response_version(&frame),
3668            since,
3669            ctx.egress.clone(),
3670        ) {
3671            Ok(()) => Ok(Vec::new()),
3672            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
3673                Ok(vec![control_error_body_frame(
3674                    &frame,
3675                    ErrorBody {
3676                        code: "spawn_cursor_incarnation_mismatch".to_string(),
3677                        message: "spawn cursor belongs to a different daemon incarnation"
3678                            .to_string(),
3679                        detail: Some(serde_json::json!({
3680                            "current_daemon_incarnation": current
3681                        })),
3682                    },
3683                )?])
3684            }
3685            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
3686                &frame,
3687                ErrorBody {
3688                    code: "spawn_cursor_too_old".to_string(),
3689                    message: "spawn cursor predates the retained event ring".to_string(),
3690                    detail: Some(serde_json::json!({
3691                        "oldest_retained_cursor": oldest
3692                    })),
3693                },
3694            )?]),
3695            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
3696                0,
3697                frame.header.corr,
3698                format!("failed to open supervisor spawn subscription: {error}"),
3699            )),
3700        }
3701    }
3702
3703    async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3704        let generation = self
3705            .registry
3706            .generation()
3707            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3708        let mut modules = Vec::new();
3709        for module in self.supervisor.list() {
3710            let status = module.status_for_control("list").map_err(|err| {
3711                RouterError::backend(
3712                    0,
3713                    frame.header.corr,
3714                    format!("failed to read supervisor status: {err}"),
3715                )
3716            })?;
3717            let (configured, _) = module.configuration().map_err(|err| {
3718                RouterError::backend(
3719                    0,
3720                    frame.header.corr,
3721                    format!("failed to read module configuration: {err}"),
3722                )
3723            })?;
3724            // Status and configuration snapshots release their locks before the image probe awaits.
3725            let image = module.running_image_agreement().await;
3726            // Read per request so the figure is current when the operator asks;
3727            // the daemon samples nothing in between.
3728            let resources = Some(module.child_resource_usage());
3729            let pending_reload = Some(reload_verdict(
3730                &configured.program,
3731                status.spawned_from.as_deref(),
3732                image,
3733            ));
3734            modules.push(SupervisorEntry {
3735                // Keep the retired policy field on the wire for one release so
3736                // existing status consumers still receive the platform policy.
3737                launch_nonce_env: Some(!cfg!(unix)),
3738                module_id: status.module_id,
3739                state: status.state.to_string(),
3740                enabled: status.enabled,
3741                live: status.live,
3742                protocol: status.protocol,
3743                health: status.health.status,
3744                pending_reload,
3745                last_probe_ms: status.health.last_probe_ms,
3746                last_exit_code: status.last_exit.as_ref().and_then(|e| e.code),
3747                last_exit_signal: status.last_exit.as_ref().and_then(|e| e.signal),
3748                last_exit_ms: status.last_exit.as_ref().map(|e| e.at_ms),
3749                last_exit_kind: status.last_exit.as_ref().map(|e| e.kind.into()),
3750                restart_count: Some(status.restart_count),
3751                max_restarts: Some(status.max_restarts),
3752                lifetime_restarts: Some(status.lifetime_restarts),
3753                spawn_generation: Some(status.spawn_generation),
3754                restart_window_secs: Some(status.restart_window.as_secs()),
3755                drain_timeout_ms: Some(status.drain_timeout.as_millis() as u64),
3756                restart_backoff_ms: Some(status.restart_backoff.as_millis() as u64),
3757                restart_max_backoff_ms: Some(status.restart_max_backoff.as_millis() as u64),
3758                resources,
3759            });
3760        }
3761        let response = ClientControlResponse::SupervisorList {
3762            generation,
3763            modules,
3764        };
3765        Ok(vec![control_response_body_frame(
3766            &frame,
3767            &response,
3768            "ClientControlResponse::SupervisorList",
3769        )?])
3770    }
3771
3772    fn handle_supervisor_stderr_tail(
3773        &self,
3774        frame: Frame,
3775        module_id: String,
3776        max_lines: Option<u32>,
3777        max_bytes: Option<u32>,
3778    ) -> Result<Vec<Frame>, RouterError> {
3779        let Some(module) = self.supervisor.get(&module_id) else {
3780            return Ok(vec![control_error_frame(
3781                &frame,
3782                "unknown_module",
3783                format!("module_id '{module_id}' is not supervised"),
3784            )?]);
3785        };
3786
3787        let snapshot = module.stderr_tail(
3788            max_lines.map(|value| value as usize),
3789            max_bytes.map(|value| value as usize),
3790        );
3791
3792        let response = ClientControlResponse::SupervisorStderrTail {
3793            module_id,
3794            tail: StderrTail {
3795                capture: match snapshot.capture {
3796                    CaptureState::Captured => StderrCaptureState::Captured,
3797                    CaptureState::Incomplete { reason } => {
3798                        StderrCaptureState::Incomplete { reason }
3799                    }
3800                    CaptureState::NotCaptured { reason } => {
3801                        StderrCaptureState::NotCaptured { reason }
3802                    }
3803                },
3804                entries: snapshot
3805                    .entries
3806                    .into_iter()
3807                    .map(|entry| match entry {
3808                        TailEntry::Line {
3809                            text,
3810                            truncated,
3811                            at_ms,
3812                        } => StderrTailEntry::Line {
3813                            text,
3814                            truncated,
3815                            at_ms,
3816                        },
3817                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
3818                    })
3819                    .collect(),
3820                dropped_lines: snapshot.dropped_lines,
3821            },
3822        };
3823        Ok(vec![control_response_body_frame(
3824            &frame,
3825            &response,
3826            "ClientControlResponse::SupervisorStderrTail",
3827        )?])
3828    }
3829
3830    async fn handle_supervisor_terminals(
3831        &self,
3832        frame: Frame,
3833        module_id: String,
3834    ) -> Result<Vec<Frame>, RouterError> {
3835        let Some(module) = self.supervisor.get(&module_id) else {
3836            return Ok(vec![control_error_frame(
3837                &frame,
3838                "unknown_module",
3839                format!("module_id '{module_id}' is not supervised"),
3840            )?]);
3841        };
3842
3843        // The journal read runs on a blocking thread: it can be megabytes of
3844        // file I/O and must not occupy a runtime worker.
3845        let terminals = module
3846            .read_durable_terminal_history()
3847            .await
3848            .map_err(|error| {
3849                RouterError::backend(
3850                    0,
3851                    frame.header.corr,
3852                    format!("failed to read terminal history: {error}"),
3853                )
3854            })?;
3855        let response = ClientControlResponse::SupervisorTerminals {
3856            module_id,
3857            terminals,
3858        };
3859        Ok(vec![control_response_body_frame(
3860            &frame,
3861            &response,
3862            "ClientControlResponse::SupervisorTerminals",
3863        )?])
3864    }
3865
3866    fn handle_supervisor_routes(
3867        &self,
3868        frame: Frame,
3869        module_id: Option<String>,
3870    ) -> Result<Vec<Frame>, RouterError> {
3871        let modules = self
3872            .forwarding
3873            .route_census(module_id.as_deref())
3874            .map_err(RouterError::Forwarding)?
3875            .into_iter()
3876            .map(|(module_id, routes)| SupervisorRouteModule {
3877                module_id,
3878                routes: routes
3879                    .into_iter()
3880                    .map(|route| SupervisorRoute {
3881                        consumer: match route.principal {
3882                            Principal::Reserved { module_id } => {
3883                                SupervisorRouteConsumer::Reserved { module_id }
3884                            }
3885                            Principal::Direct | Principal::Unverified => {
3886                                SupervisorRouteConsumer::Direct {
3887                                    connection_id: route.goodbye_target.connection_id.get(),
3888                                }
3889                            }
3890                        },
3891                        age_ms: Instant::now()
3892                            .saturating_duration_since(route.bound_at)
3893                            .as_millis()
3894                            .try_into()
3895                            .unwrap_or(u64::MAX),
3896                        draining: route.draining,
3897                        drain_reason: route.drain_reason,
3898                    })
3899                    .collect(),
3900            })
3901            .collect();
3902        let response = ClientControlResponse::SupervisorRoutes { modules };
3903        Ok(vec![control_response_body_frame(
3904            &frame,
3905            &response,
3906            "ClientControlResponse::SupervisorRoutes",
3907        )?])
3908    }
3909
3910    async fn handle_supervisor_provenance(
3911        &self,
3912        frame: Frame,
3913        module_id: Option<String>,
3914    ) -> Result<Vec<Frame>, RouterError> {
3915        let mut selected = if let Some(module_id) = module_id {
3916            let Some(module) = self.supervisor.get(&module_id) else {
3917                return Ok(vec![control_error_frame(
3918                    &frame,
3919                    "unknown_module",
3920                    format!("module_id '{module_id}' is not supervised"),
3921                )?]);
3922            };
3923            vec![module]
3924        } else {
3925            self.supervisor.list()
3926        };
3927
3928        let mut modules = Vec::with_capacity(selected.len());
3929        for module in selected.drain(..) {
3930            let status = module.status().map_err(|err| {
3931                RouterError::backend(
3932                    0,
3933                    frame.header.corr,
3934                    format!("failed to read supervisor status: {err}"),
3935                )
3936            })?;
3937            let module_declared = self
3938                .registry
3939                .get_module(&status.module_id)
3940                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3941                .and_then(|registration| registration.manifest.provenance)
3942                .map(|build| ModuleDeclaredProvenance::Reported { build })
3943                .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
3944            #[cfg(test)]
3945            let running_image = match &self.provenance_probe_override {
3946                Some(result) => result.clone(),
3947                None => module.running_image_agreement().await,
3948            };
3949            #[cfg(not(test))]
3950            let running_image = module.running_image_agreement().await;
3951            modules.push(SupervisorModuleProvenance {
3952                module_id: status.module_id,
3953                module_declared,
3954                daemon_observed: SupervisorObservedProcess {
3955                    pid: status.pid,
3956                    spawned_at_ms: status.spawned_at_ms,
3957                    spawned_from: status.spawned_from,
3958                    running_image,
3959                },
3960            });
3961        }
3962        let daemon = SupervisorDaemonProvenance {
3963            daemon_build: self.daemon_provenance.build.clone(),
3964            daemon_observed: DaemonObservedProcess {
3965                pid: self.daemon_provenance.pid,
3966                started_at_ms: self
3967                    .daemon_provenance
3968                    .start_clock
3969                    .map(|clock| clock.started_at_ms())
3970                    .or(self.daemon_provenance.started_at_ms),
3971                running_image: self
3972                    .daemon_provenance
3973                    .probe
3974                    .observe(
3975                        self.daemon_provenance.pid,
3976                        self.daemon_provenance.executable_path.as_deref(),
3977                        self.daemon_provenance.executable_identity,
3978                        self.daemon_provenance.process_start_time,
3979                    )
3980                    .await,
3981            },
3982        };
3983        let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
3984        Ok(vec![control_response_body_frame(
3985            &frame,
3986            &response,
3987            "ClientControlResponse::SupervisorProvenance",
3988        )?])
3989    }
3990
3991    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3992        self.refresh_capability_requirements();
3993        let generation = self
3994            .registry
3995            .generation()
3996            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3997        let modules = self
3998            .supervisor
3999            .list()
4000            .into_iter()
4001            .map(|module| {
4002                let status = module.status_for_control("health").map_err(|err| {
4003                    RouterError::backend(
4004                        0,
4005                        frame.header.corr,
4006                        format!("failed to read supervisor health: {err}"),
4007                    )
4008                })?;
4009                let module_id = status.module_id;
4010                let capability_detail = self
4011                    .capability_evaluator
4012                    .required_problem_detail(&module_id);
4013                Ok(SupervisorHealthEntry {
4014                    module_id,
4015                    status: status.health.status,
4016                    detail: append_capability_problem_detail(
4017                        status.health.detail,
4018                        capability_detail,
4019                    ),
4020                    metrics: status.health.metrics,
4021                    consecutive_failures: status.health.consecutive_failures,
4022                    late_answer_count: status.health.late_answer_count,
4023                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
4024                    last_action: status.health.last_action,
4025                    last_action_ms: status.health.last_action_ms,
4026                    last_probe_ms: status.health.last_probe_ms,
4027                })
4028            })
4029            .collect::<Result<Vec<_>, RouterError>>()?;
4030        let response = ClientControlResponse::SupervisorHealth {
4031            generation,
4032            modules,
4033        };
4034        Ok(vec![control_response_body_frame(
4035            &frame,
4036            &response,
4037            "ClientControlResponse::SupervisorHealth",
4038        )?])
4039    }
4040
4041    async fn handle_supervisor_restart(
4042        &self,
4043        frame: Frame,
4044        module_id: String,
4045        drain_timeout_ms: Option<u64>,
4046    ) -> Result<Vec<Frame>, RouterError> {
4047        let operation_lock = self.supervisor.operation_lock();
4048        let _operation_guard = operation_lock.lock().await;
4049        let Some(module) = self.supervisor.get(&module_id) else {
4050            return Ok(vec![control_error_frame(
4051                &frame,
4052                "unknown_module",
4053                format!("module_id '{module_id}' is not supervised"),
4054            )?]);
4055        };
4056
4057        self.route_outages.mark_operator_action(&module_id);
4058        if let Err(err) = module.restart(drain_timeout_ms).await {
4059            self.route_outages
4060                .operator_action_ended_unrefused(&module_id);
4061            let (code, message) = match err {
4062                crate::supervise::SuperviseError::Disabled { .. } => {
4063                    ("module_disabled", err.to_string())
4064                }
4065                crate::supervise::SuperviseError::SwapInProgress { .. } => {
4066                    ("swap_in_progress", err.to_string())
4067                }
4068                _ => (
4069                    "target_unavailable",
4070                    format!("failed to restart module_id '{module_id}': {err}"),
4071                ),
4072            };
4073            return Ok(vec![control_error_frame(&frame, code, message)?]);
4074        }
4075
4076        let response = ClientControlResponse::SupervisorAck {
4077            module_id,
4078            applied: true,
4079        };
4080        Ok(vec![control_response_body_frame(
4081            &frame,
4082            &response,
4083            "ClientControlResponse::SupervisorAck",
4084        )?])
4085    }
4086
4087    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
4088    /// when the old process has finished draining: a caller whose own lane
4089    /// rides the old process must get its reply before that drain waits on it.
4090    async fn handle_supervisor_swap(
4091        &self,
4092        frame: Frame,
4093        module_id: String,
4094        ready_timeout_ms: Option<u64>,
4095    ) -> Result<Vec<Frame>, RouterError> {
4096        // The daemon-wide operation lock is held only to resolve the handle,
4097        // not across the swap. The swap can take its whole readiness budget,
4098        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
4099        // holding it here would park an operator's stop behind the swap it is
4100        // meant to abort. A rescan or stop that reaches the module during the
4101        // swap is served by the swap itself (see `supervise_swap`).
4102        let module = {
4103            let operation_lock = self.supervisor.operation_lock();
4104            let _operation_guard = operation_lock.lock().await;
4105            self.supervisor.get(&module_id)
4106        };
4107        let Some(module) = module else {
4108            return Ok(vec![control_error_frame(
4109                &frame,
4110                "unknown_module",
4111                format!("module_id '{module_id}' is not supervised"),
4112            )?]);
4113        };
4114
4115        self.route_outages.mark_operator_action(&module_id);
4116        if let Err(err) = module
4117            .swap(ready_timeout_ms.map(Duration::from_millis))
4118            .await
4119        {
4120            self.route_outages
4121                .operator_action_ended_unrefused(&module_id);
4122            use crate::supervise::SuperviseError;
4123            let message = err.to_string();
4124            let error = match err {
4125                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
4126                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
4127                    code: "swap_refused".to_string(),
4128                    message,
4129                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
4130                },
4131                SuperviseError::SwapFailed {
4132                    arm,
4133                    candidate_exit,
4134                    ..
4135                } => ErrorBody {
4136                    code: "swap_failed".to_string(),
4137                    message,
4138                    detail: Some(serde_json::json!({
4139                        "arm": arm.as_str(),
4140                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
4141                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
4142                    })),
4143                },
4144                _ => ErrorBody::new(
4145                    "target_unavailable",
4146                    format!("failed to swap module_id '{module_id}': {message}"),
4147                ),
4148            };
4149            return Ok(vec![control_error_body_frame(&frame, error)?]);
4150        }
4151        // A completed swap kept the incumbent serving until cutover, so it
4152        // usually opened no outage; a mark left behind would make the next,
4153        // unrelated outage read as requested.
4154        self.route_outages
4155            .operator_action_ended_unrefused(&module_id);
4156
4157        let response = ClientControlResponse::SupervisorAck {
4158            module_id,
4159            applied: true,
4160        };
4161        Ok(vec![control_response_body_frame(
4162            &frame,
4163            &response,
4164            "ClientControlResponse::SupervisorAck",
4165        )?])
4166    }
4167
4168    async fn handle_supervisor_reload(
4169        &self,
4170        frame: Frame,
4171        module_id: String,
4172    ) -> Result<Vec<Frame>, RouterError> {
4173        let operation_lock = self.supervisor.operation_lock();
4174        let _operation_guard = operation_lock.lock().await;
4175        let Some(module) = self.supervisor.get(&module_id) else {
4176            return Ok(vec![control_error_frame(
4177                &frame,
4178                "unknown_module",
4179                format!("module_id '{module_id}' is not supervised"),
4180            )?]);
4181        };
4182
4183        self.route_outages.mark_operator_action(&module_id);
4184        if let Err(err) = module.reload().await {
4185            self.route_outages
4186                .operator_action_ended_unrefused(&module_id);
4187            let (code, message) = match err {
4188                crate::supervise::SuperviseError::Disabled { .. } => {
4189                    ("module_disabled", err.to_string())
4190                }
4191                crate::supervise::SuperviseError::SwapInProgress { .. } => {
4192                    ("swap_in_progress", err.to_string())
4193                }
4194                _ => (
4195                    "reload_failed",
4196                    format!("failed to reload module_id '{module_id}': {err}"),
4197                ),
4198            };
4199            return Ok(vec![control_error_frame(&frame, code, message)?]);
4200        }
4201
4202        let response = ClientControlResponse::SupervisorAck {
4203            module_id,
4204            applied: true,
4205        };
4206        Ok(vec![control_response_body_frame(
4207            &frame,
4208            &response,
4209            "ClientControlResponse::SupervisorAck",
4210        )?])
4211    }
4212
4213    async fn handle_supervisor_rescan(
4214        &self,
4215        frame: Frame,
4216        preview: bool,
4217    ) -> Result<Vec<Frame>, RouterError> {
4218        let Some(context) = self.rescan.clone() else {
4219            return Ok(vec![control_error_frame(
4220                &frame,
4221                "rescan_unavailable",
4222                "the daemon was not started with a reloadable config path".to_string(),
4223            )?]);
4224        };
4225
4226        let operation_lock = self.supervisor.operation_lock();
4227        let _operation_guard = operation_lock.lock().await;
4228        let loaded = match crate::daemon_config::load(&context.config_path) {
4229            Ok(config) => config,
4230            Err(err) => {
4231                return Ok(vec![control_error_frame(
4232                    &frame,
4233                    "invalid_daemon_config",
4234                    format!("supervisor rescan rejected daemon config: {err}"),
4235                )?])
4236            }
4237        };
4238        // `load` reports a missing file as Ok(None), which is correct at boot
4239        // (no config, nothing to supervise) and catastrophic here: rescan treats
4240        // "not in the config" as "remove it", so an absent file would read as an
4241        // empty module list and retire the entire running fleet. An editor
4242        // writing via write-new-then-rename, or a half-finished edit, is enough
4243        // to open that window. Refuse instead: a config that cannot be read
4244        // carries no instruction to remove anything.
4245        let Some(config) = loaded else {
4246            return Ok(vec![control_error_frame(
4247                &frame,
4248                "invalid_daemon_config",
4249                format!(
4250                    "daemon config not found at {}; refusing to rescan (an absent config would \
4251                     retire every supervised module)",
4252                    context.config_path.display()
4253                ),
4254            )?]);
4255        };
4256        let (
4257            configured_port,
4258            storage_config,
4259            admission_facts_carrier_module_id,
4260            admission_facts_targets,
4261            scope_authority_owners,
4262            modules,
4263            reserved_capabilities,
4264        ) = (
4265            config.port,
4266            config.storage,
4267            config.admission_facts_carrier_module_id,
4268            config.admission_facts_targets,
4269            config.scope_authority_owners,
4270            config.modules,
4271            config.reserved_capabilities,
4272        );
4273
4274        // Collect the sections rescan cannot apply, so the REPLY carries them.
4275        //
4276        // The warning below has always been correct and has always gone only to
4277        // the journal -- addressed to whoever reads logs, while the person who
4278        // just edited the config is looking at the CLI. Naming each section
4279        // individually rather than setting a flag: "something outside modules
4280        // changed" sends the operator back to diffing their own file, which is
4281        // the work this is meant to save.
4282        let mut restart_required = Vec::new();
4283        for section in RestartRequiredSection::ALL {
4284            let changed = match section {
4285                RestartRequiredSection::Port => configured_port != context.configured_port,
4286                RestartRequiredSection::Storage => storage_config != context.storage_config,
4287                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
4288                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
4289                }
4290                RestartRequiredSection::AdmissionFactsTargets => {
4291                    admission_facts_targets != context.admission_facts_targets
4292                }
4293                RestartRequiredSection::ScopeAuthorityOwners => {
4294                    scope_authority_owners != context.scope_authority_owners
4295                }
4296            };
4297            if changed {
4298                restart_required.push(section.label().to_string());
4299            }
4300        }
4301        if !restart_required.is_empty() {
4302            warn!(
4303                config_path = %context.config_path.display(),
4304                sections = %restart_required.join(", "),
4305                "daemon config changed outside the modules section; restart the daemon to apply those changes"
4306            );
4307        }
4308
4309        for configured in &modules {
4310            if let Err(err) = validate_spec(&configured.module_spec()) {
4311                return Ok(vec![control_error_frame(
4312                    &frame,
4313                    "invalid_daemon_config",
4314                    format!("supervisor rescan rejected daemon config: {err}"),
4315                )?]);
4316            }
4317        }
4318
4319        let configured_capabilities = modules
4320            .iter()
4321            .map(|module| (module.module_id.clone(), module.enabled))
4322            .collect::<Vec<_>>();
4323        let preview_capability_warnings = if preview {
4324            let (_, registrations) = self.runtime_capability_snapshot()?;
4325            let current_modules = self
4326                .supervisor
4327                .list()
4328                .into_iter()
4329                .map(|module| module.module_id().to_string())
4330                .collect::<BTreeSet<_>>();
4331            let resulting_modules = configured_capabilities.clone();
4332            let removed = current_modules
4333                .into_iter()
4334                .filter(|module_id| {
4335                    !resulting_modules
4336                        .iter()
4337                        .any(|(configured_id, _)| configured_id == module_id)
4338                })
4339                .collect::<Vec<_>>();
4340            self.capability_evaluator.preview_removal_warnings(
4341                resulting_modules,
4342                &removed,
4343                &registrations,
4344            )
4345        } else {
4346            Vec::new()
4347        };
4348        let result = match self
4349            .reconcile_supervised_modules(&context.supervisor, modules, preview)
4350            .await
4351        {
4352            Ok(result) => result,
4353            Err(message) => {
4354                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
4355            }
4356        };
4357        if !preview {
4358            self.capability_evaluator
4359                .configure(configured_capabilities, reserved_capabilities);
4360            self.capability_evaluator.wake_deadline_loop();
4361            self.refresh_capability_requirements();
4362        }
4363        let mut result = result;
4364        result.restart_required = restart_required;
4365        result.capability_warnings = preview_capability_warnings;
4366        let response = ClientControlResponse::SupervisorRescan { result };
4367        Ok(vec![control_response_body_frame(
4368            &frame,
4369            &response,
4370            "ClientControlResponse::SupervisorRescan",
4371        )?])
4372    }
4373
4374    async fn handle_supervisor_release_reserved(
4375        &self,
4376        frame: Frame,
4377        module_id: String,
4378    ) -> Result<Vec<Frame>, RouterError> {
4379        let Some(context) = self.rescan.clone() else {
4380            return Ok(vec![control_error_frame(
4381                &frame,
4382                "release_unavailable",
4383                "reserved-id release requires a daemon started with a reloadable config path",
4384            )?]);
4385        };
4386        let operation_lock = self.supervisor.operation_lock();
4387        let _operation_guard = operation_lock.lock().await;
4388        let loaded = match crate::daemon_config::load(&context.config_path) {
4389            Ok(Some(config)) => config,
4390            Ok(None) => {
4391                return Ok(vec![control_error_frame(
4392                    &frame,
4393                    "invalid_daemon_config",
4394                    format!(
4395                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
4396                        context.config_path.display()
4397                    ),
4398                )?])
4399            }
4400            Err(err) => {
4401                return Ok(vec![control_error_frame(
4402                    &frame,
4403                    "invalid_daemon_config",
4404                    format!("unable to verify reserved-id release against daemon config: {err}"),
4405                )?])
4406            }
4407        };
4408        if loaded
4409            .modules
4410            .iter()
4411            .any(|configured| configured.module_id == module_id)
4412        {
4413            return Ok(vec![control_error_frame(
4414                &frame,
4415                "reserved_module_configured",
4416                format!(
4417                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
4418                ),
4419            )?]);
4420        }
4421        if !self.supervisor.release_retained_reserved_gate(&module_id) {
4422            return Ok(vec![control_error_frame(
4423                &frame,
4424                "reserved_gate_not_retained",
4425                format!(
4426                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
4427                ),
4428            )?]);
4429        }
4430
4431        let response = ClientControlResponse::SupervisorAck {
4432            module_id,
4433            applied: true,
4434        };
4435        Ok(vec![control_response_body_frame(
4436            &frame,
4437            &response,
4438            "ClientControlResponse::SupervisorAck",
4439        )?])
4440    }
4441
4442    /// Reconcile the running module set against the configured one.
4443    ///
4444    /// With `preview` set, the diff is computed and returned WITHOUT applying any
4445    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
4446    /// deliberately shares this function with the executing path rather than
4447    /// computing the same diff somewhere else -- two implementations of one
4448    /// decision agree until they do not, and the whole value of a preview is that
4449    /// it describes the operation that will actually run.
4450    async fn reconcile_supervised_modules(
4451        &self,
4452        supervisor: &Supervisor,
4453        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
4454        preview: bool,
4455    ) -> Result<SupervisorRescanResult, String> {
4456        let mut current = BTreeMap::new();
4457        for module in self.supervisor.list() {
4458            let (spec, health) = module.configuration().map_err(|err| {
4459                format!(
4460                    "failed to read configuration for module_id '{}': {err}",
4461                    module.module_id()
4462                )
4463            })?;
4464            let enabled = module
4465                .status()
4466                .map_err(|err| {
4467                    format!(
4468                        "failed to read status for module_id '{}': {err}",
4469                        module.module_id()
4470                    )
4471                })?
4472                .enabled;
4473            current.insert(
4474                module.module_id().to_string(),
4475                (module, spec, health, enabled),
4476            );
4477        }
4478        let configured = configured_modules
4479            .into_iter()
4480            .map(|module| (module.module_id.clone(), module))
4481            .collect::<BTreeMap<_, _>>();
4482
4483        let added = configured
4484            .keys()
4485            .filter(|module_id| !current.contains_key(*module_id))
4486            .cloned()
4487            .collect::<Vec<_>>();
4488        let removed = current
4489            .keys()
4490            .filter(|module_id| !configured.contains_key(*module_id))
4491            .cloned()
4492            .collect::<Vec<_>>();
4493        let mut changed_pending_reload = Vec::new();
4494        let mut configuration_changes = BTreeSet::new();
4495        let mut enabled_changes = BTreeSet::new();
4496        let mut unchanged = 0_u32;
4497
4498        for (module_id, configured_module) in &configured {
4499            let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
4500            else {
4501                continue;
4502            };
4503            let configuration_changed = *current_spec != configured_module.module_spec()
4504                || *current_health != configured_module.health;
4505            let enabled_changed = *current_enabled != configured_module.enabled;
4506            if configuration_changed {
4507                configuration_changes.insert(module_id.clone());
4508                changed_pending_reload.push(module_id.clone());
4509            }
4510            if enabled_changed {
4511                enabled_changes.insert(module_id.clone());
4512            }
4513            if !configuration_changed && !enabled_changed {
4514                unchanged = unchanged.saturating_add(1);
4515            }
4516        }
4517
4518        // Everything above this point is pure computation over two snapshots.
4519        // Everything below MUTATES. The preview returns here so the boundary is a
4520        // single early return rather than a condition repeated at each mutation
4521        // site, where one missed guard would apply part of a change the caller was
4522        // told would not happen.
4523        if preview {
4524            return Ok(SupervisorRescanResult {
4525                added,
4526                removed,
4527                changed_pending_reload,
4528                enabled_changes: enabled_changes.iter().cloned().collect(),
4529                unchanged,
4530                preview: true,
4531                // Filled by the caller on both paths, so the preview reports
4532                // restart-required sections identically to an executed rescan --
4533                // the preview is where an operator is most likely to be looking.
4534                restart_required: Vec::new(),
4535                capability_warnings: Vec::new(),
4536            });
4537        }
4538
4539        for module_id in &removed {
4540            let module = &current
4541                .get(module_id)
4542                .expect("removed module came from current supervisor state")
4543                .0;
4544            module.retire().await.map_err(|err| {
4545                format!("failed to retire module_id '{module_id}' during rescan: {err}")
4546            })?;
4547            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
4548            //
4549            // `handle_route_open` resolves an absent module in three steps:
4550            // registry, then supervisor status, then tombstone. Retiring first
4551            // opens a window where ALL THREE ARE ABSENT -- the registry entry
4552            // went with the teardown above, the supervisor entry went with
4553            // `retire`, and the tombstone does not exist yet -- so a route.open
4554            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
4555            // it") for a module that was deliberately removed and whose caller
4556            // should get `module_removed` (TERMINAL, carrying a removal age).
4557            //
4558            // Writing the tombstone first closes it: during the window the
4559            // supervisor entry still answers, so the caller gets
4560            // `target_unavailable` -- retryable, and TRUE, because the module
4561            // is mid-teardown. After both statements it is `module_removed`.
4562            // No instant remains where a removed module reads as one that
4563            // never existed.
4564            //
4565            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
4566            // the absence of a test beside a fix invites deletion: these are
4567            // two sync statements with no await between them, so reaching the
4568            // window needs a second worker thread to land exactly between them
4569            // and there is no hook to force it. MEASURED: the 25 daemon_config
4570            // tests pass identically with the old order and the new one, so
4571            // the existing suite cannot see this and a green run is not
4572            // evidence either way. What the suite does hold is the
4573            // post-condition -- a removed module answers `module_removed` --
4574            // which this preserves.
4575            //
4576            // Found by an Athena panel reading the shipped tree against a
4577            // design note (2026-09-19), as the one concrete instance of that
4578            // note's class that survived contact with source. Direction is
4579            // benign: retryable where terminal was intended, never the reverse.
4580            self.supervisor.record_rescan_removal(module_id);
4581            self.supervisor.retire(module_id);
4582            self.route_outages.forget(module_id);
4583        }
4584
4585        for module_id in configured.keys() {
4586            let Some((module, _, _, _)) = current.get(module_id) else {
4587                continue;
4588            };
4589            let configured_module = configured
4590                .get(module_id)
4591                .expect("configured module id came from configured map");
4592            if configuration_changes.contains(module_id) {
4593                module
4594                    .update_configuration(
4595                        configured_module.module_spec(),
4596                        configured_module.health,
4597                        configured_module.drain_timeout_ms,
4598                    )
4599                    .await
4600                    .map_err(|err| {
4601                        format!(
4602                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
4603                        )
4604                    })?;
4605            }
4606            if enabled_changes.contains(module_id) {
4607                // A rescan that starts or stops a module applies an operator's
4608                // edit to the config, so the resulting outage was asked for.
4609                self.route_outages.mark_operator_action(module_id);
4610                module
4611                    .set_enabled(configured_module.enabled)
4612                    .await
4613                    .map_err(|err| {
4614                        self.route_outages.operator_action_ended_unrefused(module_id);
4615                        format!(
4616                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
4617                            configured_module.enabled
4618                        )
4619                    })?;
4620            }
4621        }
4622
4623        for module_id in &added {
4624            let configured_module = configured
4625                .get(module_id)
4626                .expect("added module id came from configured map");
4627            supervisor
4628                .supervise_configured_with_health(
4629                    configured_module.module_spec(),
4630                    configured_module.enabled,
4631                    configured_module.health,
4632                    configured_module.drain_timeout_ms,
4633                    configured_module.restart,
4634                )
4635                .map_err(|err| {
4636                    format!("failed to add module_id '{module_id}' during rescan: {err}")
4637                })?;
4638        }
4639
4640        Ok(SupervisorRescanResult {
4641            added,
4642            removed,
4643            changed_pending_reload,
4644            enabled_changes: enabled_changes.iter().cloned().collect(),
4645            unchanged,
4646            preview: false,
4647            // Filled by the caller, which is the only layer that can see the
4648            // previous config to diff against.
4649            restart_required: Vec::new(),
4650            capability_warnings: Vec::new(),
4651        })
4652    }
4653
4654    async fn handle_supervisor_set_enabled(
4655        &self,
4656        frame: Frame,
4657        module_id: String,
4658        enabled: bool,
4659    ) -> Result<Vec<Frame>, RouterError> {
4660        let operation_lock = self.supervisor.operation_lock();
4661        let _operation_guard = operation_lock.lock().await;
4662        let Some(module) = self.supervisor.get(&module_id) else {
4663            return Ok(vec![control_error_frame(
4664                &frame,
4665                "unknown_module",
4666                format!("module_id '{module_id}' is not supervised"),
4667            )?]);
4668        };
4669
4670        // Enabling counts as well as disabling: a module an operator starts
4671        // is refused until it registers, and that wait was asked for.
4672        self.route_outages.mark_operator_action(&module_id);
4673        let applied = match module.set_enabled(enabled).await {
4674            Ok(applied) => applied,
4675            Err(err) => {
4676                self.route_outages
4677                    .operator_action_ended_unrefused(&module_id);
4678                return Ok(vec![control_error_frame(
4679                    &frame,
4680                    "target_unavailable",
4681                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
4682                )?]);
4683            }
4684        };
4685        if !applied {
4686            // Already in the requested state: nothing was made unavailable,
4687            // so the mark must not outlive this request.
4688            self.route_outages
4689                .operator_action_ended_unrefused(&module_id);
4690        }
4691
4692        self.capability_evaluator.wake_deadline_loop();
4693        self.refresh_capability_requirements();
4694        let response = ClientControlResponse::SupervisorAck { module_id, applied };
4695        Ok(vec![control_response_body_frame(
4696            &frame,
4697            &response,
4698            "ClientControlResponse::SupervisorAck",
4699        )?])
4700    }
4701
4702    async fn handle_supervisor_health_probe(
4703        &self,
4704        frame: Frame,
4705        module_id: String,
4706    ) -> Result<Vec<Frame>, RouterError> {
4707        self.refresh_capability_requirements();
4708        let Some(registration) = self
4709            .registry
4710            .get_module(&module_id)
4711            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4712        else {
4713            return Ok(vec![control_error_frame(
4714                &frame,
4715                "unknown_module",
4716                format!("module_id '{module_id}' is not registered"),
4717            )?]);
4718        };
4719
4720        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
4721        // named for it. Making `module_registration_grants_op` return false
4722        // unconditionally reddens five tests, and every one is named for something
4723        // else -- capability relay, probe/bind demultiplexing, supervision-only
4724        // probing. They exercise a successful advertisement check on the way to their
4725        // own subject.
4726        //
4727        // Real protection, fragile in a specific way: narrowing any of those tests to
4728        // focus on its stated subject would silently remove coverage nobody knows
4729        // they are carrying. Recorded here rather than as a sixth test, because the
4730        // useful fact is WHICH tests hold the guard up -- a new test would add
4731        // coverage without telling the next person what the existing ones quietly do.
4732        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
4733        {
4734            return Ok(vec![control_error_frame(
4735                &frame,
4736                "health_not_advertised",
4737                format!("module_id '{module_id}' did not advertise health.check"),
4738            )?]);
4739        }
4740
4741        let deadline = Instant::now() + self.health_probe_timeout;
4742        let pending = match self.forwarding.begin_module_control_rpc_for(
4743            &module_id,
4744            MODULE_CONTROL_OP_HEALTH_CHECK,
4745            deadline,
4746        ) {
4747            Ok(pending) => pending,
4748            Err(err) => {
4749                return Ok(vec![control_error_frame(
4750                    &frame,
4751                    forwarding_error_code(&err),
4752                    err.to_string(),
4753                )?])
4754            }
4755        };
4756
4757        let PendingModuleControlRpc {
4758            endpoint,
4759            module_sink,
4760            negotiated_ver,
4761            corr: probe_corr,
4762            receiver,
4763        } = pending;
4764        let mut guard =
4765            ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
4766        let probe_body =
4767            serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
4768                RouterError::backend(
4769                    0,
4770                    frame.header.corr,
4771                    format!("failed to encode health.check request: {err}"),
4772                )
4773            })?;
4774        let probe_frame = Frame::build_with_version(
4775            negotiated_ver,
4776            FrameType::Request,
4777            control_flags(),
4778            0,
4779            0,
4780            probe_corr,
4781            probe_body,
4782        )
4783        .map_err(RouterError::FrameBuild)?;
4784
4785        if let Err(err) = module_sink.send(probe_frame).await {
4786            return Ok(vec![control_error_frame(
4787                &frame,
4788                "target_unavailable",
4789                err.to_string(),
4790            )?]);
4791        }
4792
4793        match timeout_at(deadline, receiver).await {
4794            Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
4795                guard.disarm();
4796                let Some(report) = response.health_report() else {
4797                    return Ok(vec![control_error_frame(
4798                        &frame,
4799                        "invalid_control_body",
4800                        "health.check RPC returned a non-health response",
4801                    )?]);
4802                };
4803                // Metrics go out whole here. The supervisor's cached snapshot
4804                // caps this blob (see truncate_health_metrics), and this path
4805                // exists precisely to answer without that cap -- so applying it
4806                // here would leave no way to see what the cached view drops.
4807                let HealthReport {
4808                    status,
4809                    detail,
4810                    metrics,
4811                } = report;
4812                let capability_detail = self
4813                    .capability_evaluator
4814                    .required_problem_detail(&module_id);
4815                let response = ClientControlResponse::SupervisorHealthProbe {
4816                    module_id,
4817                    status,
4818                    detail: append_capability_problem_detail(detail, capability_detail),
4819                    metrics,
4820                };
4821                Ok(vec![control_response_body_frame(
4822                    &frame,
4823                    &response,
4824                    "ClientControlResponse::SupervisorHealthProbe",
4825                )?])
4826            }
4827            Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
4828                guard.disarm();
4829                Ok(vec![control_error_body_frame(&frame, body)?])
4830            }
4831            Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
4832                guard.disarm();
4833                Ok(vec![control_error_frame(
4834                    &frame,
4835                    "target_unavailable",
4836                    message,
4837                )?])
4838            }
4839            Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
4840                guard.disarm();
4841                Ok(vec![control_error_frame(
4842                    &frame,
4843                    "invalid_control_body",
4844                    message,
4845                )?])
4846            }
4847            Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
4848                guard.disarm();
4849                Ok(vec![control_error_frame(
4850                    &frame,
4851                    "invalid_control_body",
4852                    format!("expected module-control op '{expected}', got '{actual}'"),
4853                )?])
4854            }
4855            Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
4856                guard.disarm();
4857                Ok(vec![control_error_frame(
4858                    &frame,
4859                    "module_timeout",
4860                    format!(
4861                        "module_id '{module_id}' answered health.check after {:?}",
4862                        self.health_probe_timeout
4863                    ),
4864                )?])
4865            }
4866            Ok(Err(_)) => Ok(vec![control_error_frame(
4867                &frame,
4868                "target_unavailable",
4869                "health.check waiter was canceled before the module responded",
4870            )?]),
4871            Err(_) => Ok(vec![control_error_frame(
4872                &frame,
4873                "module_timeout",
4874                format!(
4875                    "module_id '{module_id}' did not answer health.check within {:?}",
4876                    self.health_probe_timeout
4877                ),
4878            )?]),
4879        }
4880    }
4881
4882    fn supervisor_status(
4883        &self,
4884        module_id: &str,
4885        corr: u64,
4886    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
4887        self.supervisor
4888            .get(module_id)
4889            .map(|module| {
4890                let warming = module.is_warming_for_control("status").map_err(|err| {
4891                    RouterError::backend(
4892                        0,
4893                        corr,
4894                        format!(
4895                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
4896                        ),
4897                    )
4898                })?;
4899                module.status_for_control("status").map_err(|err| {
4900                    RouterError::backend(
4901                        0,
4902                        corr,
4903                        format!(
4904                            "failed to read supervisor status for module_id '{module_id}': {err}"
4905                        ),
4906                    )
4907                }).map(|status| (status, warming))
4908            })
4909            .transpose()
4910    }
4911
4912    fn guard_module_control_op(
4913        &self,
4914        frame: &Frame,
4915        module_id: &str,
4916        op: &str,
4917    ) -> Result<Option<Frame>, RouterError> {
4918        if self.module_grants_op(module_id, op, frame.header.corr)? {
4919            return Ok(None);
4920        }
4921
4922        Ok(Some(control_error_frame(
4923            frame,
4924            "op_not_allowed",
4925            format!("module_id '{module_id}' did not grant control op '{op}'"),
4926        )?))
4927    }
4928
4929    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
4930        let Some(registration) = self
4931            .registry
4932            .get_module(module_id)
4933            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
4934        else {
4935            return Ok(false);
4936        };
4937        Ok(module_registration_grants_op(&registration.control_ops, op))
4938    }
4939
4940    fn handle_status_update(
4941        &self,
4942        endpoint: ModuleEndpointId,
4943        frame: Frame,
4944    ) -> Result<Vec<Frame>, RouterError> {
4945        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
4946            Ok(update) => update,
4947            Err(err) => {
4948                // Forward-compat: a newer module may push a channel-0 op this subc
4949                // version doesn't know. The control contract says unknown push ops
4950                // are IGNORED, never answered with an error. Only a malformed body
4951                // for an op we DO know is a real error worth surfacing.
4952                if is_known_module_push_op(&frame.body) {
4953                    return Ok(vec![control_error_frame(
4954                        &frame,
4955                        "invalid_control_body",
4956                        format!("malformed module control push body: {err}"),
4957                    )?]);
4958                }
4959                return Ok(Vec::new());
4960            }
4961        };
4962
4963        match update {
4964            ModuleControlPush::RouteStatus {
4965                route_channel,
4966                route_epoch,
4967                status,
4968            } => {
4969                self.forwarding
4970                    .cache_status(endpoint, route_channel, route_epoch, status)
4971                    .map_err(RouterError::Forwarding)?;
4972            }
4973        }
4974        Ok(Vec::new())
4975    }
4976
4977    fn handle_route_poll(
4978        &self,
4979        ctx: &RouteCtx,
4980        frame: Frame,
4981        route_channel: u16,
4982        route_epoch: u32,
4983        kind: PollKind,
4984    ) -> Result<Vec<Frame>, RouterError> {
4985        let snapshot = self
4986            .forwarding
4987            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
4988            .map_err(RouterError::Forwarding)?;
4989        let response = match (kind, snapshot) {
4990            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
4991                ClientControlResponse::RoutePoll {
4992                    route_channel,
4993                    route_epoch,
4994                    status,
4995                    live: None,
4996                }
4997            }
4998            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
4999                route_channel,
5000                route_epoch,
5001                status: None,
5002                live: None,
5003            },
5004            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
5005                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
5006                // is what makes reporting `true` correct rather than a
5007                // confident guess. `process_live` returns None only when the
5008                // module id has no supervisor snapshot at all -- an
5009                // externally-started module the daemon did not spawn -- and
5010                // for those the supervisor has no opinion to offer, ever. It
5011                // is never None for a supervised module in an unknown state:
5012                // a supervised module always has a snapshot, and the answer
5013                // comes from `state == Running && process_alive`.
5014                //
5015                // The route is Bound, so the module completed a HELLO on a
5016                // live connection; "the process this route points at is
5017                // running" is therefore attested by the binding rather than
5018                // assumed. Reporting `false` for an unsupervised module would
5019                // be the actual lie -- it would tell a client its healthy
5020                // route is dead because the daemon does not manage the
5021                // process.
5022                //
5023                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
5024                // module whose liveness is genuinely unknown, e.g. a snapshot
5025                // that has not been populated yet -- THIS DEFAULT BECOMES
5026                // WRONG and must split: unsupervised stays true, unknown
5027                // becomes null so the client can tell the two apart. The
5028                // response field is already `Option<bool>`, so the wire can
5029                // carry that distinction today.
5030                let live = self
5031                    .process_liveness
5032                    .as_ref()
5033                    .and_then(|source| source.process_live(&module_id))
5034                    .unwrap_or(true);
5035                ClientControlResponse::RoutePoll {
5036                    route_channel,
5037                    route_epoch,
5038                    status: None,
5039                    live: Some(live),
5040                }
5041            }
5042            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5043                route_channel,
5044                route_epoch,
5045                status: None,
5046                live: Some(false),
5047            },
5048        };
5049
5050        Ok(vec![control_response_body_frame(
5051            &frame,
5052            &response,
5053            "ClientControlResponse::RoutePoll",
5054        )?])
5055    }
5056
5057    pub(crate) fn observe_module_control_completion(
5058        &self,
5059        completion: ModuleControlRpcCompletion,
5060    ) -> bool {
5061        match completion {
5062            ModuleControlRpcCompletion::Unknown => false,
5063            ModuleControlRpcCompletion::Settled => true,
5064            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
5065                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
5066                info!(
5067                    module_id = %module_id,
5068                    latency_ms,
5069                    "late health.check answer proves the module is alive"
5070                );
5071                match self
5072                    .supervisor
5073                    .record_late_health_answer(&module_id, latency_ms)
5074                {
5075                    Ok(true) => {}
5076                    Ok(false) => debug!(
5077                        module_id = %module_id,
5078                        latency_ms,
5079                        "late health.check answer has no active supervisor snapshot"
5080                    ),
5081                    Err(err) => warn!(
5082                        module_id = %module_id,
5083                        latency_ms,
5084                        error = %err,
5085                        "failed to record late health.check answer"
5086                    ),
5087                }
5088                true
5089            }
5090        }
5091    }
5092
5093    /// Decide whether a failure while settling a relayed `route.bind` belongs to
5094    /// the module connection whose frame is being handled, or to the client that
5095    /// relay was opened for.
5096    ///
5097    /// This runs on the MODULE connection's frame handler, where returning `Err`
5098    /// ends that connection -- and a module connection carries every client's
5099    /// routes to that module, so ending it costs the whole fleet its tools.
5100    /// `ConnectionClosing` carries the id of the connection that is closing, and
5101    /// when that id is a CLIENT's, the condition is entirely about that one
5102    /// client's route.open. A client-scoped condition has no authority over a
5103    /// shared module connection, so it is logged and the single relay is dropped:
5104    /// the client is going away, and `complete_pending_relay` already removed the
5105    /// relay before failing, so there is nothing left to settle. Anything that
5106    /// relay still reserved is released by that client's own connection teardown,
5107    /// which is already under way -- that is what "closing" means.
5108    ///
5109    /// Every other failure is a statement about THIS connection and stays fatal:
5110    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
5111    /// id in `ConnectionClosing` all mean this connection cannot keep serving
5112    /// frames correctly.
5113    fn refuse_to_end_module_connection_for_a_client(
5114        &self,
5115        module_connection_id: ConnectionId,
5116        corr: u64,
5117        err: ForwardingError,
5118    ) -> Result<(), RouterError> {
5119        if let ForwardingError::ConnectionClosing { connection_id } = err {
5120            if connection_id != module_connection_id {
5121                warn!(
5122                    module_connection_id = module_connection_id.get(),
5123                    client_connection_id = connection_id.get(),
5124                    corr,
5125                    "dropping a route.bind response for a closing client; the module connection keeps serving"
5126                );
5127                return Ok(());
5128            }
5129        }
5130        Err(RouterError::Forwarding(err))
5131    }
5132
5133    fn handle_module_relay_response(
5134        &self,
5135        connection_id: ConnectionId,
5136        frame: Frame,
5137    ) -> Result<Vec<Frame>, RouterError> {
5138        let mut secondary_error = None;
5139        let outcome = match frame.header.ty {
5140            FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
5141                Ok(probe) if probe.op == "route.bind" => {
5142                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
5143                        Ok(ModuleControlResponse::RouteBindAck {}) => {
5144                            RouteBindRelayOutcome::Accepted
5145                        }
5146                        Ok(other) => {
5147                            let message =
5148                                format!("route.bind response carried unexpected body: {other:?}");
5149                            secondary_error = Some(control_error_frame(
5150                                &frame,
5151                                "invalid_control_body",
5152                                message.clone(),
5153                            )?);
5154                            RouteBindRelayOutcome::ModuleGone(message)
5155                        }
5156                        Err(err) => {
5157                            let message = format!("malformed route.bind response body: {err}");
5158                            secondary_error = Some(control_error_frame(
5159                                &frame,
5160                                "invalid_control_body",
5161                                message.clone(),
5162                            )?);
5163                            RouteBindRelayOutcome::ModuleGone(message)
5164                        }
5165                    }
5166                }
5167                Ok(probe) => {
5168                    let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
5169                    {
5170                        Ok(response) => ModuleControlRpcOutcome::Response(response),
5171                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5172                            "malformed {} response body: {err}",
5173                            probe.op
5174                        )),
5175                    };
5176                    let completion = self
5177                        .forwarding
5178                        .complete_module_control_rpc(
5179                            connection_id,
5180                            frame.header.corr,
5181                            Some(&probe.op),
5182                            outcome,
5183                        )
5184                        .map_err(RouterError::Forwarding)?;
5185                    if !self.observe_module_control_completion(completion) {
5186                        debug!(
5187                            connection_id = connection_id.get(),
5188                            corr = frame.header.corr,
5189                            op = %probe.op,
5190                            "dropping late or unknown module-control RPC response"
5191                        );
5192                    }
5193                    return Ok(Vec::new());
5194                }
5195                Err(err) => {
5196                    if let Some(expected_op) = self
5197                        .forwarding
5198                        .pending_module_control_op(connection_id, frame.header.corr)
5199                        .map_err(RouterError::Forwarding)?
5200                    {
5201                        let completion = self
5202                            .forwarding
5203                            .complete_module_control_rpc(
5204                                connection_id,
5205                                frame.header.corr,
5206                                None,
5207                                ModuleControlRpcOutcome::MalformedResponse(format!(
5208                                    "malformed {expected_op} response body: {err}"
5209                                )),
5210                            )
5211                            .map_err(RouterError::Forwarding)?;
5212                        if !self.observe_module_control_completion(completion) {
5213                            debug!(
5214                                connection_id = connection_id.get(),
5215                                corr = frame.header.corr,
5216                                "dropping late malformed module-control RPC response"
5217                            );
5218                        }
5219                        return Ok(Vec::new());
5220                    }
5221                    let message = format!("malformed route.bind response body: {err}");
5222                    secondary_error = Some(control_error_frame(
5223                        &frame,
5224                        "invalid_control_body",
5225                        message.clone(),
5226                    )?);
5227                    RouteBindRelayOutcome::ModuleGone(message)
5228                }
5229            },
5230            FrameType::Error => {
5231                if self
5232                    .forwarding
5233                    .pending_module_control_op(connection_id, frame.header.corr)
5234                    .map_err(RouterError::Forwarding)?
5235                    .is_some()
5236                {
5237                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
5238                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
5239                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5240                            "malformed module-control ERROR body: {err}"
5241                        )),
5242                    };
5243                    let completion = self
5244                        .forwarding
5245                        .complete_module_control_rpc(
5246                            connection_id,
5247                            frame.header.corr,
5248                            None,
5249                            outcome,
5250                        )
5251                        .map_err(RouterError::Forwarding)?;
5252                    if !self.observe_module_control_completion(completion) {
5253                        debug!(
5254                            connection_id = connection_id.get(),
5255                            corr = frame.header.corr,
5256                            "dropping late or unknown module-control RPC error"
5257                        );
5258                    }
5259                    return Ok(Vec::new());
5260                }
5261                match serde_json::from_slice::<ErrorBody>(&frame.body) {
5262                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
5263                    Err(err) => {
5264                        let message = format!("malformed route.bind ERROR body: {err}");
5265                        secondary_error = Some(control_error_frame(
5266                            &frame,
5267                            "invalid_control_body",
5268                            message.clone(),
5269                        )?);
5270                        RouteBindRelayOutcome::ModuleGone(message)
5271                    }
5272                }
5273            }
5274            ty => {
5275                return Ok(vec![control_error_frame(
5276                    &frame,
5277                    "unsupported_control_frame",
5278                    format!("unsupported module channel-0 frame {ty:?}"),
5279                )?])
5280            }
5281        };
5282
5283        let settled =
5284            self.forwarding
5285                .complete_pending_relay(connection_id, frame.header.corr, outcome);
5286        let completion = match settled {
5287            Ok(completion) => completion,
5288            Err(err) => {
5289                self.refuse_to_end_module_connection_for_a_client(
5290                    connection_id,
5291                    frame.header.corr,
5292                    err,
5293                )?;
5294                return Ok(secondary_error.into_iter().collect());
5295            }
5296        };
5297        if let Some(target) = completion.abandoned.as_ref() {
5298            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
5299        }
5300        if !completion.settled {
5301            debug!(
5302                connection_id = connection_id.get(),
5303                corr = frame.header.corr,
5304                frame_type = ?frame.header.ty,
5305                "dropping late or unknown route.bind relay response"
5306            );
5307        }
5308        Ok(secondary_error.into_iter().collect())
5309    }
5310
5311    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
5312        debug!(connection_id = connection_id.get(), "handling GOODBYE");
5313        let registrations = self
5314            .deregister_connection(connection_id)
5315            .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
5316        let released_routes = self
5317            .forwarding
5318            .cleanup_connection(connection_id)
5319            .map_err(RouterError::Forwarding)?;
5320        self.emit_route_goodbyes(released_routes);
5321        // Notify only after forwarding teardown completes (see cleanup_connection).
5322        if !registrations.is_empty() {
5323            crate::supervise::notify_registration_release();
5324        }
5325        Ok(Vec::new())
5326    }
5327}
5328
5329impl Default for ControlHandler {
5330    fn default() -> Self {
5331        Self::new(Arc::new(Registry::default()))
5332    }
5333}
5334
5335impl crate::supervise::SwapPromotionObserver for ControlHandler {
5336    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
5337        self.apply_registration_capabilities(registration);
5338    }
5339}
5340
5341fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
5342    CapabilityRequirementStatus {
5343        consumer: status.consumer,
5344        capability: status.capability,
5345        need: match status.need {
5346            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
5347            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
5348        },
5349        verdict: status.verdict.as_str().to_string(),
5350        episode_seq: status.episode_seq,
5351        config_satisfiable: status.config_satisfiable,
5352        runtime_available: status.runtime_available,
5353        detail: status.detail,
5354    }
5355}
5356
5357fn append_capability_problem_detail(
5358    detail: Option<String>,
5359    capability_detail: Option<String>,
5360) -> Option<String> {
5361    match (detail, capability_detail) {
5362        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
5363        (Some(detail), None) => Some(detail),
5364        (None, Some(capability_detail)) => Some(capability_detail),
5365        (None, None) => None,
5366    }
5367}
5368
5369fn subc_ops() -> Vec<String> {
5370    SUBC_CONTROL_OPS
5371        .iter()
5372        .map(|op| (*op).to_string())
5373        .collect()
5374}
5375
5376fn module_subc_ops() -> Vec<String> {
5377    SUBC_CONTROL_OPS
5378        .iter()
5379        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
5380        .map(|op| (*op).to_string())
5381        .collect()
5382}
5383
5384#[cfg(test)]
5385fn module_baseline_control_ops() -> Vec<String> {
5386    MODULE_BASELINE_CONTROL_OPS
5387        .iter()
5388        .map(|op| (*op).to_string())
5389        .collect()
5390}
5391
5392fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
5393    let mut seen = HashSet::new();
5394    let mut effective = Vec::new();
5395    for op in MODULE_BASELINE_CONTROL_OPS {
5396        if seen.insert((*op).to_string()) {
5397            effective.push((*op).to_string());
5398        }
5399    }
5400    for op in declared.unwrap_or_default() {
5401        if seen.insert(op.clone()) {
5402            effective.push(op);
5403        }
5404    }
5405    effective
5406}
5407
5408fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
5409    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
5410}
5411
5412fn target_module_id(target: &RouteTarget) -> &str {
5413    match target {
5414        RouteTarget::ToolProvider { module_id }
5415        | RouteTarget::ManagementSurface { module_id }
5416        | RouteTarget::InternalService { module_id, .. } => module_id,
5417    }
5418}
5419
5420fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
5421    roles.iter().any(|role| match (target, role) {
5422        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
5423        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
5424        (
5425            RouteTarget::InternalService { service_id, .. },
5426            ProviderRole::InternalService {
5427                service_id: provided,
5428                ..
5429            },
5430        ) => service_id == provided,
5431        _ => false,
5432    })
5433}
5434
5435fn is_routable_role(role: &ProviderRole) -> bool {
5436    matches!(
5437        role,
5438        ProviderRole::ToolProvider { .. }
5439            | ProviderRole::ManagementSurface { .. }
5440            | ProviderRole::InternalService { .. }
5441    )
5442}
5443
5444#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5445enum ControlRequestBodyError {
5446    UnknownOp,
5447    InvalidBody,
5448}
5449
5450#[derive(Debug, Deserialize)]
5451struct ControlOpProbe {
5452    op: String,
5453}
5454
5455/// Channel-0 push ops this subc version understands. A push whose `op` is not in
5456/// this set is treated as a forward-compat unknown and ignored rather than errored.
5457const MODULE_PUSH_OPS: &[&str] = &["route.status"];
5458
5459fn is_known_module_push_op(body: &[u8]) -> bool {
5460    serde_json::from_slice::<ControlOpProbe>(body)
5461        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
5462        .unwrap_or(false)
5463}
5464
5465fn is_known_module_request_op(body: &[u8]) -> bool {
5466    serde_json::from_slice::<ControlOpProbe>(body)
5467        .map(|probe| is_module_to_subc_op(&probe.op))
5468        .unwrap_or(false)
5469}
5470
5471fn is_module_to_subc_op(op: &str) -> bool {
5472    MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
5473}
5474
5475fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
5476    debug!(
5477        op = %op,
5478        connection_id = connection_id.get(),
5479        corr,
5480        "control dispatch"
5481    );
5482}
5483
5484fn log_slow_control_dispatch(
5485    dispatch_started_at: Option<StdInstant>,
5486    op: &'static str,
5487    connection_id: ConnectionId,
5488    corr: u64,
5489) {
5490    let Some(dispatch_started_at) = dispatch_started_at else {
5491        return;
5492    };
5493    let elapsed = dispatch_started_at.elapsed();
5494    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
5495        warn!(
5496            op = %op,
5497            connection_id = connection_id.get(),
5498            corr,
5499            elapsed_ms = elapsed.as_millis() as u64,
5500            "slow control dispatch"
5501        );
5502    }
5503}
5504
5505fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
5506    match request {
5507        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
5508        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
5509        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
5510        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
5511        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
5512        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
5513        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
5514        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
5515        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
5516        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
5517        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
5518        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
5519        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
5520        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
5521        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
5522        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
5523        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
5524        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
5525        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
5526    }
5527}
5528
5529fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
5530    match request {
5531        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
5532        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
5533        ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
5534        ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
5535    }
5536}
5537
5538fn parse_client_control_request(
5539    body: &[u8],
5540) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
5541    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
5542        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5543            Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5544                ControlRequestBodyError::InvalidBody
5545            }
5546            Ok(_) => ControlRequestBodyError::UnknownOp,
5547            Err(_) => ControlRequestBodyError::InvalidBody,
5548        };
5549        (err, classification)
5550    })
5551}
5552
5553fn parse_module_control_request_from_module(
5554    body: &[u8],
5555) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
5556    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
5557        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5558            Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
5559            Ok(_) => ControlRequestBodyError::UnknownOp,
5560            Err(_) => ControlRequestBodyError::InvalidBody,
5561        };
5562        (err, classification)
5563    })
5564}
5565
5566#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
5567enum ProviderRoleKind {
5568    ToolProvider,
5569    PipelineStage,
5570    ManagementSurface,
5571    InternalService,
5572}
5573
5574fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
5575    match role {
5576        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
5577        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
5578        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
5579        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
5580    }
5581}
5582
5583fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
5584    roles.iter().map(provider_role_kind).collect()
5585}
5586
5587/// Most refused scope records named individually in the log per sync; the
5588/// `refused` count on the accepted line is always complete.
5589const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
5590
5591/// Per-outcome counts of one accepted `scope.sync`, for its log line.
5592#[derive(Debug, Default, PartialEq, Eq)]
5593struct ScopeOutcomeCounts {
5594    created: usize,
5595    replaced: usize,
5596    updated: usize,
5597    unchanged: usize,
5598    refused: usize,
5599}
5600
5601impl ScopeOutcomeCounts {
5602    fn of(results: &[ScopeRecordResult]) -> Self {
5603        let mut counts = Self::default();
5604        for result in results {
5605            let slot = match result.outcome {
5606                ScopeRecordOutcome::Created => &mut counts.created,
5607                ScopeRecordOutcome::Replaced => &mut counts.replaced,
5608                ScopeRecordOutcome::Updated => &mut counts.updated,
5609                ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
5610                ScopeRecordOutcome::Refused => &mut counts.refused,
5611            };
5612            *slot += 1;
5613        }
5614        counts
5615    }
5616}
5617
5618#[cfg(test)]
5619mod scope_outcome_count_tests {
5620    use super::*;
5621
5622    fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
5623        ScopeRecordResult {
5624            scope_ref: "r".to_string(),
5625            scope_epoch: 1,
5626            outcome,
5627            code: None,
5628            message: None,
5629            version: None,
5630            parent_state: None,
5631        }
5632    }
5633
5634    /// Each outcome lands in its own count, so a refused record can never be
5635    /// hidden inside the total the log already printed.
5636    #[test]
5637    fn every_outcome_is_counted_in_its_own_field() {
5638        let results = [
5639            result(ScopeRecordOutcome::Created),
5640            result(ScopeRecordOutcome::Created),
5641            result(ScopeRecordOutcome::Replaced),
5642            result(ScopeRecordOutcome::Updated),
5643            result(ScopeRecordOutcome::Unchanged),
5644            result(ScopeRecordOutcome::Refused),
5645            result(ScopeRecordOutcome::Refused),
5646            result(ScopeRecordOutcome::Refused),
5647        ];
5648        assert_eq!(
5649            ScopeOutcomeCounts::of(&results),
5650            ScopeOutcomeCounts {
5651                created: 2,
5652                replaced: 1,
5653                updated: 1,
5654                unchanged: 1,
5655                refused: 3,
5656            }
5657        );
5658    }
5659}
5660
5661/// Return whether a catalog change can create a newly violating live route.
5662/// Removing an attested claim is intentionally excluded: it makes fewer routes
5663/// forbidden and therefore must leave the existing route census untouched.
5664fn capability_census_trigger(
5665    old: Option<&CapabilityDeclarations>,
5666    new: Option<&CapabilityDeclarations>,
5667) -> bool {
5668    let old_provides = old
5669        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
5670        .unwrap_or_default();
5671    let old_denies = old
5672        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
5673        .unwrap_or_default();
5674    let new = new.cloned().unwrap_or(CapabilityDeclarations {
5675        provides: Vec::new(),
5676        requires: Vec::new(),
5677        must_never_reach: Vec::new(),
5678    });
5679
5680    new.provides
5681        .iter()
5682        .any(|capability| !old_provides.contains(capability))
5683        || new
5684            .must_never_reach
5685            .iter()
5686            .any(|capability| !old_denies.contains(capability))
5687}
5688
5689/// Find the first capability an attested opener denies that an attested target
5690/// claims. Both manifests are live registry records, never cached or client data.
5691fn denied_capability<'a>(
5692    opening_manifest: &'a ModuleManifest,
5693    target_manifest: &ModuleManifest,
5694) -> Option<&'a str> {
5695    let opening_capabilities = opening_manifest.capabilities.as_ref()?;
5696    let target_capabilities = target_manifest.capabilities.as_ref()?;
5697    opening_capabilities
5698        .must_never_reach
5699        .iter()
5700        .find(|denied| {
5701            target_capabilities
5702                .provides
5703                .iter()
5704                .any(|provided| provided == *denied)
5705        })
5706        .map(String::as_str)
5707}
5708
5709fn catalog_update_frozen_field_message(
5710    registered: &ModuleManifest,
5711    provides: &[ProviderRole],
5712) -> Option<String> {
5713    let old_has_provides = !registered.provides.is_empty();
5714    let new_has_provides = !provides.is_empty();
5715    if old_has_provides != new_has_provides {
5716        return Some(format!(
5717            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
5718            registered.module_id
5719        ));
5720    }
5721
5722    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
5723        return Some(format!(
5724            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
5725            registered.module_id
5726        ));
5727    }
5728
5729    let registered_concurrency = manifest_concurrency(registered);
5730    let mut candidate = registered.clone();
5731    candidate.provides = provides.to_vec();
5732    let candidate_concurrency = manifest_concurrency(&candidate);
5733    if candidate_concurrency != registered_concurrency {
5734        return Some(format!(
5735            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
5736            registered.module_id, registered_concurrency, candidate_concurrency
5737        ));
5738    }
5739
5740    // control_ops live beside the manifest in the HELLO body, not inside
5741    // ModuleManifest, so a provides-only catalog.update cannot change them.
5742    None
5743}
5744
5745fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
5746    manifest.provides.iter().any(is_routable_role)
5747}
5748
5749/// Returns the routable-provider concurrency subc should enforce for this manifest.
5750///
5751/// ToolProvider and ManagementSurface store their delivery concurrency directly.
5752/// InternalService has no role-specific concurrency field, so it retains the
5753/// existing ModuleManaged default for backward compatibility.
5754fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
5755    manifest
5756        .provides
5757        .iter()
5758        .find_map(|provider| match provider {
5759            ProviderRole::ToolProvider { concurrency, .. }
5760            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
5761            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
5762        })
5763        .unwrap_or(Concurrency::ModuleManaged)
5764}
5765
5766/// True when the manifest carries a ManagementSurface role whose concurrency
5767/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
5768/// bytes because the typed manifest deliberately erases that distinction: the
5769/// default exists for wire compatibility, and this probe exists so the default
5770/// stays observable. Any parse irregularity returns false -- the caller only
5771/// logs, and a malformed body already failed registration upstream.
5772fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
5773    let has_management_surface = manifest
5774        .provides
5775        .iter()
5776        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
5777    if !has_management_surface {
5778        return false;
5779    }
5780    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
5781        return false;
5782    };
5783    let Some(provides) = raw
5784        .get("manifest")
5785        .and_then(|manifest| manifest.get("provides"))
5786        .and_then(serde_json::Value::as_array)
5787    else {
5788        return false;
5789    };
5790    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
5791    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
5792    // against the management_surface_manifest_without_concurrency golden, not
5793    // recalled (the externally-tagged guess was this function's first bug).
5794    provides.iter().any(|role| {
5795        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
5796            && role.get("concurrency").is_none()
5797    })
5798}
5799
5800fn negotiate_version(peer_version: u8) -> Result<u8, String> {
5801    if peer_version != PROTOCOL_VERSION {
5802        return Err(format!(
5803            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
5804        ));
5805    }
5806    Ok(PROTOCOL_VERSION)
5807}
5808
5809fn pong(frame: &Frame) -> Result<Frame, RouterError> {
5810    Frame::build_with_version(
5811        response_version(frame),
5812        FrameType::Pong,
5813        frame.header.flags,
5814        0,
5815        0,
5816        frame.header.corr,
5817        Vec::new(),
5818    )
5819    .map_err(RouterError::FrameBuild)
5820}
5821
5822fn control_error_frame(
5823    frame: &Frame,
5824    code: &'static str,
5825    message: impl Into<String>,
5826) -> Result<Frame, RouterError> {
5827    control_error_body_frame(
5828        frame,
5829        ErrorBody {
5830            code: code.to_string(),
5831            message: message.into(),
5832            detail: None,
5833        },
5834    )
5835}
5836
5837fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
5838    let body = serde_json::to_vec(&error).map_err(|err| {
5839        RouterError::backend(
5840            0,
5841            frame.header.corr,
5842            format!("failed to encode control ERROR: {err}"),
5843        )
5844    })?;
5845
5846    Frame::build_with_version(
5847        response_version(frame),
5848        FrameType::Error,
5849        control_flags(),
5850        0,
5851        0,
5852        frame.header.corr,
5853        body,
5854    )
5855    .map_err(RouterError::FrameBuild)
5856}
5857
5858fn control_response_body_frame<T: Serialize>(
5859    frame: &Frame,
5860    reply: &T,
5861    label: &'static str,
5862) -> Result<Frame, RouterError> {
5863    let body = serde_json::to_vec(reply).map_err(|err| {
5864        RouterError::backend(
5865            0,
5866            frame.header.corr,
5867            format!("failed to encode {label}: {err}"),
5868        )
5869    })?;
5870
5871    Frame::build_with_version(
5872        response_version(frame),
5873        FrameType::Response,
5874        control_flags(),
5875        0,
5876        0,
5877        frame.header.corr,
5878        body,
5879    )
5880    .map_err(RouterError::FrameBuild)
5881}
5882
5883/// Map a forwarding failure to the wire code a client sees.
5884///
5885/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
5886/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
5887/// chosen here decides whether a caller retries or gives up.
5888///
5889/// That makes attribution the load-bearing property, not merely having a code. A
5890/// permanent fault published as a retryable one produces a fleet-wide retry storm
5891/// against something that can never recover; a transient fault published as
5892/// permanent gives up on work that would have succeeded. Both look correct in a
5893/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
5894/// the mapping per variant rather than merely asserting that some code exists.
5895///
5896/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
5897/// two codes on the same side of the boundary passes it. Measured rather than
5898/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
5899/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
5900/// a test named for something else that happens to assert the string.
5901///
5902/// That accidental coverage is deliberately left alone rather than promoted to a
5903/// named test, because it guards a property this function does not promise.
5904/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
5905/// specific code within a class, so identity is free to change and only the
5906/// partition is a contract. Splitting it out would assert a guarantee nothing
5907/// depends on — and a suite that promises more than the code does is the harder
5908/// thing to correct later, because the next reader cannot tell which assertions
5909/// are load-bearing.
5910///
5911/// Pin identity here the moment a consumer branches on a specific code.
5912fn forwarding_error_code(err: &ForwardingError) -> &'static str {
5913    match err {
5914        ForwardingError::NoModuleConnection => "target_unavailable",
5915        ForwardingError::ModuleReloading { .. } => "module_reloading",
5916        ForwardingError::ClientRouteChannelExhausted { .. }
5917        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
5918        ForwardingError::StaleModuleEndpoint
5919        | ForwardingError::UnknownReservation { .. }
5920        | ForwardingError::ConnectionClosing { .. }
5921        | ForwardingError::ClientEgressClosed { .. }
5922        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
5923        // Only a swap candidate's registration can produce this, and it means
5924        // exactly what a second active HELLO for a live id means.
5925        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
5926        ForwardingError::RelayCorrelationExhausted
5927        | ForwardingError::RouteOpenBuild(_)
5928        | ForwardingError::Poisoned => "forwarding_error",
5929    }
5930}
5931
5932fn response_version(frame: &Frame) -> u8 {
5933    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
5934        frame.header.ver
5935    } else {
5936        PROTOCOL_VERSION
5937    }
5938}
5939
5940fn control_flags() -> Flags {
5941    Flags::new(false, Priority::Passive, false)
5942}
5943
5944/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
5945/// channel. The target is the module (a client never saw the route), so this
5946/// takes the module path: delivered late rather than dropped when the module's
5947/// queue is momentarily full, and never closing its connection.
5948fn send_goodbye_target_best_effort(
5949    counters: &DaemonCounters,
5950    target: &GoodbyeTarget,
5951    context: &'static str,
5952) {
5953    let Ok(frame) = Frame::build_with_version(
5954        target.negotiated_ver,
5955        FrameType::Goodbye,
5956        control_flags(),
5957        target.channel,
5958        target.epoch,
5959        0,
5960        Vec::new(),
5961    ) else {
5962        return;
5963    };
5964    crate::forwarding::send_module_route_goodbye(
5965        counters,
5966        &target.sink,
5967        frame,
5968        target.module_id.as_deref(),
5969        context,
5970    );
5971}
5972
5973pub(crate) fn send_route_control_pushes(
5974    forwarding: &ForwardingTable,
5975    routes: Vec<EndpointRoute>,
5976    push: ClientControlPush,
5977) {
5978    let body = match serde_json::to_vec(&push) {
5979        Ok(body) => body,
5980        Err(err) => {
5981            warn!(error = %err, "failed to serialize route lifecycle control PUSH");
5982            return;
5983        }
5984    };
5985    let mut targets = Vec::new();
5986    for route in routes {
5987        let target = route.goodbye_target;
5988        if let Some(existing) = targets
5989            .iter()
5990            .find(|existing: &&GoodbyeTarget| existing.connection_id == target.connection_id)
5991        {
5992            debug_assert_eq!(
5993                existing.negotiated_ver, target.negotiated_ver,
5994                "one connection cannot negotiate multiple frame versions"
5995            );
5996            continue;
5997        }
5998        targets.push(target);
5999    }
6000    for target in targets {
6001        let frame = match Frame::build_with_version(
6002            target.negotiated_ver,
6003            FrameType::Push,
6004            control_flags(),
6005            0,
6006            0,
6007            0,
6008            body.clone(),
6009        ) {
6010            Ok(frame) => frame,
6011            Err(err) => {
6012                warn!(
6013                    route_channel = target.channel,
6014                    error = %err,
6015                    "failed to build route lifecycle control PUSH frame"
6016                );
6017                continue;
6018            }
6019        };
6020        if let Err(err) = target.sink.try_send(frame) {
6021            if target.close_on_delivery_failure() {
6022                warn!(
6023                    target_connection_id = target.connection_id.get(),
6024                    route_channel = target.channel,
6025                    error = %err,
6026                    "route lifecycle control PUSH was not delivered to client; closing target connection"
6027                );
6028                let _ = forwarding.escalate_client_delivery_failure(
6029                    target.connection_id,
6030                    target.channel,
6031                    target.epoch,
6032                    CloseReason::new(
6033                        "route_lifecycle_push_delivery_failed",
6034                        format!(
6035                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
6036                            target.channel
6037                        ),
6038                    ),
6039                    crate::forwarding::UndeliveredFrame {
6040                        module_id: target.module_id.as_deref(),
6041                        sink: &target.sink,
6042                    },
6043                );
6044            }
6045        }
6046    }
6047}
6048
6049#[cfg(test)]
6050mod tests {
6051    use std::{
6052        collections::BTreeMap,
6053        fmt,
6054        path::PathBuf,
6055        sync::{Arc, Mutex},
6056        time::Duration,
6057    };
6058    use subc_test_support::TestTempDir;
6059
6060    use serde_json::{json, Value};
6061    use subc_protocol::{
6062        manifest::{
6063            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
6064            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
6065        },
6066        session::HealthStatus,
6067        FrameType,
6068    };
6069
6070    use super::*;
6071    use crate::{
6072        forwarding::{DataRoute, DataRouteState},
6073        registry::ChannelState,
6074        router::FrameSink,
6075        stderr_tail::DEFAULT_MAX_LINE_BYTES,
6076        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
6077        RouteCtx, Router,
6078    };
6079    use tokio::{
6080        sync::mpsc,
6081        time::{sleep, Instant},
6082    };
6083    use tracing::{
6084        field::{Field, Visit},
6085        Event, Subscriber,
6086    };
6087    use tracing_subscriber::{layer::Context, prelude::*, Layer};
6088
6089    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
6090    ///
6091    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
6092    /// is only populated for `tests/*.rs` integration test binaries -- this file
6093    /// compiles as part of the library target, which gets neither. This test's
6094    /// own executable path is `<target-dir>/<profile>/deps/subc_core-<hash>`,
6095    /// and the sibling binary lives two directories up at
6096    /// `<target-dir>/<profile>/fake-aft-stub`.
6097    ///
6098    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
6099    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
6100    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
6101    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
6102    /// `NotFound`, which reads as a broken test rather than an unbuilt
6103    /// dependency -- so state the cause and the remedy instead. Deliberately a
6104    /// panic and not a silent skip: a test that quietly passes when it could not
6105    /// run is worse than one that fails, because it reports health it never
6106    /// verified.
6107    fn fake_aft_stub_path() -> PathBuf {
6108        let mut path = std::env::current_exe().expect("current_exe available in tests");
6109        path.pop(); // .../deps/
6110        path.pop(); // .../<profile>/
6111        path.push(if cfg!(windows) {
6112            "fake-aft-stub.exe"
6113        } else {
6114            "fake-aft-stub"
6115        });
6116        assert!(
6117            path.exists(),
6118            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
6119             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
6120            path.display()
6121        );
6122        path
6123    }
6124
6125    /// Whether clients retry `code` in place: the predicate itself, never a copy
6126    /// of its set. A copied list breaks silently when a code is added to or
6127    /// removed from the real one, and a stale copy here would let exactly the
6128    /// failure this test exists to catch pass.
6129    fn client_retries(code: &str) -> bool {
6130        subc_protocol::error_codes::is_retryable_route_open(code)
6131    }
6132
6133    /// A code is not a label — clients branch on it, so publishing the wrong KIND
6134    /// of failure is worse than publishing none. A permanent fault dressed as
6135    /// retryable makes every client in the fleet retry forever against something
6136    /// that cannot recover; a transient fault dressed as permanent abandons work
6137    /// that would have succeeded.
6138    ///
6139    /// Asserting "a code exists" cannot catch either, because the string is free
6140    /// to say anything. This enumerates every variant and pins which side of the
6141    /// retry boundary it lands on, so a new variant must be classified here
6142    /// deliberately rather than inheriting whichever arm it was appended to.
6143    #[test]
6144    fn retryability_of_forwarding_codes_matches_the_failure() {
6145        // Transient by nature: the target is booting, reloading, or its endpoint
6146        // was swapped mid-flight. Retrying is how these resolve.
6147        let transient = [
6148            ForwardingError::NoModuleConnection,
6149            ForwardingError::ModuleReloading {
6150                module_id: "m".into(),
6151            },
6152            ForwardingError::StaleModuleEndpoint,
6153            ForwardingError::UnknownReservation {
6154                client_channel: 1,
6155                module_channel: 1,
6156            },
6157            ForwardingError::ConnectionClosing {
6158                connection_id: ConnectionId::new(1),
6159            },
6160            ForwardingError::ClientEgressClosed {
6161                connection_id: ConnectionId::new(1),
6162            },
6163            ForwardingError::ModuleEgressUnavailable {
6164                connection_id: ConnectionId::new(1),
6165            },
6166        ];
6167        for err in transient {
6168            let code = forwarding_error_code(&err);
6169            assert!(
6170                client_retries(code),
6171                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
6172            );
6173        }
6174
6175        // Not fixed by retrying. Channel and correlation exhaustion need the
6176        // caller to close routes, and a poisoned lock is a daemon that cannot
6177        // recover at all — the worst thing to advertise as retryable, since every
6178        // client would storm a daemon that will never answer.
6179        let permanent = [
6180            ForwardingError::ClientRouteChannelExhausted {
6181                connection_id: ConnectionId::new(1),
6182            },
6183            ForwardingError::ModuleRouteChannelExhausted {
6184                endpoint: ModuleEndpointId {
6185                    connection_id: ConnectionId::new(1),
6186                    generation: 1,
6187                },
6188            },
6189            ForwardingError::RelayCorrelationExhausted,
6190            ForwardingError::RouteOpenBuild("x".into()),
6191            ForwardingError::Poisoned,
6192        ];
6193        for err in permanent {
6194            let code = forwarding_error_code(&err);
6195            assert!(
6196                !client_retries(code),
6197                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
6198            );
6199        }
6200    }
6201
6202    /// The principal is the daemon's answer to "who is calling", and modules
6203    /// branch on it: aft gates bash on it, cerebellum gates browser control,
6204    /// plexus gates connector invocation. So a stamp is an authorization input in
6205    /// another process, not a label — and both possible answers SUCCEED, which is
6206    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
6207    /// first-party capability to something that never proved it; a supervised one
6208    /// stamped `Direct` silently strips a module of capability it is entitled to.
6209    ///
6210    /// Neither shows up in a test that only checks the bind succeeded. Before this
6211    /// test the only coverage was accidental —
6212    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
6213    /// stamped principal on its way past, so narrowing that wire-shape test to its
6214    /// stated subject would have deleted the last assertion on this value. It
6215    /// still asserts the stamp, which is now redundancy rather than the only
6216    /// guard: both fail under the same mutation, and this one names the reason.
6217    /// SCOPE: this handler's supervisor has spawned nothing, so
6218    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
6219    /// is unreachable here. Both assertions below are refusals, and a mutant that
6220    /// refuses everything would satisfy them.
6221    ///
6222    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
6223    /// spawns a supervised consumer, reads its live nonce, and asserts the module
6224    /// observed `principal.kind == "reserved"` carrying that module_id — verified
6225    /// at source rather than assumed, since a citation is a claim about another
6226    /// file and ages like one. Recorded because a harness that structurally
6227    /// cannot reach an arm reports "none" for that arm identically to one that
6228    /// covers it and found nothing.
6229    #[tokio::test]
6230    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
6231        let handler = ControlHandler::default();
6232        let frame =
6233            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
6234
6235        // Absent consumer_identity is the ordinary case: a human at a terminal, or
6236        // any process holding the connection file. Nothing was proved, so nothing
6237        // may be granted beyond the unattested floor.
6238        let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
6239        assert_eq!(
6240            stamped,
6241            Principal::Direct,
6242            "a caller that proved nothing must not be stamped as a supervised module"
6243        );
6244
6245        // A claimed module_id with a nonce no supervised child was given is a
6246        // forgery attempt, not a weaker caller: it must be REFUSED rather than
6247        // quietly demoted to Direct, or an impersonation attempt looks identical
6248        // to an ordinary unattested connection.
6249        let forged = handler
6250            .route_open_principal(
6251                &frame,
6252                Some(ConsumerIdentity {
6253                    module_id: "aft".to_string(),
6254                    launch_nonce: "not-a-real-nonce".to_string(),
6255                }),
6256            )
6257            .unwrap();
6258        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
6259        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
6260    }
6261
6262    /// The test above hands `route_open_principal` an identity it built itself,
6263    /// which proves the stamping rule and nothing about where the identity comes
6264    /// from. The real producer is a wire body, and the two are joined by a serde
6265    /// field name that nothing else asserts.
6266    ///
6267    /// That join fails quietly in one specific way: an unrecognised key is simply
6268    /// absent after parsing, so a renamed or misspelled `consumer_identity`
6269    /// yields `None` and every supervised module silently drops to `Direct`.
6270    /// Capability-wise that is the safe direction, but it surfaces far from its
6271    /// cause — as a module mysteriously refused bash — and it would pass every
6272    /// test that builds its own input.
6273    ///
6274    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
6275    /// would break every client the moment the daemon gains a field, trading a
6276    /// quiet demotion for a hard refusal on additive change. Asserting the join
6277    /// instead means a rename breaks a test here rather than the fleet.
6278    #[test]
6279    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
6280        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
6281        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
6282        let ClientControlRequest::RouteOpen {
6283            consumer_identity, ..
6284        } = parsed
6285        else {
6286            panic!("route.open body must parse as RouteOpen");
6287        };
6288        assert_eq!(
6289            consumer_identity,
6290            Some(ConsumerIdentity {
6291                module_id: "aft".to_string(),
6292                launch_nonce: "n".to_string(),
6293            }),
6294            "the wire field name must reach the value route_open_principal reads"
6295        );
6296    }
6297
6298    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
6299        ModuleManifest::builder(module_id, "0.1.0")
6300            .protocol_ver(protocol_ver)
6301            .provides(vec![ProviderRole::ToolProvider {
6302                tools: vec![Tool {
6303                    name: "read".to_string(),
6304                    description: None,
6305                    execution_mode: ExecutionMode::Pure,
6306                    schema: json!({"type": "object"}),
6307                }],
6308                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
6309                concurrency: Concurrency::ModuleManaged,
6310                emits_push: true,
6311                sub_supervises: true,
6312            }])
6313            .build()
6314    }
6315
6316    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
6317        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
6318    }
6319
6320    fn hello_frame_with_control_ops(
6321        module_id: &str,
6322        protocol_ver: u8,
6323        corr: u64,
6324        control_ops: Option<Vec<String>>,
6325    ) -> Frame {
6326        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
6327    }
6328
6329    fn hello_frame_with_nonce(
6330        module_id: &str,
6331        protocol_ver: u8,
6332        corr: u64,
6333        launch_nonce: Option<&str>,
6334    ) -> Frame {
6335        hello_frame_full(
6336            module_id,
6337            protocol_ver,
6338            corr,
6339            None,
6340            launch_nonce.map(ToOwned::to_owned),
6341        )
6342    }
6343
6344    fn hello_frame_full(
6345        module_id: &str,
6346        protocol_ver: u8,
6347        corr: u64,
6348        control_ops: Option<Vec<String>>,
6349        launch_nonce: Option<String>,
6350    ) -> Frame {
6351        let body = serde_json::to_vec(&ModuleHelloBody {
6352            manifest: manifest(module_id, protocol_ver),
6353            protocol_ver,
6354            control_ops,
6355            launch_nonce,
6356        })
6357        .unwrap();
6358        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6359    }
6360
6361    fn non_routable_hello_frame_with_control_ops(
6362        module_id: &str,
6363        corr: u64,
6364        control_ops: Option<Vec<String>>,
6365    ) -> Frame {
6366        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
6367        manifest.provides.clear();
6368        let body = serde_json::to_vec(&ModuleHelloBody {
6369            manifest,
6370            protocol_ver: PROTOCOL_VERSION,
6371            control_ops,
6372            launch_nonce: None,
6373        })
6374        .unwrap();
6375        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6376    }
6377
6378    fn capability_grammar_hello_frame(
6379        capabilities: Value,
6380        runtime_computed: Option<Value>,
6381        corr: u64,
6382    ) -> Frame {
6383        let mut body = serde_json::to_value(ModuleHelloBody {
6384            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
6385            protocol_ver: PROTOCOL_VERSION,
6386            control_ops: None,
6387            launch_nonce: None,
6388        })
6389        .expect("HELLO body serializes");
6390        body["manifest"]["capabilities"] = capabilities;
6391        if let Some(runtime_computed) = runtime_computed {
6392            body["runtime_computed"] = runtime_computed;
6393        }
6394        Frame::build(
6395            FrameType::Hello,
6396            control_flags(),
6397            0,
6398            0,
6399            corr,
6400            serde_json::to_vec(&body).expect("HELLO body reserializes"),
6401        )
6402        .expect("HELLO frame builds")
6403    }
6404
6405    fn channel_request(channel: u16, corr: u64) -> Frame {
6406        Frame::build(
6407            FrameType::Request,
6408            Flags::new(true, Priority::Interactive, false),
6409            channel,
6410            0,
6411            corr,
6412            b"opaque".to_vec(),
6413        )
6414        .unwrap()
6415    }
6416
6417    fn route_ctx(
6418        connection_id: ConnectionId,
6419    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
6420        let (tx, rx) = mpsc::channel(8);
6421        (
6422            RouteCtx {
6423                connection_id,
6424                egress: FrameSink::new(tx),
6425            },
6426            rx,
6427        )
6428    }
6429
6430    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
6431        serde_json::from_slice(&frame.body).unwrap()
6432    }
6433
6434    /// Register a module over a connection that has a sink and return the
6435    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
6436    /// module's own sink rather than returning it as a reply, so the ack is
6437    /// taken off `rx` here and whatever the test reads next is what followed it.
6438    async fn hello_via_sink(
6439        handler: &ControlHandler,
6440        ctx: &RouteCtx,
6441        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
6442        hello: Frame,
6443    ) -> Frame {
6444        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
6445        assert!(
6446            replies.is_empty(),
6447            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
6448        );
6449        let ack = rx
6450            .try_recv()
6451            .expect("HELLO_ACK is queued on the module sink")
6452            .frame;
6453        assert_eq!(ack.header.ty, FrameType::HelloAck);
6454        ack
6455    }
6456
6457    fn parse_error(frame: &Frame) -> Value {
6458        serde_json::from_slice(&frame.body).unwrap()
6459    }
6460
6461    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
6462        serde_json::from_slice(&frame.body).unwrap()
6463    }
6464
6465    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
6466        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
6467            route_channel,
6468            route_epoch: 0,
6469            kind,
6470        })
6471        .unwrap();
6472        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6473    }
6474
6475    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
6476        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
6477            module_id: module_id.to_string(),
6478        })
6479        .unwrap();
6480        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6481    }
6482
6483    fn route_open_frame(corr: u64, module_id: &str, project_root: TestTempDir) -> Frame {
6484        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
6485    }
6486
6487    fn route_open_frame_with_consumer_capabilities(
6488        corr: u64,
6489        module_id: &str,
6490        project_root: TestTempDir,
6491        consumer_capabilities: Option<Vec<String>>,
6492    ) -> Frame {
6493        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6494            target: RouteTarget::ToolProvider {
6495                module_id: module_id.to_string(),
6496            },
6497            identity: BindIdentity::new(
6498                project_root.path().to_path_buf(),
6499                "unit".to_string(),
6500                "session".to_string(),
6501            ),
6502            consumer_identity: None,
6503            consumer_capabilities,
6504            role_versions: None,
6505            admission_facts: None,
6506            scope: None,
6507        })
6508        .unwrap();
6509        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6510    }
6511
6512    fn route_open_frame_with_role_versions(
6513        corr: u64,
6514        module_id: &str,
6515        project_root: TestTempDir,
6516        role_versions: Option<BTreeMap<String, String>>,
6517    ) -> Frame {
6518        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6519            target: RouteTarget::ToolProvider {
6520                module_id: module_id.to_string(),
6521            },
6522            identity: BindIdentity::new(
6523                project_root.path().to_path_buf(),
6524                "unit".to_string(),
6525                format!("session-{corr}"),
6526            ),
6527            consumer_identity: None,
6528            consumer_capabilities: None,
6529            role_versions,
6530            admission_facts: None,
6531            scope: None,
6532        })
6533        .unwrap();
6534        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6535    }
6536
6537    fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
6538        entries
6539            .iter()
6540            .map(|(role, version)| (role.to_string(), version.to_string()))
6541            .collect()
6542    }
6543
6544    fn route_open_frame_with_admission_facts(
6545        corr: u64,
6546        module_id: &str,
6547        project_root: TestTempDir,
6548        consumer_identity: Option<subc_control::ConsumerIdentity>,
6549        facts: Option<Value>,
6550    ) -> Frame {
6551        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6552            target: RouteTarget::ToolProvider {
6553                module_id: module_id.to_string(),
6554            },
6555            identity: BindIdentity::new(
6556                project_root.path().to_path_buf(),
6557                "unit".to_string(),
6558                format!("session-{corr}"),
6559            ),
6560            consumer_identity,
6561            consumer_capabilities: None,
6562            role_versions: None,
6563            admission_facts: facts,
6564            scope: None,
6565        })
6566        .unwrap();
6567        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6568    }
6569
6570    #[derive(Clone, Default)]
6571    struct EventCapture {
6572        events: Arc<Mutex<Vec<CapturedEvent>>>,
6573    }
6574
6575    #[derive(Clone, Debug)]
6576    struct CapturedEvent {
6577        target: String,
6578        level: tracing::Level,
6579        fields: BTreeMap<String, String>,
6580    }
6581
6582    impl EventCapture {
6583        fn events(&self) -> Vec<CapturedEvent> {
6584            self.events.lock().unwrap().clone()
6585        }
6586    }
6587
6588    impl<S> Layer<S> for EventCapture
6589    where
6590        S: Subscriber,
6591    {
6592        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
6593            let mut visitor = EventFieldVisitor::default();
6594            event.record(&mut visitor);
6595            self.events.lock().unwrap().push(CapturedEvent {
6596                target: event.metadata().target().to_string(),
6597                level: *event.metadata().level(),
6598                fields: visitor.fields,
6599            });
6600        }
6601    }
6602
6603    #[derive(Default)]
6604    struct EventFieldVisitor {
6605        fields: BTreeMap<String, String>,
6606    }
6607
6608    impl Visit for EventFieldVisitor {
6609        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
6610            self.fields
6611                .insert(field.name().to_string(), format!("{value:?}"));
6612        }
6613    }
6614
6615    fn health_response(corr: u64, status: HealthStatus) -> Frame {
6616        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
6617            status,
6618            detail: Some("warming".to_string()),
6619            metrics: Some(json!({"queue_depth": 3})),
6620        })
6621        .unwrap();
6622        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6623    }
6624
6625    fn route_bind_ack(corr: u64) -> Frame {
6626        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
6627        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6628    }
6629
6630    fn unique_project_root(label: &str) -> TestTempDir {
6631        TestTempDir::new(label)
6632    }
6633
6634    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
6635        match parse_route_poll(frame) {
6636            ClientControlResponse::RoutePoll {
6637                status: None,
6638                live: Some(live),
6639                ..
6640            } => assert_eq!(live, expected_live),
6641            other => panic!("unexpected route.poll response: {other:?}"),
6642        }
6643    }
6644
6645    fn bind_liveness_route(
6646        registry: &Registry,
6647        forwarding: &ForwardingTable,
6648        module_id: &str,
6649    ) -> (RouteCtx, u16, u32) {
6650        let module_connection = ConnectionId::new(101);
6651        let client_connection = ConnectionId::new(202);
6652        let registration = registry
6653            .register_with_control_ops(
6654                manifest(module_id, PROTOCOL_VERSION),
6655                PROTOCOL_VERSION,
6656                module_connection,
6657                module_baseline_control_ops(),
6658            )
6659            .unwrap();
6660        let (module_tx, _module_rx) = mpsc::channel(8);
6661        let endpoint = forwarding
6662            .register_module_connection(
6663                module_connection,
6664                module_id.to_string(),
6665                PROTOCOL_VERSION,
6666                manifest_concurrency(&registration.manifest),
6667                FrameSink::new(module_tx),
6668            )
6669            .unwrap();
6670        let (client_ctx, _client_rx) = route_ctx(client_connection);
6671        let pending = forwarding
6672            .begin_route_bind_relay_for_test(
6673                client_connection,
6674                client_ctx.egress.clone(),
6675                1,
6676                module_id,
6677            )
6678            .unwrap();
6679        assert_eq!(pending.endpoint, endpoint);
6680        let route_channel = pending.client_channel;
6681        let route_epoch = pending.client_epoch;
6682        forwarding
6683            .complete_pending_relay(
6684                module_connection,
6685                pending.corr,
6686                RouteBindRelayOutcome::Accepted,
6687            )
6688            .unwrap();
6689        (client_ctx, route_channel, route_epoch)
6690    }
6691
6692    struct FakeProcessLiveness {
6693        live: Option<bool>,
6694    }
6695
6696    impl ModuleProcessLiveness for FakeProcessLiveness {
6697        fn process_live(&self, _module_id: &str) -> Option<bool> {
6698            self.live
6699        }
6700    }
6701
6702    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6703    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
6704    {
6705        let registry = Arc::new(Registry::default());
6706        let supervisor_handle = SupervisorHandle::new();
6707        let supervisor = Supervisor::new(
6708            Arc::clone(&registry),
6709            RestartPolicy::new(1, Duration::from_millis(10)),
6710        )
6711        .with_handle(supervisor_handle.clone());
6712        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
6713        let module = supervisor
6714            .spawn(ModuleSpec {
6715                module_id: "stderr-tail-wire".to_string(),
6716                program: fake_aft_stub_path(),
6717                args: Vec::new(),
6718                env: vec![
6719                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
6720                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
6721                ],
6722                reserved: false,
6723                reserved_prefixes: Vec::new(),
6724                protocol: ModuleProtocol::Subc,
6725                overlap: Default::default(),
6726            })
6727            .unwrap();
6728
6729        let deadline = Instant::now() + Duration::from_secs(5);
6730        loop {
6731            let tail = module.stderr_tail(None, None);
6732            if tail
6733                .entries
6734                .iter()
6735                .any(|entry| matches!(entry, TailEntry::ProcessStart))
6736                && tail.entries.iter().any(|entry| {
6737                    matches!(
6738                        entry,
6739                        TailEntry::Line {
6740                            truncated: true,
6741                            ..
6742                        }
6743                    )
6744                })
6745            {
6746                break;
6747            }
6748            assert!(
6749                Instant::now() < deadline,
6750                "module did not produce a truncated line and restart boundary: {tail:?}"
6751            );
6752            sleep(Duration::from_millis(10)).await;
6753        }
6754
6755        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
6756        let request = ClientControlRequest::SupervisorStderrTail {
6757            module_id: "stderr-tail-wire".to_string(),
6758            max_lines: None,
6759            max_bytes: None,
6760        };
6761        let frame = Frame::build(
6762            FrameType::Request,
6763            control_flags(),
6764            0,
6765            0,
6766            1,
6767            serde_json::to_vec(&request).unwrap(),
6768        )
6769        .unwrap();
6770        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6771        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6772        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
6773            serde_json::from_slice(&responses[0].body).unwrap()
6774        else {
6775            panic!("expected supervisor.stderr_tail response");
6776        };
6777
6778        assert!(
6779            tail.entries
6780                .iter()
6781                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
6782            "the control response lost the restart boundary"
6783        );
6784        let Some(StderrTailEntry::Line {
6785            text,
6786            truncated,
6787            at_ms,
6788        }) = tail.entries.iter().find(|entry| {
6789            matches!(
6790                entry,
6791                StderrTailEntry::Line {
6792                    truncated: true,
6793                    ..
6794                }
6795            )
6796        })
6797        else {
6798            panic!("the control response lost the truncated line");
6799        };
6800        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
6801        assert!(*truncated);
6802        assert!(
6803            at_ms.is_some(),
6804            "the control response lost the line's capture time"
6805        );
6806    }
6807
6808    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
6809    /// read done on the worker thread would stall every other task until it
6810    /// finished; the read must run off the worker so this test's own task keeps
6811    /// running while the read is paused.
6812    #[tokio::test(flavor = "current_thread")]
6813    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
6814        let dir = TestTempDir::new("terminals-off-worker");
6815        let journal_path = dir.join("terminals.jsonl");
6816        let registry = Arc::new(Registry::default());
6817        let supervisor_handle = SupervisorHandle::new();
6818        let supervisor =
6819            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
6820                .with_handle(supervisor_handle.clone())
6821                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
6822        let module = supervisor
6823            .spawn(ModuleSpec {
6824                module_id: "terminal-off-worker".to_string(),
6825                program: fake_aft_stub_path(),
6826                args: Vec::new(),
6827                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6828                reserved: false,
6829                reserved_prefixes: Vec::new(),
6830                protocol: ModuleProtocol::Subc,
6831                overlap: Default::default(),
6832            })
6833            .unwrap();
6834        let deadline = Instant::now() + Duration::from_secs(5);
6835        while module.terminal_history().entries.len() != 2 {
6836            assert!(Instant::now() < deadline, "module did not record two exits");
6837            sleep(Duration::from_millis(10)).await;
6838        }
6839
6840        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
6841        let handler =
6842            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
6843        let frame = Frame::build(
6844            FrameType::Request,
6845            control_flags(),
6846            0,
6847            0,
6848            1,
6849            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
6850                module_id: "terminal-off-worker".to_string(),
6851            })
6852            .unwrap(),
6853        )
6854        .unwrap();
6855        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6856        let spawned_at = std::time::Instant::now();
6857        let read = tokio::spawn({
6858            let handler = Arc::clone(&handler);
6859            async move { handler.handle_control_frame(&ctx, frame).await }
6860        });
6861        // Waiting for the pause from a blocking thread keeps this task pending,
6862        // so the runtime's single worker is free to run the read task.
6863        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
6864            .await
6865            .unwrap()
6866            .expect("the history read reached its pause");
6867        let elapsed = spawned_at.elapsed();
6868        assert!(
6869            elapsed < Duration::from_secs(2) && !read.is_finished(),
6870            "this task could not run while the history read was paused \
6871             (resumed after {elapsed:?}, read finished: {})",
6872            read.is_finished()
6873        );
6874
6875        drop(release);
6876        let responses = read.await.unwrap().unwrap();
6877        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6878        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
6879            panic!("expected supervisor.terminals response");
6880        };
6881        assert_eq!(terminals.entries.len(), 2);
6882        assert_eq!(terminals.journal_skipped_lines, 0);
6883        assert_eq!(terminals.journal_read_errors, 0);
6884    }
6885
6886    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6887    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
6888        let registry = Arc::new(Registry::default());
6889        let supervisor_handle = SupervisorHandle::new();
6890        let supervisor =
6891            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
6892                .with_handle(supervisor_handle.clone());
6893        let module = supervisor
6894            .spawn(ModuleSpec {
6895                module_id: "terminal-golden".to_string(),
6896                program: fake_aft_stub_path(),
6897                args: Vec::new(),
6898                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6899                reserved: false,
6900                reserved_prefixes: Vec::new(),
6901                protocol: ModuleProtocol::Subc,
6902                overlap: Default::default(),
6903            })
6904            .unwrap();
6905
6906        let deadline = Instant::now() + Duration::from_secs(5);
6907        while module.terminal_history().entries.len() != 2 {
6908            assert!(
6909                Instant::now() < deadline,
6910                "module did not retain two terminal exits: {:?}",
6911                module.terminal_history()
6912            );
6913            sleep(Duration::from_millis(10)).await;
6914        }
6915
6916        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
6917        let request = ClientControlRequest::SupervisorTerminals {
6918            module_id: "terminal-golden".to_string(),
6919        };
6920        let frame = Frame::build(
6921            FrameType::Request,
6922            control_flags(),
6923            0,
6924            0,
6925            1,
6926            serde_json::to_vec(&request).unwrap(),
6927        )
6928        .unwrap();
6929        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6930        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6931        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6932        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
6933            panic!("expected supervisor.terminals response");
6934        };
6935        assert_eq!(terminals.entries.len(), 2);
6936        assert_eq!(terminals.dropped, 0);
6937
6938        let mut rendered = serde_json::to_value(response).unwrap();
6939        // Wall-clock fields are the observation contract, but not stable fixture
6940        // bytes; normalize only them after the real handler has shaped the response.
6941        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
6942        for (index, entry) in rendered["entries"]
6943            .as_array_mut()
6944            .expect("terminal response entries array")
6945            .iter_mut()
6946            .enumerate()
6947        {
6948            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
6949        }
6950
6951        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
6952            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
6953        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
6954        if std::env::var_os("UPDATE_GOLDEN").is_some() {
6955            std::fs::write(&golden_path, &serialized).unwrap();
6956        }
6957        let expected: Value =
6958            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
6959        assert_eq!(rendered, expected);
6960    }
6961
6962    #[test]
6963    fn hello_registers_manifest_and_returns_ack() {
6964        let registry = Arc::new(Registry::default());
6965        let handler = ControlHandler::new(Arc::clone(&registry));
6966        let conn = ConnectionId::new(1);
6967
6968        let responses = handler
6969            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
6970            .unwrap();
6971
6972        assert_eq!(responses.len(), 1);
6973        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6974        assert_eq!(responses[0].header.channel, 0);
6975        assert_eq!(responses[0].header.corr, 7);
6976        let ack = parse_ack(&responses[0]);
6977        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
6978        assert!(ack
6979            .subc_capabilities
6980            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
6981        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
6982        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
6983        assert!(ack
6984            .subc_ops
6985            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
6986        assert!(ack
6987            .subc_ops
6988            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
6989
6990        let registration = registry.get_module("aft").unwrap().unwrap();
6991        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
6992        assert_eq!(registration.state, ChannelState::Active);
6993        assert_eq!(registration.connection_id, conn);
6994        assert_eq!(registration.control_ops, module_baseline_control_ops());
6995    }
6996
6997    #[test]
6998    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
6999        let invalid_identifiers = [
7000            ("case_change", "credentials-Provider/v1"),
7001            ("leading_zero", "credentials-provider/v01"),
7002            ("trailing_hyphen", "credentials-provider-/v1"),
7003            ("consecutive_hyphens", "credentials--provider/v1"),
7004            ("uppercase", "Credentials-provider/v1"),
7005            ("missing_v", "credentials-provider/1"),
7006            ("whitespace", "credentials provider/v1"),
7007            ("zero_version", "credentials-provider/v0"),
7008            ("out_of_range_version", "credentials-provider/v4294967296"),
7009            (
7010                "overlength_name",
7011                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
7012            ),
7013        ];
7014        let mut cases = invalid_identifiers
7015            .into_iter()
7016            .map(|(name, identifier)| {
7017                (
7018                    format!("identifier_{name}"),
7019                    "capabilities.provides[0]".to_string(),
7020                    identifier.to_string(),
7021                    json!({ "provides": [identifier] }),
7022                    None,
7023                )
7024            })
7025            .collect::<Vec<_>>();
7026        cases.extend([
7027            (
7028                "unknown_need".to_string(),
7029                "capabilities.requires[0].need".to_string(),
7030                "deferred".to_string(),
7031                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
7032                None,
7033            ),
7034            (
7035                "duplicate_provides".to_string(),
7036                "capabilities.provides[1]".to_string(),
7037                "credentials-provider/v1".to_string(),
7038                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
7039                None,
7040            ),
7041            (
7042                "duplicate_must_never_reach".to_string(),
7043                "capabilities.must_never_reach[1]".to_string(),
7044                "credentials-provider/v1".to_string(),
7045                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
7046                None,
7047            ),
7048            (
7049                "duplicate_requires_same_need".to_string(),
7050                "capabilities.requires[1]".to_string(),
7051                "credentials-provider/v1".to_string(),
7052                json!({ "requires": [
7053                    { "capability": "credentials-provider/v1", "need": "required" },
7054                    { "capability": "credentials-provider/v1", "need": "required" }
7055                ] }),
7056                None,
7057            ),
7058            (
7059                "duplicate_requires_conflicting_need".to_string(),
7060                "capabilities.requires[1]".to_string(),
7061                "credentials-provider/v1".to_string(),
7062                json!({ "requires": [
7063                    { "capability": "credentials-provider/v1", "need": "required" },
7064                    { "capability": "credentials-provider/v1", "need": "optional" }
7065                ] }),
7066                None,
7067            ),
7068            (
7069                "capabilities_root_pointer".to_string(),
7070                "runtime_computed[0]".to_string(),
7071                "/capabilities".to_string(),
7072                json!({}),
7073                Some(json!(["/capabilities"])),
7074            ),
7075            (
7076                "capabilities_descendant_pointer".to_string(),
7077                "runtime_computed[0]".to_string(),
7078                "/capabilities/provides".to_string(),
7079                json!({}),
7080                Some(json!(["/capabilities/provides"])),
7081            ),
7082            (
7083                "malformed_pointer_without_leading_slash".to_string(),
7084                "runtime_computed[0]".to_string(),
7085                "capabilities".to_string(),
7086                json!({}),
7087                Some(json!(["capabilities"])),
7088            ),
7089            (
7090                "malformed_pointer_escape".to_string(),
7091                "runtime_computed[0]".to_string(),
7092                "/roles/~2/tools".to_string(),
7093                json!({}),
7094                Some(json!(["/roles/~2/tools"])),
7095            ),
7096            (
7097                "unknown_capabilities_field".to_string(),
7098                "capabilities.future".to_string(),
7099                "<array>".to_string(),
7100                json!({ "future": [] }),
7101                None,
7102            ),
7103        ]);
7104
7105        for (index, (name, field, value, capabilities, runtime_computed)) in
7106            cases.into_iter().enumerate()
7107        {
7108            let registry = Arc::new(Registry::default());
7109            let handler = ControlHandler::new(Arc::clone(&registry));
7110            let response = handler
7111                .handle_control(
7112                    ConnectionId::new((index + 1) as u64),
7113                    capability_grammar_hello_frame(
7114                        capabilities,
7115                        runtime_computed,
7116                        index as u64 + 1,
7117                    ),
7118                )
7119                .expect("invalid HELLO returns a refusal");
7120
7121            assert_eq!(response.len(), 1, "{name} must emit one refusal");
7122            let error = parse_error(&response[0]);
7123            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
7124            let message = error["message"]
7125                .as_str()
7126                .expect("error message is a string");
7127            assert!(
7128                message.contains(&field),
7129                "{name}: field missing from {message}"
7130            );
7131            assert!(
7132                message.contains(&value),
7133                "{name}: value missing from {message}"
7134            );
7135            assert_eq!(
7136                registry
7137                    .active_registration_count()
7138                    .expect("registry reads"),
7139                0,
7140                "{name}: refused HELLO must not create a catalog entry"
7141            );
7142        }
7143    }
7144
7145    #[test]
7146    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
7147        let registry = Arc::new(Registry::default());
7148        let handler = ControlHandler::new(Arc::clone(&registry));
7149        let capabilities = json!({
7150            "provides": ["credentials-provider/v1"],
7151            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
7152            "must_never_reach": ["federation-transport/v1"]
7153        });
7154        let response = handler
7155            .handle_control(
7156                ConnectionId::new(99),
7157                capability_grammar_hello_frame(
7158                    capabilities.clone(),
7159                    Some(json!(["/roles/0/tools"])),
7160                    99,
7161                ),
7162            )
7163            .expect("valid HELLO registers");
7164        assert_eq!(response[0].header.ty, FrameType::HelloAck);
7165
7166        let request = Frame::build(
7167            FrameType::Request,
7168            control_flags(),
7169            0,
7170            0,
7171            100,
7172            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7173                .expect("catalog request serializes"),
7174        )
7175        .expect("catalog request frame builds");
7176        let response = handler
7177            .handle_catalog_list(request, None)
7178            .expect("catalog list succeeds");
7179        let ClientControlResponse::CatalogList { modules, .. } =
7180            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
7181        else {
7182            panic!("catalog request must return catalog.list");
7183        };
7184        assert_eq!(modules.len(), 1);
7185        assert_eq!(
7186            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
7187            capabilities
7188        );
7189    }
7190
7191    #[test]
7192    fn catalog_list_mirrors_management_operation_description() {
7193        let registry = Arc::new(Registry::default());
7194        let handler = ControlHandler::new(Arc::clone(&registry));
7195        let description = "List managed records and return their identifiers and metadata.";
7196        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
7197        manifest.provides = vec![ProviderRole::ManagementSurface {
7198            operations: vec![ManagementOperation {
7199                name: "records.list".to_string(),
7200                kind: ManagementOperationKind::Query,
7201                description: Some(description.to_string()),
7202            }],
7203            config_schema: json!({"type": "object"}),
7204            observability: vec![ObservabilitySurface {
7205                name: "records.stats".to_string(),
7206                kind: ObservabilityKind::Snapshot,
7207            }],
7208            identity_scope: vec![IdentityScope::Project],
7209            concurrency: Concurrency::ModuleManaged,
7210        }];
7211        registry
7212            .register_with_control_ops(
7213                manifest,
7214                PROTOCOL_VERSION,
7215                ConnectionId::new(99),
7216                Vec::new(),
7217            )
7218            .expect("described management manifest registers");
7219
7220        let request = Frame::build(
7221            FrameType::Request,
7222            control_flags(),
7223            0,
7224            0,
7225            100,
7226            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7227                .expect("catalog request serializes"),
7228        )
7229        .expect("catalog request frame builds");
7230        let response = handler
7231            .handle_catalog_list(request, None)
7232            .expect("catalog list succeeds");
7233        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7234        assert_eq!(
7235            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
7236            "catalog.list must preserve the declared operation description verbatim"
7237        );
7238    }
7239
7240    #[test]
7241    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
7242        let registry = Arc::new(Registry::default());
7243        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
7244            [("vault".to_string(), true), ("squatter".to_string(), true)],
7245            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
7246        );
7247        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
7248        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7249            provides: vec!["credentials-provider/v1".to_string()],
7250            requires: Vec::new(),
7251            must_never_reach: Vec::new(),
7252        });
7253        let frame = Frame::build(
7254            FrameType::Hello,
7255            control_flags(),
7256            0,
7257            0,
7258            77,
7259            serde_json::to_vec(&ModuleHelloBody {
7260                manifest: squatter,
7261                protocol_ver: PROTOCOL_VERSION,
7262                control_ops: None,
7263                launch_nonce: None,
7264            })
7265            .expect("HELLO serializes"),
7266        )
7267        .expect("HELLO frame builds");
7268        let response = handler
7269            .handle_control(ConnectionId::new(77), frame)
7270            .expect("reserved claim receives a typed refusal");
7271        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
7272        assert_eq!(
7273            registry
7274                .active_registration_count()
7275                .expect("registry reads"),
7276            0,
7277            "a reserved capability refusal must not leave a catalog entry"
7278        );
7279    }
7280
7281    #[test]
7282    fn server_describe_surfaces_required_capability_verdict_fields() {
7283        let registry = Arc::new(Registry::default());
7284        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
7285            [
7286                ("consumer".to_string(), true),
7287                ("provider".to_string(), false),
7288            ],
7289            BTreeMap::new(),
7290        );
7291        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
7292        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7293            provides: Vec::new(),
7294            requires: vec![subc_protocol::manifest::CapabilityRequirement {
7295                capability: "credentials-provider/v1".to_string(),
7296                need: subc_protocol::manifest::CapabilityNeed::Required,
7297            }],
7298            must_never_reach: Vec::new(),
7299        });
7300        let hello = Frame::build(
7301            FrameType::Hello,
7302            control_flags(),
7303            0,
7304            0,
7305            78,
7306            serde_json::to_vec(&ModuleHelloBody {
7307                manifest: consumer,
7308                protocol_ver: PROTOCOL_VERSION,
7309                control_ops: None,
7310                launch_nonce: None,
7311            })
7312            .expect("HELLO serializes"),
7313        )
7314        .expect("HELLO frame builds");
7315        handler
7316            .handle_control(ConnectionId::new(78), hello)
7317            .expect("consumer registers");
7318        let describe = Frame::build(
7319            FrameType::Request,
7320            control_flags(),
7321            0,
7322            0,
7323            79,
7324            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
7325                .expect("request serializes"),
7326        )
7327        .expect("describe frame builds");
7328        let response = handler
7329            .handle_server_describe(describe)
7330            .expect("server.describe succeeds");
7331        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
7332        let requirement = &rendered["capability_requirements"][0];
7333        assert_eq!(requirement["consumer"], "consumer");
7334        assert_eq!(requirement["verdict"], "never_provided");
7335        assert_eq!(requirement["episode_seq"], 1);
7336        assert_eq!(requirement["config_satisfiable"], false);
7337        assert_eq!(requirement["runtime_available"], false);
7338        assert!(requirement["detail"]
7339            .as_str()
7340            .expect("detail string")
7341            .contains("credentials-provider/v1"));
7342    }
7343
7344    #[test]
7345    fn catalog_list_omits_capabilities_for_legacy_manifest() {
7346        let registry = Arc::new(Registry::default());
7347        let handler = ControlHandler::new(Arc::clone(&registry));
7348        let hello = handler
7349            .handle_control(
7350                ConnectionId::new(101),
7351                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
7352            )
7353            .expect("legacy HELLO registers");
7354        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
7355
7356        let request = Frame::build(
7357            FrameType::Request,
7358            control_flags(),
7359            0,
7360            0,
7361            102,
7362            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7363                .expect("catalog request serializes"),
7364        )
7365        .expect("catalog request frame builds");
7366        let response = handler
7367            .handle_catalog_list(request, None)
7368            .expect("catalog list succeeds");
7369        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7370        assert!(
7371            body["modules"][0].get("capabilities").is_none(),
7372            "legacy manifest must retain an absent capabilities field on catalog.list"
7373        );
7374    }
7375
7376    #[test]
7377    fn hello_ack_omits_storage_when_no_storage_config() {
7378        let registry = Arc::new(Registry::default());
7379        let handler = ControlHandler::new(Arc::clone(&registry));
7380        let responses = handler
7381            .handle_control(
7382                ConnectionId::new(1),
7383                hello_frame("aft", PROTOCOL_VERSION, 7),
7384            )
7385            .unwrap();
7386        let ack = parse_ack(&responses[0]);
7387        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
7388        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
7389    }
7390
7391    #[tokio::test]
7392    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
7393        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
7394        let registry = Arc::new(Registry::default());
7395        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
7396        let responses = handler
7397            .handle_control(
7398                ConnectionId::new(1),
7399                hello_frame("aft", PROTOCOL_VERSION, 7),
7400            )
7401            .unwrap();
7402        let ack = parse_ack(&responses[0]);
7403        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
7404
7405        let described = handler
7406            .handle_control_frame(
7407                &route_ctx(ConnectionId::new(2)).0,
7408                Frame::build(
7409                    FrameType::Request,
7410                    control_flags(),
7411                    0,
7412                    0,
7413                    9,
7414                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
7415                )
7416                .unwrap(),
7417            )
7418            .await
7419            .unwrap();
7420        let ClientControlResponse::ServerDescribe { machine_id, .. } =
7421            serde_json::from_slice(&described[0].body).unwrap()
7422        else {
7423            panic!("server.describe answered with another shape");
7424        };
7425        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
7426    }
7427
7428    #[test]
7429    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
7430        // With a central sqlite storage policy, each registering module gets its
7431        // own resolved descriptor in HELLO_ACK, keyed by its module id.
7432        let registry = Arc::new(Registry::default());
7433        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
7434            crate::daemon_config::StorageConfig::Sqlite {
7435                data_home: std::path::PathBuf::from("/data"),
7436            },
7437        ));
7438
7439        let responses = handler
7440            .handle_control(
7441                ConnectionId::new(1),
7442                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
7443            )
7444            .unwrap();
7445        let ack = parse_ack(&responses[0]);
7446        assert_eq!(
7447            ack.storage,
7448            Some(serde_json::json!({
7449                "module_id": "alfonso-routing",
7450                "storage_namespace": "default",
7451                "isolation": { "kind": "module" },
7452                "backend": {
7453                    "backend": "sqlite",
7454                    "path": "/data/cortexkit/alfonso-routing/store.db"
7455                }
7456            })),
7457            "the delivered descriptor is the module's own sqlite store path"
7458        );
7459    }
7460
7461    #[test]
7462    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
7463        let registry = Arc::new(Registry::default());
7464        let handler = ControlHandler::new(Arc::clone(&registry));
7465        let conn = ConnectionId::new(1);
7466        let responses = handler
7467            .handle_control(
7468                conn,
7469                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
7470            )
7471            .unwrap();
7472        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
7473        let registration = registry.get_module("aft").unwrap().unwrap();
7474        assert_eq!(registration.control_ops, module_baseline_control_ops());
7475
7476        let frame =
7477            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
7478        assert!(handler
7479            .guard_module_control_op(&frame, "aft", "route.bind")
7480            .unwrap()
7481            .is_none());
7482        let error = handler
7483            .guard_module_control_op(&frame, "aft", "test.synthetic")
7484            .unwrap()
7485            .expect("synthetic ungranted op should be rejected");
7486        assert_eq!(error.header.ty, FrameType::Error);
7487        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
7488    }
7489
7490    #[test]
7491    fn hello_control_ops_some_adds_optional_grants() {
7492        let registry = Arc::new(Registry::default());
7493        let handler = ControlHandler::new(Arc::clone(&registry));
7494        handler
7495            .handle_control(
7496                ConnectionId::new(1),
7497                hello_frame_with_control_ops(
7498                    "aft",
7499                    PROTOCOL_VERSION,
7500                    7,
7501                    Some(vec![
7502                        "future.synthetic".to_string(),
7503                        "route.bind".to_string(),
7504                    ]),
7505                ),
7506            )
7507            .unwrap();
7508        let registration = registry.get_module("aft").unwrap().unwrap();
7509        assert_eq!(
7510            registration.control_ops,
7511            vec![
7512                "route.bind".to_string(),
7513                "route.status".to_string(),
7514                "future.synthetic".to_string(),
7515            ]
7516        );
7517        let frame =
7518            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
7519        assert!(handler
7520            .guard_module_control_op(&frame, "aft", "future.synthetic")
7521            .unwrap()
7522            .is_none());
7523    }
7524
7525    #[tokio::test]
7526    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
7527        let registry = Arc::new(Registry::default());
7528        let forwarding = Arc::new(ForwardingTable::default());
7529        let handler =
7530            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
7531        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
7532        hello_via_sink(
7533            &handler,
7534            &module_ctx,
7535            &mut module_rx,
7536            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
7537        )
7538        .await;
7539
7540        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
7541        let responses = handler
7542            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
7543            .await
7544            .unwrap();
7545        assert_eq!(responses.len(), 1);
7546        assert_eq!(responses[0].header.ty, FrameType::Error);
7547        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
7548        assert!(module_rx.try_recv().is_err());
7549    }
7550
7551    #[tokio::test]
7552    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
7553        let registry = Arc::new(Registry::default());
7554        let forwarding = Arc::new(ForwardingTable::default());
7555        let handler =
7556            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
7557        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
7558        hello_via_sink(
7559            &handler,
7560            &module_ctx,
7561            &mut module_rx,
7562            hello_frame_with_control_ops(
7563                "aft",
7564                PROTOCOL_VERSION,
7565                7,
7566                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
7567            ),
7568        )
7569        .await;
7570
7571        let project_root = unique_project_root("demux");
7572        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
7573        let route_handler = handler.clone();
7574        let route_task = tokio::spawn(async move {
7575            route_handler
7576                .handle_control_frame(
7577                    &route_client_ctx,
7578                    route_open_frame(100, "aft", project_root),
7579                )
7580                .await
7581                .unwrap()
7582        });
7583        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7584            .await
7585            .unwrap()
7586            .unwrap();
7587        assert!(matches!(
7588            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
7589            ModuleControlRequest::RouteBind { .. }
7590        ));
7591
7592        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
7593        let health_handler = handler.clone();
7594        let health_task = tokio::spawn(async move {
7595            health_handler
7596                .handle_control_frame(
7597                    &health_client_ctx,
7598                    supervisor_health_probe_frame(101, "aft"),
7599                )
7600                .await
7601                .unwrap()
7602        });
7603        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7604            .await
7605            .unwrap()
7606            .unwrap();
7607        assert_eq!(
7608            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
7609            ModuleControlRequest::HealthCheck {}
7610        );
7611
7612        handler
7613            .handle_control_frame(
7614                &module_ctx,
7615                health_response(health_frame.header.corr, HealthStatus::Degraded),
7616            )
7617            .await
7618            .unwrap();
7619        let health_response = health_task.await.unwrap();
7620        assert_eq!(health_response.len(), 1);
7621        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
7622            ClientControlResponse::SupervisorHealthProbe {
7623                module_id,
7624                status,
7625                detail,
7626                metrics,
7627            } => {
7628                assert_eq!(module_id, "aft");
7629                assert_eq!(status, HealthStatus::Degraded);
7630                assert_eq!(detail.as_deref(), Some("warming"));
7631                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
7632            }
7633            other => panic!("unexpected health response: {other:?}"),
7634        }
7635
7636        handler
7637            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
7638            .await
7639            .unwrap();
7640        let route_response = route_task.await.unwrap();
7641        assert!(route_response.is_empty());
7642        let published = route_client_rx.recv().await.unwrap();
7643        assert!(matches!(
7644            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
7645            ClientControlResponse::RouteOpen { .. }
7646        ));
7647    }
7648
7649    /// Start one `route.open` on `client_connection` and return its still-running
7650    /// handler task together with the `route.bind` the module received for it.
7651    /// The handler blocks until the module answers, so it has to run as a task
7652    /// while the test drives the module side.
7653    async fn relay_route_open(
7654        handler: &ControlHandler,
7655        client_connection: ConnectionId,
7656        client_egress: &FrameSink,
7657        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7658        corr: u64,
7659        module_id: &str,
7660        project_root_label: &str,
7661    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
7662        let ctx = RouteCtx {
7663            connection_id: client_connection,
7664            egress: client_egress.clone(),
7665        };
7666        let handler = handler.clone();
7667        let project_root = unique_project_root(project_root_label);
7668        let module_id = module_id.to_string();
7669        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
7670        let task = tokio::spawn(async move {
7671            let _guard = tracing::dispatcher::set_default(&dispatch);
7672            handler
7673                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
7674                .await
7675                .unwrap()
7676        });
7677        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
7678            .await
7679            .expect("module receives the relayed route.bind")
7680            .expect("module egress is open");
7681        (task, bind.frame)
7682    }
7683
7684    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
7685        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
7686            ModuleControlRequest::RouteBind {
7687                route_channel,
7688                epoch,
7689                ..
7690            } => (route_channel, epoch),
7691            other => panic!("expected a route.bind request, got {other:?}"),
7692        }
7693    }
7694
7695    fn published_route(frame: &Frame) -> (u16, u32) {
7696        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
7697            ClientControlResponse::RouteOpen {
7698                route_channel,
7699                route_epoch,
7700            } => (route_channel, route_epoch),
7701            other => panic!("expected a route.open response, got {other:?}"),
7702        }
7703    }
7704
7705    /// Reproduction of a production outage. A client had `route.open`s in
7706    /// flight to a module and was already marked closing -- its egress had refused a
7707    /// module frame, so the daemon asked its connection to end -- while its sink
7708    /// was still open. When the module acked those binds, the daemon refused to
7709    /// commit a route for a closing client, and that refusal was returned from
7710    /// the MODULE connection's frame handler, where a router error that has no
7711    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
7712    /// the supervisor correctly did not respawn a clean exit, and every seat lost
7713    /// its tools for hours -- one client's teardown took down a connection
7714    /// carrying ~170 other routes.
7715    ///
7716    /// The window is opened here by calling the production path that opens it
7717    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
7718    /// state that matters is "in `closing_connections`, sink still open, relay
7719    /// still pending", and it lasts only from the close request until the
7720    /// connection loop reacts to it; a socket-level test can flood a client into
7721    /// that escalation but cannot pin the module's ack inside the window. Closing
7722    /// the socket instead takes the other path entirely -- connection teardown
7723    /// removes the pending relay under the same lock, so the ack finds nothing.
7724    #[tokio::test]
7725    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
7726        let registry = Arc::new(Registry::default());
7727        let forwarding = Arc::new(ForwardingTable::default());
7728        let handler =
7729            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
7730
7731        let module_connection = ConnectionId::new(30);
7732        let (module_ctx, mut module_rx) = route_ctx(module_connection);
7733        hello_via_sink(
7734            &handler,
7735            &module_ctx,
7736            &mut module_rx,
7737            hello_frame("aft", PROTOCOL_VERSION, 7),
7738        )
7739        .await;
7740
7741        let dying_client = ConnectionId::new(31);
7742        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
7743
7744        // A published route on the dying client. The escalation below only marks
7745        // a connection closing for a route it has already published.
7746        let (first_task, first_bind) = relay_route_open(
7747            &handler,
7748            dying_client,
7749            &dying_ctx.egress,
7750            &mut module_rx,
7751            100,
7752            "aft",
7753            "closing-first",
7754        )
7755        .await;
7756        handler
7757            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
7758            .await
7759            .unwrap();
7760        assert!(first_task.await.unwrap().is_empty());
7761        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
7762
7763        // A second route.open from the same client, relayed and awaiting its ack.
7764        let (second_task, second_bind) = relay_route_open(
7765            &handler,
7766            dying_client,
7767            &dying_ctx.egress,
7768            &mut module_rx,
7769            101,
7770            "aft",
7771            "closing-second",
7772        )
7773        .await;
7774        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
7775
7776        // The window: the client is closing, its sink is still open, and its
7777        // second bind is still pending.
7778        assert!(forwarding
7779            .escalate_client_delivery_failure(
7780                dying_client,
7781                first_channel,
7782                first_epoch,
7783                CloseReason::new(
7784                    "module_to_client_delivery_failed",
7785                    "client egress refused a module frame",
7786                ),
7787                crate::forwarding::UndeliveredFrame {
7788                    module_id: None,
7789                    sink: &dying_ctx.egress,
7790                },
7791            )
7792            .unwrap());
7793        assert!(!dying_ctx.egress.is_closed());
7794
7795        // The frame that used to end the module connection.
7796        let ack = handler
7797            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
7798            .await;
7799        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
7800        if module_loop_error.is_some() {
7801            // What the server's connection loop does with a router error that has
7802            // no ERROR-frame translation: end the connection, which releases the
7803            // module's registration and every route on it.
7804            handler.cleanup_connection(module_connection).unwrap();
7805        }
7806        // Read the module's next frame before opening the co-tenant's route, so
7807        // the GOODBYE assertion below is about THIS ack and not about later
7808        // traffic. `None` means the module was told nothing.
7809        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7810            .await
7811            .ok()
7812            .flatten();
7813
7814        // 1. The module connection is still registered.
7815        assert!(
7816            registry
7817                .get_module_by_connection(module_connection)
7818                .unwrap()
7819                .is_some(),
7820            "one client's closing connection ended the shared module connection: \
7821             {module_loop_error:?}"
7822        );
7823        // ...and still serving: another client can open and use a route on it.
7824        let cotenant = ConnectionId::new(32);
7825        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
7826        let (cotenant_task, cotenant_bind) = relay_route_open(
7827            &handler,
7828            cotenant,
7829            &cotenant_ctx.egress,
7830            &mut module_rx,
7831            102,
7832            "aft",
7833            "closing-cotenant",
7834        )
7835        .await;
7836        handler
7837            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
7838            .await
7839            .unwrap();
7840        assert!(cotenant_task.await.unwrap().is_empty());
7841        let (cotenant_channel, cotenant_epoch) =
7842            published_route(&cotenant_rx.recv().await.unwrap());
7843        assert!(matches!(
7844            forwarding
7845                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
7846                .unwrap(),
7847            DataRoute::Client(DataRouteState::Bound(_))
7848        ));
7849
7850        // 2. The module was told to drop the binding it created for the route
7851        //    that will never be published.
7852        let goodbye = post_ack_module_frame
7853            .expect("module receives a GOODBYE for the abandoned route channel");
7854        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
7855        assert_eq!(goodbye.header.channel, abandoned_channel);
7856        assert_eq!(goodbye.header.epoch, abandoned_epoch);
7857
7858        // 3. The dying client received nothing: no route was ever published to
7859        //    it. Its route.open is answered as unavailable, which the connection
7860        //    loop would write to a socket that is already going away.
7861        assert!(dying_rx.try_recv().is_err());
7862        let second_response = second_task.await.unwrap();
7863        assert_eq!(second_response.len(), 1);
7864        assert_eq!(
7865            parse_error(&second_response[0])["code"],
7866            "target_unavailable"
7867        );
7868    }
7869
7870    /// The fence at the module-loop boundary, stated as its own contract: which
7871    /// forwarding failures are allowed to end the module connection that is being
7872    /// served. A `ConnectionClosing` naming some client is about that client, and
7873    /// a module connection is shared; the same error naming the module's own
7874    /// connection is about this connection and must stay fatal, as must failures
7875    /// that are about the forwarding table itself.
7876    #[test]
7877    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
7878        let handler = ControlHandler::default();
7879        let module_connection = ConnectionId::new(30);
7880        let client_connection = ConnectionId::new(31);
7881
7882        handler
7883            .refuse_to_end_module_connection_for_a_client(
7884                module_connection,
7885                77,
7886                ForwardingError::ConnectionClosing {
7887                    connection_id: client_connection,
7888                },
7889            )
7890            .expect("a closing client must never end the module connection");
7891
7892        assert!(matches!(
7893            handler.refuse_to_end_module_connection_for_a_client(
7894                module_connection,
7895                78,
7896                ForwardingError::ConnectionClosing {
7897                    connection_id: module_connection,
7898                },
7899            ),
7900            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
7901                connection_id
7902            })) if connection_id == module_connection
7903        ));
7904        assert!(matches!(
7905            handler.refuse_to_end_module_connection_for_a_client(
7906                module_connection,
7907                79,
7908                ForwardingError::Poisoned,
7909            ),
7910            Err(RouterError::Forwarding(ForwardingError::Poisoned))
7911        ));
7912        assert!(matches!(
7913            handler.refuse_to_end_module_connection_for_a_client(
7914                module_connection,
7915                80,
7916                ForwardingError::StaleModuleEndpoint,
7917            ),
7918            Err(RouterError::Forwarding(
7919                ForwardingError::StaleModuleEndpoint
7920            ))
7921        ));
7922    }
7923
7924    /// The spawn-attestation guard is what stops a connected module from claiming
7925    /// another module's identity and being stamped `Reserved` for it. Every other
7926    /// test that supplies a consumer_identity supplies a CORRECT one, because a
7927    /// correct one is what the rest of the flow needs -- so the guard's rejection
7928    /// branch was never the subject of an assertion, only its acceptance branch.
7929    ///
7930    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
7931    /// whole subc-core library suite green; only the forwarding integration tests
7932    /// notice, and they notice for unrelated reasons. This test exists so the
7933    /// refusal itself is asserted where the guard lives: it fails if the identity
7934    /// check stops refusing, which is the direction that matters, since a guard
7935    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
7936    #[tokio::test]
7937    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
7938        let registry = Arc::new(Registry::default());
7939        let forwarding = Arc::new(ForwardingTable::default());
7940        let supervisor = SupervisorHandle::new();
7941        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7942        let handler =
7943            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
7944                .with_supervisor(supervisor);
7945
7946        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
7947        hello_via_sink(
7948            &handler,
7949            &target_ctx,
7950            &mut target_rx,
7951            hello_frame("target", PROTOCOL_VERSION, 1),
7952        )
7953        .await;
7954
7955        // A real supervised module id presenting the wrong nonce. This is the
7956        // impersonation case: the attacker knows a privileged module_id, which is
7957        // public, and guesses at the nonce, which is not.
7958        let wrong_nonce = handler
7959            .handle_control_frame(
7960                &route_ctx(ConnectionId::new(91)).0,
7961                route_open_frame_with_admission_facts(
7962                    20,
7963                    "target",
7964                    unique_project_root("admission-facts"),
7965                    Some(subc_control::ConsumerIdentity {
7966                        module_id: "fed".to_string(),
7967                        launch_nonce: "not-the-real-nonce".to_string(),
7968                    }),
7969                    None,
7970                ),
7971            )
7972            .await
7973            .unwrap();
7974        assert_eq!(
7975            parse_error(&wrong_nonce[0])["code"],
7976            "bad_consumer_identity",
7977            "a mismatched launch nonce must be refused, not stamped Reserved"
7978        );
7979
7980        // A module id the supervisor never spawned at all, so no nonce exists to
7981        // compare against. An implementation that treats "no record" as "nothing
7982        // to check" fails open here while passing the case above.
7983        let never_spawned = handler
7984            .handle_control_frame(
7985                &route_ctx(ConnectionId::new(92)).0,
7986                route_open_frame_with_admission_facts(
7987                    21,
7988                    "target",
7989                    unique_project_root("admission-facts"),
7990                    Some(subc_control::ConsumerIdentity {
7991                        module_id: "never-spawned".to_string(),
7992                        launch_nonce: "any-nonce".to_string(),
7993                    }),
7994                    None,
7995                ),
7996            )
7997            .await
7998            .unwrap();
7999        assert_eq!(
8000            parse_error(&never_spawned[0])["code"],
8001            "bad_consumer_identity",
8002            "an unspawned module_id must be refused rather than accepted for lack of a record"
8003        );
8004    }
8005
8006    /// The refusal test above proves the guard says NO. Nothing proved it can say
8007    /// YES, and the difference is not academic: replacing the whole authorization
8008    /// with `false` -- admitting no consumer identity at all, revoking Reserved
8009    /// standing for every supervised module in the fleet -- leaves 110 of the 111
8010    /// library tests GREEN. The one that notices does so by HANGING, because it
8011    /// waits for a bind that can no longer happen.
8012    ///
8013    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
8014    /// or flaky test, invites a RETRY rather than an investigation, and the retry
8015    /// hangs too and gets blamed on the runner. So a total revocation of the
8016    /// daemon's trust grant would have shipped behind a symptom nobody attributes
8017    /// to code.
8018    ///
8019    /// The bias is structural rather than accidental. A REFUSAL looks like a
8020    /// failure someone writes a test for; a GRANT looks like the happy path. Every
8021    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
8022    /// suite for that reason, and this one is the purest case in the daemon.
8023    ///
8024    /// This test asserts the EFFECT rather than the absence of an error: the module
8025    /// receives a RouteBind and it carries `Reserved` naming the attested module.
8026    /// A guard that admitted nobody would produce no bind at all; one that admitted
8027    /// everybody would stamp the wrong principal, which the refusal test catches.
8028    #[tokio::test]
8029    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
8030        let registry = Arc::new(Registry::default());
8031        let forwarding = Arc::new(ForwardingTable::default());
8032        let supervisor = SupervisorHandle::new();
8033        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8034        let handler =
8035            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8036                .with_supervisor(supervisor);
8037
8038        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
8039        hello_via_sink(
8040            &handler,
8041            &target_ctx,
8042            &mut target_rx,
8043            hello_frame("target", PROTOCOL_VERSION, 1),
8044        )
8045        .await;
8046
8047        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
8048        let route_handler = handler.clone();
8049        let route_task = tokio::spawn(async move {
8050            route_handler
8051                .handle_control_frame(
8052                    &client_ctx,
8053                    route_open_frame_with_admission_facts(
8054                        30,
8055                        "target",
8056                        unique_project_root("admission-facts"),
8057                        Some(subc_control::ConsumerIdentity {
8058                            module_id: "fed".to_string(),
8059                            launch_nonce: "fed-nonce".to_string(),
8060                        }),
8061                        None,
8062                    ),
8063                )
8064                .await
8065                .unwrap()
8066        });
8067
8068        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
8069        // the very mutation it exists to catch -- a guard that admits nobody -- no
8070        // bind is ever sent, so it HUNG rather than failing. That reproduces the
8071        // exact defect being fixed: a total revocation detected only as a stalled
8072        // suite, which reads as flakiness and invites a retry. An acceptance test
8073        // that waits for an effect must bound the wait, or a red becomes a hang.
8074        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8075            .await
8076            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
8077            .expect("module control channel closed before route.bind");
8078        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8079        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
8080            panic!("expected route.bind")
8081        };
8082        assert_eq!(
8083            principal,
8084            Some(Principal::Reserved {
8085                module_id: "fed".to_string()
8086            }),
8087            "a correctly attested consumer must be stamped Reserved for its own id"
8088        );
8089
8090        handler
8091            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8092            .await
8093            .unwrap();
8094        assert!(route_task.await.unwrap().is_empty());
8095        assert!(
8096            matches!(
8097                serde_json::from_slice::<ClientControlResponse>(
8098                    &client_rx.recv().await.unwrap().body
8099                )
8100                .unwrap(),
8101                ClientControlResponse::RouteOpen { .. }
8102            ),
8103            "the route must actually open, not merely avoid an error"
8104        );
8105    }
8106
8107    #[tokio::test(start_paused = true)]
8108    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
8109        let registry = Arc::new(Registry::default());
8110        let forwarding = Arc::new(ForwardingTable::default());
8111        let supervisor = SupervisorHandle::new();
8112        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8113        let handler =
8114            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8115                .with_supervisor(supervisor);
8116
8117        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
8118        hello_via_sink(
8119            &handler,
8120            &target_ctx,
8121            &mut target_rx,
8122            hello_frame("target", PROTOCOL_VERSION, 1),
8123        )
8124        .await;
8125
8126        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
8127        let direct_handler = handler.clone();
8128        let direct_open = tokio::spawn(async move {
8129            direct_handler
8130                .handle_control_frame(
8131                    &direct_ctx,
8132                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
8133                )
8134                .await
8135                .unwrap()
8136        });
8137        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8138            .await
8139            .expect("no direct route.bind within 5s")
8140            .expect("target control channel closed before direct route.bind");
8141        handler
8142            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
8143            .await
8144            .unwrap();
8145        assert!(direct_open.await.unwrap().is_empty());
8146        let _ = direct_rx.recv().await.unwrap();
8147
8148        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
8149        let reserved_handler = handler.clone();
8150        let reserved_open = tokio::spawn(async move {
8151            reserved_handler
8152                .handle_control_frame(
8153                    &reserved_ctx,
8154                    route_open_frame_with_admission_facts(
8155                        3,
8156                        "target",
8157                        unique_project_root("admission-facts"),
8158                        Some(ConsumerIdentity {
8159                            module_id: "fed".to_string(),
8160                            launch_nonce: "fed-nonce".to_string(),
8161                        }),
8162                        None,
8163                    ),
8164                )
8165                .await
8166                .unwrap()
8167        });
8168        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8169            .await
8170            .expect("no reserved route.bind within 5s")
8171            .expect("target control channel closed before reserved route.bind");
8172        handler
8173            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
8174            .await
8175            .unwrap();
8176        assert!(reserved_open.await.unwrap().is_empty());
8177        let _ = reserved_rx.recv().await.unwrap();
8178
8179        forwarding
8180            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
8181            .unwrap();
8182        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
8183        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
8184            module_id: Some("target".to_string()),
8185        })
8186        .unwrap();
8187        let census_frame =
8188            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
8189        let response = handler
8190            .handle_control_frame(&census_ctx, census_frame)
8191            .await
8192            .unwrap()
8193            .pop()
8194            .unwrap();
8195        let actual: Value = serde_json::from_slice(&response.body).unwrap();
8196        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
8197        assert!(matches!(
8198            decoded,
8199            ClientControlResponse::SupervisorRoutes { .. }
8200        ));
8201        let routes = actual["modules"][0]["routes"].as_array().unwrap();
8202        assert_eq!(routes.len(), 2);
8203        assert!(routes.iter().all(|route| route["draining"] == true));
8204        // The census carries WHY: the reason the drain was begun with, in the
8205        // route.closing vocabulary, on every draining route this drain marked.
8206        assert!(
8207            routes.iter().all(|route| route["drain_reason"] == "reload"),
8208            "draining routes must name the drain's reason: {routes:?}"
8209        );
8210        assert!(routes.iter().any(|route| {
8211            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
8212        }));
8213        assert!(routes.iter().any(|route| {
8214            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
8215        }));
8216
8217        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8218            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
8219        if std::env::var_os("UPDATE_GOLDEN").is_some() {
8220            std::fs::write(
8221                &golden_path,
8222                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
8223            )
8224            .unwrap();
8225        }
8226        let expected: Value =
8227            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
8228        assert_eq!(actual, expected);
8229    }
8230
8231    async fn query_live_roots(
8232        handler: &ControlHandler,
8233        module_ctx: &RouteCtx,
8234    ) -> ModuleControlResponseToModule {
8235        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
8236        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
8237        let response = handler
8238            .handle_control_frame(module_ctx, frame)
8239            .await
8240            .unwrap()
8241            .pop()
8242            .unwrap();
8243        serde_json::from_slice(&response.body).unwrap()
8244    }
8245
8246    #[tokio::test(start_paused = true)]
8247    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
8248        let registry = Arc::new(Registry::default());
8249        let forwarding = Arc::new(ForwardingTable::default());
8250        let handler = ControlHandler::with_forwarding(registry, forwarding);
8251        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
8252        hello_via_sink(
8253            &handler,
8254            &target_ctx,
8255            &mut target_rx,
8256            hello_frame("target", PROTOCOL_VERSION, 1),
8257        )
8258        .await;
8259        let root = unique_project_root("live-roots-known");
8260        let path = ProjectRootId::from_path_allowing_missing(root.path())
8261            .unwrap()
8262            .as_path()
8263            .to_path_buf();
8264        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
8265        let open_handler = handler.clone();
8266        let opened = tokio::spawn(async move {
8267            open_handler
8268                .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
8269                .await
8270                .unwrap()
8271        });
8272        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8273            .await
8274            .unwrap()
8275            .unwrap();
8276        handler
8277            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
8278            .await
8279            .unwrap();
8280        assert!(opened.await.unwrap().is_empty());
8281        let _ = client_rx.recv().await.unwrap();
8282
8283        let root = unique_project_root("live-roots-pending");
8284        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
8285            .unwrap()
8286            .as_path()
8287            .to_path_buf();
8288        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
8289        let open_handler = handler.clone();
8290        let pending = tokio::spawn(async move {
8291            open_handler
8292                .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
8293                .await
8294                .unwrap()
8295        });
8296        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8297            .await
8298            .unwrap()
8299            .unwrap();
8300        let actual = query_live_roots(&handler, &target_ctx).await;
8301        let ModuleControlResponseToModule::LiveRoots {
8302            roots,
8303            unknown_root_bindings,
8304            total_bindings,
8305        } = actual
8306        else {
8307            panic!("expected live roots")
8308        };
8309        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
8310        assert_eq!(unknown_root_bindings, 0);
8311        assert_eq!(
8312            roots.len(),
8313            2,
8314            "root-known arm must retain each canonical root"
8315        );
8316        assert_eq!(
8317            total_bindings,
8318            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8319        );
8320        let counts = roots
8321            .iter()
8322            .map(|root| (root.project_root.clone(), root.bound, root.pending))
8323            .collect::<Vec<_>>();
8324        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
8325        expected.sort_by(|a, b| a.0.cmp(&b.0));
8326        assert_eq!(
8327            counts, expected,
8328            "roots must sort by path and count pending separately"
8329        );
8330        handler
8331            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
8332            .await
8333            .unwrap();
8334        assert!(pending.await.unwrap().is_empty());
8335    }
8336
8337    #[tokio::test(start_paused = true)]
8338    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
8339        let forwarding = Arc::new(ForwardingTable::default());
8340        let handler =
8341            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
8342        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
8343        hello_via_sink(
8344            &handler,
8345            &target_ctx,
8346            &mut target_rx,
8347            hello_frame("target", PROTOCOL_VERSION, 1),
8348        )
8349        .await;
8350        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
8351        let pending = forwarding
8352            .begin_route_bind_relay_for_test(
8353                client_ctx.connection_id,
8354                client_ctx.egress.clone(),
8355                2,
8356                "target",
8357            )
8358            .unwrap();
8359        forwarding
8360            .complete_pending_relay(
8361                target_ctx.connection_id,
8362                pending.corr,
8363                RouteBindRelayOutcome::Accepted,
8364            )
8365            .unwrap();
8366        let actual = query_live_roots(&handler, &target_ctx).await;
8367        let ModuleControlResponseToModule::LiveRoots {
8368            roots,
8369            unknown_root_bindings,
8370            total_bindings,
8371        } = actual
8372        else {
8373            panic!("expected live roots")
8374        };
8375        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
8376        assert_eq!(
8377            unknown_root_bindings, 1,
8378            "unknown-root arm must not read as no bindings"
8379        );
8380        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
8381        assert_eq!(
8382            total_bindings,
8383            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8384        );
8385    }
8386
8387    /// A module reads HELLO_ACK as its first frame and exits on anything else,
8388    /// so the ack has to be on its outbound queue before the module is
8389    /// routable. The connection loop writes a handler's replies only after the
8390    /// handler returns; this test stops in exactly that gap, runs a real
8391    /// route.open from another connection, and only then writes whatever the
8392    /// HELLO handler returned, the way the loop would. If the ack were still a
8393    /// reply, the route.bind request would reach the module first.
8394    #[tokio::test(start_paused = true)]
8395    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
8396        let forwarding = Arc::new(ForwardingTable::default());
8397        let handler =
8398            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
8399        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
8400        let replies = handler
8401            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
8402            .await
8403            .unwrap();
8404        let queued_by_hello = module_rx.len();
8405
8406        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
8407        let open_handler = handler.clone();
8408        let open = tokio::spawn(async move {
8409            open_handler
8410                .handle_control_frame(
8411                    &client_ctx,
8412                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
8413                )
8414                .await
8415                .unwrap()
8416        });
8417        // Let the route.open run until its route.bind is on the module's queue.
8418        let mut spins = 0;
8419        while module_rx.len() == queued_by_hello {
8420            spins += 1;
8421            assert!(spins < 10_000, "route.open never queued a route.bind");
8422            tokio::task::yield_now().await;
8423        }
8424
8425        // Now the connection loop's half: write the HELLO handler's replies.
8426        for reply in replies {
8427            module_ctx.egress.send(reply).await.unwrap();
8428        }
8429
8430        let first = module_rx.recv().await.unwrap().frame;
8431        assert_eq!(
8432            first.header.ty,
8433            FrameType::HelloAck,
8434            "the first frame a registering module reads must be its HELLO_ACK"
8435        );
8436        assert_eq!(first.header.corr, 7);
8437        let second = module_rx.recv().await.unwrap().frame;
8438        assert_eq!(second.header.ty, FrameType::Request);
8439        assert!(
8440            matches!(
8441                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
8442                ModuleControlRequest::RouteBind { .. }
8443            ),
8444            "the route.bind follows the ack"
8445        );
8446        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
8447
8448        handler
8449            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
8450            .await
8451            .unwrap();
8452        assert!(open.await.unwrap().is_empty());
8453        let _ = client_rx.recv().await.unwrap();
8454    }
8455
8456    #[tokio::test(start_paused = true)]
8457    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
8458        let handler = ControlHandler::with_forwarding(
8459            Arc::new(Registry::default()),
8460            Arc::new(ForwardingTable::default()),
8461        );
8462        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
8463        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
8464        hello_via_sink(
8465            &handler,
8466            &first_ctx,
8467            &mut first_rx,
8468            hello_frame("first", PROTOCOL_VERSION, 1),
8469        )
8470        .await;
8471        hello_via_sink(
8472            &handler,
8473            &second_ctx,
8474            &mut second_rx,
8475            hello_frame("second", PROTOCOL_VERSION, 2),
8476        )
8477        .await;
8478        let root = unique_project_root("second-only");
8479        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
8480        let cloned = handler.clone();
8481        let open = tokio::spawn(async move {
8482            cloned
8483                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
8484                .await
8485                .unwrap()
8486        });
8487        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
8488            .await
8489            .unwrap()
8490            .unwrap();
8491        let first = query_live_roots(&handler, &first_ctx).await;
8492        let second = query_live_roots(&handler, &second_ctx).await;
8493        assert!(
8494            matches!(
8495                first,
8496                ModuleControlResponseToModule::LiveRoots {
8497                    total_bindings: 0,
8498                    ..
8499                }
8500            ),
8501            "cross-module scope must not expose another module's roots"
8502        );
8503        assert!(
8504            matches!(
8505                second,
8506                ModuleControlResponseToModule::LiveRoots {
8507                    total_bindings: 1,
8508                    ..
8509                }
8510            ),
8511            "second module must see its pending route"
8512        );
8513        handler
8514            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
8515            .await
8516            .unwrap();
8517        assert!(open.await.unwrap().is_empty());
8518    }
8519
8520    #[tokio::test(start_paused = true)]
8521    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
8522        let handler = ControlHandler::with_forwarding(
8523            Arc::new(Registry::default()),
8524            Arc::new(ForwardingTable::default()),
8525        );
8526        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
8527        hello_via_sink(
8528            &handler,
8529            &target_ctx,
8530            &mut target_rx,
8531            hello_frame("target", PROTOCOL_VERSION, 1),
8532        )
8533        .await;
8534        let actual = query_live_roots(&handler, &target_ctx).await;
8535        let ModuleControlResponseToModule::LiveRoots {
8536            roots,
8537            unknown_root_bindings,
8538            total_bindings,
8539        } = actual
8540        else {
8541            panic!("expected live roots")
8542        };
8543        assert!(roots.is_empty());
8544        assert_eq!(unknown_root_bindings, 0);
8545        assert_eq!(total_bindings, 0);
8546        assert_eq!(
8547            total_bindings,
8548            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8549        );
8550    }
8551
8552    /// Read the vendored fed corpus rather than hand-building a package.
8553    ///
8554    /// A hand-built object encodes what the test author believed the carrier
8555    /// emits. These vectors are what it actually emits, and one of them exists
8556    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
8557    /// ignores additive unknown fields at the traversal emit terminus."
8558    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
8559        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8560            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
8561        let text = std::fs::read_to_string(&path)
8562            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
8563        let vectors: Vec<(String, Value)> = text
8564            .lines()
8565            .filter(|line| !line.trim().is_empty())
8566            .map(|line| {
8567                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
8568                let id = entry["corpus_id"]
8569                    .as_str()
8570                    .expect("every vector carries a corpus_id")
8571                    .to_string();
8572                (id, entry["package"].clone())
8573            })
8574            .collect();
8575        // Pin the count: a corpus that silently shrinks would take its coverage
8576        // with it, and a suite reading N-1 vectors reports the same clean pass
8577        // as one reading N.
8578        assert_eq!(
8579            vectors.len(),
8580            3,
8581            "vendored fed corpus changed size; re-sync from subc-federation"
8582        );
8583
8584        // Pin what makes the corpus DISCRIMINATING, not just present.
8585        //
8586        // The relay test below takes its expected value from the corpus, so the
8587        // corpus supplies the test's power to detect a lossy relay rather than
8588        // its correctness. A relay that dropped unrecognised fields would still
8589        // be caught -- but only by a package carrying fields it does not know.
8590        // Shrink every package to the handful of keys any implementation would
8591        // recognise and the test keeps passing over an input that can no longer
8592        // fail, which is the same clean green as a corpus that shrank away.
8593        //
8594        // So assert the precondition rather than duplicating the packages here:
8595        // at least one vector must carry a field beyond the small common set.
8596        // That is one claim to maintain instead of nine, and it fails loudly if
8597        // a re-sync ever flattens the corpus.
8598        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
8599        let richest = vectors
8600            .iter()
8601            .filter_map(|(_, package)| package.as_object())
8602            .map(|object| {
8603                object
8604                    .keys()
8605                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
8606                    .count()
8607            })
8608            .max()
8609            .unwrap_or(0);
8610        assert!(
8611            richest >= 2,
8612            "vendored corpus no longer carries a package with unmodelled fields, \
8613             so the relay test can no longer distinguish a verbatim relay from a lossy one"
8614        );
8615
8616        vectors
8617    }
8618
8619    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
8620    ///
8621    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
8622    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
8623    /// three-key object, so a relay that quietly dropped fields it did not
8624    /// recognise would satisfy it. These vectors carry nine keys including ones
8625    /// this crate has no type for, so a typed relay fails here and only here.
8626    #[tokio::test]
8627    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
8628        for (corpus_id, package) in fed_admission_facts_vectors() {
8629            let registry = Arc::new(Registry::default());
8630            let forwarding = Arc::new(ForwardingTable::default());
8631            let supervisor = SupervisorHandle::new();
8632            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8633            let handler =
8634                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8635                    .with_supervisor(supervisor)
8636                    .with_admission_facts_config(
8637                        Some("fed".to_string()),
8638                        Some(vec!["target".to_string()]),
8639                    );
8640
8641            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
8642            hello_via_sink(
8643                &handler,
8644                &target_ctx,
8645                &mut target_rx,
8646                hello_frame("target", PROTOCOL_VERSION, 1),
8647            )
8648            .await;
8649
8650            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
8651            let route_handler = handler.clone();
8652            let expected = package.clone();
8653            let route_task = tokio::spawn(async move {
8654                route_handler
8655                    .handle_control_frame(
8656                        &client_ctx,
8657                        route_open_frame_with_admission_facts(
8658                            20,
8659                            "target",
8660                            unique_project_root("admission-facts"),
8661                            Some(subc_control::ConsumerIdentity {
8662                                module_id: "fed".to_string(),
8663                                launch_nonce: "fed-nonce".to_string(),
8664                            }),
8665                            Some(package),
8666                        ),
8667                    )
8668                    .await
8669                    .unwrap()
8670            });
8671
8672            let bind_frame = target_rx.recv().await.unwrap();
8673            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8674            let ModuleControlRequest::RouteBind {
8675                admission_facts, ..
8676            } = bind
8677            else {
8678                panic!("{corpus_id}: expected route.bind")
8679            };
8680            assert_eq!(
8681                admission_facts,
8682                Some(expected),
8683                "{corpus_id}: relay must not add, drop or reshape any field"
8684            );
8685
8686            handler
8687                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8688                .await
8689                .unwrap();
8690            route_task.await.unwrap();
8691        }
8692    }
8693
8694    #[tokio::test]
8695    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
8696        let registry = Arc::new(Registry::default());
8697        let forwarding = Arc::new(ForwardingTable::default());
8698        let supervisor = SupervisorHandle::new();
8699        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8700        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
8701        let handler =
8702            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8703                .with_supervisor(supervisor)
8704                .with_admission_facts_config(
8705                    Some("fed".to_string()),
8706                    Some(vec!["target".to_string()]),
8707                );
8708
8709        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
8710        hello_via_sink(
8711            &handler,
8712            &target_ctx,
8713            &mut target_rx,
8714            hello_frame("target", PROTOCOL_VERSION, 1),
8715        )
8716        .await;
8717        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
8718        hello_via_sink(
8719            &handler,
8720            &other_ctx,
8721            &mut other_rx,
8722            hello_frame("other", PROTOCOL_VERSION, 2),
8723        )
8724        .await;
8725
8726        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
8727        let expected_facts = facts.clone();
8728        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
8729        let route_handler = handler.clone();
8730        let route_task = tokio::spawn(async move {
8731            route_handler
8732                .handle_control_frame(
8733                    &client_ctx,
8734                    route_open_frame_with_admission_facts(
8735                        10,
8736                        "target",
8737                        unique_project_root("admission-facts"),
8738                        Some(subc_control::ConsumerIdentity {
8739                            module_id: "fed".to_string(),
8740                            launch_nonce: "fed-nonce".to_string(),
8741                        }),
8742                        Some(facts.clone()),
8743                    ),
8744                )
8745                .await
8746                .unwrap()
8747        });
8748        let bind_frame = target_rx.recv().await.unwrap();
8749        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8750        let ModuleControlRequest::RouteBind {
8751            admission_facts, ..
8752        } = bind
8753        else {
8754            panic!("expected route.bind")
8755        };
8756        assert_eq!(admission_facts, Some(expected_facts));
8757        handler
8758            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8759            .await
8760            .unwrap();
8761        assert!(route_task.await.unwrap().is_empty());
8762        assert!(matches!(
8763            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
8764                .unwrap(),
8765            ClientControlResponse::RouteOpen { .. }
8766        ));
8767
8768        let direct = handler
8769            .handle_control_frame(
8770                &route_ctx(ConnectionId::new(73)).0,
8771                route_open_frame_with_admission_facts(
8772                    11,
8773                    "target",
8774                    unique_project_root("admission-facts"),
8775                    None,
8776                    Some(json!({"x": 1})),
8777                ),
8778            )
8779            .await
8780            .unwrap();
8781        assert_eq!(
8782            parse_error(&direct[0])["code"],
8783            "admission_facts_not_permitted"
8784        );
8785
8786        let different_reserved = handler
8787            .handle_control_frame(
8788                &route_ctx(ConnectionId::new(77)).0,
8789                route_open_frame_with_admission_facts(
8790                    15,
8791                    "target",
8792                    unique_project_root("admission-facts"),
8793                    Some(subc_control::ConsumerIdentity {
8794                        module_id: "other".to_string(),
8795                        launch_nonce: "other-nonce".to_string(),
8796                    }),
8797                    Some(json!({"x": 1})),
8798                ),
8799            )
8800            .await
8801            .unwrap();
8802        assert_eq!(
8803            parse_error(&different_reserved[0])["code"],
8804            "admission_facts_not_permitted"
8805        );
8806
8807        let other_target = handler
8808            .handle_control_frame(
8809                &route_ctx(ConnectionId::new(74)).0,
8810                route_open_frame_with_admission_facts(
8811                    12,
8812                    "other",
8813                    unique_project_root("admission-facts"),
8814                    Some(subc_control::ConsumerIdentity {
8815                        module_id: "fed".to_string(),
8816                        launch_nonce: "fed-nonce".to_string(),
8817                    }),
8818                    Some(json!({"x": 1})),
8819                ),
8820            )
8821            .await
8822            .unwrap();
8823        assert_eq!(
8824            parse_error(&other_target[0])["code"],
8825            "admission_facts_target_not_allowed"
8826        );
8827
8828        let nonexistent = handler
8829            .handle_control_frame(
8830                &route_ctx(ConnectionId::new(75)).0,
8831                route_open_frame_with_admission_facts(
8832                    13,
8833                    "missing",
8834                    unique_project_root("admission-facts"),
8835                    None,
8836                    Some(json!({"x": 1})),
8837                ),
8838            )
8839            .await
8840            .unwrap();
8841        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
8842
8843        let described = handler
8844            .handle_control_frame(
8845                &route_ctx(ConnectionId::new(76)).0,
8846                Frame::build(
8847                    FrameType::Request,
8848                    control_flags(),
8849                    0,
8850                    0,
8851                    14,
8852                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8853                )
8854                .unwrap(),
8855            )
8856            .await
8857            .unwrap();
8858        let ClientControlResponse::ServerDescribe { capabilities, .. } =
8859            serde_json::from_slice(&described[0].body).unwrap()
8860        else {
8861            panic!("expected server.describe response")
8862        };
8863        assert!(capabilities
8864            .iter()
8865            .any(|cap| cap == "admission_facts_relay_v1"));
8866    }
8867
8868    #[tokio::test]
8869    async fn admission_facts_without_configured_carrier_are_rejected() {
8870        let registry = Arc::new(Registry::default());
8871        let forwarding = Arc::new(ForwardingTable::default());
8872        let handler = ControlHandler::with_forwarding(registry, forwarding);
8873        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
8874        hello_via_sink(
8875            &handler,
8876            &target_ctx,
8877            &mut target_rx,
8878            hello_frame("target", PROTOCOL_VERSION, 1),
8879        )
8880        .await;
8881
8882        let responses = handler
8883            .handle_control_frame(
8884                &route_ctx(ConnectionId::new(79)).0,
8885                route_open_frame_with_admission_facts(
8886                    16,
8887                    "target",
8888                    unique_project_root("admission-facts"),
8889                    None,
8890                    Some(json!({"x": 1})),
8891                ),
8892            )
8893            .await
8894            .unwrap();
8895        assert_eq!(
8896            parse_error(&responses[0])["code"],
8897            "admission_facts_not_permitted"
8898        );
8899    }
8900
8901    #[tokio::test]
8902    async fn route_open_relays_consumer_capabilities_verbatim() {
8903        let registry = Arc::new(Registry::default());
8904        let forwarding = Arc::new(ForwardingTable::default());
8905        let handler =
8906            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
8907        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
8908        hello_via_sink(
8909            &handler,
8910            &module_ctx,
8911            &mut module_rx,
8912            hello_frame("aft", PROTOCOL_VERSION, 7),
8913        )
8914        .await;
8915
8916        let expected = vec!["elicitation".to_string(), "roots".to_string()];
8917        let expected_for_request = expected.clone();
8918        let project_root = unique_project_root("consumer-capabilities-present");
8919        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
8920        let route_handler = handler.clone();
8921        let route_task = tokio::spawn(async move {
8922            route_handler
8923                .handle_control_frame(
8924                    &client_ctx,
8925                    route_open_frame_with_consumer_capabilities(
8926                        401,
8927                        "aft",
8928                        project_root,
8929                        Some(expected_for_request),
8930                    ),
8931                )
8932                .await
8933                .unwrap()
8934        });
8935        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8936            .await
8937            .unwrap()
8938            .unwrap();
8939        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8940        let ModuleControlRequest::RouteBind {
8941            consumer_capabilities,
8942            ..
8943        } = bind
8944        else {
8945            panic!("expected route.bind request, got {bind:?}");
8946        };
8947        assert_eq!(consumer_capabilities, Some(expected.clone()));
8948
8949        handler
8950            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8951            .await
8952            .unwrap();
8953        let route_response = route_task.await.unwrap();
8954        assert!(route_response.is_empty());
8955        let published = client_rx.recv().await.unwrap();
8956        assert!(matches!(
8957            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8958            ClientControlResponse::RouteOpen { .. }
8959        ));
8960    }
8961
8962    #[tokio::test]
8963    async fn route_open_without_consumer_capabilities_relays_none() {
8964        let registry = Arc::new(Registry::default());
8965        let forwarding = Arc::new(ForwardingTable::default());
8966        let handler =
8967            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
8968        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
8969        hello_via_sink(
8970            &handler,
8971            &module_ctx,
8972            &mut module_rx,
8973            hello_frame("aft", PROTOCOL_VERSION, 7),
8974        )
8975        .await;
8976
8977        let project_root = unique_project_root("consumer-capabilities-absent");
8978        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
8979        let route_handler = handler.clone();
8980        let route_task = tokio::spawn(async move {
8981            route_handler
8982                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
8983                .await
8984                .unwrap()
8985        });
8986        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8987            .await
8988            .unwrap()
8989            .unwrap();
8990        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8991        let ModuleControlRequest::RouteBind {
8992            consumer_capabilities,
8993            ..
8994        } = bind
8995        else {
8996            panic!("expected route.bind request, got {bind:?}");
8997        };
8998        assert_eq!(consumer_capabilities, None);
8999
9000        handler
9001            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9002            .await
9003            .unwrap();
9004        let route_response = route_task.await.unwrap();
9005        assert!(route_response.is_empty());
9006        let published = client_rx.recv().await.unwrap();
9007        assert!(matches!(
9008            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9009            ClientControlResponse::RouteOpen { .. }
9010        ));
9011    }
9012
9013    /// Opens a route to a freshly registered `aft` with `sent` as the
9014    /// route.open's role_versions, acks the bind, and returns the role_versions
9015    /// the module's bind carried.
9016    async fn bind_role_versions_for(
9017        sent: Option<BTreeMap<String, String>>,
9018        connection: u64,
9019    ) -> Option<BTreeMap<String, String>> {
9020        let registry = Arc::new(Registry::default());
9021        let forwarding = Arc::new(ForwardingTable::default());
9022        let handler =
9023            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9024        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
9025        hello_via_sink(
9026            &handler,
9027            &module_ctx,
9028            &mut module_rx,
9029            hello_frame("aft", PROTOCOL_VERSION, 7),
9030        )
9031        .await;
9032        let project_root = unique_project_root("role-versions");
9033        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
9034        let route_handler = handler.clone();
9035        let route_task = tokio::spawn(async move {
9036            route_handler
9037                .handle_control_frame(
9038                    &client_ctx,
9039                    route_open_frame_with_role_versions(403, "aft", project_root, sent),
9040                )
9041                .await
9042                .unwrap()
9043        });
9044        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9045            .await
9046            .expect("a well-formed route.open reaches the module as a bind")
9047            .unwrap();
9048        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9049        let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
9050            panic!("expected route.bind request, got {bind:?}");
9051        };
9052        handler
9053            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9054            .await
9055            .unwrap();
9056        assert!(route_task.await.unwrap().is_empty());
9057        let published = client_rx.recv().await.unwrap();
9058        assert!(matches!(
9059            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9060            ClientControlResponse::RouteOpen { .. }
9061        ));
9062        role_versions
9063    }
9064
9065    #[tokio::test]
9066    async fn route_open_relays_role_versions_verbatim() {
9067        let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
9068        assert_eq!(
9069            bind_role_versions_for(Some(sent.clone()), 141).await,
9070            Some(sent)
9071        );
9072    }
9073
9074    /// An empty map declares nothing, so the provider sees no field rather
9075    /// than an empty object it would have to treat as a second "none".
9076    #[tokio::test]
9077    async fn route_open_with_empty_or_absent_role_versions_relays_none() {
9078        assert_eq!(bind_role_versions_for(None, 143).await, None);
9079        assert_eq!(
9080            bind_role_versions_for(Some(BTreeMap::new()), 145).await,
9081            None
9082        );
9083    }
9084
9085    #[tokio::test]
9086    async fn route_open_refuses_malformed_role_versions_before_any_bind() {
9087        let registry = Arc::new(Registry::default());
9088        let forwarding = Arc::new(ForwardingTable::default());
9089        let handler =
9090            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9091        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
9092        hello_via_sink(
9093            &handler,
9094            &module_ctx,
9095            &mut module_rx,
9096            hello_frame("aft", PROTOCOL_VERSION, 7),
9097        )
9098        .await;
9099
9100        let nine: BTreeMap<String, String> = (0..9)
9101            .map(|index| (format!("role-{index}"), "v1".to_string()))
9102            .collect();
9103        for (label, malformed) in [
9104            (
9105                "invalid role name",
9106                role_versions(&[("Tool_Provider", "v1")]),
9107            ),
9108            ("invalid version", role_versions(&[("tool-provider", "v0")])),
9109            ("nine entries", nine),
9110        ] {
9111            // A refused open answers at once; one that reached the module
9112            // would wait for its bind ack and trip this timeout.
9113            let responses = tokio::time::timeout(
9114                Duration::from_secs(1),
9115                handler.handle_control_frame(
9116                    &route_ctx(ConnectionId::new(148)).0,
9117                    route_open_frame_with_role_versions(
9118                        404,
9119                        "aft",
9120                        unique_project_root("role-versions-malformed"),
9121                        Some(malformed),
9122                    ),
9123                ),
9124            )
9125            .await
9126            .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
9127            .unwrap();
9128            assert_eq!(responses.len(), 1, "{label}");
9129            assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
9130            let error = parse_error(&responses[0]);
9131            assert_eq!(error["code"], "invalid_request", "{label}: {error}");
9132            assert_eq!(
9133                error["detail"]["field"], "role_versions",
9134                "{label}: {error}"
9135            );
9136            assert!(
9137                !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
9138                "{label}: a malformed declaration is terminal"
9139            );
9140            assert!(
9141                module_rx.try_recv().is_err(),
9142                "{label}: the module must never see a bind"
9143            );
9144        }
9145    }
9146
9147    /// `route-role-versions/v1` is in HELLO_ACK and `server.describe`, so a
9148    /// consumer can tell this daemon forwards the field from one that would
9149    /// drop it.
9150    #[tokio::test]
9151    async fn route_role_versions_capability_is_advertised() {
9152        let handler = ControlHandler::new(Arc::new(Registry::default()));
9153        let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
9154        let ack = hello_via_sink(
9155            &handler,
9156            &ctx,
9157            &mut rx,
9158            hello_frame("m", PROTOCOL_VERSION, 1),
9159        )
9160        .await;
9161        let ack = parse_ack(&ack);
9162        assert!(
9163            ack.subc_capabilities
9164                .iter()
9165                .any(|c| c == "route-role-versions/v1"),
9166            "{:?}",
9167            ack.subc_capabilities
9168        );
9169
9170        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
9171        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
9172        let reply = handler
9173            .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
9174            .await
9175            .unwrap()
9176            .pop()
9177            .unwrap();
9178        let ClientControlResponse::ServerDescribe { capabilities, .. } =
9179            serde_json::from_slice(&reply.body).unwrap()
9180        else {
9181            panic!("not a server.describe reply");
9182        };
9183        assert!(
9184            capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
9185            "{capabilities:?}"
9186        );
9187    }
9188
9189    #[tokio::test]
9190    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
9191        let registry = Arc::new(Registry::default());
9192        let forwarding = Arc::new(ForwardingTable::default());
9193        let handler =
9194            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9195                .with_health_probe_timeout(Duration::from_secs(5));
9196        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
9197        hello_via_sink(
9198            &handler,
9199            &module_ctx,
9200            &mut module_rx,
9201            non_routable_hello_frame_with_control_ops(
9202                "mcp",
9203                300,
9204                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9205            ),
9206        )
9207        .await;
9208        assert!(registry
9209            .get_module("mcp")
9210            .unwrap()
9211            .unwrap()
9212            .manifest
9213            .provides
9214            .is_empty());
9215
9216        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
9217        let route_response = handler
9218            .handle_control_frame(
9219                &route_client_ctx,
9220                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
9221            )
9222            .await
9223            .unwrap();
9224        assert_eq!(route_response[0].header.ty, FrameType::Error);
9225        assert_eq!(
9226            parse_error(&route_response[0])["code"],
9227            "target_unavailable"
9228        );
9229        assert!(parse_error(&route_response[0])["message"]
9230            .as_str()
9231            .unwrap()
9232            .contains("does not provide the requested target"));
9233        assert!(module_rx.try_recv().is_err());
9234
9235        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
9236        let health_handler = handler.clone();
9237        let health_task = tokio::spawn(async move {
9238            health_handler
9239                .handle_control_frame(
9240                    &health_client_ctx,
9241                    supervisor_health_probe_frame(302, "mcp"),
9242                )
9243                .await
9244                .unwrap()
9245        });
9246        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9247            .await
9248            .unwrap()
9249            .unwrap();
9250        assert_eq!(
9251            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
9252            ModuleControlRequest::HealthCheck {}
9253        );
9254        handler
9255            .handle_control_frame(
9256                &module_ctx,
9257                health_response(health_frame.header.corr, HealthStatus::Ok),
9258            )
9259            .await
9260            .unwrap();
9261        let health_response = health_task.await.unwrap();
9262        assert_eq!(health_response[0].header.ty, FrameType::Response);
9263        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
9264            ClientControlResponse::SupervisorHealthProbe {
9265                module_id, status, ..
9266            } => {
9267                assert_eq!(module_id, "mcp");
9268                assert_eq!(status, HealthStatus::Ok);
9269            }
9270            other => panic!("unexpected health response: {other:?}"),
9271        }
9272
9273        // Exercise the forwarding cleanup path directly while leaving the registry
9274        // advertisement in place. If cleanup leaves a stale control sink behind,
9275        // the next probe will enqueue onto it and wait for the long probe timeout
9276        // instead of returning an immediate no-connection error.
9277        forwarding
9278            .cleanup_connection(module_ctx.connection_id)
9279            .unwrap();
9280        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
9281        let cleanup_response = tokio::time::timeout(
9282            Duration::from_millis(200),
9283            handler.handle_control_frame(
9284                &cleanup_probe_ctx,
9285                supervisor_health_probe_frame(303, "mcp"),
9286            ),
9287        )
9288        .await
9289        .expect("probe should fail immediately when the control lane is gone")
9290        .unwrap();
9291        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
9292        assert_eq!(
9293            parse_error(&cleanup_response[0])["code"],
9294            "target_unavailable"
9295        );
9296        assert!(parse_error(&cleanup_response[0])["message"]
9297            .as_str()
9298            .unwrap()
9299            .contains("no module connection"));
9300
9301        handler
9302            .cleanup_connection(module_ctx.connection_id)
9303            .unwrap();
9304    }
9305
9306    #[tokio::test]
9307    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
9308        let registry = Arc::new(Registry::default());
9309        let supervisor_handle = SupervisorHandle::new();
9310        let supervisor =
9311            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
9312                .with_handle(supervisor_handle.clone())
9313                .with_connection_file_path(
9314                    std::env::temp_dir()
9315                        .join(format!("subc-route-open-warming-{}", std::process::id())),
9316                );
9317        let module = supervisor
9318            .supervise_configured(
9319                ModuleSpec {
9320                    module_id: "warming".to_string(),
9321                    program: fake_aft_stub_path(),
9322                    args: Vec::new(),
9323                    env: Vec::new(),
9324                    reserved: false,
9325                    reserved_prefixes: Vec::new(),
9326                    protocol: ModuleProtocol::Subc,
9327                    overlap: Default::default(),
9328                },
9329                true,
9330            )
9331            .unwrap();
9332        assert_eq!(module.state().unwrap(), ModuleState::Running);
9333
9334        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9335        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
9336        let response = handler
9337            .handle_control_frame(
9338                &ctx,
9339                route_open_frame(304, "warming", unique_project_root("warming")),
9340            )
9341            .await
9342            .unwrap();
9343        module.stop().await.unwrap();
9344
9345        assert_eq!(response[0].header.ty, FrameType::Error);
9346        let error = parse_error(&response[0]);
9347        assert_eq!(error["code"], "module_warming");
9348        assert!(error["message"]
9349            .as_str()
9350            .unwrap()
9351            .contains("state=running, enabled=true, live=false"));
9352    }
9353
9354    #[test]
9355    fn route_open_connection_cap_logs_admission_reason_and_capacity() {
9356        let handler = ControlHandler::new(Arc::new(Registry::default()));
9357        let capture = EventCapture::default();
9358        let _subscriber =
9359            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9360        let (ctx, _rx) = route_ctx(ConnectionId::new(96));
9361        let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
9362        let pending = (0..limit).collect::<Vec<_>>();
9363        let response = handler
9364            .route_open_capacity_refusal(
9365                &ctx,
9366                &route_open_frame(396, "busy", unique_project_root("connection-cap")),
9367                "busy",
9368                pending.len(),
9369                limit,
9370            )
9371            .unwrap();
9372        assert_eq!(parse_error(&response)["code"], "target_unavailable");
9373        let event = capture
9374            .events()
9375            .into_iter()
9376            .find(|event| {
9377                event.target == "control"
9378                    && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
9379            })
9380            .expect("connection admission refusal event");
9381        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
9382        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
9383    }
9384
9385    #[test]
9386    fn route_open_target_cap_logs_admission_reason_and_capacity() {
9387        let handler = ControlHandler::new(Arc::new(Registry::default()));
9388        let capture = EventCapture::default();
9389        let _subscriber =
9390            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9391        let (ctx, _rx) = route_ctx(ConnectionId::new(97));
9392        let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
9393        let guards = (0..limit)
9394            .map(|_| {
9395                handler
9396                    .route_bind_concurrency
9397                    .try_admit("busy", limit)
9398                    .unwrap()
9399            })
9400            .collect::<Vec<_>>();
9401        let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
9402            Err(in_flight) => in_flight,
9403            Ok(_) => panic!("target cap must refuse after {limit} admissions"),
9404        };
9405        let response = handler
9406            .route_open_target_capacity_refusal(
9407                &ctx,
9408                &route_open_frame(397, "busy", unique_project_root("target-cap")),
9409                "busy",
9410                in_flight,
9411            )
9412            .unwrap();
9413        assert_eq!(parse_error(&response)["code"], "target_unavailable");
9414        let event = capture
9415            .events()
9416            .into_iter()
9417            .find(|event| {
9418                event.target == "control"
9419                    && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
9420            })
9421            .expect("target admission refusal event");
9422        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
9423        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
9424        drop(guards);
9425    }
9426
9427    /// One wire code has several senders, so the refusal line names the check
9428    /// that refused. This drives the shared refusal path for ordinary refusals
9429    /// with an unregistered
9430    /// target and requires the branch label on the event.
9431    #[tokio::test]
9432    async fn route_open_refusal_names_the_check_that_refused() {
9433        let handler = ControlHandler::new(Arc::new(Registry::default()));
9434        let capture = EventCapture::default();
9435        let _subscriber =
9436            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9437        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
9438        let response = handler
9439            .handle_control_frame(
9440                &ctx,
9441                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
9442            )
9443            .await
9444            .unwrap();
9445
9446        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9447        let event = capture
9448            .events()
9449            .into_iter()
9450            .find(|event| {
9451                event.target == "control"
9452                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
9453            })
9454            .expect("route.open refusal event");
9455        assert_eq!(
9456            event.fields.get("reason"),
9457            Some(&"\"not_registered\"".to_string())
9458        );
9459    }
9460
9461    #[tokio::test]
9462    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
9463        let registry = Arc::new(Registry::default());
9464        let supervisor_handle = SupervisorHandle::new();
9465        let supervisor =
9466            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
9467                .with_handle(supervisor_handle.clone())
9468                .with_connection_file_path(std::env::temp_dir().join(format!(
9469                    "subc-route-open-refusal-info-{}",
9470                    std::process::id()
9471                )));
9472        let module = supervisor
9473            .supervise_configured(
9474                ModuleSpec {
9475                    module_id: "warming".to_string(),
9476                    program: fake_aft_stub_path(),
9477                    args: Vec::new(),
9478                    env: Vec::new(),
9479                    reserved: false,
9480                    reserved_prefixes: Vec::new(),
9481                    protocol: ModuleProtocol::Subc,
9482                    overlap: Default::default(),
9483                },
9484                true,
9485            )
9486            .unwrap();
9487        assert_eq!(module.state().unwrap(), ModuleState::Running);
9488
9489        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9490        assert!(handler
9491            .counters()
9492            .snapshot()
9493            .get("route_open_refused_by_code")
9494            .is_none());
9495        let capture = EventCapture::default();
9496        let _subscriber =
9497            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9498        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
9499        let response = handler
9500            .handle_control_frame(
9501                &ctx,
9502                route_open_frame(394, "warming", unique_project_root("refusal-info")),
9503            )
9504            .await
9505            .unwrap();
9506        module.stop().await.unwrap();
9507
9508        assert_eq!(parse_error(&response[0])["code"], "module_warming");
9509        let event = capture
9510            .events()
9511            .into_iter()
9512            .find(|event| {
9513                event.target == "control"
9514                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
9515            })
9516            .expect("route.open refusal event");
9517        assert_eq!(
9518            event.fields.get("module_id"),
9519            Some(&"\"warming\"".to_string())
9520        );
9521        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
9522        assert_eq!(
9523            event.fields.get("reason"),
9524            Some(&"\"supervised_not_registered\"".to_string())
9525        );
9526        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
9527        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
9528        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
9529        assert_eq!(
9530            handler.counters().snapshot()["route_open_refused_by_code"],
9531            json!({ "module_warming": 1 })
9532        );
9533    }
9534
9535    const OUTAGE_START: &str = "route.open refusing module: not serving";
9536    const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
9537
9538    fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
9539        capture
9540            .events()
9541            .into_iter()
9542            .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
9543            .collect()
9544    }
9545
9546    fn supervise_stub(
9547        registry: &Arc<Registry>,
9548        module_id: &str,
9549        enabled: bool,
9550    ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
9551        let supervisor_handle = SupervisorHandle::new();
9552        let supervisor =
9553            Supervisor::new(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
9554                .with_handle(supervisor_handle.clone())
9555                .with_connection_file_path(std::env::temp_dir().join(format!(
9556                    "subc-route-outage-{module_id}-{}",
9557                    std::process::id()
9558                )));
9559        let module = supervisor
9560            .supervise_configured(
9561                ModuleSpec {
9562                    module_id: module_id.to_string(),
9563                    program: fake_aft_stub_path(),
9564                    args: Vec::new(),
9565                    env: Vec::new(),
9566                    reserved: false,
9567                    reserved_prefixes: Vec::new(),
9568                    protocol: ModuleProtocol::Subc,
9569                    overlap: Default::default(),
9570                },
9571                enabled,
9572            )
9573            .unwrap();
9574        (supervisor_handle, module)
9575    }
9576
9577    fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
9578        let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
9579            module_id: module_id.to_string(),
9580            drain_timeout_ms: Some(50),
9581        })
9582        .unwrap();
9583        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9584    }
9585
9586    /// Two handlers built over one forwarding table must share one outage
9587    /// tracker; separate trackers would each log their own opening line for
9588    /// the same outage.
9589    #[test]
9590    fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
9591        let registry = Arc::new(Registry::default());
9592        let forwarding = Arc::new(ForwardingTable::default());
9593        let first = ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9594        let second = ControlHandler::with_forwarding(registry, forwarding);
9595        assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
9596    }
9597
9598    /// A client can name any module id it likes. Refusing an unknown one,
9599    /// however often, must not create outage state or outage lines, or the
9600    /// tracker would be a memory sink any client could fill.
9601    #[tokio::test(flavor = "current_thread")]
9602    async fn route_open_unknown_module_refusals_add_no_outage_state() {
9603        let handler = ControlHandler::new(Arc::new(Registry::default()));
9604        let capture = EventCapture::default();
9605        let _subscriber =
9606            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9607        let (ctx, _rx) = route_ctx(ConnectionId::new(90));
9608        for corr in 0..8 {
9609            let response = handler
9610                .handle_control_frame(
9611                    &ctx,
9612                    route_open_frame(
9613                        380 + corr,
9614                        &format!("nobody-{corr}"),
9615                        unique_project_root("outage-unknown"),
9616                    ),
9617                )
9618                .await
9619                .unwrap();
9620            assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9621        }
9622
9623        assert_eq!(handler.route_outages.tracked_module_count(), 0);
9624        assert!(outage_lines(&capture, OUTAGE_START).is_empty());
9625        assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
9626    }
9627
9628    /// Drives the refusal path end to end: a supervised module that served
9629    /// before and stopped being registered with no instruction to stop is a
9630    /// WARN, and the same module refused after an operator `supervisor.restart`
9631    /// is an INFO.
9632    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9633    async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
9634        let registry = Arc::new(Registry::default());
9635        let (supervisor_handle, module) = supervise_stub(&registry, "outage-restart", true);
9636        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9637        let capture = EventCapture::default();
9638        let _subscriber =
9639            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9640        let (ctx, _rx) = route_ctx(ConnectionId::new(91));
9641        // The stub never registers, so pretend it served once: otherwise every
9642        // refusal would fall in its startup window.
9643        handler.route_outages.record_accepted("outage-restart");
9644
9645        let response = handler
9646            .handle_control_frame(
9647                &ctx,
9648                route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
9649            )
9650            .await
9651            .unwrap();
9652        assert_eq!(response[0].header.ty, FrameType::Error);
9653        let starts = outage_lines(&capture, OUTAGE_START);
9654        assert_eq!(starts.len(), 1, "{starts:?}");
9655        assert_eq!(starts[0].level, tracing::Level::WARN);
9656        assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
9657        assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
9658        assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
9659        handler.route_outages.record_accepted("outage-restart");
9660        assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
9661
9662        let restart = handler
9663            .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
9664            .await
9665            .unwrap();
9666        assert_eq!(
9667            restart[0].header.ty,
9668            FrameType::Response,
9669            "{:?}",
9670            parse_error(&restart[0])
9671        );
9672        handler
9673            .handle_control_frame(
9674                &ctx,
9675                route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
9676            )
9677            .await
9678            .unwrap();
9679        module.stop().await.unwrap();
9680
9681        let starts = outage_lines(&capture, OUTAGE_START);
9682        assert_eq!(starts.len(), 2, "{starts:?}");
9683        assert_eq!(starts[1].level, tracing::Level::INFO);
9684        assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
9685    }
9686
9687    /// A restart refused before it touched the module (here: the module is
9688    /// disabled) must clear its operator mark, so the next real outage is
9689    /// still reported as a warning.
9690    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9691    async fn failed_operator_restart_leaves_no_operator_mark() {
9692        let registry = Arc::new(Registry::default());
9693        let (supervisor_handle, _module) = supervise_stub(&registry, "outage-disabled", false);
9694        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9695        let capture = EventCapture::default();
9696        let _subscriber =
9697            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9698        let (ctx, _rx) = route_ctx(ConnectionId::new(92));
9699        handler.route_outages.record_accepted("outage-disabled");
9700
9701        let restart = handler
9702            .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
9703            .await
9704            .unwrap();
9705        assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
9706        assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
9707
9708        handler
9709            .handle_control_frame(
9710                &ctx,
9711                route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
9712            )
9713            .await
9714            .unwrap();
9715        let starts = outage_lines(&capture, OUTAGE_START);
9716        assert_eq!(starts.len(), 1, "{starts:?}");
9717        assert_eq!(starts[0].level, tracing::Level::WARN);
9718    }
9719
9720    #[tokio::test(flavor = "current_thread")]
9721    async fn route_open_unknown_module_escapes_target_module_id() {
9722        let handler = ControlHandler::new(Arc::new(Registry::default()));
9723        let capture = EventCapture::default();
9724        let _subscriber =
9725            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9726        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
9727        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
9728        let response = handler
9729            .handle_control_frame(
9730                &ctx,
9731                route_open_frame(
9732                    395,
9733                    hostile_module_id,
9734                    unique_project_root("hostile-target-module-id"),
9735                ),
9736            )
9737            .await
9738            .unwrap();
9739
9740        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9741        let event = capture
9742            .events()
9743            .into_iter()
9744            .find(|event| {
9745                event.target == "control"
9746                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
9747            })
9748            .expect("route.open unknown-module refusal event");
9749        let logged = event.fields.get("module_id").expect("module_id field");
9750        assert!(!logged.bytes().any(|byte| byte < 0x20));
9751        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
9752    }
9753
9754    #[tokio::test(flavor = "current_thread")]
9755    async fn route_open_module_rejection_uses_daemon_counter_key() {
9756        let registry = Arc::new(Registry::default());
9757        let forwarding = Arc::new(ForwardingTable::default());
9758        let handler =
9759            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9760        let module_connection = ConnectionId::new(95);
9761        let (module_ctx, mut module_rx) = route_ctx(module_connection);
9762        hello_via_sink(
9763            &handler,
9764            &module_ctx,
9765            &mut module_rx,
9766            hello_frame("aft", PROTOCOL_VERSION, 395),
9767        )
9768        .await;
9769
9770        let client_connection = ConnectionId::new(96);
9771        let (client_ctx, _client_rx) = route_ctx(client_connection);
9772        let capture = EventCapture::default();
9773        let _subscriber =
9774            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9775        let (route_task, bind) = relay_route_open(
9776            &handler,
9777            client_connection,
9778            &client_ctx.egress,
9779            &mut module_rx,
9780            396,
9781            "aft",
9782            "hostile-module-code",
9783        )
9784        .await;
9785        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
9786        let rejection = Frame::build(
9787            FrameType::Error,
9788            control_flags(),
9789            0,
9790            0,
9791            bind.header.corr,
9792            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
9793        )
9794        .unwrap();
9795        handler
9796            .handle_control_frame(&module_ctx, rejection)
9797            .await
9798            .unwrap();
9799
9800        let response = route_task.await.unwrap();
9801        assert_eq!(parse_error(&response[0])["code"], hostile_code);
9802        let counters = handler.counters().snapshot();
9803        assert_eq!(
9804            counters["route_open_refused_by_code"],
9805            json!({ "module_rejected": 1 })
9806        );
9807        assert!(counters["route_open_refused_by_code"]
9808            .get(hostile_code)
9809            .is_none());
9810
9811        let event = capture
9812            .events()
9813            .into_iter()
9814            .find(|event| {
9815                event.target == "control"
9816                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
9817            })
9818            .expect("route.open module-rejection refusal event");
9819        let logged = event.fields.get("module_code").expect("module_code field");
9820        assert!(!logged.bytes().any(|byte| byte < 0x20));
9821        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
9822    }
9823
9824    #[tokio::test]
9825    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
9826        let registry = Arc::new(Registry::default());
9827        let supervisor_handle = SupervisorHandle::new();
9828        let missing_program = std::env::temp_dir().join(format!(
9829            "subc-route-open-missing-program-{}",
9830            std::process::id()
9831        ));
9832        let supervisor =
9833            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
9834                .with_handle(supervisor_handle.clone());
9835        let module = supervisor
9836            .supervise_configured(
9837                ModuleSpec {
9838                    module_id: "failed".to_string(),
9839                    program: missing_program,
9840                    args: Vec::new(),
9841                    env: Vec::new(),
9842                    reserved: false,
9843                    reserved_prefixes: Vec::new(),
9844                    protocol: ModuleProtocol::Subc,
9845                    overlap: Default::default(),
9846                },
9847                true,
9848            )
9849            .unwrap();
9850        assert_eq!(module.state().unwrap(), ModuleState::Failed);
9851
9852        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9853        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
9854        let response = handler
9855            .handle_control_frame(
9856                &ctx,
9857                route_open_frame(305, "failed", unique_project_root("failed")),
9858            )
9859            .await
9860            .unwrap();
9861
9862        assert_eq!(response[0].header.ty, FrameType::Error);
9863        let error = parse_error(&response[0]);
9864        assert_eq!(error["code"], "target_unavailable");
9865        assert!(error["message"]
9866            .as_str()
9867            .unwrap()
9868            .contains("state=failed, enabled=true, live=false"));
9869    }
9870
9871    #[tokio::test]
9872    async fn route_open_role_mismatch_remains_target_unavailable() {
9873        let registry = Arc::new(Registry::default());
9874        let handler = ControlHandler::new(Arc::clone(&registry));
9875        handler
9876            .handle_control(
9877                ConnectionId::new(41),
9878                non_routable_hello_frame_with_control_ops("health-only", 306, None),
9879            )
9880            .unwrap();
9881
9882        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
9883        let response = handler
9884            .handle_control_frame(
9885                &ctx,
9886                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
9887            )
9888            .await
9889            .unwrap();
9890
9891        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9892        assert!(parse_error(&response[0])["message"]
9893            .as_str()
9894            .unwrap()
9895            .contains("does not provide the requested target"));
9896    }
9897
9898    #[tokio::test]
9899    async fn route_open_inactive_registration_remains_target_unavailable() {
9900        let registry = Arc::new(Registry::default());
9901        let handler = ControlHandler::new(Arc::clone(&registry));
9902        handler
9903            .handle_control(
9904                ConnectionId::new(43),
9905                hello_frame("inactive", PROTOCOL_VERSION, 308),
9906            )
9907            .unwrap();
9908        assert!(registry
9909            .set_module_state_for_test("inactive", ChannelState::Closed)
9910            .unwrap());
9911
9912        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
9913        let response = handler
9914            .handle_control_frame(
9915                &ctx,
9916                route_open_frame(309, "inactive", unique_project_root("inactive")),
9917            )
9918            .await
9919            .unwrap();
9920
9921        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9922        assert!(parse_error(&response[0])["message"]
9923            .as_str()
9924            .unwrap()
9925            .contains("is not active"));
9926    }
9927
9928    #[tokio::test]
9929    async fn late_health_reply_is_recorded_through_the_module_response_path() {
9930        let registry = Arc::new(Registry::default());
9931        let forwarding = Arc::new(ForwardingTable::default());
9932        let supervisor_handle = SupervisorHandle::new();
9933        let supervisor = Supervisor::new(Arc::clone(&registry), crate::RestartPolicy::default())
9934            .with_forwarding(Arc::clone(&forwarding))
9935            .with_handle(supervisor_handle.clone());
9936        let module = supervisor
9937            .supervise_configured(
9938                crate::ModuleSpec {
9939                    module_id: "late-health-response".to_string(),
9940                    program: PathBuf::from("disabled-module"),
9941                    args: Vec::new(),
9942                    env: Vec::new(),
9943                    reserved: false,
9944                    reserved_prefixes: Vec::new(),
9945                    protocol: ModuleProtocol::Subc,
9946                    overlap: Default::default(),
9947                },
9948                false,
9949            )
9950            .unwrap();
9951        let handler =
9952            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9953                .with_supervisor(supervisor_handle);
9954        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
9955        handler
9956            .handle_control_frame(
9957                &module_ctx,
9958                hello_frame_with_control_ops(
9959                    "late-health-response",
9960                    PROTOCOL_VERSION,
9961                    7,
9962                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9963                ),
9964            )
9965            .await
9966            .unwrap();
9967        let probe_started_at = Instant::now() - Duration::from_millis(80);
9968        let pending = forwarding
9969            .begin_health_probe_rpc_for(
9970                "late-health-response",
9971                MODULE_CONTROL_OP_HEALTH_CHECK,
9972                probe_started_at,
9973                Instant::now() - Duration::from_millis(1),
9974            )
9975            .unwrap();
9976        assert!(forwarding
9977            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
9978            .unwrap());
9979
9980        let responses = handler
9981            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
9982            .await
9983            .unwrap();
9984
9985        assert!(responses.is_empty());
9986        let health = module.status().unwrap().health;
9987        assert_eq!(health.late_answer_count, 1);
9988        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
9989    }
9990
9991    #[tokio::test]
9992    async fn health_probe_timeout_and_module_death_are_typed() {
9993        let registry = Arc::new(Registry::default());
9994        let forwarding = Arc::new(ForwardingTable::default());
9995        let handler =
9996            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9997                .with_health_probe_timeout(Duration::from_millis(50));
9998        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
9999        hello_via_sink(
10000            &handler,
10001            &module_ctx,
10002            &mut module_rx,
10003            hello_frame_with_control_ops(
10004                "aft",
10005                PROTOCOL_VERSION,
10006                7,
10007                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10008            ),
10009        )
10010        .await;
10011
10012        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10013        let responses = handler
10014            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
10015            .await
10016            .unwrap();
10017        assert_eq!(responses[0].header.ty, FrameType::Error);
10018        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
10019        let _ = module_rx.try_recv();
10020
10021        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
10022        let health_handler = handler.clone();
10023        let death_task = tokio::spawn(async move {
10024            health_handler
10025                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
10026                .await
10027                .unwrap()
10028        });
10029        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10030            .await
10031            .unwrap()
10032            .unwrap();
10033        handler
10034            .cleanup_connection(module_ctx.connection_id)
10035            .unwrap();
10036        let responses = death_task.await.unwrap();
10037        assert_eq!(responses[0].header.ty, FrameType::Error);
10038        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
10039    }
10040
10041    #[test]
10042    fn hello_requires_exact_protocol_version() {
10043        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
10044            let registry = Arc::new(Registry::default());
10045            let handler = ControlHandler::new(Arc::clone(&registry));
10046            let responses = handler
10047                .handle_control(
10048                    ConnectionId::new(connection),
10049                    hello_frame("aft", offered, 9),
10050                )
10051                .unwrap();
10052
10053            assert_eq!(responses.len(), 1);
10054            assert_eq!(responses[0].header.ty, FrameType::Error);
10055            let error = parse_error(&responses[0]);
10056            assert_eq!(error["code"], "version_unsupported");
10057            assert!(registry.get_module("aft").unwrap().is_none());
10058            assert_eq!(registry.active_registration_count().unwrap(), 0);
10059        }
10060    }
10061
10062    #[test]
10063    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
10064        let registry = Arc::new(Registry::default());
10065        let forwarding = Arc::new(ForwardingTable::default());
10066        let handler =
10067            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10068        let module_connection = ConnectionId::new(301);
10069        let registration = registry
10070            .register_with_control_ops(
10071                manifest("aft-push", PROTOCOL_VERSION),
10072                PROTOCOL_VERSION,
10073                module_connection,
10074                module_baseline_control_ops(),
10075            )
10076            .unwrap();
10077        let (module_tx, _module_rx) = mpsc::channel(8);
10078        let endpoint = forwarding
10079            .register_module_connection(
10080                module_connection,
10081                "aft-push".to_string(),
10082                PROTOCOL_VERSION,
10083                manifest_concurrency(&registration.manifest),
10084                FrameSink::new(module_tx),
10085            )
10086            .unwrap();
10087
10088        // A push op this version does not know is ignored (forward-compat), not errored.
10089        let unknown = Frame::build(
10090            FrameType::Push,
10091            control_flags(),
10092            0,
10093            0,
10094            5,
10095            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
10096        )
10097        .unwrap();
10098        let out = handler.handle_status_update(endpoint, unknown).unwrap();
10099        assert!(
10100            out.is_empty(),
10101            "unknown push op must be ignored, got {out:?}"
10102        );
10103
10104        // A malformed body for a KNOWN op is a real error worth surfacing.
10105        let malformed = Frame::build(
10106            FrameType::Push,
10107            control_flags(),
10108            0,
10109            0,
10110            6,
10111            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
10112        )
10113        .unwrap();
10114        let out = handler.handle_status_update(endpoint, malformed).unwrap();
10115        assert_eq!(out.len(), 1);
10116        assert_eq!(out[0].header.ty, FrameType::Error);
10117        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
10118    }
10119
10120    #[test]
10121    fn hello_rejected_when_connection_already_owns_client_routes() {
10122        let registry = Arc::new(Registry::default());
10123        let forwarding = Arc::new(ForwardingTable::default());
10124        let handler =
10125            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10126        // Commits a client route on connection 202 (bound to a module on conn 101).
10127        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
10128        let client_connection = ConnectionId::new(202);
10129
10130        // That same connection now tries to register as a module: rejected, so one
10131        // connection never holds both client-route and module-endpoint state.
10132        let responses = handler
10133            .handle_control(
10134                client_connection,
10135                hello_frame("aft-second", PROTOCOL_VERSION, 9),
10136            )
10137            .unwrap();
10138        assert_eq!(responses[0].header.ty, FrameType::Error);
10139        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
10140        assert!(registry.get_module("aft-second").unwrap().is_none());
10141    }
10142
10143    #[test]
10144    fn reserved_module_hello_requires_matching_launch_nonce() {
10145        let registry = Arc::new(Registry::default());
10146        let supervisor = SupervisorHandle::new();
10147        // The supervisor recorded the nonce it injected when it spawned the reserved
10148        // module; the HELLO verifier checks against the same shared handle.
10149        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
10150        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
10151
10152        // A HELLO with NO nonce is rejected.
10153        let no_nonce = handler
10154            .handle_control(
10155                ConnectionId::new(1),
10156                hello_frame("vault", PROTOCOL_VERSION, 1),
10157            )
10158            .unwrap();
10159        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
10160        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
10161        assert!(registry.get_module("vault").unwrap().is_none());
10162
10163        // A HELLO with the WRONG nonce is rejected.
10164        let wrong = handler
10165            .handle_control(
10166                ConnectionId::new(2),
10167                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
10168            )
10169            .unwrap();
10170        assert_eq!(wrong[0].header.ty, FrameType::Error);
10171        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
10172        assert!(registry.get_module("vault").unwrap().is_none());
10173
10174        // A HELLO with the CORRECT nonce registers.
10175        let ok = handler
10176            .handle_control(
10177                ConnectionId::new(3),
10178                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
10179            )
10180            .unwrap();
10181        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
10182        assert!(registry.get_module("vault").unwrap().is_some());
10183    }
10184
10185    #[test]
10186    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
10187        let registry = Arc::new(Registry::default());
10188        let supervisor = SupervisorHandle::new();
10189        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
10190        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
10191        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
10192
10193        let squat = handler
10194            .handle_control(
10195                ConnectionId::new(1),
10196                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
10197            )
10198            .unwrap();
10199        assert_eq!(squat[0].header.ty, FrameType::Error);
10200        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
10201        assert!(parse_error(&squat[0])["message"]
10202            .as_str()
10203            .unwrap()
10204            .contains("fed:"));
10205
10206        let accepted_peer = handler
10207            .handle_control(
10208                ConnectionId::new(2),
10209                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
10210            )
10211            .unwrap();
10212        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
10213
10214        let accepted_short = handler
10215            .handle_control(
10216                ConnectionId::new(3),
10217                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
10218            )
10219            .unwrap();
10220        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
10221
10222        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
10223            let response = handler
10224                .handle_control(
10225                    ConnectionId::new(conn),
10226                    hello_frame(module_id, PROTOCOL_VERSION, conn),
10227                )
10228                .unwrap();
10229            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
10230        }
10231    }
10232
10233    #[test]
10234    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
10235        let registry = Arc::new(Registry::default());
10236        let supervisor = SupervisorHandle::new();
10237        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
10238        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
10239        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
10240        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
10241
10242        let owner_nonce = handler
10243            .handle_control(
10244                ConnectionId::new(1),
10245                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
10246            )
10247            .unwrap();
10248        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
10249        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
10250        assert!(registry.get_module("fed:special").unwrap().is_none());
10251
10252        let exact_nonce = handler
10253            .handle_control(
10254                ConnectionId::new(2),
10255                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
10256            )
10257            .unwrap();
10258        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
10259        assert!(registry.get_module("fed:special").unwrap().is_some());
10260    }
10261
10262    #[test]
10263    fn non_reserved_module_ignores_launch_nonce() {
10264        let registry = Arc::new(Registry::default());
10265        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
10266        // registration succeeds whether a spawned process echoes a nonce or not.
10267        let handler = ControlHandler::new(Arc::clone(&registry));
10268        let no_nonce = handler
10269            .handle_control(
10270                ConnectionId::new(1),
10271                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
10272            )
10273            .unwrap();
10274        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
10275        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
10276
10277        let echoed_nonce = handler
10278            .handle_control(
10279                ConnectionId::new(2),
10280                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
10281            )
10282            .unwrap();
10283        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
10284        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
10285    }
10286
10287    #[test]
10288    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
10289        let handler = ControlHandler::default();
10290        let conn = ConnectionId::new(1);
10291        let malformed = Frame::build(
10292            FrameType::Hello,
10293            control_flags(),
10294            0,
10295            0,
10296            3,
10297            b"{not json".to_vec(),
10298        )
10299        .unwrap();
10300
10301        let error = handler.handle_control(conn, malformed).unwrap();
10302        assert_eq!(error[0].header.ty, FrameType::Error);
10303        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
10304
10305        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
10306        let pong = handler.handle_control(conn, ping).unwrap();
10307        assert_eq!(pong[0].header.ty, FrameType::Pong);
10308        assert_eq!(pong[0].header.corr, 4);
10309    }
10310
10311    #[test]
10312    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
10313        let registry = Arc::new(Registry::default());
10314        let handler = ControlHandler::new(Arc::clone(&registry));
10315
10316        handler
10317            .handle_control(
10318                ConnectionId::new(1),
10319                hello_frame("aft", PROTOCOL_VERSION, 1),
10320            )
10321            .unwrap();
10322        let duplicate = handler
10323            .handle_control(
10324                ConnectionId::new(2),
10325                hello_frame("aft", PROTOCOL_VERSION, 2),
10326            )
10327            .unwrap();
10328
10329        assert_eq!(duplicate[0].header.ty, FrameType::Error);
10330        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10331        let registration = registry.get_module("aft").unwrap().unwrap();
10332        assert_eq!(registration.connection_id, ConnectionId::new(1));
10333    }
10334
10335    #[test]
10336    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
10337        let registry = Arc::new(Registry::default());
10338        let forwarding = Arc::new(ForwardingTable::default());
10339        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
10340        let handler =
10341            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10342                .with_process_liveness(process_liveness);
10343        let (ctx, route_channel, route_epoch) =
10344            bind_liveness_route(&registry, &forwarding, "aft-dead");
10345        let responses = handler
10346            .handle_route_poll(
10347                &ctx,
10348                route_poll_frame(41, PollKind::Liveness, route_channel),
10349                route_channel,
10350                route_epoch,
10351                PollKind::Liveness,
10352            )
10353            .unwrap();
10354
10355        assert_eq!(responses.len(), 1);
10356        assert_eq!(responses[0].header.ty, FrameType::Response);
10357        assert_route_poll_liveness(&responses[0], false);
10358    }
10359
10360    #[test]
10361    fn liveness_poll_without_process_source_uses_bound_route() {
10362        let registry = Arc::new(Registry::default());
10363        let forwarding = Arc::new(ForwardingTable::default());
10364        let handler =
10365            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10366        let (ctx, route_channel, route_epoch) =
10367            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
10368        let responses = handler
10369            .handle_route_poll(
10370                &ctx,
10371                route_poll_frame(42, PollKind::Liveness, route_channel),
10372                route_channel,
10373                route_epoch,
10374                PollKind::Liveness,
10375            )
10376            .unwrap();
10377
10378        assert_route_poll_liveness(&responses[0], true);
10379    }
10380
10381    #[test]
10382    fn liveness_poll_untracked_process_source_uses_bound_route() {
10383        let registry = Arc::new(Registry::default());
10384        let forwarding = Arc::new(ForwardingTable::default());
10385        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
10386        let handler =
10387            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10388                .with_process_liveness(process_liveness);
10389        let (ctx, route_channel, route_epoch) =
10390            bind_liveness_route(&registry, &forwarding, "aft-untracked");
10391        let responses = handler
10392            .handle_route_poll(
10393                &ctx,
10394                route_poll_frame(43, PollKind::Liveness, route_channel),
10395                route_channel,
10396                route_epoch,
10397                PollKind::Liveness,
10398            )
10399            .unwrap();
10400
10401        assert_route_poll_liveness(&responses[0], true);
10402    }
10403
10404    #[tokio::test]
10405    async fn unknown_op_returns_unknown_control_op() {
10406        let handler = ControlHandler::default();
10407        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
10408        let request = Frame::build(
10409            FrameType::Request,
10410            control_flags(),
10411            0,
10412            0,
10413            55,
10414            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
10415        )
10416        .unwrap();
10417
10418        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10419
10420        assert_eq!(response.len(), 1);
10421        assert_eq!(response[0].header.ty, FrameType::Error);
10422        assert_eq!(response[0].header.corr, 55);
10423        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
10424    }
10425
10426    #[tokio::test]
10427    async fn supervisor_provenance_rejects_unknown_exact_module() {
10428        let handler = ControlHandler::default();
10429        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
10430        let request = Frame::build(
10431            FrameType::Request,
10432            control_flags(),
10433            0,
10434            0,
10435            57,
10436            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
10437        )
10438        .unwrap();
10439
10440        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10441
10442        assert_eq!(response.len(), 1);
10443        assert_eq!(response[0].header.ty, FrameType::Error);
10444        assert_eq!(response[0].header.corr, 57);
10445        let error = parse_error(&response[0]);
10446        assert_eq!(error["code"], "unknown_module");
10447        assert_eq!(error["message"], "module_id 'missing' is not supervised");
10448    }
10449
10450    #[test]
10451    fn provenance_probe_override_keeps_handler_tests_deterministic() {
10452        let expected = subc_control::RunningImageAgreement::Unavailable {
10453            reason: subc_control::RunningImageUnavailableReason::HashFailed,
10454        };
10455        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
10456        assert_eq!(handler.provenance_probe_override, Some(expected));
10457    }
10458
10459    #[test]
10460    fn reload_verdict_detects_configured_program_different_from_spawned_path() {
10461        let verdict = reload_verdict(
10462            std::path::Path::new("/bin/new"),
10463            Some(std::path::Path::new("/bin/old")),
10464            subc_control::RunningImageAgreement::Unavailable {
10465                reason: subc_control::RunningImageUnavailableReason::HashFailed,
10466            },
10467        );
10468        assert!(matches!(
10469            verdict.path,
10470            subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
10471                if configured == std::path::Path::new("/bin/new")
10472                    && spawned_from == std::path::Path::new("/bin/old")
10473        ));
10474    }
10475
10476    #[test]
10477    fn reload_verdict_detects_replaced_image_at_same_path() {
10478        let image = subc_control::RunningImageAgreement::Mismatch {
10479            running: subc_control::RunningImageEvidence::LinuxProcSha256 {
10480                digest: "old".into(),
10481            },
10482            disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
10483                digest: "new".into(),
10484            },
10485        };
10486        let verdict = reload_verdict(
10487            std::path::Path::new("/bin/same"),
10488            Some(std::path::Path::new("/bin/same")),
10489            image.clone(),
10490        );
10491        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10492        assert_eq!(verdict.image, image);
10493    }
10494
10495    #[test]
10496    fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
10497        let image = subc_control::RunningImageAgreement::Unavailable {
10498            reason: subc_control::RunningImageUnavailableReason::NotRunning,
10499        };
10500        let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
10501        assert_eq!(
10502            verdict.path,
10503            subc_control::ReloadPathAgreement::Unavailable {
10504                reason: subc_control::ReloadPathUnavailableReason::NotRunning,
10505            }
10506        );
10507        assert_eq!(verdict.image, image);
10508
10509        let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
10510            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
10511        };
10512        let verdict = reload_verdict(
10513            std::path::Path::new("/bin/same"),
10514            Some(std::path::Path::new("/bin/same")),
10515            unconfirmed.clone(),
10516        );
10517        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10518        assert_eq!(verdict.image, unconfirmed);
10519    }
10520
10521    #[test]
10522    fn reload_verdict_preserves_each_image_unavailability_reason() {
10523        use subc_control::RunningImageUnavailableReason as Reason;
10524
10525        for reason in [
10526            Reason::NotRunning,
10527            Reason::UnsupportedPlatform,
10528            Reason::RunningExecutableUnreadable,
10529            Reason::SpawnedPathUnreadable,
10530            Reason::HashFailed,
10531            Reason::ProcessIdentityUnconfirmed,
10532            Reason::Unknown("future_probe_reason".to_string()),
10533        ] {
10534            let image = subc_control::RunningImageAgreement::Unavailable {
10535                reason: reason.clone(),
10536            };
10537            let verdict = reload_verdict(
10538                std::path::Path::new("/bin/same"),
10539                Some(std::path::Path::new("/bin/same")),
10540                image.clone(),
10541            );
10542            assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10543            assert_eq!(verdict.image, image, "{reason:?}");
10544        }
10545    }
10546
10547    #[tokio::test]
10548    async fn malformed_control_bodies_return_invalid_control_body() {
10549        let handler = ControlHandler::default();
10550        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
10551
10552        for (corr, body) in [
10553            (56, br#"{"route_channel":1}"#.as_slice()),
10554            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
10555            (
10556                58,
10557                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
10558            ),
10559        ] {
10560            let request = Frame::build(
10561                FrameType::Request,
10562                control_flags(),
10563                0,
10564                0,
10565                corr,
10566                body.to_vec(),
10567            )
10568            .unwrap();
10569            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10570
10571            assert_eq!(response.len(), 1);
10572            assert_eq!(response[0].header.ty, FrameType::Error);
10573            assert_eq!(response[0].header.corr, corr);
10574            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
10575        }
10576    }
10577
10578    #[tokio::test]
10579    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
10580        let registry = Arc::new(Registry::default());
10581        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
10582        let router = Router::with_control_handler(Arc::clone(&control));
10583        let connection = router.begin_connection();
10584        let (ctx, mut rx) = route_ctx(connection.id());
10585
10586        router
10587            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
10588            .await
10589            .unwrap();
10590        let response = rx.recv().await.unwrap();
10591        let ack = parse_ack(&response);
10592        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
10593        let channel = 1;
10594
10595        let goodbye =
10596            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
10597        router.route_for_connection(&ctx, goodbye).await.unwrap();
10598        assert!(rx.try_recv().is_err());
10599        assert!(registry.get_module("aft").unwrap().is_none());
10600
10601        router
10602            .route_for_connection(&ctx, channel_request(channel, 13))
10603            .await
10604            .unwrap();
10605        let error_frame = rx.recv().await.unwrap();
10606        assert_eq!(error_frame.header.ty, FrameType::Error);
10607        assert_eq!(error_frame.header.channel, channel);
10608    }
10609
10610    #[tokio::test]
10611    async fn dropping_router_connection_releases_registration() {
10612        let registry = Arc::new(Registry::default());
10613        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
10614        let router = Router::with_control_handler(control);
10615        let connection = router.begin_connection();
10616        let (ctx, mut rx) = route_ctx(connection.id());
10617
10618        router
10619            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
10620            .await
10621            .unwrap();
10622        let response = rx.recv().await.unwrap();
10623        let ack = parse_ack(&response);
10624        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
10625        assert!(registry.get_module("aft").unwrap().is_some());
10626
10627        drop(connection);
10628
10629        assert!(registry.get_module("aft").unwrap().is_none());
10630        assert_eq!(registry.active_registration_count().unwrap(), 0);
10631    }
10632
10633    fn capability_manifest(
10634        module_id: &str,
10635        provides: &[&str],
10636        must_never_reach: &[&str],
10637    ) -> ModuleManifest {
10638        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
10639        manifest.capabilities = Some(CapabilityDeclarations {
10640            provides: provides
10641                .iter()
10642                .map(|capability| (*capability).to_string())
10643                .collect(),
10644            requires: Vec::new(),
10645            must_never_reach: must_never_reach
10646                .iter()
10647                .map(|capability| (*capability).to_string())
10648                .collect(),
10649        });
10650        manifest
10651    }
10652
10653    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
10654        Frame::build(
10655            FrameType::Hello,
10656            control_flags(),
10657            0,
10658            0,
10659            corr,
10660            serde_json::to_vec(&ModuleHelloBody {
10661                protocol_ver: manifest.protocol_ver,
10662                manifest,
10663                control_ops: None,
10664                launch_nonce: None,
10665            })
10666            .expect("capability test HELLO serializes"),
10667        )
10668        .expect("capability test HELLO frame builds")
10669    }
10670
10671    fn catalog_update_with_capabilities_frame(
10672        corr: u64,
10673        capabilities: CapabilityDeclarations,
10674    ) -> Frame {
10675        Frame::build(
10676            FrameType::Request,
10677            control_flags(),
10678            0,
10679            0,
10680            corr,
10681            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
10682                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
10683                capabilities: Some(capabilities),
10684                ready: None,
10685            })
10686            .expect("capability catalog.update serializes"),
10687        )
10688        .expect("capability catalog.update frame builds")
10689    }
10690
10691    async fn register_capability_manifest(
10692        handler: &ControlHandler,
10693        ctx: &RouteCtx,
10694        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10695        manifest: ModuleManifest,
10696        corr: u64,
10697    ) {
10698        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
10699    }
10700
10701    async fn open_route_for_capability_test(
10702        handler: &ControlHandler,
10703        target_ctx: &RouteCtx,
10704        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10705        client_connection_id: u64,
10706        corr: u64,
10707        target_module_id: &str,
10708        consumer_identity: Option<ConsumerIdentity>,
10709    ) -> (
10710        mpsc::Receiver<crate::router::OutboundFrame>,
10711        ModuleControlRequest,
10712    ) {
10713        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
10714        let route_handler = handler.clone();
10715        let target_module_id = target_module_id.to_string();
10716        let route_task = tokio::spawn(async move {
10717            route_handler
10718                .handle_control_frame(
10719                    &client_ctx,
10720                    route_open_frame_with_admission_facts(
10721                        corr,
10722                        &target_module_id,
10723                        unique_project_root("admission-facts"),
10724                        consumer_identity,
10725                        None,
10726                    ),
10727                )
10728                .await
10729                .expect("capability test route.open succeeds")
10730        });
10731        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
10732            .await
10733            .expect("capability test route.open must reach route.bind")
10734            .expect("target control receiver stays open");
10735        let bind_request: ModuleControlRequest =
10736            serde_json::from_slice(&bind.body).expect("route.bind decodes");
10737        handler
10738            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
10739            .await
10740            .expect("capability test route.bind ACK succeeds");
10741        assert!(route_task.await.expect("route.open task joins").is_empty());
10742        let opened = client_rx
10743            .recv()
10744            .await
10745            .expect("successful route.open publishes a response");
10746        assert!(matches!(
10747            serde_json::from_slice::<ClientControlResponse>(&opened.body),
10748            Ok(ClientControlResponse::RouteOpen { .. })
10749        ));
10750        (client_rx, bind_request)
10751    }
10752
10753    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
10754        assert_eq!(frame.header.ty, FrameType::Push);
10755        assert_eq!(frame.header.channel, 0);
10756        assert_eq!(
10757            serde_json::from_slice::<ClientControlPush>(&frame.body)
10758                .expect("route.closed control push decodes"),
10759            ClientControlPush::RouteClosed {
10760                module_id: target_module_id.to_string(),
10761                reason: RouteCloseReason::CapabilityDenied,
10762                drained: false,
10763                abandoned: 0,
10764                excluded_subscriptions: 0,
10765                terminal: Some(false),
10766            }
10767        );
10768    }
10769
10770    #[tokio::test]
10771    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
10772        let registry = Arc::new(Registry::default());
10773        let forwarding = Arc::new(ForwardingTable::default());
10774        let supervisor = SupervisorHandle::new();
10775        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10776        let handler =
10777            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10778                .with_supervisor(supervisor);
10779        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
10780        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
10781        register_capability_manifest(
10782            &handler,
10783            &target_ctx,
10784            &mut target_rx,
10785            capability_manifest("target", &["credentials-provider/v1"], &[]),
10786            1,
10787        )
10788        .await;
10789        register_capability_manifest(
10790            &handler,
10791            &opener_ctx,
10792            &mut opener_rx,
10793            capability_manifest("opener", &[], &["credentials-provider/v1"]),
10794            2,
10795        )
10796        .await;
10797
10798        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
10799        let replies = handler
10800            .handle_control_frame(
10801                &client_ctx,
10802                route_open_frame_with_admission_facts(
10803                    3,
10804                    "target",
10805                    unique_project_root("admission-facts"),
10806                    Some(ConsumerIdentity {
10807                        module_id: "opener".to_string(),
10808                        launch_nonce: "opener-nonce".to_string(),
10809                    }),
10810                    None,
10811                ),
10812            )
10813            .await
10814            .expect("denied route.open returns a typed frame");
10815        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
10816        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10817        assert!(
10818            target_rx.try_recv().is_err(),
10819            "forbidden route.open must not relay route.bind"
10820        );
10821    }
10822
10823    #[tokio::test]
10824    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
10825        let registry = Arc::new(Registry::default());
10826        let forwarding = Arc::new(ForwardingTable::default());
10827        let supervisor = SupervisorHandle::new();
10828        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10829        let handler =
10830            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10831                .with_supervisor(supervisor);
10832        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
10833        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
10834        register_capability_manifest(
10835            &handler,
10836            &target_ctx,
10837            &mut target_rx,
10838            capability_manifest("target", &["credentials-provider/v1"], &[]),
10839            1,
10840        )
10841        .await;
10842        register_capability_manifest(
10843            &handler,
10844            &old_opener_ctx,
10845            &mut old_opener_rx,
10846            capability_manifest("opener", &[], &[]),
10847            2,
10848        )
10849        .await;
10850        let (mut client_rx, _) = open_route_for_capability_test(
10851            &handler,
10852            &target_ctx,
10853            &mut target_rx,
10854            712,
10855            3,
10856            "target",
10857            Some(ConsumerIdentity {
10858                module_id: "opener".to_string(),
10859                launch_nonce: "opener-nonce".to_string(),
10860            }),
10861        )
10862        .await;
10863        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10864
10865        handler
10866            .cleanup_connection(old_opener_ctx.connection_id)
10867            .expect("old opener registration cleans up");
10868        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
10869        register_capability_manifest(
10870            &handler,
10871            &new_opener_ctx,
10872            &mut new_opener_rx,
10873            capability_manifest("opener", &[], &["credentials-provider/v1"]),
10874            4,
10875        )
10876        .await;
10877
10878        assert_capability_denied_push(
10879            client_rx
10880                .try_recv()
10881                .expect("HELLO deny addition must emit route.closed")
10882                .frame,
10883            "target",
10884        );
10885        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10886        assert!(matches!(
10887            target_rx.try_recv(),
10888            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
10889        ));
10890    }
10891
10892    #[tokio::test]
10893    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
10894        let registry = Arc::new(Registry::default());
10895        let forwarding = Arc::new(ForwardingTable::default());
10896        let supervisor = SupervisorHandle::new();
10897        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10898        let handler =
10899            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10900                .with_supervisor(supervisor);
10901        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
10902        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
10903        register_capability_manifest(
10904            &handler,
10905            &target_ctx,
10906            &mut target_rx,
10907            capability_manifest("target", &[], &[]),
10908            1,
10909        )
10910        .await;
10911        register_capability_manifest(
10912            &handler,
10913            &opener_ctx,
10914            &mut opener_rx,
10915            capability_manifest("opener", &[], &["credentials-provider/v1"]),
10916            2,
10917        )
10918        .await;
10919        let (mut client_rx, _) = open_route_for_capability_test(
10920            &handler,
10921            &target_ctx,
10922            &mut target_rx,
10923            722,
10924            3,
10925            "target",
10926            Some(ConsumerIdentity {
10927                module_id: "opener".to_string(),
10928                launch_nonce: "opener-nonce".to_string(),
10929            }),
10930        )
10931        .await;
10932        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10933
10934        let replies = handler
10935            .handle_control_frame(
10936                &target_ctx,
10937                catalog_update_with_capabilities_frame(
10938                    4,
10939                    CapabilityDeclarations {
10940                        provides: vec!["credentials-provider/v1".to_string()],
10941                        requires: Vec::new(),
10942                        must_never_reach: Vec::new(),
10943                    },
10944                ),
10945            )
10946            .await
10947            .expect("claim catalog.update succeeds");
10948        assert!(matches!(
10949            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
10950            Ok(ModuleControlResponseToModule::CatalogUpdate {})
10951        ));
10952        assert_capability_denied_push(
10953            client_rx
10954                .try_recv()
10955                .expect("claim addition must emit route.closed")
10956                .frame,
10957            "target",
10958        );
10959        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10960        assert!(matches!(
10961            target_rx.try_recv(),
10962            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
10963        ));
10964    }
10965
10966    #[tokio::test]
10967    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
10968        let registry = Arc::new(Registry::default());
10969        let forwarding = Arc::new(ForwardingTable::default());
10970        let supervisor = SupervisorHandle::new();
10971        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10972        let handler =
10973            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10974                .with_supervisor(supervisor);
10975        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
10976        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
10977        register_capability_manifest(
10978            &handler,
10979            &target_ctx,
10980            &mut target_rx,
10981            capability_manifest("target", &["credentials-provider/v1"], &[]),
10982            1,
10983        )
10984        .await;
10985        register_capability_manifest(
10986            &handler,
10987            &opener_ctx,
10988            &mut opener_rx,
10989            capability_manifest("opener", &[], &[]),
10990            2,
10991        )
10992        .await;
10993        let (mut client_rx, _) = open_route_for_capability_test(
10994            &handler,
10995            &target_ctx,
10996            &mut target_rx,
10997            732,
10998            3,
10999            "target",
11000            Some(ConsumerIdentity {
11001                module_id: "opener".to_string(),
11002                launch_nonce: "opener-nonce".to_string(),
11003            }),
11004        )
11005        .await;
11006        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11007
11008        handler
11009            .handle_control_frame(
11010                &target_ctx,
11011                catalog_update_with_capabilities_frame(
11012                    4,
11013                    CapabilityDeclarations {
11014                        provides: Vec::new(),
11015                        requires: Vec::new(),
11016                        must_never_reach: Vec::new(),
11017                    },
11018                ),
11019            )
11020            .await
11021            .expect("claim removal catalog.update succeeds");
11022        assert_eq!(
11023            forwarding.active_binding_count().unwrap(),
11024            1,
11025            "removing an attested target claim must leave the route census unchanged"
11026        );
11027        assert!(
11028            client_rx.try_recv().is_err(),
11029            "claim removal must not emit route.closed capability_denied"
11030        );
11031        assert!(
11032            target_rx.try_recv().is_err(),
11033            "claim removal must not send the target a route GOODBYE"
11034        );
11035    }
11036
11037    /// A direct client may open a route to a denied capability provider; this
11038    /// policy applies only to attested supervised module origins, not to direct clients.
11039    #[tokio::test]
11040    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
11041        let registry = Arc::new(Registry::default());
11042        let forwarding = Arc::new(ForwardingTable::default());
11043        let supervisor = SupervisorHandle::new();
11044        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11045        let handler =
11046            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11047                .with_supervisor(supervisor);
11048        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
11049        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
11050        register_capability_manifest(
11051            &handler,
11052            &target_ctx,
11053            &mut target_rx,
11054            capability_manifest("target", &["credentials-provider/v1"], &[]),
11055            1,
11056        )
11057        .await;
11058        register_capability_manifest(
11059            &handler,
11060            &opener_ctx,
11061            &mut opener_rx,
11062            capability_manifest("opener", &[], &["credentials-provider/v1"]),
11063            2,
11064        )
11065        .await;
11066
11067        let (_client_rx, bind) = open_route_for_capability_test(
11068            &handler,
11069            &target_ctx,
11070            &mut target_rx,
11071            742,
11072            3,
11073            "target",
11074            None,
11075        )
11076        .await;
11077        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11078            panic!("direct scope-honesty route must bind");
11079        };
11080        assert_eq!(principal, Some(Principal::Direct));
11081        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11082    }
11083
11084    /// A module that denies a capability receives no self-route exemption when it
11085    /// also attestedly provides that capability.
11086    #[tokio::test]
11087    async fn must_never_reach_self_route_is_capability_forbidden() {
11088        let registry = Arc::new(Registry::default());
11089        let forwarding = Arc::new(ForwardingTable::default());
11090        let supervisor = SupervisorHandle::new();
11091        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
11092        let handler =
11093            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11094                .with_supervisor(supervisor);
11095        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
11096        register_capability_manifest(
11097            &handler,
11098            &self_ctx,
11099            &mut self_rx,
11100            capability_manifest(
11101                "self-provider",
11102                &["credentials-provider/v1"],
11103                &["credentials-provider/v1"],
11104            ),
11105            1,
11106        )
11107        .await;
11108
11109        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
11110        let replies = handler
11111            .handle_control_frame(
11112                &client_ctx,
11113                route_open_frame_with_admission_facts(
11114                    2,
11115                    "self-provider",
11116                    unique_project_root("admission-facts"),
11117                    Some(ConsumerIdentity {
11118                        module_id: "self-provider".to_string(),
11119                        launch_nonce: "self-nonce".to_string(),
11120                    }),
11121                    None,
11122                ),
11123            )
11124            .await
11125            .expect("self-route refusal returns a typed frame");
11126        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
11127        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11128        assert!(
11129            self_rx.try_recv().is_err(),
11130            "self denial must not relay route.bind"
11131        );
11132    }
11133
11134    #[test]
11135    fn unsupported_channel_zero_frame_returns_error() {
11136        let handler = ControlHandler::default();
11137        let request = Frame::build(
11138            FrameType::Request,
11139            control_flags(),
11140            0,
11141            0,
11142            21,
11143            b"opaque".to_vec(),
11144        )
11145        .unwrap();
11146
11147        let response = handler
11148            .handle_control(ConnectionId::new(1), request)
11149            .unwrap();
11150
11151        assert_eq!(response[0].header.ty, FrameType::Error);
11152        assert_eq!(
11153            parse_error(&response[0])["code"],
11154            "unsupported_control_frame"
11155        );
11156    }
11157
11158    /// Blue/green swap at the control-plane boundary. The supervisor that opens
11159    /// a swap is not wired yet, so the candidate is registered here directly
11160    /// into the registry and forwarding candidate slots, the way the swap's
11161    /// HELLO admission will.
11162    mod swap {
11163        use super::*;
11164
11165        const INCUMBENT: ConnectionId = ConnectionId::new(30);
11166        const CANDIDATE: ConnectionId = ConnectionId::new(40);
11167
11168        struct Swap {
11169            registry: Arc<Registry>,
11170            forwarding: Arc<ForwardingTable>,
11171            handler: ControlHandler,
11172            incumbent_ctx: RouteCtx,
11173            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
11174            candidate_ctx: RouteCtx,
11175            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
11176        }
11177
11178        async fn swap_with_incumbent() -> Swap {
11179            let registry = Arc::new(Registry::default());
11180            let forwarding = Arc::new(ForwardingTable::default());
11181            let handler =
11182                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11183            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
11184            hello_via_sink(
11185                &handler,
11186                &incumbent_ctx,
11187                &mut incumbent_rx,
11188                hello_frame("aft", PROTOCOL_VERSION, 7),
11189            )
11190            .await;
11191            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
11192            Swap {
11193                registry,
11194                forwarding,
11195                handler,
11196                incumbent_ctx,
11197                incumbent_rx,
11198                candidate_ctx,
11199                candidate_rx,
11200            }
11201        }
11202
11203        fn register_candidate(swap: &Swap, ready: Option<bool>) {
11204            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
11205            candidate_manifest.ready = ready;
11206            let registration = swap
11207                .registry
11208                .register_candidate_with_control_ops(
11209                    candidate_manifest,
11210                    PROTOCOL_VERSION,
11211                    CANDIDATE,
11212                    module_baseline_control_ops(),
11213                )
11214                .unwrap();
11215            swap.forwarding
11216                .register_candidate_module_connection(
11217                    CANDIDATE,
11218                    "aft".to_string(),
11219                    PROTOCOL_VERSION,
11220                    manifest_concurrency(&registration.manifest),
11221                    swap.candidate_ctx.egress.clone(),
11222                )
11223                .unwrap();
11224        }
11225
11226        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
11227            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
11228            swap.registry.promote_candidate("aft").unwrap().unwrap();
11229            cutover.incumbent.unwrap()
11230        }
11231
11232        fn keyed_total(counters: &Value, key: &str) -> u64 {
11233            counters[key]
11234                .as_object()
11235                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
11236                .unwrap_or(0)
11237        }
11238
11239        /// An ack from the incumbent for a bind it was sent before cutover,
11240        /// arriving before the incumbent is drained. The incumbent is the live
11241        /// connection carrying every other client's routes, so the ack must
11242        /// not end it: the waiting client is told to retry, the reservation is
11243        /// given back, and the incumbent is told to drop just that binding.
11244        #[tokio::test]
11245        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
11246            let mut swap = swap_with_incumbent().await;
11247            let handler = swap.handler.clone();
11248
11249            // A co-tenant route, bound on the incumbent before the swap.
11250            let cotenant = ConnectionId::new(31);
11251            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
11252            let (cotenant_task, cotenant_bind) = relay_route_open(
11253                &handler,
11254                cotenant,
11255                &cotenant_ctx.egress,
11256                &mut swap.incumbent_rx,
11257                100,
11258                "aft",
11259                "swap-cotenant",
11260            )
11261            .await;
11262            handler
11263                .handle_control_frame(
11264                    &swap.incumbent_ctx,
11265                    route_bind_ack(cotenant_bind.header.corr),
11266                )
11267                .await
11268                .unwrap();
11269            assert!(cotenant_task.await.unwrap().is_empty());
11270            let (cotenant_channel, cotenant_epoch) =
11271                published_route(&cotenant_rx.recv().await.unwrap());
11272
11273            // A second route.open, relayed to the incumbent and not yet acked.
11274            let caller = ConnectionId::new(32);
11275            let (caller_ctx, mut caller_rx) = route_ctx(caller);
11276            let (caller_task, caller_bind) = relay_route_open(
11277                &handler,
11278                caller,
11279                &caller_ctx.egress,
11280                &mut swap.incumbent_rx,
11281                101,
11282                "aft",
11283                "swap-caller",
11284            )
11285            .await;
11286            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
11287
11288            register_candidate(&swap, None);
11289            cutover(&swap);
11290
11291            // The incumbent acks after promotion and before any drain.
11292            let ack = handler
11293                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
11294                .await;
11295            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
11296            if module_loop_error.is_some() {
11297                // What the connection loop does with an untranslated router
11298                // error: end the connection, releasing every route on it.
11299                handler.cleanup_connection(INCUMBENT).unwrap();
11300            }
11301
11302            // 1. The incumbent's other routes survive.
11303            assert!(
11304                cotenant_rx.try_recv().is_err(),
11305                "the co-tenant route on the incumbent was torn down by one late ack: \
11306                 {module_loop_error:?}"
11307            );
11308            assert!(matches!(
11309                swap.forwarding
11310                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
11311                    .unwrap(),
11312                DataRoute::Client(DataRouteState::Bound(_))
11313            ));
11314            assert_eq!(module_loop_error, None);
11315            assert!(swap
11316                .registry
11317                .get_module_by_connection(INCUMBENT)
11318                .unwrap()
11319                .is_some());
11320
11321            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
11322            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
11323                .await
11324                .expect("the incumbent is told to drop the abandoned binding")
11325                .unwrap()
11326                .frame;
11327            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11328            assert_eq!(goodbye.header.channel, abandoned_channel);
11329            assert_eq!(goodbye.header.epoch, abandoned_epoch);
11330            assert!(swap.incumbent_rx.try_recv().is_err());
11331
11332            // 3. The waiting client gets a retryable refusal and no route.
11333            let response = caller_task.await.unwrap();
11334            assert_eq!(response.len(), 1);
11335            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
11336            assert!(caller_rx.try_recv().is_err());
11337
11338            // 4. The reservation pair is given back, and the pending bind
11339            //    settled exactly once: one accepted open (the co-tenant) and one
11340            //    refused open (the caller), nothing counted twice.
11341            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
11342            let counters = handler.counters().snapshot();
11343            assert_eq!(
11344                keyed_total(&counters, "route_open_accepted_by_principal"),
11345                1
11346            );
11347            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
11348            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
11349        }
11350
11351        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
11352        /// id would resolve to the promoted candidate and every new route.open
11353        /// would be refused as reloading, leaving neither process routable.
11354        #[tokio::test]
11355        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
11356            let mut swap = swap_with_incumbent().await;
11357            register_candidate(&swap, None);
11358            let incumbent = cutover(&swap);
11359            swap.forwarding
11360                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
11361                .unwrap()
11362                .expect("the incumbent is still registered");
11363
11364            let client = ConnectionId::new(33);
11365            let (client_ctx, mut client_rx) = route_ctx(client);
11366            let route_handler = swap.handler.clone();
11367            let open_ctx = RouteCtx {
11368                connection_id: client,
11369                egress: client_ctx.egress.clone(),
11370            };
11371            let mut route_task = tokio::spawn(async move {
11372                route_handler
11373                    .handle_control_frame(
11374                        &open_ctx,
11375                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
11376                    )
11377                    .await
11378                    .unwrap()
11379            });
11380            let bind = tokio::select! {
11381                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
11382                response = &mut route_task => {
11383                    let response = response.unwrap();
11384                    panic!(
11385                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
11386                        parse_error(&response[0])["code"]
11387                    );
11388                }
11389            };
11390            swap.handler
11391                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
11392                .await
11393                .unwrap();
11394            assert!(route_task.await.unwrap().is_empty());
11395            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
11396            match swap
11397                .forwarding
11398                .lookup_data_route(client, channel, epoch)
11399                .unwrap()
11400            {
11401                DataRoute::Client(DataRouteState::Bound(route)) => {
11402                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
11403                }
11404                other => panic!("expected a bound route on the candidate, got {other:?}"),
11405            }
11406            assert!(swap.incumbent_rx.try_recv().is_err());
11407        }
11408
11409        /// A candidate declares itself ready with `catalog.update` on its own
11410        /// connection. If the connection-keyed registry lookups searched only the
11411        /// active slot, this would answer `not_registered` and the candidate
11412        /// would never become ready.
11413        #[tokio::test]
11414        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
11415            let swap = swap_with_incumbent().await;
11416            register_candidate(&swap, Some(false));
11417            let update = Frame::build(
11418                FrameType::Request,
11419                control_flags(),
11420                0,
11421                0,
11422                55,
11423                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
11424                    provides: manifest("aft", PROTOCOL_VERSION).provides,
11425                    capabilities: None,
11426                    ready: Some(true),
11427                })
11428                .unwrap(),
11429            )
11430            .unwrap();
11431
11432            let replies = swap
11433                .handler
11434                .handle_control_frame(&swap.candidate_ctx, update)
11435                .await
11436                .unwrap();
11437
11438            assert_eq!(replies.len(), 1);
11439            assert_eq!(
11440                replies[0].header.ty,
11441                FrameType::Response,
11442                "candidate catalog.update was refused: {:?}",
11443                serde_json::from_slice::<Value>(&replies[0].body).ok()
11444            );
11445            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
11446            assert_eq!(
11447                swap.registry
11448                    .get_module("aft")
11449                    .unwrap()
11450                    .unwrap()
11451                    .connection_id,
11452                INCUMBENT
11453            );
11454        }
11455    }
11456
11457    /// The HELLO gate while the supervisor has a swap open: only the nonce it
11458    /// minted for the candidate admits a second process, into the candidate
11459    /// slot, and that check runs ahead of the reserved-module gate.
11460    mod swap_admission {
11461        use super::*;
11462
11463        const INCUMBENT_NONCE: &str = "incumbent-nonce";
11464        const CANDIDATE_NONCE: &str = "candidate-nonce";
11465
11466        fn handler_with_incumbent(
11467            module_id: &str,
11468            reserved: bool,
11469        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
11470            let registry = Arc::new(Registry::default());
11471            let supervisor = SupervisorHandle::new();
11472            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
11473            if reserved {
11474                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
11475            }
11476            let handler =
11477                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
11478            let incumbent = handler
11479                .handle_control(
11480                    ConnectionId::new(1),
11481                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
11482                )
11483                .unwrap();
11484            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
11485            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
11486            (registry, supervisor, handler)
11487        }
11488
11489        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
11490        /// admits every nonce, so while a swap is open the swap gate is the only
11491        /// thing between a key-holder and the candidate slot. A nonce the
11492        /// supervisor did not mint, or none at all, is refused, and neither the
11493        /// incumbent's registration nor the candidate slot moves.
11494        #[test]
11495        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
11496            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
11497
11498            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
11499                let replies = handler
11500                    .handle_control(
11501                        ConnectionId::new(connection),
11502                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
11503                    )
11504                    .unwrap();
11505                assert_eq!(replies[0].header.ty, FrameType::Error);
11506                assert_eq!(
11507                    parse_error(&replies[0])["code"],
11508                    "swap_token_invalid",
11509                    "nonce {nonce:?}"
11510                );
11511            }
11512            assert!(registry.get_candidate("aft").unwrap().is_none());
11513            assert_eq!(
11514                registry.get_module("aft").unwrap().unwrap().connection_id,
11515                ConnectionId::new(1)
11516            );
11517
11518            // Control: the minted token is admitted, into the candidate slot,
11519            // and only once.
11520            let admitted = handler
11521                .handle_control(
11522                    ConnectionId::new(4),
11523                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
11524                )
11525                .unwrap();
11526            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
11527            assert_eq!(
11528                registry
11529                    .get_candidate("aft")
11530                    .unwrap()
11531                    .unwrap()
11532                    .connection_id,
11533                ConnectionId::new(4)
11534            );
11535            assert_eq!(
11536                registry.get_module("aft").unwrap().unwrap().connection_id,
11537                ConnectionId::new(1),
11538                "the candidate must not take the active slot"
11539            );
11540            let replayed = handler
11541                .handle_control(
11542                    ConnectionId::new(5),
11543                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
11544                )
11545                .unwrap();
11546            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
11547
11548            // The case only this gate covers: the incumbent has died mid-swap,
11549            // so its duplicate refusal is gone too, and without the gate a
11550            // key-holder would take the id's ACTIVE slot.
11551            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
11552            let squatter = handler
11553                .handle_control(
11554                    ConnectionId::new(6),
11555                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
11556                )
11557                .unwrap();
11558            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
11559            assert!(
11560                registry.get_module("aft").unwrap().is_none(),
11561                "a squatter took the active slot of an id being swapped"
11562            );
11563        }
11564
11565        /// Design mutation arm (iii). A reserved module's candidate presents a
11566        /// nonce the reserved gate has never seen (that gate holds the
11567        /// incumbent's), so the swap gate must run first or the candidate is
11568        /// refused `reserved_module` and a reserved module can never be swapped.
11569        #[test]
11570        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
11571            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
11572
11573            let replies = handler
11574                .handle_control(
11575                    ConnectionId::new(2),
11576                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
11577                )
11578                .unwrap();
11579
11580            assert_eq!(
11581                replies[0].header.ty,
11582                FrameType::HelloAck,
11583                "reserved candidate refused: {:?}",
11584                serde_json::from_slice::<Value>(&replies[0].body).ok()
11585            );
11586            assert_eq!(
11587                registry
11588                    .get_candidate("vault")
11589                    .unwrap()
11590                    .unwrap()
11591                    .connection_id,
11592                ConnectionId::new(2)
11593            );
11594        }
11595
11596        /// With no swap open the gate is inert: the incumbent's reserved gate
11597        /// and duplicate refusal behave exactly as before.
11598        #[test]
11599        fn without_an_open_swap_the_ordinary_gates_decide() {
11600            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
11601            supervisor.close_swap("vault");
11602
11603            let candidate = handler
11604                .handle_control(
11605                    ConnectionId::new(2),
11606                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
11607                )
11608                .unwrap();
11609            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
11610            let duplicate = handler
11611                .handle_control(
11612                    ConnectionId::new(3),
11613                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
11614                )
11615                .unwrap();
11616            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11617            assert!(registry.get_candidate("vault").unwrap().is_none());
11618        }
11619    }
11620
11621    /// `scope.sync` and `scope.describe` through the real control handler: who
11622    /// may sync is decided by the registration and launch nonce of the module
11623    /// connection, never by the request body.
11624    mod scopes {
11625        use subc_protocol::scope::{
11626            ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
11627            ScopeStatus,
11628        };
11629
11630        use super::*;
11631
11632        const OWNER: &str = "prefrontal-core";
11633
11634        fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
11635            ScopeRecord {
11636                scope_ref: scope_ref.to_string(),
11637                scope_epoch,
11638                kind: ScopeKind::Head,
11639                parent: None,
11640                child_owners: Vec::new(),
11641                carriers: Vec::new(),
11642                attributes: Default::default(),
11643            }
11644        }
11645
11646        async fn call(
11647            handler: &ControlHandler,
11648            ctx: &RouteCtx,
11649            request: &ModuleControlRequestFromModule,
11650        ) -> Frame {
11651            let body = serde_json::to_vec(request).unwrap();
11652            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
11653            let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
11654            assert_eq!(replies.len(), 1, "{replies:?}");
11655            replies.pop().unwrap()
11656        }
11657
11658        async fn sync(
11659            handler: &ControlHandler,
11660            ctx: &RouteCtx,
11661            generation: u64,
11662            scopes: Vec<ScopeRecord>,
11663        ) -> Result<ModuleControlResponseToModule, String> {
11664            let reply = call(
11665                handler,
11666                ctx,
11667                &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
11668            )
11669            .await;
11670            match reply.header.ty {
11671                FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
11672                _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
11673            }
11674        }
11675
11676        async fn describe(
11677            handler: &ControlHandler,
11678            ctx: &RouteCtx,
11679            owner: &str,
11680            scope_ref: &str,
11681        ) -> ModuleControlResponseToModule {
11682            let reply = call(
11683                handler,
11684                ctx,
11685                &ModuleControlRequestFromModule::ScopeDescribe {
11686                    owner: Principal::Reserved {
11687                        module_id: owner.to_string(),
11688                    },
11689                    scope_ref: scope_ref.to_string(),
11690                },
11691            )
11692            .await;
11693            assert_eq!(
11694                reply.header.ty,
11695                FrameType::Response,
11696                "{:?}",
11697                parse_error(&reply)
11698            );
11699            serde_json::from_slice(&reply.body).unwrap()
11700        }
11701
11702        /// Register `module_id` on `connection` with `nonce`, returning its ctx.
11703        async fn module(
11704            handler: &ControlHandler,
11705            connection: u64,
11706            module_id: &str,
11707            nonce: Option<&str>,
11708        ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
11709            let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
11710            hello_via_sink(
11711                handler,
11712                &ctx,
11713                &mut rx,
11714                hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
11715            )
11716            .await;
11717            (ctx, rx)
11718        }
11719
11720        /// `direct` and every other client connection has no registration, so
11721        /// it can neither sync nor own a scope.
11722        #[tokio::test]
11723        async fn a_client_connection_cannot_sync_or_describe() {
11724            let handler = ControlHandler::new(Arc::new(Registry::default()));
11725            let (ctx, _rx) = route_ctx(ConnectionId::new(9));
11726            for request in [
11727                ModuleControlRequestFromModule::ScopeSync {
11728                    generation: 1,
11729                    scopes: vec![head("s", 1)],
11730                },
11731                ModuleControlRequestFromModule::ScopeDescribe {
11732                    owner: Principal::Direct,
11733                    scope_ref: "s".to_string(),
11734                },
11735            ] {
11736                let reply = call(&handler, &ctx, &request).await;
11737                assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
11738            }
11739            assert!(
11740                !handler
11741                    .scopes
11742                    .read()
11743                    .unwrap()
11744                    .describe(
11745                        &Principal::Reserved {
11746                            module_id: OWNER.to_string()
11747                        },
11748                        "s"
11749                    )
11750                    .owner_synced
11751            );
11752        }
11753
11754        /// A module the supervisor did not spawn registers without a launch
11755        /// nonce, so it is never an owner's current launch.
11756        #[tokio::test]
11757        async fn a_module_without_a_supervised_launch_cannot_sync() {
11758            let handler = ControlHandler::new(Arc::new(Registry::default()));
11759            let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
11760            assert_eq!(
11761                sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
11762                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
11763            );
11764        }
11765
11766        #[tokio::test]
11767        async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
11768            let supervisor = SupervisorHandle::new();
11769            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
11770            let handler = ControlHandler::new(Arc::new(Registry::default()))
11771                .with_supervisor(supervisor.clone());
11772            let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
11773            sync(&handler, &incumbent, 1, vec![head("s", 1)])
11774                .await
11775                .expect("the current launch syncs");
11776
11777            // A swap candidate registers with the swap token and is refused
11778            // while the incumbent keeps syncing.
11779            supervisor.open_swap(OWNER, "n2".to_string());
11780            let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
11781            assert_eq!(
11782                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
11783                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
11784            );
11785            sync(&handler, &incumbent, 2, vec![head("s", 1)])
11786                .await
11787                .expect("the serving owner syncs during the swap");
11788
11789            // The swap fails and is rolled back. The candidate never held sync
11790            // authority, and still cannot sync.
11791            supervisor.close_swap(OWNER);
11792            assert_eq!(
11793                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
11794                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
11795            );
11796            sync(&handler, &incumbent, 3, vec![head("s", 1)])
11797                .await
11798                .expect("the serving owner syncs after the rollback");
11799            handler.cleanup_connection(candidate.connection_id).unwrap();
11800
11801            // A swap that cuts over. Promotion records the candidate's nonce as
11802            // the module's spawn nonce, which is what `set_spawn_nonce` does
11803            // here; the promoted connection then takes authority at any
11804            // generation and the superseded incumbent is refused.
11805            supervisor.open_swap(OWNER, "n3".to_string());
11806            let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
11807            supervisor.set_spawn_nonce(OWNER, "n3".to_string());
11808            let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
11809                .await
11810                .expect("the promoted launch takes authority");
11811            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
11812                panic!("unexpected reply {reply:?}");
11813            };
11814            assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
11815            assert_eq!(
11816                sync(&handler, &incumbent, 4, Vec::new()).await,
11817                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
11818            );
11819        }
11820
11821        /// Authority dies with its connection: the cleanup path releases it,
11822        /// so the owner's next connection takes it at any generation.
11823        #[tokio::test]
11824        async fn closing_the_authority_connection_frees_sync_authority() {
11825            let supervisor = SupervisorHandle::new();
11826            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
11827            let handler = ControlHandler::new(Arc::new(Registry::default()))
11828                .with_supervisor(supervisor.clone());
11829            let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
11830            sync(&handler, &first, 10, vec![head("s", 1)])
11831                .await
11832                .unwrap();
11833            handler.cleanup_connection(first.connection_id).unwrap();
11834
11835            let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
11836            sync(&handler, &second, 1, vec![head("s", 1)])
11837                .await
11838                .expect("the next connection takes the released authority");
11839        }
11840
11841        #[tokio::test]
11842        async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
11843            let registry = Arc::new(Registry::default());
11844            let supervisor_handle = SupervisorHandle::new();
11845            let supervisor = Supervisor::new(Arc::clone(&registry), RestartPolicy::default())
11846                .with_handle(supervisor_handle.clone())
11847                .with_daemon_incarnation("incarnation-7".to_string());
11848            // Configured with enabled: false, so the supervisor lists the
11849            // module without spawning a process for it.
11850            supervisor
11851                .supervise_configured(
11852                    ModuleSpec {
11853                        module_id: OWNER.to_string(),
11854                        program: PathBuf::from("/nonexistent/prefrontal-core"),
11855                        args: Vec::new(),
11856                        env: Vec::new(),
11857                        reserved: false,
11858                        reserved_prefixes: Vec::new(),
11859                        protocol: ModuleProtocol::Subc,
11860                        overlap: Default::default(),
11861                    },
11862                    false,
11863                )
11864                .unwrap();
11865            supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
11866            let handler =
11867                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
11868            let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
11869
11870            // Configured but not yet synced: a reader waits for the owner.
11871            let ModuleControlResponseToModule::ScopeDescribe {
11872                status,
11873                daemon_incarnation,
11874                owner_synced,
11875                owner_configured,
11876                scope,
11877                ..
11878            } = describe(&handler, &reader, OWNER, "s").await
11879            else {
11880                panic!("not a describe reply");
11881            };
11882            assert_eq!(status, ScopeStatus::NotLive);
11883            assert_eq!(daemon_incarnation, "incarnation-7");
11884            assert!(!owner_synced);
11885            assert!(owner_configured);
11886            assert!(scope.is_none());
11887
11888            // Not a supervised module: the owner will never sync, and a reader
11889            // refuses rather than waits.
11890            let ModuleControlResponseToModule::ScopeDescribe {
11891                status,
11892                owner_configured,
11893                ..
11894            } = describe(&handler, &reader, "ghost", "s").await
11895            else {
11896                panic!("not a describe reply");
11897            };
11898            assert_eq!(status, ScopeStatus::NotLive);
11899            assert!(!owner_configured);
11900
11901            // Live, with the stamp fields and the computed owner_authorized.
11902            let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
11903            sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
11904            let ModuleControlResponseToModule::ScopeDescribe {
11905                status,
11906                scope_epoch,
11907                owner_synced,
11908                scope,
11909                ..
11910            } = describe(&handler, &reader, OWNER, "s").await
11911            else {
11912                panic!("not a describe reply");
11913            };
11914            assert_eq!(status, ScopeStatus::Live);
11915            assert_eq!(scope_epoch, Some(4));
11916            assert!(owner_synced);
11917            let stamp = scope.expect("a live scope carries its stamp");
11918            assert!(
11919                stamp.owner_authorized,
11920                "prefrontal-core is the default authority"
11921            );
11922            assert_eq!(stamp.kind, ScopeKind::Head);
11923        }
11924
11925        #[tokio::test]
11926        async fn scope_authority_owners_decides_owner_authorized() {
11927            let supervisor = SupervisorHandle::new();
11928            supervisor.set_spawn_nonce("broca", "b1".to_string());
11929            let handler = ControlHandler::new(Arc::new(Registry::default()))
11930                .with_supervisor(supervisor)
11931                .with_scope_authority_owners(vec!["broca".to_string()]);
11932            let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
11933            let mut gated = head("s", 1);
11934            gated.attributes.agent_id = Some("agent".to_string());
11935            sync(&handler, &broca, 1, vec![gated]).await.unwrap();
11936            let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
11937                describe(&handler, &broca, "broca", "s").await
11938            else {
11939                panic!("not a describe reply");
11940            };
11941            assert!(scope.unwrap().owner_authorized);
11942        }
11943
11944        /// With route admission, the stamp, the commit re-check and drains in
11945        /// place, the feature is advertised: the module ops in HELLO_ACK, and
11946        /// `scopes/v1` in HELLO_ACK and `server.describe`.
11947        #[tokio::test]
11948        async fn scope_ops_and_the_scopes_capability_are_advertised() {
11949            let handler = ControlHandler::new(Arc::new(Registry::default()));
11950            let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
11951            let ack = hello_via_sink(
11952                &handler,
11953                &ctx,
11954                &mut rx,
11955                hello_frame("m", PROTOCOL_VERSION, 1),
11956            )
11957            .await;
11958            let ack = parse_ack(&ack);
11959            for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
11960                assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
11961            }
11962            assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
11963
11964            let (client, _client_rx) = route_ctx(ConnectionId::new(2));
11965            let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11966            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11967            let reply = handler
11968                .handle_control_frame(&client, frame)
11969                .await
11970                .unwrap()
11971                .pop()
11972                .unwrap();
11973            let ClientControlResponse::ServerDescribe { capabilities, .. } =
11974                serde_json::from_slice(&reply.body).unwrap()
11975            else {
11976                panic!("not a server.describe reply");
11977            };
11978            assert!(
11979                capabilities.iter().any(|c| c == CAP_SCOPES_V1),
11980                "{capabilities:?}"
11981            );
11982        }
11983
11984        // ---- route admission, stamps, commit re-check and drains ----------
11985
11986        const PLEXUS: &str = "plexus";
11987        const OTHER: &str = "other";
11988        const AFT: &str = "aft";
11989        const BROCA: &str = "broca";
11990        const MAGIC: &str = "magic-context";
11991
11992        fn nonce(module_id: &str) -> String {
11993            format!("nonce-{module_id}")
11994        }
11995
11996        fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
11997            let (tx, rx) = mpsc::channel(64);
11998            (
11999                RouteCtx {
12000                    connection_id: ConnectionId::new(connection),
12001                    egress: FrameSink::new(tx),
12002                },
12003                rx,
12004            )
12005        }
12006
12007        /// A daemon with a configured owner (prefrontal-core) registered on its
12008        /// own module connection, two routable targets (plexus, other), and
12009        /// launch nonces minted for the modules that open routes as carriers.
12010        struct Rig {
12011            handler: ControlHandler,
12012            forwarding: Arc<ForwardingTable>,
12013            owner: RouteCtx,
12014            _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12015            modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
12016            generation: u64,
12017            next_connection: u64,
12018            _supervisor: Supervisor,
12019        }
12020
12021        async fn rig() -> Rig {
12022            let registry = Arc::new(Registry::default());
12023            let forwarding = Arc::new(ForwardingTable::default());
12024            let supervisor_handle = SupervisorHandle::new();
12025            let supervisor = Supervisor::new(Arc::clone(&registry), RestartPolicy::default())
12026                .with_handle(supervisor_handle.clone());
12027            supervisor
12028                .supervise_configured(
12029                    ModuleSpec {
12030                        module_id: OWNER.to_string(),
12031                        program: PathBuf::from("/nonexistent/prefrontal-core"),
12032                        args: Vec::new(),
12033                        env: Vec::new(),
12034                        reserved: false,
12035                        reserved_prefixes: Vec::new(),
12036                        protocol: ModuleProtocol::Subc,
12037                        overlap: Default::default(),
12038                    },
12039                    false,
12040                )
12041                .unwrap();
12042            for module_id in [OWNER, AFT, BROCA, MAGIC] {
12043                supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
12044            }
12045            let handler =
12046                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12047                    .with_supervisor(supervisor_handle);
12048            let (owner, mut owner_rx) = wide_ctx(1);
12049            hello_via_sink(
12050                &handler,
12051                &owner,
12052                &mut owner_rx,
12053                hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
12054            )
12055            .await;
12056            let mut modules = BTreeMap::new();
12057            for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
12058                let (ctx, mut rx) = wide_ctx(connection);
12059                hello_via_sink(
12060                    &handler,
12061                    &ctx,
12062                    &mut rx,
12063                    hello_frame(module_id, PROTOCOL_VERSION, connection),
12064                )
12065                .await;
12066                modules.insert(module_id.to_string(), (ctx, rx));
12067            }
12068            Rig {
12069                handler,
12070                forwarding,
12071                owner,
12072                _owner_rx: owner_rx,
12073                modules,
12074                generation: 0,
12075                next_connection: 100,
12076                _supervisor: supervisor,
12077            }
12078        }
12079
12080        fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
12081            ScopeCarrier {
12082                principal: Principal::Reserved {
12083                    module_id: module_id.to_string(),
12084                },
12085                targets: targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()),
12086            }
12087        }
12088
12089        /// The scope most tests open under: aft carries to any module, broca
12090        /// only to plexus and other, and the owner delegates as agent-1.
12091        fn session(scope_epoch: u64) -> ScopeRecord {
12092            let mut record = head("s", scope_epoch);
12093            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
12094            record.attributes.agent_id = Some("agent-1".to_string());
12095            record.attributes.delegates = true;
12096            record
12097        }
12098
12099        impl Rig {
12100            async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
12101                self.generation += 1;
12102                sync(&self.handler, &self.owner, self.generation, scopes)
12103                    .await
12104                    .expect("the owner's sync is accepted");
12105            }
12106
12107            fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
12108                ScopeSelector {
12109                    owner: Principal::Reserved {
12110                        module_id: OWNER.to_string(),
12111                    },
12112                    scope_ref: scope_ref.to_string(),
12113                    scope_epoch,
12114                }
12115            }
12116
12117            fn open_frame(
12118                &mut self,
12119                opener: Option<&str>,
12120                target: &str,
12121                scope: Option<ScopeSelector>,
12122            ) -> (
12123                RouteCtx,
12124                mpsc::Receiver<crate::router::OutboundFrame>,
12125                Frame,
12126            ) {
12127                self.next_connection += 1;
12128                let (ctx, rx) = wide_ctx(self.next_connection);
12129                let root = unique_project_root("scoped-open");
12130                let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
12131                    target: RouteTarget::ToolProvider {
12132                        module_id: target.to_string(),
12133                    },
12134                    identity: BindIdentity::new(
12135                        root.path().to_path_buf(),
12136                        "unit".to_string(),
12137                        "session".to_string(),
12138                    ),
12139                    consumer_identity: opener.map(|module_id| ConsumerIdentity {
12140                        module_id: module_id.to_string(),
12141                        launch_nonce: nonce(module_id),
12142                    }),
12143                    consumer_capabilities: None,
12144                    role_versions: None,
12145                    admission_facts: None,
12146                    scope,
12147                })
12148                .unwrap();
12149                let frame = Frame::build(
12150                    FrameType::Request,
12151                    control_flags(),
12152                    0,
12153                    0,
12154                    self.next_connection,
12155                    body,
12156                )
12157                .unwrap();
12158                (ctx, rx, frame)
12159            }
12160
12161            /// Open and expect a refusal before anything is relayed.
12162            async fn refused(
12163                &mut self,
12164                opener: Option<&str>,
12165                target: &str,
12166                scope: Option<ScopeSelector>,
12167            ) -> String {
12168                let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
12169                let replies = self
12170                    .handler
12171                    .handle_control_frame(&ctx, frame)
12172                    .await
12173                    .unwrap();
12174                assert_eq!(replies.len(), 1, "{replies:?}");
12175                assert_eq!(replies[0].header.ty, FrameType::Error);
12176                let (_, module_rx) = self.modules.get_mut(target).unwrap();
12177                assert!(
12178                    module_rx.try_recv().is_err(),
12179                    "a refused open relays nothing"
12180                );
12181                parse_error(&replies[0])["code"]
12182                    .as_str()
12183                    .unwrap()
12184                    .to_string()
12185            }
12186
12187            /// Start an open and return its task and the bind the target got.
12188            async fn relayed(
12189                &mut self,
12190                opener: Option<&str>,
12191                target: &str,
12192                scope: Option<ScopeSelector>,
12193            ) -> Relayed {
12194                let (ctx, rx, frame) = self.open_frame(opener, target, scope);
12195                let handler = self.handler.clone();
12196                let task_ctx = ctx.clone();
12197                let task = tokio::spawn(async move {
12198                    handler
12199                        .handle_control_frame(&task_ctx, frame)
12200                        .await
12201                        .unwrap()
12202                });
12203                let (_, module_rx) = self.modules.get_mut(target).unwrap();
12204                let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
12205                    .await
12206                    .expect("the target receives the relayed route.bind")
12207                    .unwrap()
12208                    .frame;
12209                Relayed {
12210                    target: target.to_string(),
12211                    client: ctx,
12212                    client_rx: rx,
12213                    task,
12214                    bind,
12215                }
12216            }
12217
12218            async fn ack(&self, relayed: &Relayed) {
12219                let (module, _) = &self.modules[&relayed.target];
12220                self.handler
12221                    .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
12222                    .await
12223                    .unwrap();
12224            }
12225
12226            /// Open, ack and return the bound route.
12227            async fn bound(
12228                &mut self,
12229                opener: Option<&str>,
12230                target: &str,
12231                scope: Option<ScopeSelector>,
12232            ) -> Bound {
12233                let relayed = self.relayed(opener, target, scope).await;
12234                self.ack(&relayed).await;
12235                let Relayed {
12236                    target,
12237                    client,
12238                    mut client_rx,
12239                    task,
12240                    bind,
12241                } = relayed;
12242                assert!(
12243                    task.await.unwrap().is_empty(),
12244                    "the open is answered by commit"
12245                );
12246                let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
12247                Bound {
12248                    target,
12249                    client,
12250                    client_rx,
12251                    channel,
12252                    epoch,
12253                    bind,
12254                }
12255            }
12256
12257            fn live(&self, route: &Bound) -> bool {
12258                matches!(
12259                    self.forwarding
12260                        .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
12261                        .unwrap(),
12262                    DataRoute::Client(DataRouteState::Bound(_))
12263                )
12264            }
12265        }
12266
12267        struct Relayed {
12268            target: String,
12269            client: RouteCtx,
12270            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12271            task: tokio::task::JoinHandle<Vec<Frame>>,
12272            bind: Frame,
12273        }
12274
12275        struct Bound {
12276            target: String,
12277            client: RouteCtx,
12278            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12279            channel: u16,
12280            epoch: u32,
12281            bind: Frame,
12282        }
12283
12284        impl Bound {
12285            /// The reason of the `route.closed` this client was sent, after
12286            /// checking it also got a GOODBYE on exactly this route.
12287            fn closed_reason(&mut self) -> RouteCloseReason {
12288                let mut reason = None;
12289                let mut goodbye = false;
12290                while let Ok(outbound) = self.client_rx.try_recv() {
12291                    let frame = outbound.frame;
12292                    match frame.header.ty {
12293                        FrameType::Goodbye => {
12294                            assert_eq!(
12295                                (frame.header.channel, frame.header.epoch),
12296                                (self.channel, self.epoch)
12297                            );
12298                            goodbye = true;
12299                        }
12300                        FrameType::Push => {
12301                            let ClientControlPush::RouteClosed {
12302                                reason: r,
12303                                module_id,
12304                                ..
12305                            } = serde_json::from_slice(&frame.body).unwrap()
12306                            else {
12307                                panic!("unexpected push");
12308                            };
12309                            assert_eq!(module_id, self.target);
12310                            reason = Some(r);
12311                        }
12312                        other => panic!("unexpected frame {other:?}"),
12313                    }
12314                }
12315                assert!(goodbye, "the client is sent a GOODBYE for the closed route");
12316                reason.expect("the client is told why the route closed")
12317            }
12318
12319            fn untouched(&mut self) -> bool {
12320                self.client_rx.try_recv().is_err()
12321            }
12322
12323            fn stamp(&self) -> Option<ScopeStamp> {
12324                match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
12325                    ModuleControlRequest::RouteBind { scope, .. } => scope,
12326                    other => panic!("expected a route.bind, got {other:?}"),
12327                }
12328            }
12329        }
12330
12331        #[tokio::test]
12332        async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
12333        ) {
12334            let mut rig = rig().await;
12335            let mut record = session(1);
12336            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
12337            record.child_owners = vec![Principal::Reserved {
12338                module_id: MAGIC.to_string(),
12339            }];
12340            rig.sync(vec![record]).await;
12341            let scope = || Some(rig_selector("s", Some(1)));
12342
12343            // Admitted: the owner, a bare carrier to any module, a targeted
12344            // carrier to its listed module.
12345            rig.bound(Some(OWNER), PLEXUS, scope()).await;
12346            rig.bound(Some(AFT), OTHER, scope()).await;
12347            rig.bound(Some(BROCA), PLEXUS, scope()).await;
12348
12349            // Refused scope_not_carrier: a targeted carrier to an unlisted
12350            // module, a module that is not listed at all (a child owner is not
12351            // a carrier), and a direct key-holder.
12352            for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
12353                assert_eq!(
12354                    rig.refused(opener, target, scope()).await,
12355                    error_codes::SCOPE_NOT_CARRIER,
12356                    "{opener:?} -> {target}"
12357                );
12358            }
12359        }
12360
12361        fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
12362            ScopeSelector {
12363                owner: Principal::Reserved {
12364                    module_id: OWNER.to_string(),
12365                },
12366                scope_ref: scope_ref.to_string(),
12367                scope_epoch,
12368            }
12369        }
12370
12371        #[tokio::test]
12372        async fn an_open_without_an_epoch_is_refused_the_owners_included() {
12373            let mut rig = rig().await;
12374            rig.sync(vec![session(1)]).await;
12375            for opener in [OWNER, AFT] {
12376                assert_eq!(
12377                    rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
12378                        .await,
12379                    error_codes::SCOPE_EPOCH_REQUIRED,
12380                    "{opener}"
12381                );
12382            }
12383        }
12384
12385        #[tokio::test]
12386        async fn admission_separates_not_synced_not_live_and_ended() {
12387            let mut rig = rig().await;
12388            // Before the configured owner's first sync: retryable.
12389            let code = rig
12390                .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12391                .await;
12392            assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
12393            assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
12394
12395            // An owner that is not configured will never sync: terminal.
12396            let ghost = ScopeSelector {
12397                owner: Principal::Reserved {
12398                    module_id: "ghost".to_string(),
12399                },
12400                scope_ref: "s".to_string(),
12401                scope_epoch: Some(1),
12402            };
12403            assert_eq!(
12404                rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
12405                error_codes::SCOPE_NOT_LIVE
12406            );
12407
12408            rig.sync(vec![session(2)]).await;
12409            assert_eq!(
12410                rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
12411                    .await,
12412                error_codes::SCOPE_NOT_LIVE
12413            );
12414            for epoch in [1, 3] {
12415                assert_eq!(
12416                    rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
12417                        .await,
12418                    error_codes::SCOPE_ENDED,
12419                    "epoch {epoch}"
12420                );
12421            }
12422            // Control: the live epoch is admitted.
12423            rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
12424                .await;
12425        }
12426
12427        #[tokio::test]
12428        async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
12429            let mut rig = rig().await;
12430            rig.sync(vec![session(1)]).await;
12431            let route = rig
12432                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12433                .await;
12434            let stamp = route.stamp().expect("a scoped bind carries the stamp");
12435            assert_eq!(stamp.scope_ref, "s");
12436            assert_eq!(stamp.scope_epoch, 1);
12437            assert_eq!(stamp.kind, ScopeKind::Head);
12438            assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
12439            assert!(stamp.attributes.delegates);
12440            assert!(stamp.owner_authorized);
12441            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
12442            assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
12443
12444            // broca owns a scope of its own on its own module connection; it is
12445            // not in scope_authority_owners, so its stamp is not authorized.
12446            let (broca, mut broca_rx) = wide_ctx(50);
12447            hello_via_sink(
12448                &rig.handler,
12449                &broca,
12450                &mut broca_rx,
12451                hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
12452            )
12453            .await;
12454            sync(&rig.handler, &broca, 1, vec![head("b", 1)])
12455                .await
12456                .unwrap();
12457            let own = ScopeSelector {
12458                owner: Principal::Reserved {
12459                    module_id: BROCA.to_string(),
12460                },
12461                scope_ref: "b".to_string(),
12462                scope_epoch: Some(1),
12463            };
12464            let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
12465            assert!(!route.stamp().unwrap().owner_authorized);
12466        }
12467
12468        /// The owner's sync lands between admission and the module's ack. The
12469        /// open is refused by name, the module's other routes stay up, and the
12470        /// reserved pair is released. Changed content is retryable; an ended
12471        /// scope is not.
12472        #[tokio::test]
12473        async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
12474            let mut rig = rig().await;
12475            rig.sync(vec![session(1)]).await;
12476            let mut cotenant = rig
12477                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
12478                .await;
12479
12480            let mut changed = session(1);
12481            changed.child_owners.push(Principal::Reserved {
12482                module_id: MAGIC.to_string(),
12483            });
12484            let mut ended = None;
12485            for (code, next) in [
12486                (error_codes::SCOPE_CHANGED, vec![changed]),
12487                (error_codes::SCOPE_ENDED, Vec::new()),
12488            ] {
12489                let relayed = rig
12490                    .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12491                    .await;
12492                let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
12493                ended = Some(next.is_empty());
12494                rig.sync(next).await;
12495                rig.ack(&relayed).await;
12496                let replies = relayed.task.await.unwrap();
12497                assert_eq!(replies.len(), 1, "{replies:?}");
12498                assert_eq!(parse_error(&replies[0])["code"], code);
12499                assert_eq!(
12500                    subc_protocol::error_codes::is_retryable_route_open(code),
12501                    code == error_codes::SCOPE_CHANGED
12502                );
12503                assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
12504                // The module is told to drop just the binding it created.
12505                let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
12506                // Collected, because ending the scope also closes the co-tenant
12507                // route, whose GOODBYE comes first.
12508                let mut goodbyes = Vec::new();
12509                while let Ok(outbound) = plexus_rx.try_recv() {
12510                    assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
12511                    goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
12512                }
12513                assert!(
12514                    goodbyes.contains(&(bind_channel, bind_epoch)),
12515                    "{goodbyes:?}"
12516                );
12517                assert!(rig
12518                    .handler
12519                    .registry
12520                    .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
12521                    .unwrap()
12522                    .is_some());
12523            }
12524            assert_eq!(ended, Some(true));
12525            // The co-tenant stayed up through the change, and closed only when
12526            // the scope ended, by the drain rule rather than by the commit.
12527            assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
12528        }
12529
12530        /// Each row of the drain table on one set of routes: the owner's, a
12531        /// bare carrier's, and a targeted carrier's to each of its targets.
12532        #[tokio::test]
12533        async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
12534            struct Case {
12535                name: &'static str,
12536                change: fn(&mut ScopeRecord),
12537                /// Closed routes by index: owner->plexus, aft->plexus,
12538                /// broca->plexus, broca->other.
12539                closed: [Option<RouteCloseReason>; 4],
12540            }
12541            use RouteCloseReason::*;
12542            let cases = [
12543                Case {
12544                    name: "a carrier entry removed",
12545                    change: |r| {
12546                        r.carriers.retain(|c| {
12547                            c.principal
12548                                != Principal::Reserved {
12549                                    module_id: AFT.to_string(),
12550                                }
12551                        })
12552                    },
12553                    closed: [None, Some(ScopeCarrierRemoved), None, None],
12554                },
12555                Case {
12556                    name: "a target removed from a carrier",
12557                    change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
12558                    closed: [None, None, None, Some(ScopeCarrierRemoved)],
12559                },
12560                Case {
12561                    name: "a bare carrier narrowed to targets",
12562                    change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
12563                    closed: [None, Some(ScopeCarrierRemoved), None, None],
12564                },
12565                Case {
12566                    name: "delegates turned off",
12567                    change: |r| r.attributes.delegates = false,
12568                    closed: [Some(ScopeDelegationChanged); 4],
12569                },
12570                Case {
12571                    name: "agent_id changed",
12572                    change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
12573                    closed: [Some(ScopeDelegationChanged); 4],
12574                },
12575                Case {
12576                    name: "a carrier added, child owners changed, the record re-sent",
12577                    change: |r| {
12578                        r.carriers.push(carrier(MAGIC, None));
12579                        r.child_owners.push(Principal::Reserved {
12580                            module_id: MAGIC.to_string(),
12581                        });
12582                    },
12583                    closed: [None; 4],
12584                },
12585                Case {
12586                    name: "a target added",
12587                    change: |r| {
12588                        r.carriers[1]
12589                            .targets
12590                            .as_mut()
12591                            .unwrap()
12592                            .push("third".to_string())
12593                    },
12594                    closed: [None; 4],
12595                },
12596                Case {
12597                    name: "delegates turned on",
12598                    change: |r| r.attributes.delegates = true,
12599                    closed: [None; 4],
12600                },
12601            ];
12602            for case in cases {
12603                let mut rig = rig().await;
12604                rig.sync(vec![session(1)]).await;
12605                let scope = || Some(rig_selector("s", Some(1)));
12606                let mut routes = [
12607                    rig.bound(Some(OWNER), PLEXUS, scope()).await,
12608                    rig.bound(Some(AFT), PLEXUS, scope()).await,
12609                    rig.bound(Some(BROCA), PLEXUS, scope()).await,
12610                    rig.bound(Some(BROCA), OTHER, scope()).await,
12611                ];
12612                let mut record = session(1);
12613                (case.change)(&mut record);
12614                rig.sync(vec![record]).await;
12615                for (index, expected) in case.closed.iter().enumerate() {
12616                    let route = &mut routes[index];
12617                    match expected {
12618                        Some(reason) => {
12619                            assert!(!rig.live(route), "{}: route {index} still live", case.name);
12620                            assert_eq!(
12621                                route.closed_reason(),
12622                                *reason,
12623                                "{}: route {index}",
12624                                case.name
12625                            );
12626                        }
12627                        None => {
12628                            assert!(rig.live(route), "{}: route {index} closed", case.name);
12629                            assert!(
12630                                route.untouched(),
12631                                "{}: route {index} was told something",
12632                                case.name
12633                            );
12634                        }
12635                    }
12636                }
12637            }
12638        }
12639
12640        #[tokio::test]
12641        async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
12642            // Removed, and replaced by a higher epoch.
12643            for next in [Vec::new(), vec![session(2)]] {
12644                let mut rig = rig().await;
12645                rig.sync(vec![session(1)]).await;
12646                let mut route = rig
12647                    .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12648                    .await;
12649                rig.sync(next).await;
12650                assert!(!rig.live(&route));
12651                assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
12652            }
12653
12654            // A child whose parent ends: its routes close as parent-ended, the
12655            // child stays live, and routes under the parent close as ended.
12656            let mut rig = rig().await;
12657            let mut child = session(1);
12658            child.scope_ref = "child".to_string();
12659            child.kind = ScopeKind::Worker;
12660            child.parent = Some(ScopeParent {
12661                owner: Principal::Reserved {
12662                    module_id: OWNER.to_string(),
12663                },
12664                scope_ref: "s".to_string(),
12665                scope_epoch: 1,
12666            });
12667            rig.sync(vec![session(1), child.clone()]).await;
12668            let mut child_route = rig
12669                .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
12670                .await;
12671            assert_eq!(
12672                child_route.stamp().unwrap().parent_state,
12673                Some(ParentState::Linked)
12674            );
12675            rig.sync(vec![child]).await;
12676            assert!(!rig.live(&child_route));
12677            assert_eq!(
12678                child_route.closed_reason(),
12679                RouteCloseReason::ScopeParentEnded
12680            );
12681        }
12682
12683        #[tokio::test]
12684        async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
12685        ) {
12686            let mut rig = rig().await;
12687            rig.sync(vec![session(1)]).await;
12688            let mut route = rig
12689                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12690                .await;
12691            let before = rig.forwarding.published_scope_tag(OWNER, "s");
12692            rig.sync(vec![session(1)]).await;
12693            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
12694            assert!(rig.live(&route) && route.untouched());
12695
12696            // A call in flight on the route when another carrier is added. A
12697            // forwarded REQUEST holds one credit on the route's flow until the
12698            // module answers; the router takes it exactly like this.
12699            let DataRoute::Client(DataRouteState::Bound(binding)) = rig
12700                .forwarding
12701                .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
12702                .unwrap()
12703            else {
12704                panic!("the route is bound");
12705            };
12706            binding.flow.acquire_tagged(9, false).await.unwrap();
12707            let mut widened = session(1);
12708            widened.carriers.push(carrier(MAGIC, None));
12709            rig.sync(vec![widened]).await;
12710            assert!(rig.live(&route) && route.untouched());
12711            let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
12712            assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
12713            // The call's credit is still held on an open flow, so its answer
12714            // will be delivered: closing the route would have closed the flow.
12715            assert_eq!(binding.flow.in_flight(), 1);
12716            binding
12717                .flow
12718                .acquire_tagged(10, false)
12719                .await
12720                .expect("the flow is still open");
12721        }
12722
12723        /// A swap's superseded endpoint keeps its routes until drained; ending
12724        /// the scope closes them there too.
12725        #[tokio::test]
12726        async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
12727            let mut rig = rig().await;
12728            rig.sync(vec![session(1)]).await;
12729            let mut on_incumbent = rig
12730                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12731                .await;
12732
12733            // Swap plexus: register a candidate and cut over, leaving the
12734            // incumbent superseded with the route still on it.
12735            let (candidate, _candidate_rx) = wide_ctx(9);
12736            let registration = rig
12737                .handler
12738                .registry
12739                .register_candidate_with_control_ops(
12740                    manifest(PLEXUS, PROTOCOL_VERSION),
12741                    PROTOCOL_VERSION,
12742                    candidate.connection_id,
12743                    module_baseline_control_ops(),
12744                )
12745                .unwrap();
12746            rig.forwarding
12747                .register_candidate_module_connection(
12748                    candidate.connection_id,
12749                    PLEXUS.to_string(),
12750                    PROTOCOL_VERSION,
12751                    manifest_concurrency(&registration.manifest),
12752                    candidate.egress.clone(),
12753                )
12754                .unwrap();
12755            rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
12756            rig.handler
12757                .registry
12758                .promote_candidate(PLEXUS)
12759                .unwrap()
12760                .unwrap();
12761            assert!(rig.live(&on_incumbent), "cutover alone does not drain");
12762
12763            rig.sync(Vec::new()).await;
12764            assert!(!rig.live(&on_incumbent));
12765            assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
12766            let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
12767            let goodbye = incumbent_rx
12768                .try_recv()
12769                .expect("the superseded endpoint is told")
12770                .frame;
12771            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12772        }
12773    }
12774}
12775
12776#[cfg(test)]
12777mod concurrency_default_exposure_tests {
12778    use super::*;
12779
12780    fn hello_body(role_json: &str) -> Vec<u8> {
12781        format!(
12782            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
12783        )
12784        .into_bytes()
12785    }
12786
12787    fn manifest_from(body: &[u8]) -> ModuleManifest {
12788        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
12789        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
12790            .expect("manifest parses")
12791    }
12792
12793    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
12794
12795    #[test]
12796    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
12797        let body = hello_body(&format!(
12798            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
12799        ));
12800        let manifest = manifest_from(&body);
12801        // Precondition: serde really resolved it to the default, so the typed
12802        // manifest alone cannot answer the question this probe exists for.
12803        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
12804        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
12805    }
12806
12807    #[test]
12808    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
12809        let body = hello_body(&format!(
12810            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
12811        ));
12812        let manifest = manifest_from(&body);
12813        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
12814        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
12815    }
12816
12817    #[test]
12818    fn non_management_roles_are_never_reported() {
12819        let body = hello_body(
12820            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
12821        );
12822        let manifest = manifest_from(&body);
12823        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
12824    }
12825}