Skip to main content

subc_daemon/
control.rs

1use std::{
2    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3    fmt,
4    path::{Path, PathBuf},
5    sync::{Arc, Mutex, RwLock},
6    time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14    ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20    error_codes,
21    manifest::{
22        validate_hello_capability_grammar, validate_hello_self_signal_declarations,
23        CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
24        ProviderRole,
25    },
26    scope::{
27        ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
28        CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_DESCRIBE_OP, SCOPE_SYNC_OP,
29    },
30    session::{
31        validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
32        ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
33        MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE, ROLE_VERSIONS_FIELD,
34    },
35    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
36    Priority, RouteTarget, PROTOCOL_VERSION,
37};
38use tokio::time::{timeout_at, Instant};
39use tracing::{debug, info, warn};
40
41use crate::{
42    capability_requirements::{
43        log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
44        CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
45        RuntimeModule,
46    },
47    daemon_config::RestartRequiredSection,
48    forwarding::{
49        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
50        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
51        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
52    },
53    observability::{
54        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
55    },
56    provenance::{
57        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
58    },
59    registry::{ChannelState, ConnectionId, Registry, RegistryError},
60    router::{RouteCtx, RouterError},
61    scopes::{BoundScope, HelloLaunchNonces, ScopeTable},
62    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
63    stderr_tail::{CaptureState, TailEntry},
64    supervise::{
65        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
66        SupervisorHandle, SwapHelloAdmission,
67    },
68    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
69};
70
71/// Lowest envelope version this subc build will negotiate.
72///
73/// Module HELLO negotiation is exact: peers must use the daemon's locked
74/// protocol version. Older and newer peers receive `version_unsupported` and
75/// are not registered.
76pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
77
78const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
79const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
80const CAP_PING_PONG: &str = "ping_pong_v1";
81const CAP_SESSION_ATTACH: &str = "session_attach_v1";
82const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
83
84const SUBC_CONTROL_OPS: &[&str] = &[
85    ops::SERVER_DESCRIBE,
86    ops::CATALOG_LIST,
87    ops::ROUTE_OPEN,
88    ops::ROUTE_POLL,
89    ops::ROUTE_CLOSING,
90    ops::ROUTE_CLOSED,
91    ops::SUPERVISOR_LIST,
92    ops::SUPERVISOR_RESTART,
93    ops::SUPERVISOR_SWAP,
94    ops::SUPERVISOR_RELOAD,
95    ops::SUPERVISOR_RESCAN,
96    ops::SUPERVISOR_RELEASE_RESERVED,
97    ops::SUPERVISOR_SET_ENABLED,
98    ops::SUPERVISOR_HEALTH_PROBE,
99    ops::SUPERVISOR_HEALTH,
100    ops::SUPERVISOR_STDERR_TAIL,
101    ops::SUPERVISOR_TERMINALS,
102    ops::SUPERVISOR_ROUTES,
103    ops::SUPERVISOR_PROVENANCE,
104    ops::SUPERVISOR_SPAWN_SNAPSHOT,
105    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
106];
107
108const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
109    MODULE_TO_SUBC_OP_CATALOG_UPDATE,
110    "supervisor.live_roots",
111    SCOPE_SYNC_OP,
112    SCOPE_DESCRIBE_OP,
113];
114
115/// Module-originated ops the daemon answers but does not advertise in
116/// `HELLO_ACK`. Empty today; an op is served from here while the feature it
117/// belongs to is incomplete, so no module is told it works before it does.
118const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
119
120const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
121
122/// How long subc waits for a module to ack a relayed route.bind before returning
123/// `module_timeout`. The ack waits on the module's own configure, which for AFT
124/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
125/// and DB-open work — on a cold page cache or a large repo that legitimately
126/// exceeds a couple of seconds. The default is generous because rejecting a VALID
127/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
128/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
129pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
130
131/// How many CONSECUTIVE full-budget relay timeouts against one target module
132/// open that module's bind-relay breaker.
133///
134/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
135/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
136/// so three consecutive full-budget timeouts take ~36s to observe: every client
137/// whose open contributed to opening the breaker had already given up on its
138/// own. That is what makes opening the breaker unable to turn a call that would
139/// have succeeded into a refusal — it can only make an already-failing module
140/// fail faster.
141///
142/// Two would be reachable inside one default deadline. One would convict a
143/// module on a single cold-cache bind, which is exactly the valid-but-slow case
144/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
145pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
146
147/// How long a module's bind-relay breaker stays open before exactly one
148/// `route.open` is let through as a probe.
149///
150/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
151/// re-pays a full-budget stall almost continuously, and the breaker stops being
152/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
153/// deadline: a client that starts retrying after the module recovers has to get
154/// a probe opportunity inside its own deadline, or the breaker converts a
155/// recovered module into a failed call — the failure it exists to prevent,
156/// pointed the other way.
157///
158/// 20s sits between those with room on both sides, and it caps what a wedged
159/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
160/// rather than one per `route.open` per connection. The stall that motivated
161/// this, with its measurements, is written up in
162/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
163/// waited the whole budget out.
164pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
165
166const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
167const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
168
169fn reload_verdict(
170    configured: &Path,
171    spawned_from: Option<&Path>,
172    image: subc_control::RunningImageAgreement,
173) -> PendingReloadVerdict {
174    let path = match spawned_from {
175        Some(spawned_from) if configured == spawned_from => ReloadPathAgreement::Match,
176        Some(spawned_from) => ReloadPathAgreement::Mismatch {
177            configured: configured.to_path_buf(),
178            spawned_from: spawned_from.to_path_buf(),
179        },
180        None => ReloadPathAgreement::Unavailable {
181            reason: if matches!(
182                image,
183                subc_control::RunningImageAgreement::Unavailable {
184                    reason: subc_control::RunningImageUnavailableReason::NotRunning
185                }
186            ) {
187                ReloadPathUnavailableReason::NotRunning
188            } else {
189                ReloadPathUnavailableReason::SpawnedPathUnavailable
190            },
191        },
192    };
193    PendingReloadVerdict { path, image }
194}
195
196#[derive(Clone)]
197struct DaemonProvenanceFacts {
198    build: DaemonBuildProvenance,
199    pid: Option<u32>,
200    started_at_ms: Option<u64>,
201    start_clock: Option<crate::clock::StartClock>,
202    executable_path: Option<PathBuf>,
203    executable_identity: Option<SpawnedFileIdentity>,
204    process_start_time: Option<u64>,
205    probe: ExecutableIdentityProbe,
206}
207
208impl Default for DaemonProvenanceFacts {
209    fn default() -> Self {
210        Self {
211            build: DaemonBuildProvenance {
212                build_git_sha: None,
213                build_lock_digest: None,
214            },
215            pid: None,
216            started_at_ms: None,
217            start_clock: None,
218            executable_path: None,
219            executable_identity: None,
220            process_start_time: None,
221            probe: ExecutableIdentityProbe::default(),
222        }
223    }
224}
225
226#[derive(Debug, Clone)]
227struct SupervisorRescanContext {
228    supervisor: Supervisor,
229    config_path: PathBuf,
230    configured_port: Option<u16>,
231    storage_config: Option<crate::daemon_config::StorageConfig>,
232    admission_facts_carrier_module_id: Option<String>,
233    admission_facts_targets: Option<Vec<String>>,
234    scope_authority_owners: Vec<String>,
235}
236
237/// Refusal labels passed to `observe_route_open_refusal` that mean the target
238/// module is not serving right now, and so open or extend an outage in the
239/// route outage tracker. Every one of them is only reachable after the target
240/// was found in the registry, which is what keeps an arbitrary client-chosen
241/// id from ever creating tracker state.
242///
243/// Deliberately absent: `not_registered` and `removed` (the id may be
244/// anything a client sent, and a removed module is gone on purpose),
245/// `protocol_none` (such a module never serves routes, so nothing is out),
246/// `role_not_provided`, `op_not_allowed`, `bad_consumer_identity`, the
247/// capability and admission-facts refusals (they refuse the caller, not a
248/// module outage), and `relay_reservation_failed` (its code ranges over
249/// capacity limits as well as a vanished connection). Capacity, breaker,
250/// relay-timeout and module-rejection refusals do not pass through that
251/// function at all; the breaker logs its own transitions.
252///
253/// The two not-serving refusals that bypass that function record themselves
254/// at their own sites: `supervised_not_registered` and `declared_not_ready`.
255/// `required_capability_unprovided` is not tracked: the module itself is up,
256/// and the outage belongs to the missing provider.
257const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
258    "reloading",
259    "supervisor_not_live",
260    "registration_not_active",
261    "no_forwarding_connection",
262    "relay_send_failed",
263];
264
265/// Real channel-0 control handler for subc itself.
266#[derive(Clone)]
267pub struct ControlHandler {
268    registry: Arc<Registry>,
269    forwarding: Arc<ForwardingTable>,
270    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
271    supervisor: SupervisorHandle,
272    subc_capabilities: Arc<[String]>,
273    /// Daemon-wide route.bind relay budget. Used as the fallback when the
274    /// target module has no per-module override in
275    /// `route_bind_relay_timeouts`.
276    route_bind_relay_timeout: Duration,
277    /// Per-module route.bind relay budget overrides, keyed by module id. When
278    /// `handle_route_open` resolves the deadline for a target module, a
279    /// per-module entry wins over the daemon-wide value above.
280    route_bind_relay_timeouts: BTreeMap<String, Duration>,
281    /// Per-target-module bind-relay breaker state. Shared with the forwarding
282    /// table, which is where a new module connection resets it.
283    route_bind_breakers: RouteBindBreakers,
284    /// Live relay admissions keyed by target module. Shared through the
285    /// forwarding table so cloned or separately built handlers enforce one cap.
286    route_bind_concurrency: RouteBindConcurrency,
287    /// Start and end of each module's not-serving period as seen by
288    /// `route.open`, so an outage gets one line at each edge instead of only
289    /// the per-refusal INFO lines. Taken from the forwarding table, so every
290    /// handler built over one table shares it.
291    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
292    /// Consecutive relay timeouts that open a module's breaker.
293    route_bind_breaker_threshold: u32,
294    /// How long a breaker stays open before one probe is admitted.
295    route_bind_breaker_cooldown: Duration,
296    health_probe_timeout: Duration,
297    /// Central storage policy. When set, each registering module receives its
298    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
299    storage_config: Option<crate::daemon_config::StorageConfig>,
300    /// The machine id established at boot, served on every HELLO_ACK and on
301    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
302    /// changes the file, never this value. `None` serves no id.
303    machine_id: Option<crate::machine_id::MachineId>,
304    admission_facts_carrier_module_id: Option<String>,
305    admission_facts_targets: Option<Vec<String>>,
306    /// Scope records with their sync authorities and tombstones; see
307    /// `crate::scopes`. Shared by clones of this handler, so every connection
308    /// reads and writes one table.
309    scopes: Arc<RwLock<ScopeTable>>,
310    /// The configured `scope_authority_owners`, kept so a rescan can report a
311    /// changed value as needing a daemon restart; rescan never applies it.
312    scope_authority_owners: Vec<String>,
313    /// The launch nonce each module connection presented at HELLO, which is how
314    /// a `scope.sync` is matched to the owner's current launch.
315    hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
316    rescan: Option<SupervisorRescanContext>,
317    connected_clients: ConnectedClients,
318    counters: DaemonCounters,
319    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
320    daemon_provenance: DaemonProvenanceFacts,
321    #[cfg(test)]
322    control_dispatch_delay: Option<Duration>,
323    #[cfg(test)]
324    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
325}
326
327impl fmt::Debug for ControlHandler {
328    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
329        f.debug_struct("ControlHandler")
330            .field("registry", &self.registry)
331            .field("forwarding", &self.forwarding)
332            .field("process_liveness", &self.process_liveness.is_some())
333            .field("supervisor", &self.supervisor)
334            .field("subc_capabilities", &self.subc_capabilities)
335            .finish()
336    }
337}
338
339struct RouteOpenRequest {
340    target: RouteTarget,
341    identity: BindIdentity,
342    consumer_identity: Option<ConsumerIdentity>,
343    consumer_capabilities: Option<Vec<String>>,
344    role_versions: Option<BTreeMap<String, String>>,
345    admission_facts: Option<serde_json::Value>,
346    scope: Option<ScopeSelector>,
347}
348
349struct RouteBindReservationGuard {
350    forwarding: Arc<ForwardingTable>,
351    endpoint: ModuleEndpointId,
352    relay_corr: u64,
353    armed: bool,
354}
355
356struct ModuleControlRpcGuard {
357    forwarding: Arc<ForwardingTable>,
358    endpoint: ModuleEndpointId,
359    corr: u64,
360    armed: bool,
361}
362
363impl ModuleControlRpcGuard {
364    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
365        Self {
366            forwarding,
367            endpoint,
368            corr,
369            armed: true,
370        }
371    }
372
373    fn disarm(&mut self) {
374        self.armed = false;
375    }
376}
377
378impl Drop for ModuleControlRpcGuard {
379    fn drop(&mut self) {
380        if self.armed {
381            let _ = self
382                .forwarding
383                .cancel_module_control_rpc(self.endpoint, self.corr);
384        }
385    }
386}
387
388impl RouteBindReservationGuard {
389    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
390        Self {
391            forwarding,
392            endpoint,
393            relay_corr,
394            armed: true,
395        }
396    }
397
398    fn release_and_disarm(&mut self) {
399        if !self.armed {
400            return;
401        }
402        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
403            self.endpoint,
404            self.relay_corr,
405            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
406        ) {
407            send_goodbye_target_best_effort(
408                &self.forwarding.counters(),
409                &target,
410                "canceled route.bind",
411            );
412        }
413        self.armed = false;
414    }
415
416    fn disarm(&mut self) {
417        self.armed = false;
418    }
419}
420
421impl Drop for RouteBindReservationGuard {
422    fn drop(&mut self) {
423        self.release_and_disarm();
424    }
425}
426
427/// Per-target-module circuit breaker around the `route.bind` relay.
428///
429/// The connection reader is serial per connection, so a module whose `on_bind`
430/// sits on the ack blocks every LATER frame on the connections that call it,
431/// including calls to unrelated modules. This does not make any module's bind
432/// fast; it stops the daemon paying the full budget again and again for a
433/// condition it has already observed.
434///
435/// State is keyed by TARGET MODULE and shared by every connection: a wedged
436/// module wedges everyone, so what one connection learned should protect the
437/// rest.
438///
439/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
440/// relay to that module has actually timed out, and is removed again when a
441/// relay is accepted or the module reconnects, so it cannot grow with traffic
442/// or with modules that behave.
443///
444/// # Why a `std` mutex here is not the head-of-line defect again
445///
446/// Acquisition never awaits. The critical section is a hash lookup plus a few
447/// integer updates, with no I/O and no `.await` inside it, so a reader task
448/// cannot be descheduled behind it the way it can behind
449/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
450/// primitive, held for the same kind of work, as the refusal counter this very
451/// path already increments.
452///
453/// It is also NOT on the data-plane splice path: only `route.open` and module
454/// registration touch it, so bound-route frames gain no state check and no
455/// contention.
456#[derive(Debug, Clone, Default)]
457pub(crate) struct RouteBindBreakers {
458    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
459}
460
461#[derive(Debug, Clone, Default)]
462pub(crate) struct RouteBindConcurrency {
463    modules: Arc<Mutex<HashMap<String, usize>>>,
464}
465
466struct RouteBindConcurrencyGuard {
467    concurrency: RouteBindConcurrency,
468    module_id: String,
469}
470
471impl RouteBindConcurrency {
472    /// Admit without waiting. Waiting here would move the bind stall from the
473    /// module reply to a semaphore and restore reader head-of-line blocking.
474    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
475        let mut modules = self
476            .modules
477            .lock()
478            .expect("route.bind concurrency mutex poisoned");
479        let in_flight = modules.entry(module_id.to_string()).or_default();
480        if *in_flight >= limit {
481            return Err(*in_flight);
482        }
483        *in_flight += 1;
484        Ok(RouteBindConcurrencyGuard {
485            concurrency: self.clone(),
486            module_id: module_id.to_string(),
487        })
488    }
489}
490
491impl Drop for RouteBindConcurrencyGuard {
492    fn drop(&mut self) {
493        let mut modules = self
494            .concurrency
495            .modules
496            .lock()
497            .expect("route.bind concurrency mutex poisoned");
498        let remove = {
499            let in_flight = modules
500                .get_mut(&self.module_id)
501                .expect("admitted route.bind has a concurrency entry");
502            *in_flight -= 1;
503            *in_flight == 0
504        };
505        if remove {
506            modules.remove(&self.module_id);
507        }
508    }
509}
510
511#[derive(Debug, Default)]
512struct ModuleBreakerState {
513    /// Relay timeouts observed with no accepted relay in between.
514    consecutive_timeouts: u32,
515    /// `Some` while the breaker is open: the instant the cooldown expires and
516    /// the next arrival may probe. `None` means closed.
517    cooldown_until: Option<Instant>,
518    /// A half-open probe has been admitted and has not settled yet. This is
519    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
520    /// that read the cooldown, so concurrent opens arriving at the moment the
521    /// cooldown expires cannot all decide that they are the probe.
522    probe_in_flight: bool,
523}
524
525/// What the breaker decided for one `route.open`, before any relay work.
526enum RouteBindAdmission<'a> {
527    Admitted {
528        guard: RouteBindBreakerGuard<'a>,
529        /// This open is the single half-open probe, so the transition is worth
530        /// one log line.
531        probe: bool,
532    },
533    Refused {
534        consecutive_timeouts: u32,
535        /// What is left of the cooldown. Zero when the refusal is because the
536        /// one probe is already in flight rather than because the cooldown has
537        /// not elapsed.
538        retry_in: Duration,
539        probe_in_flight: bool,
540    },
541}
542
543/// An outstanding admission, which must be told how its relay settled.
544///
545/// `Drop` settles it as inconclusive, so an early return between admission and
546/// the relay -- or the whole handler being cancelled when the client
547/// disconnects -- releases a half-open probe slot instead of leaving the
548/// breaker wedged half-open with no further probes.
549struct RouteBindBreakerGuard<'a> {
550    breakers: RouteBindBreakers,
551    module_id: &'a str,
552    settled: bool,
553}
554
555impl RouteBindBreakerGuard<'_> {
556    /// The module answered within the budget and took the bind. THE ONLY
557    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
558    /// breaker, which is a transition worth logging.
559    fn record_accepted(&mut self) -> bool {
560        self.settled = true;
561        self.breakers.record_accepted(self.module_id)
562    }
563
564    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
565    /// COUNTS TOWARD OPENING.
566    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
567        self.settled = true;
568        self.breakers
569            .record_timeout(self.module_id, threshold, cooldown)
570    }
571
572    /// Everything else: the module REJECTED the bind, its connection went away
573    /// mid-relay, or the waiter was cancelled.
574    ///
575    /// None of these is evidence that a module is slow, and each already has
576    /// its own refusal with its own code. A module that rejects a bind in
577    /// microseconds is healthy and must never be convicted for it; a module
578    /// that died has said nothing about the module that replaces it. So these
579    /// neither increment nor reset the count -- they only release a probe slot.
580    fn record_inconclusive(&mut self) {
581        self.settled = true;
582        self.breakers.record_inconclusive(self.module_id);
583    }
584}
585
586impl Drop for RouteBindBreakerGuard<'_> {
587    fn drop(&mut self) {
588        if !self.settled {
589            self.breakers.record_inconclusive(self.module_id);
590        }
591    }
592}
593
594/// The breaker moved to open, reported so the caller can log it outside the
595/// lock. Opening is rare and load-bearing; the refusals that follow are
596/// frequent and are counted rather than logged.
597struct BreakerOpened {
598    consecutive_timeouts: u32,
599    /// True when a failed probe re-opened an already-open breaker, which reads
600    /// very differently in a log from a first opening.
601    reopened_after_probe: bool,
602}
603
604impl RouteBindBreakers {
605    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
606        self.modules
607            .lock()
608            .expect("route.bind breaker mutex poisoned")
609    }
610
611    /// Decide whether this `route.open` may attempt its relay. Takes the map
612    /// lock and nothing else, and never awaits.
613    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
614        let admitted = |probe| RouteBindAdmission::Admitted {
615            guard: RouteBindBreakerGuard {
616                breakers: self.clone(),
617                module_id,
618                settled: false,
619            },
620            probe,
621        };
622
623        let mut modules = self.lock();
624        let Some(state) = modules.get_mut(module_id) else {
625            return admitted(false);
626        };
627        let Some(cooldown_until) = state.cooldown_until else {
628            return admitted(false);
629        };
630        if state.probe_in_flight {
631            return RouteBindAdmission::Refused {
632                consecutive_timeouts: state.consecutive_timeouts,
633                retry_in: Duration::ZERO,
634                probe_in_flight: true,
635            };
636        }
637        let now = Instant::now();
638        if now < cooldown_until {
639            return RouteBindAdmission::Refused {
640                consecutive_timeouts: state.consecutive_timeouts,
641                retry_in: cooldown_until - now,
642                probe_in_flight: false,
643            };
644        }
645        state.probe_in_flight = true;
646        admitted(true)
647    }
648
649    fn record_accepted(&self, module_id: &str) -> bool {
650        self.lock()
651            .remove(module_id)
652            .is_some_and(|state| state.cooldown_until.is_some())
653    }
654
655    fn record_timeout(
656        &self,
657        module_id: &str,
658        threshold: u32,
659        cooldown: Duration,
660    ) -> Option<BreakerOpened> {
661        let mut modules = self.lock();
662        let state = modules.entry(module_id.to_string()).or_default();
663        let was_open = state.cooldown_until.is_some();
664        let was_probe = state.probe_in_flight;
665        state.probe_in_flight = false;
666        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
667        if state.consecutive_timeouts < threshold {
668            return None;
669        }
670        state.cooldown_until = Some(Instant::now() + cooldown);
671        Some(BreakerOpened {
672            consecutive_timeouts: state.consecutive_timeouts,
673            reopened_after_probe: was_open && was_probe,
674        })
675    }
676
677    fn record_inconclusive(&self, module_id: &str) {
678        if let Some(state) = self.lock().get_mut(module_id) {
679            state.probe_in_flight = false;
680        }
681    }
682
683    /// Discard what was learned about a module, because the process it was
684    /// learned about is gone. Returns the discarded count when it was non-zero.
685    ///
686    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
687    /// `module_id` is a configuration identity that outlives any particular
688    /// child; what the breaker observed was the process behind the module
689    /// connection of the moment. When a new connection registers under that id
690    /// the verdict's subject no longer exists, so the verdict is stale by
691    /// construction rather than merely likely to be wrong. Keeping it would
692    /// apply a dead process's record to a live one, which is the same defect
693    /// class this breaker exists to stop the daemon committing.
694    ///
695    /// A half-open probe in flight is discarded with the rest: it was a
696    /// question about the old process.
697    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
698        self.lock()
699            .remove(module_id)
700            .map(|state| state.consecutive_timeouts)
701            .filter(|discarded| *discarded > 0)
702    }
703
704    /// Open breakers, for the `server.describe` counters object. `None` when
705    /// none is open, so the key stays absent rather than present-and-empty.
706    ///
707    /// This is the operator's answer to "is this module refusing instantly or
708    /// is it fine?", which look identical from a client that retries and then
709    /// succeeds.
710    fn open_snapshot(&self) -> Option<serde_json::Value> {
711        let now = Instant::now();
712        let modules = self.lock();
713        let open = modules
714            .iter()
715            .filter_map(|(module_id, state)| {
716                let cooldown_until = state.cooldown_until?;
717                Some((
718                    module_id.clone(),
719                    serde_json::json!({
720                        "consecutive_timeouts": state.consecutive_timeouts,
721                        "cooldown_remaining_ms":
722                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
723                        "probe_in_flight": state.probe_in_flight,
724                    }),
725                ))
726            })
727            .collect::<serde_json::Map<String, serde_json::Value>>();
728        (!open.is_empty()).then_some(serde_json::Value::Object(open))
729    }
730}
731
732impl ControlHandler {
733    pub fn new(registry: Arc<Registry>) -> Self {
734        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
735    }
736
737    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
738        let counters = forwarding.counters();
739        // Taken from the forwarding table rather than created here, so that the
740        // breaker a `route.open` consults is the same one a module's
741        // registration resets, however many handlers are built over one table.
742        let route_bind_breakers = forwarding.route_bind_breakers();
743        let route_bind_concurrency = forwarding.route_bind_concurrency();
744        let route_outages = forwarding.route_outages();
745        Self {
746            registry,
747            forwarding,
748            process_liveness: None,
749            supervisor: SupervisorHandle::new(),
750            subc_capabilities: Arc::from([
751                CAP_MANIFEST_REGISTRATION.to_string(),
752                CAP_CHANNEL_LIFECYCLE.to_string(),
753                CAP_PING_PONG.to_string(),
754                CAP_SESSION_ATTACH.to_string(),
755                CAP_ADMISSION_FACTS_RELAY.to_string(),
756                CAP_SCOPES_V1.to_string(),
757                CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
758            ]),
759            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
760            route_bind_relay_timeouts: BTreeMap::new(),
761            route_bind_breakers,
762            route_bind_concurrency,
763            route_outages,
764            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
765            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
766            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
767            storage_config: None,
768            machine_id: None,
769            admission_facts_carrier_module_id: None,
770            admission_facts_targets: None,
771            scopes: Arc::new(RwLock::new(ScopeTable::new(
772                crate::daemon_config::default_scope_authority_owners(),
773            ))),
774            scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
775            hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
776            rescan: None,
777            connected_clients: ConnectedClients::new(),
778            counters,
779            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
780            daemon_provenance: DaemonProvenanceFacts::default(),
781            #[cfg(test)]
782            control_dispatch_delay: None,
783            #[cfg(test)]
784            provenance_probe_override: None,
785        }
786    }
787
788    /// Set the central storage policy: registering modules then receive their
789    /// resolved storage descriptor in HELLO_ACK.
790    pub fn with_storage_config(
791        mut self,
792        storage_config: Option<crate::daemon_config::StorageConfig>,
793    ) -> Self {
794        self.storage_config = storage_config;
795        self
796    }
797
798    /// Set the machine id served to every registering module (HELLO_ACK) and on
799    /// `server.describe`.
800    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
801        self.machine_id = machine_id;
802        self
803    }
804
805    /// Configure the exact reserved module and target ids permitted to relay
806    /// opaque admission facts. Config-file loading validates this authority;
807    /// this builder keeps the same policy available to embedded test daemons.
808    pub fn with_admission_facts_config(
809        mut self,
810        carrier_module_id: Option<String>,
811        targets: Option<Vec<String>>,
812    ) -> Self {
813        self.admission_facts_carrier_module_id = carrier_module_id;
814        self.admission_facts_targets = targets;
815        self
816    }
817
818    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
819    /// Replaces the scope table with an empty one under the new list, so call it
820    /// while building the handler, before any module can sync.
821    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
822        self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
823        self.scope_authority_owners = owners;
824        self
825    }
826
827    /// Override the route.bind relay timeout. Used by tests that assert the
828    /// timeout path so they don't block on the production-safe default.
829    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
830        self.route_bind_relay_timeout = timeout;
831        self
832    }
833
834    /// Install per-module route.bind relay budget overrides. A module id
835    /// listed here wins over the daemon-wide default set via
836    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
837    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
838    /// the same `Duration` the bind path will use.
839    pub fn with_route_bind_relay_timeouts(
840        mut self,
841        timeouts: impl IntoIterator<Item = (String, Duration)>,
842    ) -> Self {
843        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
844        self
845    }
846
847    /// Resolve the route.bind relay budget for a specific target module id.
848    /// Per-module overrides win; the daemon-wide value (set via
849    /// `with_route_bind_relay_timeout` or the built-in default) is the
850    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
851    /// the same resolution `handle_route_open` will use.
852    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
853        self.route_bind_relay_timeouts
854            .get(module_id)
855            .copied()
856            .unwrap_or(self.route_bind_relay_timeout)
857    }
858
859    /// Override the per-module bind-relay breaker policy.
860    ///
861    /// Used by tests, which cannot spend three production budgets opening a
862    /// breaker or twenty seconds waiting for its cooldown. The production
863    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
864    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
865    /// reasoning for the numbers.
866    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
867        self.route_bind_breaker_threshold = threshold.max(1);
868        self.route_bind_breaker_cooldown = cooldown;
869        self
870    }
871
872    #[cfg(test)]
873    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
874        self.health_probe_timeout = timeout;
875        self
876    }
877
878    #[cfg(test)]
879    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
880        self.control_dispatch_delay = Some(delay);
881        self
882    }
883
884    pub fn with_process_liveness(
885        mut self,
886        process_liveness: Arc<dyn ModuleProcessLiveness>,
887    ) -> Self {
888        self.process_liveness = Some(process_liveness);
889        self
890    }
891
892    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
893        self.supervisor = supervisor;
894        self
895    }
896
897    pub fn with_daemon_provenance(
898        mut self,
899        pid: u32,
900        started_at_ms: u64,
901        executable_path: Option<PathBuf>,
902        build_git_sha: Option<String>,
903        build_lock_digest: Option<String>,
904    ) -> Self {
905        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
906        let process_start_time = process_start_time(pid);
907        self.daemon_provenance = DaemonProvenanceFacts {
908            build: DaemonBuildProvenance {
909                build_git_sha,
910                build_lock_digest,
911            },
912            pid: Some(pid),
913            started_at_ms: Some(started_at_ms),
914            start_clock: None,
915            executable_path,
916            executable_identity,
917            process_start_time,
918            probe: ExecutableIdentityProbe::default(),
919        };
920        self
921    }
922
923    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
924        self.daemon_provenance.start_clock = Some(clock);
925        self
926    }
927
928    #[cfg(test)]
929    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
930        self.provenance_probe_override = Some(result);
931        self
932    }
933
934    /// Install the configured module set and its reserved capability bindings.
935    /// Bindings are configuration-scoped and may point at a provider that has not
936    /// been installed yet, so this does not require the bound module to exist.
937    pub fn with_capability_config(
938        self,
939        modules: impl IntoIterator<Item = (String, bool)>,
940        reserved_capabilities: BTreeMap<String, String>,
941    ) -> Self {
942        self.capability_evaluator
943            .configure(modules, reserved_capabilities);
944        self
945    }
946
947    pub fn with_supervisor_rescan(
948        mut self,
949        supervisor: Supervisor,
950        config_path: impl Into<PathBuf>,
951        configured_port: Option<u16>,
952    ) -> Self {
953        self.rescan = Some(SupervisorRescanContext {
954            supervisor,
955            config_path: config_path.into(),
956            configured_port,
957            storage_config: self.storage_config.clone(),
958            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
959            admission_facts_targets: self.admission_facts_targets.clone(),
960            scope_authority_owners: self.scope_authority_owners.clone(),
961        });
962        self
963    }
964
965    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
966        self.connected_clients = connected_clients;
967        self
968    }
969
970    pub fn forwarding(&self) -> Arc<ForwardingTable> {
971        Arc::clone(&self.forwarding)
972    }
973
974    pub(crate) fn counters(&self) -> DaemonCounters {
975        self.counters.clone()
976    }
977
978    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
979    /// requirement event without depending on an operator polling a status command.
980    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
981        tokio::spawn(async move {
982            loop {
983                self.capability_evaluator
984                    .wait_for_change_or_deadline()
985                    .await;
986                self.refresh_capability_requirements();
987            }
988        });
989    }
990
991    fn runtime_capability_snapshot(
992        &self,
993    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
994        let runtime = self
995            .supervisor
996            .list()
997            .into_iter()
998            .map(|module| {
999                let status = module.status().map_err(|err| {
1000                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1001                })?;
1002                Ok(RuntimeModule {
1003                    module_id: status.module_id,
1004                    state: status.state,
1005                    enabled: status.enabled,
1006                })
1007            })
1008            .collect::<Result<Vec<_>, RouterError>>()?;
1009        let (_, registrations) = self.registry.list_modules().map_err(|err| {
1010            RouterError::backend(
1011                0,
1012                0,
1013                format!("failed to list capability registrations: {err}"),
1014            )
1015        })?;
1016        let registrations = registrations
1017            .into_iter()
1018            .map(|registration| RegisteredModule {
1019                module_id: registration.manifest.module_id,
1020                module_version: registration.manifest.module_version,
1021                capabilities: registration.manifest.capabilities,
1022            })
1023            .collect();
1024        Ok((runtime, registrations))
1025    }
1026
1027    /// The capability side effects of a module becoming the active registration
1028    /// for its id: cache its manifest (warning if its claims drifted), run the
1029    /// deny census when its declarations call for one, and recompute the
1030    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
1031    /// candidate's does not, and the supervisor does it at promotion instead,
1032    /// through [`crate::supervise::SwapPromotionObserver`].
1033    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1034        let cached_registration = RegisteredModule {
1035            module_id: registration.manifest.module_id.clone(),
1036            module_version: registration.manifest.module_version.clone(),
1037            capabilities: registration.manifest.capabilities.clone(),
1038        };
1039        if self.capability_evaluator.record_hello(&cached_registration) {
1040            warn!(
1041                module_id = %cached_registration.module_id,
1042                "capability claims drifted from the cached manifest"
1043            );
1044        }
1045        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1046            self.enforce_capability_denies();
1047        }
1048        self.refresh_capability_requirements();
1049    }
1050
1051    /// Point the shared supervisor handle at this handler for swap promotions.
1052    /// Called wherever a handler is put behind the `Arc` the router serves, so
1053    /// it can be held weakly.
1054    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1055        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1056            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1057        self.supervisor.set_swap_promotion_observer(observer);
1058    }
1059
1060    pub fn refresh_capability_requirements(&self) {
1061        match self.runtime_capability_snapshot() {
1062            Ok((runtime, registrations)) => {
1063                log_requirement_events(
1064                    self.capability_evaluator
1065                        .evaluate_now(&runtime, &registrations),
1066                );
1067            }
1068            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1069        }
1070    }
1071
1072    /// Reconcile only live, attested route bindings after a capability deny edge
1073    /// or target claim was added. This is deliberately a control-plane census:
1074    /// the opaque forwarding hot path must not grow a per-frame capability check.
1075    fn enforce_capability_denies(&self) {
1076        let (_, registrations) = match self.registry.list_modules() {
1077            Ok(snapshot) => snapshot,
1078            Err(err) => {
1079                warn!(error = %err, "failed to read registrations for capability deny census");
1080                return;
1081            }
1082        };
1083        let manifests = registrations
1084            .into_iter()
1085            .map(|registration| {
1086                (
1087                    registration.manifest.module_id.clone(),
1088                    registration.manifest,
1089                )
1090            })
1091            .collect::<BTreeMap<_, _>>();
1092        let census = match self.forwarding.route_census(None) {
1093            Ok(census) => census,
1094            Err(err) => {
1095                warn!(error = %err, "failed to read route census for capability deny enforcement");
1096                return;
1097            }
1098        };
1099
1100        for (target_module_id, routes) in census {
1101            let Some(target_manifest) = manifests.get(&target_module_id) else {
1102                continue;
1103            };
1104            let mut closed_routes = Vec::new();
1105            let mut module_goodbyes = Vec::new();
1106            for route in routes {
1107                let Principal::Reserved {
1108                    module_id: opening_module_id,
1109                } = &route.principal
1110                else {
1111                    continue;
1112                };
1113                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1114                    continue;
1115                };
1116                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1117                    continue;
1118                };
1119
1120                match self.forwarding.release_client_route(
1121                    route.goodbye_target.connection_id,
1122                    route.goodbye_target.channel,
1123                    route.goodbye_target.epoch,
1124                ) {
1125                    Ok(RouteRelease::Removed(module_goodbye)) => {
1126                        warn!(
1127                            opening_module_id,
1128                            target_module_id,
1129                            capability,
1130                            "force-closing route because an attested capability deny edge now matches"
1131                        );
1132                        closed_routes.push(route);
1133                        module_goodbyes.push(module_goodbye);
1134                    }
1135                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1136                    Err(err) => warn!(
1137                        opening_module_id,
1138                        target_module_id,
1139                        capability,
1140                        error = %err,
1141                        "failed to force-close capability-denied route"
1142                    ),
1143                }
1144            }
1145
1146            if closed_routes.is_empty() {
1147                continue;
1148            }
1149            send_route_control_pushes(
1150                &self.forwarding,
1151                closed_routes,
1152                ClientControlPush::RouteClosed {
1153                    module_id: target_module_id,
1154                    reason: RouteCloseReason::CapabilityDenied,
1155                    drained: false,
1156                    abandoned: 0,
1157                    excluded_subscriptions: 0,
1158                    terminal: Some(false),
1159                },
1160            );
1161            self.emit_route_goodbyes(module_goodbyes);
1162        }
1163    }
1164
1165    /// Why a registered module is not accepting new route binds, or `None` when
1166    /// it is. This is the module's effective readiness: its declared readiness
1167    /// first, then every `need: required` capability it declares evaluating to
1168    /// `provided`. `route.open` and `catalog.list` both read it here so the
1169    /// catalog never reports a module routable that `route.open` would refuse.
1170    fn not_ready_reason(
1171        &self,
1172        registration: &crate::registry::ModuleRegistration,
1173    ) -> Option<NotReadyReason> {
1174        if !registration.ready {
1175            return Some(NotReadyReason {
1176                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1177                capability: None,
1178            });
1179        }
1180        self.first_unprovided_required_capability(registration)
1181            .map(|capability| NotReadyReason {
1182                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1183                capability: Some(capability),
1184            })
1185    }
1186
1187    /// The lexicographically first capability this registration declares
1188    /// `need: required` whose evaluator verdict is not `provided`.
1189    ///
1190    /// The verdicts are the capability evaluator's own; nothing here decides
1191    /// what "provided" means. The evaluator counts a capability provided as
1192    /// soon as a module claiming it has REGISTERED, not once that module is
1193    /// ready. That distinction is what keeps two modules that require each
1194    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1195    /// is ready", each would wait for the other to become ready first and
1196    /// neither ever would. Do not tighten it to readiness.
1197    ///
1198    /// A required capability with no verdict at all means this registration's
1199    /// HELLO or catalog.update landed after the last recompute; recompute once
1200    /// rather than let a missing verdict read as either answer. If it is still
1201    /// missing (the recompute itself failed) the capability counts as
1202    /// unprovided: the refusal is retryable, and routing a module whose
1203    /// required provider is unknown is the outcome this check exists to stop.
1204    fn first_unprovided_required_capability(
1205        &self,
1206        registration: &crate::registry::ModuleRegistration,
1207    ) -> Option<String> {
1208        let required = registration
1209            .manifest
1210            .capabilities
1211            .iter()
1212            .flat_map(|declarations| declarations.requires.iter())
1213            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1214            .map(|requirement| requirement.capability.as_str())
1215            .collect::<BTreeSet<_>>();
1216        if required.is_empty() {
1217            return None;
1218        }
1219        let module_id = registration.manifest.module_id.as_str();
1220        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1221        if required
1222            .iter()
1223            .any(|capability| verdict(capability).is_none())
1224        {
1225            self.refresh_capability_requirements();
1226        }
1227        required
1228            .into_iter()
1229            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1230            .map(str::to_string)
1231    }
1232
1233    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1234        self.capability_evaluator
1235            .statuses()
1236            .into_iter()
1237            .map(capability_requirement_status)
1238            .collect()
1239    }
1240
1241    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1242    /// registration-release watch. The signal is what the supervisor waits on
1243    /// before spawning a replacement, so it must only fire once forwarding
1244    /// teardown is also done (see [`Self::cleanup_connection`] /
1245    /// [`Self::handle_goodbye`]). Used directly only where there is no forwarding
1246    /// state to tear down (a HELLO that failed before module registration).
1247    fn deregister_connection(
1248        &self,
1249        connection_id: ConnectionId,
1250    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1251        self.registry.deregister_connection(connection_id)
1252    }
1253
1254    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1255        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1256            return None;
1257        }
1258        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1259            parse_client_control_request(&frame.body)
1260        else {
1261            return None;
1262        };
1263        Some(target_module_id(&target).to_string())
1264    }
1265
1266    pub(crate) fn route_open_capacity_refusal(
1267        &self,
1268        ctx: &RouteCtx,
1269        frame: &Frame,
1270        target_module_id: &str,
1271        in_flight: usize,
1272        limit: usize,
1273    ) -> Result<Frame, RouterError> {
1274        self.route_open_admission_refusal_frame(
1275            ctx,
1276            frame,
1277            target_module_id,
1278            "open_admission_full",
1279            (in_flight, limit),
1280            format!(
1281                "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1282            ),
1283        )
1284    }
1285
1286    fn route_open_target_capacity_refusal(
1287        &self,
1288        ctx: &RouteCtx,
1289        frame: &Frame,
1290        target_module_id: &str,
1291        in_flight: usize,
1292    ) -> Result<Frame, RouterError> {
1293        self.route_open_admission_refusal_frame(
1294            ctx,
1295            frame,
1296            target_module_id,
1297            "target_binds_full",
1298            (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1299            format!(
1300                "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1301            ),
1302        )
1303    }
1304
1305    /// Admission pressure clears as existing binds settle, so its refusal must
1306    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
1307    /// `module_reloading`, `module_warming`, `target_unavailable`, or
1308    /// `module_timeout`. `target_unavailable` is honest for an attempt that
1309    /// cannot currently reach its target; `module_timeout` would falsely claim
1310    /// that a wait expired. A new, cleaner code would be terminal to deployed
1311    /// clients, so it requires a client-tolerance rollout before daemon emission.
1312    fn route_open_admission_refusal_frame(
1313        &self,
1314        ctx: &RouteCtx,
1315        frame: &Frame,
1316        target_module_id: &str,
1317        reason: &'static str,
1318        (in_flight, limit): (usize, usize),
1319        message: impl Into<String>,
1320    ) -> Result<Frame, RouterError> {
1321        let code = error_codes::TARGET_UNAVAILABLE;
1322        self.counters.increment_route_open_refused(code);
1323        info!(
1324            target: "control",
1325            code,
1326            reason,
1327            module_id = ?target_module_id,
1328            connection_id = ctx.connection_id.get(),
1329            in_flight,
1330            limit,
1331            "route.open refused"
1332        );
1333        control_error_frame(frame, code, message.into())
1334    }
1335
1336    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
1337    ///
1338    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
1339    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
1340    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
1341    #[cfg(test)]
1342    pub fn handle_control(
1343        &self,
1344        connection_id: ConnectionId,
1345        frame: Frame,
1346    ) -> Result<Vec<Frame>, RouterError> {
1347        match frame.header.ty {
1348            FrameType::Ping => Ok(vec![pong(&frame)?]),
1349            FrameType::Hello => self.handle_hello(connection_id, None, frame),
1350            FrameType::Goodbye => self.handle_goodbye(connection_id),
1351            ty => Ok(vec![control_error_frame(
1352                &frame,
1353                "unsupported_control_frame",
1354                format!("unsupported channel-0 frame {ty:?}"),
1355            )?]),
1356        }
1357    }
1358
1359    pub async fn handle_control_frame(
1360        &self,
1361        ctx: &RouteCtx,
1362        frame: Frame,
1363    ) -> Result<Vec<Frame>, RouterError> {
1364        self.handle_control_frame_timed(ctx, frame, None).await
1365    }
1366
1367    pub(crate) async fn handle_control_frame_timed(
1368        &self,
1369        ctx: &RouteCtx,
1370        frame: Frame,
1371        dispatch_started_at: Option<StdInstant>,
1372    ) -> Result<Vec<Frame>, RouterError> {
1373        match frame.header.ty {
1374            FrameType::Ping => Ok(vec![pong(&frame)?]),
1375            FrameType::Hello => {
1376                self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1377            }
1378            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1379            FrameType::Cancel => {
1380                if self
1381                    .supervisor
1382                    .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1383                {
1384                    Ok(Vec::new())
1385                } else {
1386                    Ok(vec![control_error_frame(
1387                        &frame,
1388                        "unknown_subscription",
1389                        "no supervisor spawn subscription has this correlation id",
1390                    )?])
1391                }
1392            }
1393            FrameType::Request => {
1394                if self
1395                    .forwarding
1396                    .module_endpoint_for_connection(ctx.connection_id)
1397                    .map_err(RouterError::Forwarding)?
1398                    .is_some()
1399                {
1400                    if !is_known_module_request_op(&frame.body) {
1401                        return Ok(vec![control_error_frame(
1402                            &frame,
1403                            "unsupported_control_frame",
1404                            "module-originated channel-0 REQUEST is not supported",
1405                        )?]);
1406                    }
1407                    let request = match parse_module_control_request_from_module(&frame.body) {
1408                        Ok(request) => request,
1409                        Err((err, ControlRequestBodyError::UnknownOp)) => {
1410                            return Ok(vec![control_error_frame(
1411                                &frame,
1412                                "unsupported_control_frame",
1413                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
1414                            )?])
1415                        }
1416                        Err((err, ControlRequestBodyError::InvalidBody)) => {
1417                            return Ok(vec![control_error_frame(
1418                                &frame,
1419                                "invalid_control_body",
1420                                format!("malformed module control body: {err}"),
1421                            )?])
1422                        }
1423                    };
1424                    let op = module_control_request_op(&request);
1425                    let corr = frame.header.corr;
1426                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
1427                    let result =
1428                        self.handle_module_control_request(ctx.connection_id, frame, request);
1429                    log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1430                    return result;
1431                }
1432
1433                if is_known_module_request_op(&frame.body) {
1434                    return Ok(vec![control_error_frame(
1435                        &frame,
1436                        "not_registered",
1437                        "catalog.update requires an active module registration owned by this connection",
1438                    )?]);
1439                }
1440
1441                let request = match parse_client_control_request(&frame.body) {
1442                    Ok(request) => request,
1443                    Err((err, ControlRequestBodyError::UnknownOp)) => {
1444                        return Ok(vec![control_error_frame(
1445                            &frame,
1446                            "unknown_control_op",
1447                            format!("unknown client control op: {err}"),
1448                        )?])
1449                    }
1450                    Err((err, ControlRequestBodyError::InvalidBody)) => {
1451                        return Ok(vec![control_error_frame(
1452                            &frame,
1453                            "invalid_control_body",
1454                            format!("malformed client control body: {err}"),
1455                        )?])
1456                    }
1457                };
1458                let op = client_control_request_op(&request);
1459                let corr = frame.header.corr;
1460                log_control_dispatch_arrival(op, ctx.connection_id, corr);
1461                #[cfg(test)]
1462                if let Some(delay) = self.control_dispatch_delay {
1463                    tokio::time::sleep(delay).await;
1464                }
1465                let result = self
1466                    .handle_client_control_request(ctx, frame, request)
1467                    .await;
1468                log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1469                result
1470            }
1471            FrameType::Push => {
1472                let Some(endpoint) = self
1473                    .forwarding
1474                    .module_endpoint_for_connection(ctx.connection_id)
1475                    .map_err(RouterError::Forwarding)?
1476                else {
1477                    return Ok(vec![control_error_frame(
1478                        &frame,
1479                        "unsupported_control_frame",
1480                        "client-originated channel-0 PUSH is not supported",
1481                    )?]);
1482                };
1483                self.handle_status_update(endpoint, frame)
1484            }
1485            FrameType::Response | FrameType::Error
1486                if self
1487                    .forwarding
1488                    .module_endpoint_for_connection(ctx.connection_id)
1489                    .map_err(RouterError::Forwarding)?
1490                    .is_some() =>
1491            {
1492                self.handle_module_relay_response(ctx.connection_id, frame)
1493            }
1494            ty => Ok(vec![control_error_frame(
1495                &frame,
1496                "unsupported_control_frame",
1497                format!("unsupported channel-0 frame {ty:?}"),
1498            )?]),
1499        }
1500    }
1501
1502    pub fn cleanup_connection(
1503        &self,
1504        connection_id: ConnectionId,
1505    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1506        let crash_closed = self
1507            .registry
1508            .get_module_by_connection(connection_id)?
1509            .and_then(|registration| {
1510                self.forwarding
1511                    .module_endpoint_for_connection(connection_id)
1512                    .ok()
1513                    .flatten()
1514                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1515                    .map(|routes| (registration.manifest.module_id, routes))
1516            });
1517        let crash_closed = crash_closed.map(|(module_id, routes)| {
1518            let terminal = match self.supervisor.get(&module_id) {
1519                None => false,
1520                Some(module) => match module.will_recover_after_connection_loss() {
1521                    Ok(will_recover) => !will_recover,
1522                    Err(err) => {
1523                        warn!(
1524                            %module_id,
1525                            error = %err,
1526                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
1527                        );
1528                        false
1529                    }
1530                },
1531            };
1532            // The forwarding table gates all providers at the start of daemon
1533            // shutdown, before their connections are closed. An ordinary
1534            // module disconnect still reports crash if that gate is not set.
1535            let reason = match self.forwarding.is_daemon_draining() {
1536                Ok(true) => RouteCloseReason::Restart,
1537                Ok(false) => RouteCloseReason::Crash,
1538                Err(err) => {
1539                    warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1540                    RouteCloseReason::Crash
1541                }
1542            };
1543            (module_id, routes, reason, terminal)
1544        });
1545        let registrations = self.deregister_connection(connection_id);
1546        let cleanup = self.forwarding.cleanup_connection_counted(connection_id);
1547        // The route.closed push waits for forwarding teardown because only
1548        // teardown knows how many pending route.bind relays it aborted. It still
1549        // goes out before the GOODBYEs for the released routes, and its targets
1550        // were captured above, before teardown removed those routes.
1551        if let Some((module_id, routes, reason, terminal)) = crash_closed {
1552            let abandoned = cleanup
1553                .as_ref()
1554                .map_or(0, |cleanup| cleanup.abandoned_relays);
1555            send_route_control_pushes(
1556                &self.forwarding,
1557                routes,
1558                ClientControlPush::RouteClosed {
1559                    module_id,
1560                    reason,
1561                    drained: false,
1562                    abandoned,
1563                    excluded_subscriptions: 0,
1564                    terminal: Some(terminal),
1565                },
1566            );
1567        }
1568        if let Ok(cleanup) = cleanup {
1569            self.emit_route_goodbyes(cleanup.released);
1570        }
1571        // Signal the registration-release watch only now that BOTH registry and
1572        // forwarding teardown are done, so a supervisor waiting to spawn a
1573        // replacement never observes release while old routes still exist.
1574        if matches!(&registrations, Ok(r) if !r.is_empty()) {
1575            crate::supervise::notify_registration_release();
1576            self.capability_evaluator.wake_deadline_loop();
1577            self.refresh_capability_requirements();
1578        }
1579        self.supervisor.remove_spawn_subscribers(connection_id);
1580        // Sync authority dies with its connection, so the owner's next
1581        // connection can take it; the owner's scopes stay as they are.
1582        self.hello_launch_nonces
1583            .lock()
1584            .unwrap_or_else(|poisoned| poisoned.into_inner())
1585            .forget(connection_id);
1586        self.scopes
1587            .write()
1588            .unwrap_or_else(|poisoned| poisoned.into_inner())
1589            .release_connection(connection_id);
1590        registrations
1591    }
1592
1593    pub(crate) fn handle_route_goodbye(
1594        &self,
1595        connection_id: ConnectionId,
1596        route_channel: u16,
1597        route_epoch: u32,
1598    ) -> Result<bool, RouterError> {
1599        debug!(
1600            connection_id = connection_id.get(),
1601            route_channel, route_epoch, "handling route GOODBYE"
1602        );
1603        let RouteRelease::Removed(released_route) = self
1604            .forwarding
1605            .release_client_route(connection_id, route_channel, route_epoch)
1606            .map_err(RouterError::Forwarding)?
1607        else {
1608            return Ok(false);
1609        };
1610        self.emit_route_goodbyes(vec![released_route]);
1611        Ok(true)
1612    }
1613
1614    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1615        for released in released_routes {
1616            let frame = match Frame::build_with_version(
1617                released.negotiated_ver,
1618                FrameType::Goodbye,
1619                control_flags(),
1620                released.channel,
1621                released.epoch,
1622                0,
1623                Vec::new(),
1624            ) {
1625                Ok(frame) => frame,
1626                Err(err) => {
1627                    warn!(
1628                        route_channel = released.channel,
1629                        error = %err,
1630                        "failed to build route GOODBYE frame"
1631                    );
1632                    continue;
1633                }
1634            };
1635            if !released.close_on_delivery_failure() {
1636                crate::forwarding::send_module_route_goodbye(
1637                    &self.counters,
1638                    &released.sink,
1639                    frame,
1640                    released.module_id.as_deref(),
1641                    "client route released",
1642                );
1643                continue;
1644            }
1645            if let Err(err) = released.sink.try_send(frame) {
1646                warn!(
1647                    target_connection_id = released.connection_id.get(),
1648                    route_channel = released.channel,
1649                    error = %err,
1650                    "route GOODBYE was not delivered to client; closing target connection"
1651                );
1652                if self
1653                    .forwarding
1654                    .escalate_client_delivery_failure(
1655                        released.connection_id,
1656                        released.channel,
1657                        released.epoch,
1658                        CloseReason::new(
1659                            "route_goodbye_delivery_failed",
1660                            format!(
1661                                "failed to enqueue route GOODBYE for channel {}: {err}",
1662                                released.channel
1663                            ),
1664                        ),
1665                        crate::forwarding::UndeliveredFrame {
1666                            module_id: released.module_id.as_deref(),
1667                            sink: &released.sink,
1668                        },
1669                    )
1670                    .unwrap_or(false)
1671                {
1672                    self.counters.increment_goodbye_relay_client_failed();
1673                }
1674            }
1675        }
1676    }
1677
1678    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
1679    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
1680    /// own commit failed after the module had already accepted). Without this, a
1681    /// module that accepts late keeps a binding subc has torn down, so a later
1682    /// frame on that module channel could misdeliver if the channel is reused.
1683    ///
1684    /// Never closes the shared module connection on failure: a dropped notification
1685    /// only wastes a bounded amount of warm module-side state, which the module's
1686    /// own idle reaper reclaims. Only call this once the route.bind relay was
1687    /// actually enqueued to the module — if the relay send itself failed, the
1688    /// module never created a binding and there is nothing to tear down.
1689    fn send_abandoned_route_bind_goodbye(
1690        &self,
1691        module_sink: &crate::FrameSink,
1692        negotiated_ver: u8,
1693        module_channel: u16,
1694        module_epoch: u32,
1695    ) {
1696        let frame = match Frame::build_with_version(
1697            negotiated_ver,
1698            FrameType::Goodbye,
1699            control_flags(),
1700            module_channel,
1701            module_epoch,
1702            0,
1703            Vec::new(),
1704        ) {
1705            Ok(frame) => frame,
1706            Err(err) => {
1707                warn!(
1708                    route_channel = module_channel,
1709                    error = %err,
1710                    "failed to build GOODBYE for abandoned route.bind"
1711                );
1712                return;
1713            }
1714        };
1715        crate::forwarding::send_module_route_goodbye(
1716            &self.counters,
1717            module_sink,
1718            frame,
1719            None,
1720            "abandoned route.bind",
1721        );
1722    }
1723
1724    fn handle_hello(
1725        &self,
1726        connection_id: ConnectionId,
1727        sink: Option<crate::FrameSink>,
1728        frame: Frame,
1729    ) -> Result<Vec<Frame>, RouterError> {
1730        debug!(
1731            connection_id = connection_id.get(),
1732            corr = frame.header.corr,
1733            "handling HELLO"
1734        );
1735        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1736            Ok(value) => value,
1737            Err(err) => {
1738                return Ok(vec![control_error_frame(
1739                    &frame,
1740                    "invalid_hello",
1741                    format!("malformed HELLO body: {err}"),
1742                )?])
1743            }
1744        };
1745        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1746            return Ok(vec![control_error_frame(
1747                &frame,
1748                "invalid_capability_grammar",
1749                err.to_string(),
1750            )?]);
1751        }
1752        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
1753            return Ok(vec![control_error_frame(
1754                &frame,
1755                "invalid_manifest",
1756                err.to_string(),
1757            )?]);
1758        }
1759        if let Some(provenance) = hello_value
1760            .get("manifest")
1761            .and_then(|manifest| manifest.get("provenance"))
1762        {
1763            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
1764                return Ok(vec![control_error_frame(
1765                    &frame,
1766                    "invalid_manifest",
1767                    format!("malformed manifest provenance: {err}"),
1768                )?]);
1769            }
1770        }
1771        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
1772            Ok(hello) => hello,
1773            Err(err) => {
1774                return Ok(vec![control_error_frame(
1775                    &frame,
1776                    "invalid_hello",
1777                    format!("malformed HELLO body: {err}"),
1778                )?])
1779            }
1780        };
1781
1782        if hello.protocol_ver != hello.manifest.protocol_ver {
1783            return Ok(vec![control_error_frame(
1784                &frame,
1785                "invalid_manifest",
1786                format!(
1787                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
1788                    hello.protocol_ver, hello.manifest.protocol_ver
1789                ),
1790            )?]);
1791        }
1792
1793        if hello.manifest.module_id.trim().is_empty() {
1794            return Ok(vec![control_error_frame(
1795                &frame,
1796                "invalid_manifest",
1797                "manifest module_id must not be empty",
1798            )?]);
1799        }
1800
1801        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
1802            Ok(negotiated_ver) => negotiated_ver,
1803            Err(message) => {
1804                return Ok(vec![control_error_frame(
1805                    &frame,
1806                    "version_unsupported",
1807                    message,
1808                )?])
1809            }
1810        };
1811
1812        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
1813        // swap is open for this id, the only HELLO admitted as a second process
1814        // is the one carrying the candidate's launch nonce (the swap token), and
1815        // it registers into the candidate slot rather than being refused as a
1816        // duplicate. Run after the reserved gate, a reserved module's candidate
1817        // would be refused `reserved_module` for presenting a nonce that gate
1818        // does not know. See `SupervisorHandle::swap_hello_admission`.
1819        let swap_admission = self
1820            .supervisor
1821            .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
1822        if swap_admission == SwapHelloAdmission::Refused {
1823            warn!(
1824                module_id = %hello.manifest.module_id,
1825                connection_id = connection_id.get(),
1826                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
1827            );
1828            return Ok(vec![control_error_frame(
1829                &frame,
1830                "swap_token_invalid",
1831                format!(
1832                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
1833                    hello.manifest.module_id
1834                ),
1835            )?]);
1836        }
1837        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
1838
1839        // Reserved-module identity gate: a module_id configured `reserved` may be
1840        // registered ONLY by the process subc spawned for it, proven by echoing the
1841        // one-time launch nonce subc injected. A non-reserved id has no recorded
1842        // nonce and always passes. This blocks a key-holder from impersonating a
1843        // security-boundary module (e.g. the credential vault) while the real one is
1844        // down/restarting and its registration slot is momentarily free. A swap
1845        // candidate has already proven the same thing with its own nonce above.
1846        if let Some(rejection) = (!swap_candidate)
1847            .then(|| {
1848                self.supervisor.reserved_hello_rejection(
1849                    &hello.manifest.module_id,
1850                    hello.launch_nonce.as_deref(),
1851                )
1852            })
1853            .flatten()
1854        {
1855            let message = match rejection {
1856                ReservedHelloRejection::Exact { module_id } => format!(
1857                    "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
1858                ),
1859                ReservedHelloRejection::Prefix {
1860                    prefix,
1861                    owner_module_id,
1862                } => format!(
1863                    "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
1864                    hello.manifest.module_id
1865                ),
1866            };
1867            return Ok(vec![control_error_frame(
1868                &frame,
1869                "reserved_module",
1870                message,
1871            )?]);
1872        }
1873
1874        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
1875            &hello.manifest.module_id,
1876            hello.manifest.capabilities.as_ref(),
1877        );
1878        if let Some(refusal) = reserved_capability_refusals.first() {
1879            let capability = refusal.capability.clone();
1880            let bound_module = refusal.claimants[0].clone();
1881            log_duplicate_claim_events(reserved_capability_refusals);
1882            return Ok(vec![control_error_frame(
1883                &frame,
1884                "reserved_capability",
1885                format!(
1886                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
1887                    capability, bound_module, hello.manifest.module_id
1888                ),
1889            )?]);
1890        }
1891
1892        // A connection that already opened client routes must not also register as
1893        // a module: cleanup would then release only one side and leak the other.
1894        if self
1895            .forwarding
1896            .connection_has_client_routes(connection_id)
1897            .map_err(RouterError::Forwarding)?
1898        {
1899            return Ok(vec![control_error_frame(
1900                &frame,
1901                "invalid_hello",
1902                "connection has open client routes and cannot also register as a module",
1903            )?]);
1904        }
1905
1906        // Kept for scope sync authority, which goes only to the connection that
1907        // presented the module's current launch nonce. Recorded before the
1908        // registration is attempted: a connection whose registration then fails
1909        // has no registration, so it cannot sync anyway, and cleanup forgets it.
1910        self.hello_launch_nonces
1911            .lock()
1912            .unwrap_or_else(|poisoned| poisoned.into_inner())
1913            .record(connection_id, hello.launch_nonce.as_deref());
1914        let control_ops = effective_module_control_ops(hello.control_ops);
1915        // Built before anything is registered so an encoding failure leaves no
1916        // registry or forwarding state behind.
1917        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
1918        if swap_candidate {
1919            return self.register_swap_candidate(
1920                connection_id,
1921                sink,
1922                &frame,
1923                hello.manifest,
1924                negotiated_ver,
1925                control_ops,
1926                hello_ack,
1927            );
1928        }
1929        let registration = match self.registry.register_with_control_ops(
1930            hello.manifest,
1931            negotiated_ver,
1932            connection_id,
1933            control_ops,
1934        ) {
1935            Ok(registration) => registration,
1936            Err(RegistryError::DuplicateModuleId { module_id }) => {
1937                return Ok(vec![control_error_frame(
1938                    &frame,
1939                    "duplicate_module_id",
1940                    format!(
1941                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
1942                    ),
1943                )?])
1944            }
1945            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
1946                return Ok(vec![control_error_frame(
1947                    &frame,
1948                    "invalid_module_id",
1949                    err.to_string(),
1950                )?])
1951            }
1952            Err(err) => {
1953                return Ok(vec![control_error_frame(
1954                    &frame,
1955                    "registry_error",
1956                    err.to_string(),
1957                )?])
1958            }
1959        };
1960
1961        let reply = if let Some(sink) = sink {
1962            // The forwarding table's module store is also the daemon-to-module
1963            // control-RPC lane, so every HELLO gets a live endpoint even when the
1964            // manifest has no routable provider role. Non-routable modules still
1965            // cannot receive route.bind in production: `handle_route_open` checks
1966            // the registry manifest with `target_has_required_role` before the
1967            // only production call to `begin_route_bind_relay_for` below that
1968            // route.open path. The remaining direct relay callers are unit tests
1969            // and benchmark harnesses that construct forwarding state explicitly.
1970            //
1971            // The HELLO_ACK is queued by the forwarding table itself, before the
1972            // endpoint becomes visible, and is NOT returned as a reply. A module
1973            // reads HELLO_ACK first and exits on anything else; a reply is only
1974            // written after this handler returns, by which time a route.open on
1975            // another connection could already have queued a route.bind request
1976            // for this module ahead of it.
1977            let concurrency = manifest_concurrency(&registration.manifest);
1978            if let Err(err) = self.forwarding.register_module_connection_acked(
1979                connection_id,
1980                registration.manifest.module_id.clone(),
1981                negotiated_ver,
1982                concurrency,
1983                sink,
1984                hello_ack,
1985            ) {
1986                // Forwarding registration failed, so there is no forwarding
1987                // state to tear down. Remove the registry entry and signal the
1988                // release watch directly.
1989                if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
1990                    crate::supervise::notify_registration_release();
1991                }
1992                return Ok(vec![control_error_frame(
1993                    &frame,
1994                    forwarding_error_code(&err),
1995                    err.to_string(),
1996                )?]);
1997            }
1998            Vec::new()
1999        } else {
2000            // No sink means no forwarding endpoint, so nothing can be routed
2001            // ahead of the ack; it goes out as the reply.
2002            vec![hello_ack]
2003        };
2004
2005        // Exposure over assumption: Concurrency's serde default is pinned to the
2006        // pre-field behavior (ModuleManaged), so a management surface that is
2007        // genuinely Serial and just never declared it inherits concurrent
2008        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
2009        // turns "no module has been bitten yet" into the checkable claim "no
2010        // module is exposed" -- one read of the boot log instead of a fleet
2011        // audit. Detected from the raw HELLO bytes because the serde default
2012        // deliberately erases the absent/declared distinction from the type.
2013        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
2014            info!(
2015                module_id = %registration.manifest.module_id,
2016                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2017            );
2018        }
2019
2020        self.apply_registration_capabilities(&registration);
2021
2022        info!(
2023            module_id = %registration.manifest.module_id,
2024            module_version = %registration.manifest.module_version,
2025            negotiated_ver,
2026            routable_provider = manifest_provides_routable_role(&registration.manifest),
2027            connection_id = connection_id.get(),
2028            "module registered"
2029        );
2030
2031        Ok(reply)
2032    }
2033
2034    /// Register a HELLO the swap gate admitted into the candidate slot of the
2035    /// registry and of forwarding, where it is reachable over its own
2036    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
2037    /// nothing routes to it until the supervisor cuts over.
2038    ///
2039    /// Registry first, then forwarding, the same order as an ordinary HELLO;
2040    /// a forwarding failure removes the registry entry again. The capability
2041    /// census is not run: it describes routable modules, and this one is not
2042    /// routable until promotion.
2043    #[allow(clippy::too_many_arguments)]
2044    fn register_swap_candidate(
2045        &self,
2046        connection_id: ConnectionId,
2047        sink: Option<crate::FrameSink>,
2048        frame: &Frame,
2049        manifest: ModuleManifest,
2050        negotiated_ver: u8,
2051        control_ops: Vec<String>,
2052        hello_ack: Frame,
2053    ) -> Result<Vec<Frame>, RouterError> {
2054        let module_id = manifest.module_id.clone();
2055        let registration = match self.registry.register_candidate_with_control_ops(
2056            manifest,
2057            negotiated_ver,
2058            connection_id,
2059            control_ops,
2060        ) {
2061            Ok(registration) => registration,
2062            Err(RegistryError::DuplicateModuleId { module_id }) => {
2063                return Ok(vec![control_error_frame(
2064                    frame,
2065                    "duplicate_module_id",
2066                    format!(
2067                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2068                    ),
2069                )?])
2070            }
2071            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2072                return Ok(vec![control_error_frame(
2073                    frame,
2074                    "invalid_module_id",
2075                    err.to_string(),
2076                )?])
2077            }
2078            Err(err) => {
2079                return Ok(vec![control_error_frame(
2080                    frame,
2081                    "registry_error",
2082                    err.to_string(),
2083                )?])
2084            }
2085        };
2086        let reply = if let Some(sink) = sink {
2087            // Same ordering as an ordinary HELLO: the forwarding table queues
2088            // the HELLO_ACK before the candidate endpoint is inserted, because
2089            // a module exits if its first frame after HELLO is anything else.
2090            let concurrency = manifest_concurrency(&registration.manifest);
2091            if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2092                connection_id,
2093                module_id.clone(),
2094                negotiated_ver,
2095                concurrency,
2096                sink,
2097                hello_ack,
2098            ) {
2099                if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
2100                    crate::supervise::notify_registration_release();
2101                }
2102                return Ok(vec![control_error_frame(
2103                    frame,
2104                    forwarding_error_code(&err),
2105                    err.to_string(),
2106                )?]);
2107            }
2108            Vec::new()
2109        } else {
2110            vec![hello_ack]
2111        };
2112        self.supervisor.mark_swap_candidate_admitted(&module_id);
2113        info!(
2114            module_id = %module_id,
2115            module_version = %registration.manifest.module_version,
2116            negotiated_ver,
2117            ready = registration.ready,
2118            connection_id = connection_id.get(),
2119            "swap candidate registered; not routable until cutover"
2120        );
2121        Ok(reply)
2122    }
2123
2124    fn build_hello_ack(
2125        &self,
2126        frame: &Frame,
2127        negotiated_ver: u8,
2128        module_id: &str,
2129    ) -> Result<Frame, RouterError> {
2130        let ack = ModuleHelloAckBody {
2131            negotiated_ver,
2132            subc_ops: module_subc_ops(),
2133            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2134            storage: self
2135                .storage_config
2136                .as_ref()
2137                .map(|cfg| cfg.descriptor_for(module_id)),
2138            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2139        };
2140        let body = serde_json::to_vec(&ack).map_err(|err| {
2141            RouterError::backend(
2142                0,
2143                frame.header.corr,
2144                format!("failed to encode HELLO_ACK: {err}"),
2145            )
2146        })?;
2147
2148        Frame::build_with_version(
2149            negotiated_ver,
2150            FrameType::HelloAck,
2151            control_flags(),
2152            0,
2153            0,
2154            frame.header.corr,
2155            body,
2156        )
2157        .map_err(RouterError::FrameBuild)
2158    }
2159
2160    async fn handle_client_control_request(
2161        &self,
2162        ctx: &RouteCtx,
2163        frame: Frame,
2164        request: ClientControlRequest,
2165    ) -> Result<Vec<Frame>, RouterError> {
2166        match request {
2167            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2168            ClientControlRequest::CatalogList { module_id } => {
2169                self.handle_catalog_list(frame, module_id)
2170            }
2171            ClientControlRequest::RouteOpen {
2172                target,
2173                identity,
2174                consumer_identity,
2175                consumer_capabilities,
2176                role_versions,
2177                admission_facts,
2178                scope,
2179            } => {
2180                self.handle_route_open(
2181                    ctx,
2182                    frame,
2183                    RouteOpenRequest {
2184                        target,
2185                        identity,
2186                        consumer_identity,
2187                        consumer_capabilities,
2188                        role_versions,
2189                        admission_facts,
2190                        scope,
2191                    },
2192                )
2193                .await
2194            }
2195            ClientControlRequest::RoutePoll {
2196                route_channel,
2197                route_epoch,
2198                kind,
2199            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2200            ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2201            ClientControlRequest::SupervisorSpawnSnapshot {} => {
2202                self.handle_supervisor_spawn_snapshot(frame)
2203            }
2204            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2205                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2206            }
2207            ClientControlRequest::SupervisorRestart {
2208                module_id,
2209                drain_timeout_ms,
2210            } => {
2211                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2212                    .await
2213            }
2214            ClientControlRequest::SupervisorSwap {
2215                module_id,
2216                ready_timeout_ms,
2217            } => {
2218                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2219                    .await
2220            }
2221            ClientControlRequest::SupervisorReload { module_id } => {
2222                self.handle_supervisor_reload(frame, module_id).await
2223            }
2224            ClientControlRequest::SupervisorRescan { preview } => {
2225                self.handle_supervisor_rescan(frame, preview).await
2226            }
2227            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2228                self.handle_supervisor_release_reserved(frame, module_id)
2229                    .await
2230            }
2231            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2232                self.handle_supervisor_set_enabled(frame, module_id, enabled)
2233                    .await
2234            }
2235            ClientControlRequest::SupervisorHealthProbe { module_id } => {
2236                self.handle_supervisor_health_probe(frame, module_id).await
2237            }
2238            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2239            ClientControlRequest::SupervisorRoutes { module_id } => {
2240                self.handle_supervisor_routes(frame, module_id)
2241            }
2242            ClientControlRequest::SupervisorProvenance { module_id } => {
2243                self.handle_supervisor_provenance(frame, module_id).await
2244            }
2245            ClientControlRequest::SupervisorStderrTail {
2246                module_id,
2247                max_lines,
2248                max_bytes,
2249            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2250            ClientControlRequest::SupervisorTerminals { module_id } => {
2251                self.handle_supervisor_terminals(frame, module_id).await
2252            }
2253        }
2254    }
2255
2256    fn handle_module_control_request(
2257        &self,
2258        connection_id: ConnectionId,
2259        frame: Frame,
2260        request: ModuleControlRequestFromModule,
2261    ) -> Result<Vec<Frame>, RouterError> {
2262        match request {
2263            ModuleControlRequestFromModule::CatalogUpdate {
2264                provides,
2265                capabilities,
2266                ready,
2267            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2268            ModuleControlRequestFromModule::LiveRoots {} => {
2269                let registered = self
2270                    .registry
2271                    .get_module_by_connection(connection_id)
2272                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2273                let Some(registration) = registered else {
2274                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2275                };
2276                let response = self
2277                    .forwarding
2278                    .live_roots(&registration.manifest.module_id)
2279                    .map_err(RouterError::Forwarding)?;
2280                Ok(vec![control_response_body_frame(
2281                    &frame,
2282                    &response,
2283                    "ModuleControlResponseToModule::LiveRoots",
2284                )?])
2285            }
2286            ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2287                self.handle_scope_sync(connection_id, frame, generation, scopes)
2288            }
2289            ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2290                self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2291            }
2292        }
2293    }
2294
2295    /// `scope.sync`: the owner is the module registered on this connection.
2296    /// A connection with no registration (every client connection, `direct`
2297    /// included) is refused `not_registered` before the table is consulted.
2298    fn handle_scope_sync(
2299        &self,
2300        connection_id: ConnectionId,
2301        frame: Frame,
2302        generation: u64,
2303        scopes: Vec<ScopeRecord>,
2304    ) -> Result<Vec<Frame>, RouterError> {
2305        let Some(registration) = self
2306            .registry
2307            .get_module_by_connection(connection_id)
2308            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2309        else {
2310            return Ok(vec![control_error_frame(
2311                &frame,
2312                "not_registered",
2313                "scope.sync requires an active module registration owned by this connection",
2314            )?]);
2315        };
2316        let owner = registration.manifest.module_id;
2317        let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
2318        let is_current_launch = |connection: ConnectionId| {
2319            self.hello_launch_nonces
2320                .lock()
2321                .unwrap_or_else(|poisoned| poisoned.into_inner())
2322                .presented(connection, current_nonce.as_deref())
2323        };
2324        // Lock order is the scope table, then the forwarding table: the new
2325        // tags are published, and the routes the change closes are selected,
2326        // while the scope table is still write-locked, so no admission can read
2327        // a record whose tag is not yet published.
2328        let mut table = self
2329            .scopes
2330            .write()
2331            .unwrap_or_else(|poisoned| poisoned.into_inner());
2332        let outcome = table.sync(&owner, connection_id, is_current_launch, generation, scopes);
2333        let drained = match &outcome {
2334            Ok(applied) => self
2335                .forwarding
2336                .publish_scope_changes(&applied.tag_changes)
2337                .map_err(RouterError::Forwarding)?,
2338            Err(_) => Vec::new(),
2339        };
2340        drop(table);
2341        match outcome {
2342            Ok(applied) => {
2343                let counts = ScopeOutcomeCounts::of(&applied.results);
2344                info!(
2345                    owner = %owner,
2346                    generation,
2347                    records = applied.results.len(),
2348                    created = counts.created,
2349                    replaced = counts.replaced,
2350                    updated = counts.updated,
2351                    unchanged = counts.unchanged,
2352                    refused = counts.refused,
2353                    ended = applied.ended.len(),
2354                    tag_changes = applied.tag_changes.len(),
2355                    routes_closed = drained.len(),
2356                    "scope sync accepted"
2357                );
2358                // An accepted sync can still refuse individual records, and the
2359                // owner is the only party that sees the reply. Name them here so
2360                // an operator can tell a refused session from a missing one
2361                // without the owner's logs. Capped so a sync that refuses
2362                // thousands cannot flood the log; the count above is complete.
2363                for refused in applied
2364                    .results
2365                    .iter()
2366                    .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
2367                    .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
2368                {
2369                    warn!(
2370                        owner = %owner,
2371                        generation,
2372                        scope_ref = %refused.scope_ref,
2373                        scope_epoch = refused.scope_epoch,
2374                        code = refused.code.as_deref().unwrap_or(""),
2375                        "scope record refused"
2376                    );
2377                }
2378                self.close_scope_drained_routes(drained);
2379                let response = ModuleControlResponseToModule::ScopeSync {
2380                    generation,
2381                    results: applied.results,
2382                    ended: applied.ended,
2383                };
2384                Ok(vec![control_response_body_frame(
2385                    &frame,
2386                    &response,
2387                    "ModuleControlResponseToModule::ScopeSync",
2388                )?])
2389            }
2390            Err(refusal) => {
2391                info!(
2392                    owner = %owner,
2393                    generation,
2394                    code = refusal.code,
2395                    "scope sync refused"
2396                );
2397                Ok(vec![control_error_frame(
2398                    &frame,
2399                    refusal.code,
2400                    refusal.message,
2401                )?])
2402            }
2403        }
2404    }
2405
2406    /// Tell both ends of each route a scope change closed. The module gets a
2407    /// channel-scoped GOODBYE and so does the client: the GOODBYE is what ends
2408    /// the client's route handle. The client also gets `route.closed` with the
2409    /// scope reason, one push per module and reason, so it can tell a revoked
2410    /// route from an ordinary close and not reopen it.
2411    fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
2412        if drained.is_empty() {
2413            return;
2414        }
2415        let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
2416            BTreeMap::new();
2417        let mut goodbyes = Vec::with_capacity(drained.len() * 2);
2418        for route in drained {
2419            warn!(
2420                module_id = %route.module_id,
2421                reason = ?route.reason,
2422                client_connection_id = route.client.connection_id.get(),
2423                route_channel = route.client.channel,
2424                "closing route because its scope changed"
2425            );
2426            pushes
2427                .entry((route.module_id.clone(), format!("{:?}", route.reason)))
2428                .or_insert_with(|| (route.reason, Vec::new()))
2429                .1
2430                .push(EndpointRoute {
2431                    goodbye_target: route.client.clone(),
2432                    principal: Principal::Unverified,
2433                    bound_at: Instant::now(),
2434                    draining: false,
2435                    drain_reason: None,
2436                });
2437            goodbyes.push(route.module);
2438            goodbyes.push(route.client);
2439        }
2440        for ((module_id, _), (reason, routes)) in pushes {
2441            send_route_control_pushes(
2442                &self.forwarding,
2443                routes,
2444                ClientControlPush::RouteClosed {
2445                    module_id,
2446                    reason,
2447                    drained: false,
2448                    abandoned: 0,
2449                    excluded_subscriptions: 0,
2450                    terminal: Some(false),
2451                },
2452            );
2453        }
2454        self.emit_route_goodbyes(goodbyes);
2455    }
2456
2457    /// `scope.describe`: any registered module may read any scope, because a
2458    /// provider must read the scope a route it serves is stamped with.
2459    fn handle_scope_describe(
2460        &self,
2461        connection_id: ConnectionId,
2462        frame: Frame,
2463        owner: Principal,
2464        scope_ref: String,
2465    ) -> Result<Vec<Frame>, RouterError> {
2466        let registered = self
2467            .registry
2468            .get_module_by_connection(connection_id)
2469            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2470        if registered.is_none() {
2471            return Ok(vec![control_error_frame(
2472                &frame,
2473                "not_registered",
2474                "scope.describe requires an active module registration owned by this connection",
2475            )?]);
2476        }
2477        let description = self
2478            .scopes
2479            .read()
2480            .unwrap_or_else(|poisoned| poisoned.into_inner())
2481            .describe(&owner, &scope_ref);
2482        let owner_configured = match &owner {
2483            Principal::Reserved { module_id } => self.supervisor.get(module_id).is_some(),
2484            _ => false,
2485        };
2486        let response = ModuleControlResponseToModule::ScopeDescribe {
2487            status: description.status,
2488            scope_epoch: description.scope_epoch,
2489            daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
2490            owner_synced: description.owner_synced,
2491            owner_configured,
2492            scope: description.stamp,
2493        };
2494        Ok(vec![control_response_body_frame(
2495            &frame,
2496            &response,
2497            "ModuleControlResponseToModule::ScopeDescribe",
2498        )?])
2499    }
2500
2501    fn handle_catalog_update(
2502        &self,
2503        connection_id: ConnectionId,
2504        frame: Frame,
2505        provides: Vec<ProviderRole>,
2506        capabilities: Option<CapabilityDeclarations>,
2507        ready: Option<bool>,
2508    ) -> Result<Vec<Frame>, RouterError> {
2509        self.refresh_capability_requirements();
2510        let Some(registration) = self
2511            .registry
2512            .get_module_by_connection(connection_id)
2513            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2514        else {
2515            return Ok(vec![control_error_frame(
2516                &frame,
2517                "not_registered",
2518                "catalog.update requires an active module registration owned by this connection",
2519            )?]);
2520        };
2521
2522        if let Some(message) =
2523            catalog_update_frozen_field_message(&registration.manifest, &provides)
2524        {
2525            return Ok(vec![control_error_frame(
2526                &frame,
2527                "catalog_update_frozen_field",
2528                message,
2529            )?]);
2530        }
2531
2532        let mut candidate = registration.manifest.clone();
2533        candidate.provides = provides.clone();
2534        candidate.capabilities = capabilities
2535            .clone()
2536            .or_else(|| registration.manifest.capabilities.clone());
2537        if let Err(err) = candidate.validate_capability_grammar() {
2538            return Ok(vec![control_error_frame(
2539                &frame,
2540                "invalid_capability_grammar",
2541                err.to_string(),
2542            )?]);
2543        }
2544
2545        let updated = self
2546            .registry
2547            .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
2548            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2549        if updated.is_none() {
2550            return Ok(vec![control_error_frame(
2551                &frame,
2552                "not_registered",
2553                "catalog.update requires an active module registration owned by this connection",
2554            )?]);
2555        }
2556        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
2557            log_duplicate_claim_events(
2558                self.capability_evaluator
2559                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
2560            );
2561        }
2562        if capability_census_trigger(
2563            registration.manifest.capabilities.as_ref(),
2564            updated
2565                .as_ref()
2566                .and_then(|entry| entry.manifest.capabilities.as_ref()),
2567        ) {
2568            self.enforce_capability_denies();
2569        }
2570        self.refresh_capability_requirements();
2571
2572        let response = ModuleControlResponseToModule::CatalogUpdate {};
2573        control_response_body_frame(
2574            &frame,
2575            &response,
2576            "ModuleControlResponseToModule::CatalogUpdate",
2577        )
2578        .map(|frame| vec![frame])
2579    }
2580
2581    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2582        self.refresh_capability_requirements();
2583        // A bare connection count is ambiguous between many clients holding a
2584        // route each and one client accumulating hundreds, so publish the
2585        // concentration alongside it. Route state is best-effort here: a
2586        // diagnostic endpoint must still answer if the forwarding lock is
2587        // contended.
2588        let mut counters = self.counters.snapshot();
2589        if let (Ok((connections_with_routes, max)), Some(obj)) = (
2590            self.forwarding.client_route_concentration(),
2591            counters.as_object_mut(),
2592        ) {
2593            obj.insert(
2594                "client_connections_with_routes".into(),
2595                connections_with_routes.into(),
2596            );
2597            obj.insert("max_routes_on_one_connection".into(), max.into());
2598        }
2599        // A module that is being fast-refused and a module that is fine look
2600        // identical from a client that retries and succeeds, so name the open
2601        // breakers here. This rides the existing free-form counters object
2602        // rather than a new wire field, so no sibling that deserializes
2603        // `ServerDescribe` has to be rebuilt to keep reading it.
2604        if let (Some(open_breakers), Some(obj)) = (
2605            self.route_bind_breakers.open_snapshot(),
2606            counters.as_object_mut(),
2607        ) {
2608            obj.insert("route_bind_breakers_open".into(), open_breakers);
2609        }
2610        let response = ClientControlResponse::ServerDescribe {
2611            protocol_ver: PROTOCOL_VERSION,
2612            subc_ops: subc_ops(),
2613            capabilities: self.subc_capabilities.as_ref().to_vec(),
2614            connected_clients: self.connected_clients.count(),
2615            counters: Some(counters),
2616            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
2617            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
2618            capability_requirements: self.capability_requirement_statuses(),
2619            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2620        };
2621        Ok(vec![control_response_body_frame(
2622            &frame,
2623            &response,
2624            "ClientControlResponse::ServerDescribe",
2625        )?])
2626    }
2627
2628    fn handle_catalog_list(
2629        &self,
2630        frame: Frame,
2631        module_id: Option<String>,
2632    ) -> Result<Vec<Frame>, RouterError> {
2633        let (generation, modules) = self.registry.list_modules().map_err(|err| {
2634            RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
2635        })?;
2636        let entries = modules
2637            .into_iter()
2638            .filter(|registration| {
2639                module_id
2640                    .as_deref()
2641                    .map(|wanted| registration.manifest.module_id == wanted)
2642                    .unwrap_or(true)
2643            })
2644            .map(|registration| {
2645                let not_ready = self.not_ready_reason(&registration);
2646                let roles = registration.manifest.provides;
2647                CatalogEntry {
2648                    module_id: registration.manifest.module_id,
2649                    ready: not_ready.is_none(),
2650                    not_ready,
2651                    module_version: Some(registration.manifest.module_version),
2652                    roles,
2653                    control_ops: registration.control_ops,
2654                    capabilities: registration.manifest.capabilities,
2655                    self_signals: registration.manifest.self_signals,
2656                }
2657            })
2658            .collect();
2659        let response = ClientControlResponse::CatalogList {
2660            generation,
2661            modules: entries,
2662            subc_ops: subc_ops(),
2663        };
2664        Ok(vec![control_response_body_frame(
2665            &frame,
2666            &response,
2667            "ClientControlResponse::CatalogList",
2668        )?])
2669    }
2670
2671    fn route_open_principal(
2672        &self,
2673        frame: &Frame,
2674        consumer_identity: Option<ConsumerIdentity>,
2675    ) -> Result<Result<Principal, Frame>, RouterError> {
2676        let Some(consumer_identity) = consumer_identity else {
2677            return Ok(Ok(Principal::Direct));
2678        };
2679
2680        if self.supervisor.spawned_consumer_authorized(
2681            &consumer_identity.module_id,
2682            &consumer_identity.launch_nonce,
2683        ) {
2684            return Ok(Ok(Principal::Reserved {
2685                module_id: consumer_identity.module_id,
2686            }));
2687        }
2688
2689        Ok(Err(control_error_frame(
2690            frame,
2691            "bad_consumer_identity",
2692            format!(
2693                "consumer_identity for module_id '{}' did not match a supervised launch nonce",
2694                consumer_identity.module_id
2695            ),
2696        )?))
2697    }
2698
2699    /// Ordinary `route.open` refusals go through here; admission and breaker
2700    /// refusals log separately with their capacity or breaker state. The daemon can
2701    /// attest which code it sent: without the event, a client's "the daemon
2702    /// refused me" and the daemon's own view could only be reconciled by
2703    /// argument. Malformed input (`invalid_project_root`) does not come here;
2704    /// rejecting a request that was never a valid open is not a refusal of one.
2705    fn route_open_refusal_frame(
2706        &self,
2707        ctx: &RouteCtx,
2708        frame: &Frame,
2709        module_id: &str,
2710        reason: &'static str,
2711        code: &'static str,
2712        message: impl Into<String>,
2713    ) -> Result<Frame, RouterError> {
2714        self.observe_route_open_refusal(ctx, module_id, reason, code);
2715        control_error_frame(frame, code, message.into())
2716    }
2717
2718    /// Refuse a `route.open` because the target module's bind-relay breaker is
2719    /// open, without attempting the relay.
2720    ///
2721    /// The wire code is `module_timeout`, which is the truth (the module has
2722    /// not been answering binds) and which both SDKs already classify as
2723    /// retryable with capped backoff. Reusing it is what keeps this change out
2724    /// of both SDKs; the daemon-side distinction lives in the counter key
2725    /// instead.
2726    ///
2727    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
2728    /// While a breaker is open this fires on every open to that module, and the
2729    /// stall written up in `docs/designs/route-open-head-of-line.md` already
2730    /// produced 261 lines about a single module inside 3000 lines of daemon
2731    /// log. The rare transitions are logged at warn/info instead and the volume
2732    /// is carried by the counter, so the evidence survives without the flood.
2733    /// The debug line keeps a per-refusal record reachable for whoever turns
2734    /// the level up.
2735    fn route_open_breaker_refusal_frame(
2736        &self,
2737        ctx: &RouteCtx,
2738        frame: &Frame,
2739        module_id: &str,
2740        consecutive_timeouts: u32,
2741        retry_in: Duration,
2742        probe_in_flight: bool,
2743    ) -> Result<Frame, RouterError> {
2744        self.counters
2745            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
2746        debug!(
2747            target: "control",
2748            code = "module_timeout",
2749            module_id = ?module_id,
2750            connection_id = ctx.connection_id.get(),
2751            consecutive_timeouts,
2752            retry_in_ms = retry_in.as_millis() as u64,
2753            probe_in_flight,
2754            "route.open refused by open bind-relay breaker"
2755        );
2756        let detail = if probe_in_flight {
2757            "one probe bind is already in flight; retry once it settles".to_string()
2758        } else {
2759            format!("not relaying for another {retry_in:?}")
2760        };
2761        control_error_frame(
2762            frame,
2763            "module_timeout",
2764            format!(
2765                "module_id '{module_id}' failed {consecutive_timeouts} consecutive route.bind \
2766                 relays; {detail}"
2767            ),
2768        )
2769    }
2770
2771    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
2772    /// requester's bytes (an unknown target is whatever the client sent) and
2773    /// is Debug-formatted so control characters land in the log escaped
2774    /// rather than as terminal sequences for whoever tails it.
2775    ///
2776    /// `reason` names the check that refused, because one wire code has
2777    /// several senders: after a module registers, `target_unavailable` can
2778    /// come from a missing role, an inactive registration, a supervisor that
2779    /// has not marked the process live, a missing forwarding connection, or a
2780    /// failed relay, and a log that records only the code cannot say which of
2781    /// them fired. It is a static, daemon-chosen label per branch, so it is
2782    /// safe to print plainly and stays a closed set.
2783    fn observe_route_open_refusal(
2784        &self,
2785        ctx: &RouteCtx,
2786        module_id: &str,
2787        reason: &'static str,
2788        code: &'static str,
2789    ) {
2790        self.counters.increment_route_open_refused(code);
2791        info!(
2792            target: "control",
2793            code,
2794            reason,
2795            module_id = ?module_id,
2796            connection_id = ctx.connection_id.get(),
2797            "route.open refused"
2798        );
2799        if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
2800            self.route_outages.record_not_serving(module_id, reason);
2801        }
2802    }
2803
2804    /// Record an ACCEPTED route.open.
2805    ///
2806    /// Refusals have been logged and counted since the attestation work; accepts
2807    /// were invisible, so the daemon knew every principal it stamped and wrote
2808    /// none of them down. The party that attests the identity was the only party
2809    /// not recording it, which left a credential vault unable to name the sender
2810    /// of a call that reached it (claustrum #43) and left the launch-nonce
2811    /// concurrency question unanswerable from the outside.
2812    ///
2813    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
2814    /// `code`/`module_id`/`connection_id` returns both directions of the same
2815    /// decision rather than two shapes a reader has to join by hand.
2816    ///
2817    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
2818    /// and the difference carries information rather than being an
2819    /// inconsistency. This line is only reachable after a successful bind to a
2820    /// REGISTERED module, so the value has already passed HELLO validation
2821    /// including the path-hazard refusal and cannot contain control bytes. A
2822    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
2823    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
2824    ///
2825    /// Bare is also what every other daemon line already emits (`module
2826    /// registered`, `configured module supervised`). Shipping `?module_id` here
2827    /// made this instrument the only one in the file whose ids did not answer
2828    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
2829    /// line whose whole purpose is being grepped beside its sibling.
2830    ///
2831    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
2832    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
2833    /// forwards every field type through it and a bare `&str` and a `?`-escaped
2834    /// one are recorded identically. A test written against that harness passes
2835    /// either way -- I wrote one, measured it, and deleted it rather than ship a
2836    /// green assertion that cannot fail. The same limit applies to the escaping
2837    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
2838    /// it reads as a guard on the Debug escaping and cannot detect its removal.
2839    /// Fencing either needs the real formatter, not the capture layer.
2840    ///
2841    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
2842    /// unix-socket options and subc is loopback TCP, so there is no peer identity
2843    /// to record. The ephemeral port would decay within minutes and answer only a
2844    /// live question. The identity question is instead answered by counting
2845    /// distinct live connections presenting one module's `consumer_identity` --
2846    /// "is anyone else holding this secret" rather than "is this the right
2847    /// process".
2848    fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
2849        self.route_outages.record_accepted(module_id);
2850        self.counters.increment_route_open_accepted(principal);
2851        info!(
2852            target: "control",
2853            principal,
2854            module_id,
2855            connection_id = ctx.connection_id.get(),
2856            "route.open accepted"
2857        );
2858    }
2859
2860    fn supervised_absent_route_open_refusal_frame(
2861        &self,
2862        ctx: &RouteCtx,
2863        frame: &Frame,
2864        module_id: &str,
2865        code: &'static str,
2866        status: &crate::supervise::ModuleStatus,
2867    ) -> Result<Frame, RouterError> {
2868        self.counters.increment_route_open_refused(code);
2869        info!(
2870            target: "control",
2871            code,
2872            reason = "supervised_not_registered",
2873            module_id = ?module_id,
2874            connection_id = ctx.connection_id.get(),
2875            state = %status.state,
2876            enabled = status.enabled,
2877            live = status.live,
2878            "route.open refused"
2879        );
2880        // A supervised module whose process has not registered is not
2881        // serving, whatever the reason; the supervisor knows this id, so it is
2882        // safe to track.
2883        self.route_outages
2884            .record_not_serving(module_id, "supervised_not_registered");
2885        control_error_frame(
2886            frame,
2887            code,
2888            format!(
2889                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
2890                status.state, status.enabled, status.live
2891            ),
2892        )
2893    }
2894
2895    async fn handle_route_open(
2896        &self,
2897        ctx: &RouteCtx,
2898        frame: Frame,
2899        request: RouteOpenRequest,
2900    ) -> Result<Vec<Frame>, RouterError> {
2901        let RouteOpenRequest {
2902            target,
2903            mut identity,
2904            consumer_identity,
2905            consumer_capabilities,
2906            role_versions,
2907            admission_facts,
2908            scope,
2909        } = request;
2910        let target_module_id = target_module_id(&target).to_string();
2911        debug!(
2912            connection_id = ctx.connection_id.get(),
2913            corr = frame.header.corr,
2914            module_id = %target_module_id,
2915            "handling route.open"
2916        );
2917
2918        // A malformed declaration is refused first, before anything about the
2919        // target is looked up: the same body would be refused against any
2920        // module, so the caller learns nothing by retrying or waiting. An empty
2921        // map declares nothing and travels as no field at all, so a provider
2922        // only ever sees a missing field or a non-empty one.
2923        let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
2924        if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
2925            self.observe_route_open_refusal(
2926                ctx,
2927                &target_module_id,
2928                "invalid_role_versions",
2929                error_codes::INVALID_REQUEST,
2930            );
2931            return Ok(vec![control_error_body_frame(
2932                &frame,
2933                ErrorBody {
2934                    code: error_codes::INVALID_REQUEST.to_string(),
2935                    message: error.to_string(),
2936                    detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
2937                },
2938            )?]);
2939        }
2940
2941        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
2942        // opposite. Below, a caller learns whether a module is unregistered,
2943        // supervised-but-down (with state/enabled/live), or registered without the
2944        // requested role. Elsewhere that is an enumeration leak: a probe learning
2945        // the shape of a fleet it cannot otherwise see.
2946        //
2947        // It is not one here, and the reason is the ACCESS MODEL rather than
2948        // anything about these errors. Reaching route.open requires the
2949        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
2950        // connection file, so any caller who completes it already runs as this
2951        // user -- and can read subc.jsonc for the module list and `ck module
2952        // status` for live state. The reply discloses nothing the caller cannot
2953        // read more easily from disk, while the precision is load-bearing:
2954        // `unknown_module` is retryable and a missing role is not.
2955        //
2956        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
2957        // remote transport, a sandboxed caller, a shared-host mode -- THAT
2958        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
2959        let Some(registration) = self
2960            .registry
2961            .get_module(&target_module_id)
2962            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2963        else {
2964            if let Some((status, warming)) =
2965                self.supervisor_status(&target_module_id, frame.header.corr)?
2966            {
2967                // BEFORE the two availability codes below, because for a module
2968                // that speaks no subc wire both of them are false comfort: they
2969                // say "not right now" and are retried, and this module will
2970                // never register no matter how long the caller waits. The
2971                // absence here is the declaration being honoured, not a module
2972                // that is late.
2973                if status.protocol == ModuleProtocol::None {
2974                    return Ok(vec![self.route_open_refusal_frame(
2975                        ctx,
2976                        &frame,
2977                        &target_module_id,
2978                        "protocol_none",
2979                        error_codes::MODULE_NO_PROTOCOL,
2980                        format!(
2981                            "module_id '{target_module_id}' is declared protocol: none; \
2982                             it speaks no subc wire and serves no routes"
2983                        ),
2984                    )?]);
2985                }
2986                let code = if warming {
2987                    "module_warming"
2988                } else {
2989                    "target_unavailable"
2990                };
2991                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
2992                    ctx,
2993                    &frame,
2994                    &target_module_id,
2995                    code,
2996                    &status,
2997                )?]);
2998            }
2999            if let Some(removed_ago_ms) =
3000                self.supervisor.removal_tombstone_age_ms(&target_module_id)
3001            {
3002                return Ok(vec![self.route_open_refusal_frame(
3003                    ctx,
3004                    &frame,
3005                    &target_module_id,
3006                    "removed",
3007                    error_codes::MODULE_REMOVED,
3008                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
3009                )?]);
3010            }
3011            return Ok(vec![self.route_open_refusal_frame(
3012                ctx,
3013                &frame,
3014                &target_module_id,
3015                "not_registered",
3016                error_codes::UNKNOWN_MODULE,
3017                format!("module_id '{target_module_id}' is not registered"),
3018            )?]);
3019        };
3020
3021        // Best-effort only: registry readiness and forwarding reservation use
3022        // different locks, so a module can flip readiness between this read and
3023        // the relay. Modules must still tolerate an `on_bind` while not ready.
3024        if !registration.ready {
3025            self.counters
3026                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
3027            info!(
3028                target: "control",
3029                code = error_codes::MODULE_WARMING,
3030                module_id = ?target_module_id,
3031                connection_id = ctx.connection_id.get(),
3032                reason = "declared_not_ready",
3033                "route.open refused"
3034            );
3035            // The module is registered but says it cannot take work, which is
3036            // an outage from the caller's side even though its process is up.
3037            self.route_outages
3038                .record_not_serving(&target_module_id, "declared_not_ready");
3039            return Ok(vec![control_error_body_frame(
3040                &frame,
3041                ErrorBody {
3042                    code: error_codes::MODULE_WARMING.to_string(),
3043                    message: format!(
3044                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
3045                    ),
3046                    detail: Some(serde_json::json!({
3047                        "reason": "declared_not_ready"
3048                    })),
3049                },
3050            )?]);
3051        }
3052
3053        // Effective readiness, second half: a module that declares a capability
3054        // `need: required` is not routable while that capability has no
3055        // registered provider. It is enforced HERE, as a retryable routing
3056        // refusal, and deliberately not as spawn ordering or a boot block. The
3057        // module is still started and registered and can make its own calls;
3058        // spawn ordering is a promise that cannot be kept once a provider
3059        // crashes at runtime, and refusing to boot would stop the whole
3060        // machine, including the tools needed to fix its configuration.
3061        //
3062        // "Provided" is the evaluator's verdict, which counts a provider as
3063        // soon as it has REGISTERED, not once it is ready. Two modules that
3064        // require each other's capabilities are therefore both routable once
3065        // both register; counting readiness instead would deadlock them.
3066        //
3067        // Only new opens are refused. Routes already bound when a provider
3068        // goes away stay bound: nothing here tears them down, and the module
3069        // answers them as it can. Like the readiness read above this is
3070        // best-effort against a provider registering or leaving concurrently.
3071        if let Some(capability) = self.first_unprovided_required_capability(&registration) {
3072            self.counters
3073                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
3074            info!(
3075                target: "control",
3076                code = error_codes::MODULE_WARMING,
3077                module_id = ?target_module_id,
3078                connection_id = ctx.connection_id.get(),
3079                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3080                capability = %capability,
3081                "route.open refused"
3082            );
3083            return Ok(vec![control_error_body_frame(
3084                &frame,
3085                ErrorBody {
3086                    code: error_codes::MODULE_WARMING.to_string(),
3087                    message: format!(
3088                        "module_id '{target_module_id}' requires capability '{capability}', \
3089                         which no registered module provides; retry"
3090                    ),
3091                    detail: Some(serde_json::json!({
3092                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3093                        "capability": capability,
3094                    })),
3095                },
3096            )?]);
3097        }
3098
3099        if !target_has_required_role(&target, &registration.manifest.provides) {
3100            return Ok(vec![self.route_open_refusal_frame(
3101                ctx,
3102                &frame,
3103                &target_module_id,
3104                "role_not_provided",
3105                "target_unavailable",
3106                format!("module_id '{target_module_id}' does not provide the requested target"),
3107            )?]);
3108        }
3109
3110        if registration.state != ChannelState::Active {
3111            return Ok(vec![self.route_open_refusal_frame(
3112                ctx,
3113                &frame,
3114                &target_module_id,
3115                "registration_not_active",
3116                "target_unavailable",
3117                format!("module_id '{target_module_id}' is not active"),
3118            )?]);
3119        }
3120
3121        if self
3122            .forwarding
3123            .module_is_draining(&target_module_id)
3124            .map_err(RouterError::Forwarding)?
3125        {
3126            return Ok(vec![self.route_open_refusal_frame(
3127                ctx,
3128                &frame,
3129                &target_module_id,
3130                "reloading",
3131                "module_reloading",
3132                format!("module_id '{target_module_id}' is reloading"),
3133            )?]);
3134        }
3135
3136        if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
3137            process_liveness.process_live(&target_module_id) == Some(false)
3138        }) {
3139            // A module the supervisor is restarting or reloading can still hold
3140            // a registration: the old process before its connection closes, or
3141            // a new one that registered while the supervisor was draining. The
3142            // forwarding table does not see that as draining, but the consumer
3143            // should still be told to retry soon, exactly as for the drain
3144            // above, rather than that the target is unavailable.
3145            if process_liveness.process_replacing(&target_module_id) {
3146                return Ok(vec![self.route_open_refusal_frame(
3147                    ctx,
3148                    &frame,
3149                    &target_module_id,
3150                    "reloading",
3151                    "module_reloading",
3152                    format!("module_id '{target_module_id}' is reloading"),
3153                )?]);
3154            }
3155            return Ok(vec![self.route_open_refusal_frame(
3156                ctx,
3157                &frame,
3158                &target_module_id,
3159                "supervisor_not_live",
3160                "target_unavailable",
3161                format!("module_id '{target_module_id}' is not live"),
3162            )?]);
3163        }
3164
3165        if !self
3166            .forwarding
3167            .has_live_module_connection(&target_module_id)
3168            .map_err(RouterError::Forwarding)?
3169        {
3170            return Ok(vec![self.route_open_refusal_frame(
3171                ctx,
3172                &frame,
3173                &target_module_id,
3174                "no_forwarding_connection",
3175                "target_unavailable",
3176                format!("module_id '{target_module_id}' has no live forwarding connection"),
3177            )?]);
3178        }
3179
3180        if let Some(error) =
3181            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
3182        {
3183            self.observe_route_open_refusal(
3184                ctx,
3185                &target_module_id,
3186                "op_not_allowed",
3187                "op_not_allowed",
3188            );
3189            return Ok(vec![error]);
3190        }
3191
3192        let principal = match self.route_open_principal(&frame, consumer_identity)? {
3193            Ok(principal) => principal,
3194            Err(error) => {
3195                self.observe_route_open_refusal(
3196                    ctx,
3197                    &target_module_id,
3198                    "bad_consumer_identity",
3199                    "bad_consumer_identity",
3200                );
3201                return Ok(vec![error]);
3202            }
3203        };
3204
3205        // This is attested, control-plane policy for supervised module origins.
3206        // Keep it before route reservation and out of the opaque forwarding hot
3207        // path: data frames must never acquire a per-frame capability check.
3208        if let Principal::Reserved {
3209            module_id: opening_module_id,
3210        } = &principal
3211        {
3212            if let Some(opening_registration) = self
3213                .registry
3214                .get_module(opening_module_id)
3215                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3216            {
3217                if let Some(capability) =
3218                    denied_capability(&opening_registration.manifest, &registration.manifest)
3219                {
3220                    warn!(
3221                        opening_module_id,
3222                        target_module_id,
3223                        capability,
3224                        "refusing route.open because an attested capability deny edge matches"
3225                    );
3226                    return Ok(vec![self.route_open_refusal_frame(
3227                        ctx,
3228                        &frame,
3229                        &target_module_id,
3230                        "capability_deny_edge",
3231                        "capability_forbidden",
3232                        format!(
3233                            "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
3234                        ),
3235                    )?]);
3236                }
3237            }
3238        }
3239
3240        if admission_facts.is_some() {
3241            let carrier_matches = matches!(
3242                &principal,
3243                Principal::Reserved { module_id }
3244                    if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
3245            );
3246            if !carrier_matches {
3247                return Ok(vec![self.route_open_refusal_frame(
3248                    ctx,
3249                    &frame,
3250                    &target_module_id,
3251                    "admission_facts_carrier_not_permitted",
3252                    "admission_facts_not_permitted",
3253                    "admission facts may only be carried by the configured reserved module",
3254                )?]);
3255            }
3256
3257            let target_allowed = self
3258                .admission_facts_targets
3259                .as_ref()
3260                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
3261            if !target_allowed {
3262                return Ok(vec![self.route_open_refusal_frame(
3263                    ctx,
3264                    &frame,
3265                    &target_module_id,
3266                    "admission_facts_target_not_listed",
3267                    "admission_facts_target_not_allowed",
3268                    format!(
3269                        "admission facts are not permitted for target module_id '{target_module_id}'"
3270                    ),
3271                )?]);
3272            }
3273
3274            // Keep the value opaque to subc. The downstream admission validator owns
3275            // schema and semantic checks; this daemon only enforces carrier authority
3276            // and the configured destination allowlist.
3277        }
3278
3279        // Scope admission, on the attested principal above and never on the
3280        // request body. The tag read here travels with the pending bind and is
3281        // compared with the published one at commit, so a sync between here
3282        // and the module's ack refuses the open instead of binding a stamp
3283        // that is no longer true.
3284        let (bound_scope, scope_stamp) = match scope {
3285            None => (None, None),
3286            Some(selector) => {
3287                let owner_configured = match &selector.owner {
3288                    Principal::Reserved { module_id } => self.supervisor.get(module_id).is_some(),
3289                    _ => false,
3290                };
3291                let admitted = self
3292                    .scopes
3293                    .read()
3294                    .unwrap_or_else(|poisoned| poisoned.into_inner())
3295                    .admit(&principal, &target_module_id, &selector, owner_configured);
3296                match admitted {
3297                    Ok(admission) => (
3298                        Some(BoundScope {
3299                            owner: admission.owner,
3300                            scope_ref: admission.stamp.scope_ref.clone(),
3301                            tag: admission.tag,
3302                        }),
3303                        Some(admission.stamp),
3304                    ),
3305                    Err(refusal) => {
3306                        return Ok(vec![self.route_open_refusal_frame(
3307                            ctx,
3308                            &frame,
3309                            &target_module_id,
3310                            refusal.code,
3311                            refusal.code,
3312                            refusal.message,
3313                        )?]);
3314                    }
3315                }
3316            }
3317        };
3318
3319        // Bind admits a root that no longer exists on disk, because refusing here
3320        // closes the only exit from a paused run: cancel needs a bound route, and a
3321        // renamed or reclaimed directory makes that route unopenable forever. The
3322        // run itself is intact and still addressable by its recorded identity.
3323        //
3324        // This does NOT relax the rule the strict constructor protects. That rule is
3325        // that no root is ever aliased into NEW durable state -- a missing component
3326        // can reappear as a symlink elsewhere, which would move the identity and
3327        // split a session's history across two of them. The engine now refuses the
3328        // two operations that create such state (send and import) at admission,
3329        // which is a narrower way to hold the same invariant: reads and terminations
3330        // are admitted, writes are not. That refusal had to ship before this line
3331        // changed, or there is an interval where a send commits under a provisional
3332        // identity -- the exact failure the original policy existed to prevent.
3333        //
3334        // Resolution follows realpath rather than lexical cleanup: the longest
3335        // existing ancestor is canonicalized and the missing tail re-appended, so a
3336        // live root is unchanged and a vanished leaf keeps the identity it was
3337        // admitted under. Lexical cleanup would mint a DIFFERENT identity for the
3338        // same caller the moment the directory vanished, which strands the run more
3339        // quietly than refusing it.
3340        let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
3341            Ok(project_root) => project_root,
3342            Err(err) => {
3343                return Ok(vec![control_error_frame(
3344                    &frame,
3345                    "invalid_project_root",
3346                    err.to_string(),
3347                )?])
3348            }
3349        };
3350        identity.project_root = project_root.as_path().to_path_buf();
3351
3352        // Last gate before any relay work, and deliberately after the cheap
3353        // registry and availability checks above: those name a more precise
3354        // condition (unknown, removed, reloading) and a caller is better served
3355        // by the precise code than by this one.
3356        //
3357        // Everything below this point costs an egress permit, a reserved handle
3358        // pair and, if the module does not answer, the whole relay budget. The
3359        // reader no longer waits for that budget, so cap each target explicitly;
3360        // serial dispatch used to provide the accidental cap of one relay per
3361        // connection. Admission is a mutex-protected count and never waits.
3362        let _concurrency_guard = match self
3363            .route_bind_concurrency
3364            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
3365        {
3366            Ok(guard) => guard,
3367            Err(in_flight) => {
3368                return Ok(vec![self.route_open_target_capacity_refusal(
3369                    ctx,
3370                    &frame,
3371                    &target_module_id,
3372                    in_flight,
3373                )?]);
3374            }
3375        };
3376
3377        // A module that has already burned the whole budget `threshold` times
3378        // in a row does not get to charge it again until a probe says it recovered.
3379        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
3380            RouteBindAdmission::Admitted { guard, probe } => {
3381                if probe {
3382                    info!(
3383                        module_id = %target_module_id,
3384                        connection_id = ctx.connection_id.get(),
3385                        "route.bind breaker half-open: admitting one probe"
3386                    );
3387                }
3388                guard
3389            }
3390            RouteBindAdmission::Refused {
3391                consecutive_timeouts,
3392                retry_in,
3393                probe_in_flight,
3394            } => {
3395                return Ok(vec![self.route_open_breaker_refusal_frame(
3396                    ctx,
3397                    &frame,
3398                    &target_module_id,
3399                    consecutive_timeouts,
3400                    retry_in,
3401                    probe_in_flight,
3402                )?]);
3403            }
3404        };
3405
3406        // Resolve the per-module budget here so the wait matches the operator's
3407        // intent for this specific target. A per-module override in
3408        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
3409        // daemons) wins over the daemon-wide default.
3410        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
3411        let relay_deadline = Instant::now() + route_bind_relay_timeout;
3412        let pending = match self
3413            .forwarding
3414            .begin_route_bind_relay_for(
3415                ctx.connection_id,
3416                ctx.egress.clone(),
3417                response_version(&frame),
3418                frame.header.corr,
3419                &target_module_id,
3420                principal.clone(),
3421                bound_scope,
3422                Some(project_root),
3423                relay_deadline,
3424            )
3425            .await
3426        {
3427            Ok(pending) => pending,
3428            Err(err) => {
3429                return Ok(vec![self.route_open_refusal_frame(
3430                    ctx,
3431                    &frame,
3432                    &target_module_id,
3433                    "relay_reservation_failed",
3434                    forwarding_error_code(&err),
3435                    err.to_string(),
3436                )?])
3437            }
3438        };
3439        let crate::forwarding::PendingRouteBindRelay {
3440            endpoint,
3441            module_sink,
3442            negotiated_ver,
3443            client_channel,
3444            client_epoch,
3445            module_channel,
3446            module_epoch,
3447            corr: relay_corr,
3448            receiver,
3449        } = pending;
3450        let mut reservation =
3451            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
3452
3453        debug!(
3454            connection_id = ctx.connection_id.get(),
3455            client_channel,
3456            client_epoch,
3457            module_channel,
3458            module_epoch,
3459            "reserved route handle pair"
3460        );
3461        // Rendered BEFORE the move into the relay, because the accept arm below
3462        // is where it is logged and the principal is gone by then.
3463        let principal_label = match &principal {
3464            Principal::Reserved { module_id } => format!("reserved:{module_id}"),
3465            Principal::Direct => "direct".to_string(),
3466            other => format!("{other:?}"),
3467        };
3468        let relay = ModuleControlRequest::RouteBind {
3469            route_channel: module_channel,
3470            epoch: module_epoch,
3471            target,
3472            identity,
3473            principal: Some(principal),
3474            consumer_capabilities,
3475            role_versions,
3476            admission_facts,
3477            scope: scope_stamp,
3478        };
3479        let relay_body = serde_json::to_vec(&relay).map_err(|err| {
3480            RouterError::backend(
3481                0,
3482                frame.header.corr,
3483                format!("failed to encode route.bind request: {err}"),
3484            )
3485        })?;
3486        let relay_frame = Frame::build_with_version(
3487            negotiated_ver,
3488            FrameType::Request,
3489            control_flags(),
3490            0,
3491            0,
3492            relay_corr,
3493            relay_body,
3494        )
3495        .map_err(RouterError::FrameBuild)?;
3496
3497        if let Err(err) = module_sink.send(relay_frame).await {
3498            reservation.release_and_disarm();
3499            return Ok(vec![self.route_open_refusal_frame(
3500                ctx,
3501                &frame,
3502                &target_module_id,
3503                "relay_send_failed",
3504                "target_unavailable",
3505                err.to_string(),
3506            )?]);
3507        }
3508
3509        if !self
3510            .forwarding
3511            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
3512            .map_err(RouterError::Forwarding)?
3513        {
3514            self.send_abandoned_route_bind_goodbye(
3515                &module_sink,
3516                negotiated_ver,
3517                module_channel,
3518                module_epoch,
3519            );
3520        }
3521
3522        match timeout_at(relay_deadline, receiver).await {
3523            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
3524                reservation.disarm();
3525                if breaker.record_accepted() {
3526                    info!(
3527                        module_id = %target_module_id,
3528                        "route.bind breaker closed: the probe was accepted"
3529                    );
3530                }
3531                self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
3532                Ok(Vec::new())
3533            }
3534            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
3535                reservation.release_and_disarm();
3536                // A module that says no in microseconds is healthy. Rejection
3537                // is a different condition with its own refusal and must not
3538                // move the breaker.
3539                breaker.record_inconclusive();
3540                // The daemon's own commit re-check refused the bind because the
3541                // scope ended or changed after admission. The module accepted;
3542                // counting it as a module rejection would blame the module.
3543                let scope_code = match body.code.as_str() {
3544                    error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
3545                    error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
3546                    _ => None,
3547                };
3548                if let Some(code) = scope_code {
3549                    self.observe_route_open_refusal(
3550                        ctx,
3551                        &target_module_id,
3552                        "scope_changed_before_commit",
3553                        code,
3554                    );
3555                    return Ok(vec![control_error_body_frame(&frame, body)?]);
3556                }
3557                self.counters
3558                    .increment_route_open_refused("module_rejected");
3559                info!(
3560                    target: "control",
3561                    code = "module_rejected",
3562                    module_code = ?body.code,
3563                    module_id = ?target_module_id,
3564                    connection_id = ctx.connection_id.get(),
3565                    "route.open refused"
3566                );
3567                Ok(vec![control_error_body_frame(&frame, body)?])
3568            }
3569            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
3570                reservation.release_and_disarm();
3571                breaker.record_inconclusive();
3572                // Fires when the module's connection closes while a relayed
3573                // bind is pending -- typically a caller racing a module restart
3574                // whose bind was relayed BEFORE the drain mark went up. Logged
3575                // because the caller sees only its own error and the fleet has
3576                // already spent one diagnosis round unable to tell this arm
3577                // from a relay timeout without daemon-side evidence.
3578                tracing::warn!(
3579                    module_id = %target_module_id,
3580                    "route.bind relay abandoned: {message}"
3581                );
3582                Ok(vec![self.route_open_refusal_frame(
3583                    ctx,
3584                    &frame,
3585                    &target_module_id,
3586                    "relay_abandoned",
3587                    "target_unavailable",
3588                    message,
3589                )?])
3590            }
3591            Ok(Err(_)) => {
3592                reservation.release_and_disarm();
3593                breaker.record_inconclusive();
3594                Ok(vec![self.route_open_refusal_frame(
3595                    ctx,
3596                    &frame,
3597                    &target_module_id,
3598                    "relay_waiter_canceled",
3599                    "target_unavailable",
3600                    "route.bind relay waiter was canceled before the module responded",
3601                )?])
3602            }
3603            Err(_) => {
3604                reservation.release_and_disarm();
3605                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
3606                // answer at all is the one condition a fast refusal can
3607                // usefully stand in for; every other arm already answered.
3608                if let Some(opened) = breaker.record_timeout(
3609                    self.route_bind_breaker_threshold,
3610                    self.route_bind_breaker_cooldown,
3611                ) {
3612                    warn!(
3613                        module_id = %target_module_id,
3614                        consecutive_timeouts = opened.consecutive_timeouts,
3615                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
3616                        reopened_after_probe = opened.reopened_after_probe,
3617                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
3618                    );
3619                }
3620                // The generous budget just burned to no answer: the module is
3621                // registered and its connection is up, but its bind handler sat
3622                // on the ack for the full budget (warm-on-bind, cold configure,
3623                // or a wedged handler). Every earlier unavailability shape
3624                // fast-refuses BEFORE the relay, so this arm firing means the
3625                // slowness is module-side -- log it so the per-module timeline
3626                // is reconstructable without client audit rows.
3627                tracing::warn!(
3628                    module_id = %target_module_id,
3629                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
3630                    "route.bind relay timed out: module did not ack within budget"
3631                );
3632                Ok(vec![self.route_open_refusal_frame(
3633                    ctx,
3634                    &frame,
3635                    &target_module_id,
3636                    "relay_timed_out",
3637                    "module_timeout",
3638                    format!(
3639                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
3640                        route_bind_relay_timeout
3641                    ),
3642                )?])
3643            }
3644        }
3645    }
3646
3647    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3648        let response = ClientControlResponse::SupervisorSpawnSnapshot {
3649            snapshot: self.supervisor.spawn_snapshot(),
3650        };
3651        Ok(vec![control_response_body_frame(
3652            &frame,
3653            &response,
3654            "ClientControlResponse::SupervisorSpawnSnapshot",
3655        )?])
3656    }
3657
3658    fn handle_supervisor_spawn_subscribe(
3659        &self,
3660        ctx: &RouteCtx,
3661        frame: Frame,
3662        since: Option<SpawnCursor>,
3663    ) -> Result<Vec<Frame>, RouterError> {
3664        match self.supervisor.subscribe_spawns(
3665            ctx.connection_id,
3666            frame.header.corr,
3667            response_version(&frame),
3668            since,
3669            ctx.egress.clone(),
3670        ) {
3671            Ok(()) => Ok(Vec::new()),
3672            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
3673                Ok(vec![control_error_body_frame(
3674                    &frame,
3675                    ErrorBody {
3676                        code: "spawn_cursor_incarnation_mismatch".to_string(),
3677                        message: "spawn cursor belongs to a different daemon incarnation"
3678                            .to_string(),
3679                        detail: Some(serde_json::json!({
3680                            "current_daemon_incarnation": current
3681                        })),
3682                    },
3683                )?])
3684            }
3685            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
3686                &frame,
3687                ErrorBody {
3688                    code: "spawn_cursor_too_old".to_string(),
3689                    message: "spawn cursor predates the retained event ring".to_string(),
3690                    detail: Some(serde_json::json!({
3691                        "oldest_retained_cursor": oldest
3692                    })),
3693                },
3694            )?]),
3695            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
3696                0,
3697                frame.header.corr,
3698                format!("failed to open supervisor spawn subscription: {error}"),
3699            )),
3700        }
3701    }
3702
3703    async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3704        let generation = self
3705            .registry
3706            .generation()
3707            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3708        let mut modules = Vec::new();
3709        for module in self.supervisor.list() {
3710            let status = module.status_for_control("list").map_err(|err| {
3711                RouterError::backend(
3712                    0,
3713                    frame.header.corr,
3714                    format!("failed to read supervisor status: {err}"),
3715                )
3716            })?;
3717            let (configured, _) = module.configuration().map_err(|err| {
3718                RouterError::backend(
3719                    0,
3720                    frame.header.corr,
3721                    format!("failed to read module configuration: {err}"),
3722                )
3723            })?;
3724            // Status and configuration snapshots release their locks before the image probe awaits.
3725            let image = module.running_image_agreement().await;
3726            // Read per request so the figure is current when the operator asks;
3727            // the daemon samples nothing in between.
3728            let resources = Some(module.child_resource_usage());
3729            let pending_reload = Some(reload_verdict(
3730                &configured.program,
3731                status.spawned_from.as_deref(),
3732                image,
3733            ));
3734            modules.push(SupervisorEntry {
3735                launch_nonce_env: Some(
3736                    configured.protocol != subc_control::ModuleProtocol::None
3737                        && (!cfg!(unix) || configured.launch_nonce_env),
3738                ),
3739                module_id: status.module_id,
3740                state: status.state.to_string(),
3741                enabled: status.enabled,
3742                live: status.live,
3743                protocol: status.protocol,
3744                health: status.health.status,
3745                pending_reload,
3746                last_probe_ms: status.health.last_probe_ms,
3747                last_exit_code: status.last_exit.as_ref().and_then(|e| e.code),
3748                last_exit_signal: status.last_exit.as_ref().and_then(|e| e.signal),
3749                last_exit_ms: status.last_exit.as_ref().map(|e| e.at_ms),
3750                last_exit_kind: status.last_exit.as_ref().map(|e| e.kind.into()),
3751                restart_count: Some(status.restart_count),
3752                max_restarts: Some(status.max_restarts),
3753                lifetime_restarts: Some(status.lifetime_restarts),
3754                spawn_generation: Some(status.spawn_generation),
3755                restart_window_secs: Some(status.restart_window.as_secs()),
3756                drain_timeout_ms: Some(status.drain_timeout.as_millis() as u64),
3757                restart_backoff_ms: Some(status.restart_backoff.as_millis() as u64),
3758                restart_max_backoff_ms: Some(status.restart_max_backoff.as_millis() as u64),
3759                resources,
3760            });
3761        }
3762        let response = ClientControlResponse::SupervisorList {
3763            generation,
3764            modules,
3765        };
3766        Ok(vec![control_response_body_frame(
3767            &frame,
3768            &response,
3769            "ClientControlResponse::SupervisorList",
3770        )?])
3771    }
3772
3773    fn handle_supervisor_stderr_tail(
3774        &self,
3775        frame: Frame,
3776        module_id: String,
3777        max_lines: Option<u32>,
3778        max_bytes: Option<u32>,
3779    ) -> Result<Vec<Frame>, RouterError> {
3780        let Some(module) = self.supervisor.get(&module_id) else {
3781            return Ok(vec![control_error_frame(
3782                &frame,
3783                "unknown_module",
3784                format!("module_id '{module_id}' is not supervised"),
3785            )?]);
3786        };
3787
3788        let snapshot = module.stderr_tail(
3789            max_lines.map(|value| value as usize),
3790            max_bytes.map(|value| value as usize),
3791        );
3792
3793        let response = ClientControlResponse::SupervisorStderrTail {
3794            module_id,
3795            tail: StderrTail {
3796                capture: match snapshot.capture {
3797                    CaptureState::Captured => StderrCaptureState::Captured,
3798                    CaptureState::Incomplete { reason } => {
3799                        StderrCaptureState::Incomplete { reason }
3800                    }
3801                    CaptureState::NotCaptured { reason } => {
3802                        StderrCaptureState::NotCaptured { reason }
3803                    }
3804                },
3805                entries: snapshot
3806                    .entries
3807                    .into_iter()
3808                    .map(|entry| match entry {
3809                        TailEntry::Line {
3810                            text,
3811                            truncated,
3812                            at_ms,
3813                        } => StderrTailEntry::Line {
3814                            text,
3815                            truncated,
3816                            at_ms,
3817                        },
3818                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
3819                    })
3820                    .collect(),
3821                dropped_lines: snapshot.dropped_lines,
3822            },
3823        };
3824        Ok(vec![control_response_body_frame(
3825            &frame,
3826            &response,
3827            "ClientControlResponse::SupervisorStderrTail",
3828        )?])
3829    }
3830
3831    async fn handle_supervisor_terminals(
3832        &self,
3833        frame: Frame,
3834        module_id: String,
3835    ) -> Result<Vec<Frame>, RouterError> {
3836        let Some(module) = self.supervisor.get(&module_id) else {
3837            return Ok(vec![control_error_frame(
3838                &frame,
3839                "unknown_module",
3840                format!("module_id '{module_id}' is not supervised"),
3841            )?]);
3842        };
3843
3844        // The journal read runs on a blocking thread: it can be megabytes of
3845        // file I/O and must not occupy a runtime worker.
3846        let terminals = module
3847            .read_durable_terminal_history()
3848            .await
3849            .map_err(|error| {
3850                RouterError::backend(
3851                    0,
3852                    frame.header.corr,
3853                    format!("failed to read terminal history: {error}"),
3854                )
3855            })?;
3856        let response = ClientControlResponse::SupervisorTerminals {
3857            module_id,
3858            terminals,
3859        };
3860        Ok(vec![control_response_body_frame(
3861            &frame,
3862            &response,
3863            "ClientControlResponse::SupervisorTerminals",
3864        )?])
3865    }
3866
3867    fn handle_supervisor_routes(
3868        &self,
3869        frame: Frame,
3870        module_id: Option<String>,
3871    ) -> Result<Vec<Frame>, RouterError> {
3872        let modules = self
3873            .forwarding
3874            .route_census(module_id.as_deref())
3875            .map_err(RouterError::Forwarding)?
3876            .into_iter()
3877            .map(|(module_id, routes)| SupervisorRouteModule {
3878                module_id,
3879                routes: routes
3880                    .into_iter()
3881                    .map(|route| SupervisorRoute {
3882                        consumer: match route.principal {
3883                            Principal::Reserved { module_id } => {
3884                                SupervisorRouteConsumer::Reserved { module_id }
3885                            }
3886                            Principal::Direct | Principal::Unverified => {
3887                                SupervisorRouteConsumer::Direct {
3888                                    connection_id: route.goodbye_target.connection_id.get(),
3889                                }
3890                            }
3891                        },
3892                        age_ms: Instant::now()
3893                            .saturating_duration_since(route.bound_at)
3894                            .as_millis()
3895                            .try_into()
3896                            .unwrap_or(u64::MAX),
3897                        draining: route.draining,
3898                        drain_reason: route.drain_reason,
3899                    })
3900                    .collect(),
3901            })
3902            .collect();
3903        let response = ClientControlResponse::SupervisorRoutes { modules };
3904        Ok(vec![control_response_body_frame(
3905            &frame,
3906            &response,
3907            "ClientControlResponse::SupervisorRoutes",
3908        )?])
3909    }
3910
3911    async fn handle_supervisor_provenance(
3912        &self,
3913        frame: Frame,
3914        module_id: Option<String>,
3915    ) -> Result<Vec<Frame>, RouterError> {
3916        let mut selected = if let Some(module_id) = module_id {
3917            let Some(module) = self.supervisor.get(&module_id) else {
3918                return Ok(vec![control_error_frame(
3919                    &frame,
3920                    "unknown_module",
3921                    format!("module_id '{module_id}' is not supervised"),
3922                )?]);
3923            };
3924            vec![module]
3925        } else {
3926            self.supervisor.list()
3927        };
3928
3929        let mut modules = Vec::with_capacity(selected.len());
3930        for module in selected.drain(..) {
3931            let status = module.status().map_err(|err| {
3932                RouterError::backend(
3933                    0,
3934                    frame.header.corr,
3935                    format!("failed to read supervisor status: {err}"),
3936                )
3937            })?;
3938            let module_declared = self
3939                .registry
3940                .get_module(&status.module_id)
3941                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3942                .and_then(|registration| registration.manifest.provenance)
3943                .map(|build| ModuleDeclaredProvenance::Reported { build })
3944                .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
3945            #[cfg(test)]
3946            let running_image = match &self.provenance_probe_override {
3947                Some(result) => result.clone(),
3948                None => module.running_image_agreement().await,
3949            };
3950            #[cfg(not(test))]
3951            let running_image = module.running_image_agreement().await;
3952            modules.push(SupervisorModuleProvenance {
3953                module_id: status.module_id,
3954                module_declared,
3955                daemon_observed: SupervisorObservedProcess {
3956                    pid: status.pid,
3957                    spawned_at_ms: status.spawned_at_ms,
3958                    spawned_from: status.spawned_from,
3959                    running_image,
3960                },
3961            });
3962        }
3963        let daemon = SupervisorDaemonProvenance {
3964            daemon_build: self.daemon_provenance.build.clone(),
3965            daemon_observed: DaemonObservedProcess {
3966                pid: self.daemon_provenance.pid,
3967                started_at_ms: self
3968                    .daemon_provenance
3969                    .start_clock
3970                    .map(|clock| clock.started_at_ms())
3971                    .or(self.daemon_provenance.started_at_ms),
3972                running_image: self
3973                    .daemon_provenance
3974                    .probe
3975                    .observe(
3976                        self.daemon_provenance.pid,
3977                        self.daemon_provenance.executable_path.as_deref(),
3978                        self.daemon_provenance.executable_identity,
3979                        self.daemon_provenance.process_start_time,
3980                    )
3981                    .await,
3982            },
3983        };
3984        let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
3985        Ok(vec![control_response_body_frame(
3986            &frame,
3987            &response,
3988            "ClientControlResponse::SupervisorProvenance",
3989        )?])
3990    }
3991
3992    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3993        self.refresh_capability_requirements();
3994        let generation = self
3995            .registry
3996            .generation()
3997            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3998        let modules = self
3999            .supervisor
4000            .list()
4001            .into_iter()
4002            .map(|module| {
4003                let status = module.status_for_control("health").map_err(|err| {
4004                    RouterError::backend(
4005                        0,
4006                        frame.header.corr,
4007                        format!("failed to read supervisor health: {err}"),
4008                    )
4009                })?;
4010                let module_id = status.module_id;
4011                let capability_detail = self
4012                    .capability_evaluator
4013                    .required_problem_detail(&module_id);
4014                Ok(SupervisorHealthEntry {
4015                    module_id,
4016                    status: status.health.status,
4017                    detail: append_capability_problem_detail(
4018                        status.health.detail,
4019                        capability_detail,
4020                    ),
4021                    metrics: status.health.metrics,
4022                    consecutive_failures: status.health.consecutive_failures,
4023                    late_answer_count: status.health.late_answer_count,
4024                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
4025                    last_action: status.health.last_action,
4026                    last_action_ms: status.health.last_action_ms,
4027                    last_probe_ms: status.health.last_probe_ms,
4028                })
4029            })
4030            .collect::<Result<Vec<_>, RouterError>>()?;
4031        let response = ClientControlResponse::SupervisorHealth {
4032            generation,
4033            modules,
4034        };
4035        Ok(vec![control_response_body_frame(
4036            &frame,
4037            &response,
4038            "ClientControlResponse::SupervisorHealth",
4039        )?])
4040    }
4041
4042    async fn handle_supervisor_restart(
4043        &self,
4044        frame: Frame,
4045        module_id: String,
4046        drain_timeout_ms: Option<u64>,
4047    ) -> Result<Vec<Frame>, RouterError> {
4048        let operation_lock = self.supervisor.operation_lock();
4049        let _operation_guard = operation_lock.lock().await;
4050        let Some(module) = self.supervisor.get(&module_id) else {
4051            return Ok(vec![control_error_frame(
4052                &frame,
4053                "unknown_module",
4054                format!("module_id '{module_id}' is not supervised"),
4055            )?]);
4056        };
4057
4058        self.route_outages.mark_operator_action(&module_id);
4059        if let Err(err) = module.restart(drain_timeout_ms).await {
4060            self.route_outages
4061                .operator_action_ended_unrefused(&module_id);
4062            let (code, message) = match err {
4063                crate::supervise::SuperviseError::Disabled { .. } => {
4064                    ("module_disabled", err.to_string())
4065                }
4066                crate::supervise::SuperviseError::SwapInProgress { .. } => {
4067                    ("swap_in_progress", err.to_string())
4068                }
4069                _ => (
4070                    "target_unavailable",
4071                    format!("failed to restart module_id '{module_id}': {err}"),
4072                ),
4073            };
4074            return Ok(vec![control_error_frame(&frame, code, message)?]);
4075        }
4076
4077        let response = ClientControlResponse::SupervisorAck {
4078            module_id,
4079            applied: true,
4080        };
4081        Ok(vec![control_response_body_frame(
4082            &frame,
4083            &response,
4084            "ClientControlResponse::SupervisorAck",
4085        )?])
4086    }
4087
4088    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
4089    /// when the old process has finished draining: a caller whose own lane
4090    /// rides the old process must get its reply before that drain waits on it.
4091    async fn handle_supervisor_swap(
4092        &self,
4093        frame: Frame,
4094        module_id: String,
4095        ready_timeout_ms: Option<u64>,
4096    ) -> Result<Vec<Frame>, RouterError> {
4097        // The daemon-wide operation lock is held only to resolve the handle,
4098        // not across the swap. The swap can take its whole readiness budget,
4099        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
4100        // holding it here would park an operator's stop behind the swap it is
4101        // meant to abort. A rescan or stop that reaches the module during the
4102        // swap is served by the swap itself (see `supervise_swap`).
4103        let module = {
4104            let operation_lock = self.supervisor.operation_lock();
4105            let _operation_guard = operation_lock.lock().await;
4106            self.supervisor.get(&module_id)
4107        };
4108        let Some(module) = module else {
4109            return Ok(vec![control_error_frame(
4110                &frame,
4111                "unknown_module",
4112                format!("module_id '{module_id}' is not supervised"),
4113            )?]);
4114        };
4115
4116        self.route_outages.mark_operator_action(&module_id);
4117        if let Err(err) = module
4118            .swap(ready_timeout_ms.map(Duration::from_millis))
4119            .await
4120        {
4121            self.route_outages
4122                .operator_action_ended_unrefused(&module_id);
4123            use crate::supervise::SuperviseError;
4124            let message = err.to_string();
4125            let error = match err {
4126                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
4127                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
4128                    code: "swap_refused".to_string(),
4129                    message,
4130                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
4131                },
4132                SuperviseError::SwapFailed {
4133                    arm,
4134                    candidate_exit,
4135                    ..
4136                } => ErrorBody {
4137                    code: "swap_failed".to_string(),
4138                    message,
4139                    detail: Some(serde_json::json!({
4140                        "arm": arm.as_str(),
4141                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
4142                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
4143                    })),
4144                },
4145                _ => ErrorBody::new(
4146                    "target_unavailable",
4147                    format!("failed to swap module_id '{module_id}': {message}"),
4148                ),
4149            };
4150            return Ok(vec![control_error_body_frame(&frame, error)?]);
4151        }
4152        // A completed swap kept the incumbent serving until cutover, so it
4153        // usually opened no outage; a mark left behind would make the next,
4154        // unrelated outage read as requested.
4155        self.route_outages
4156            .operator_action_ended_unrefused(&module_id);
4157
4158        let response = ClientControlResponse::SupervisorAck {
4159            module_id,
4160            applied: true,
4161        };
4162        Ok(vec![control_response_body_frame(
4163            &frame,
4164            &response,
4165            "ClientControlResponse::SupervisorAck",
4166        )?])
4167    }
4168
4169    async fn handle_supervisor_reload(
4170        &self,
4171        frame: Frame,
4172        module_id: String,
4173    ) -> Result<Vec<Frame>, RouterError> {
4174        let operation_lock = self.supervisor.operation_lock();
4175        let _operation_guard = operation_lock.lock().await;
4176        let Some(module) = self.supervisor.get(&module_id) else {
4177            return Ok(vec![control_error_frame(
4178                &frame,
4179                "unknown_module",
4180                format!("module_id '{module_id}' is not supervised"),
4181            )?]);
4182        };
4183
4184        self.route_outages.mark_operator_action(&module_id);
4185        if let Err(err) = module.reload().await {
4186            self.route_outages
4187                .operator_action_ended_unrefused(&module_id);
4188            let (code, message) = match err {
4189                crate::supervise::SuperviseError::Disabled { .. } => {
4190                    ("module_disabled", err.to_string())
4191                }
4192                crate::supervise::SuperviseError::SwapInProgress { .. } => {
4193                    ("swap_in_progress", err.to_string())
4194                }
4195                _ => (
4196                    "reload_failed",
4197                    format!("failed to reload module_id '{module_id}': {err}"),
4198                ),
4199            };
4200            return Ok(vec![control_error_frame(&frame, code, message)?]);
4201        }
4202
4203        let response = ClientControlResponse::SupervisorAck {
4204            module_id,
4205            applied: true,
4206        };
4207        Ok(vec![control_response_body_frame(
4208            &frame,
4209            &response,
4210            "ClientControlResponse::SupervisorAck",
4211        )?])
4212    }
4213
4214    async fn handle_supervisor_rescan(
4215        &self,
4216        frame: Frame,
4217        preview: bool,
4218    ) -> Result<Vec<Frame>, RouterError> {
4219        let Some(context) = self.rescan.clone() else {
4220            return Ok(vec![control_error_frame(
4221                &frame,
4222                "rescan_unavailable",
4223                "the daemon was not started with a reloadable config path".to_string(),
4224            )?]);
4225        };
4226
4227        let operation_lock = self.supervisor.operation_lock();
4228        let _operation_guard = operation_lock.lock().await;
4229        let loaded = match crate::daemon_config::load(&context.config_path) {
4230            Ok(config) => config,
4231            Err(err) => {
4232                return Ok(vec![control_error_frame(
4233                    &frame,
4234                    "invalid_daemon_config",
4235                    format!("supervisor rescan rejected daemon config: {err}"),
4236                )?])
4237            }
4238        };
4239        // `load` reports a missing file as Ok(None), which is correct at boot
4240        // (no config, nothing to supervise) and catastrophic here: rescan treats
4241        // "not in the config" as "remove it", so an absent file would read as an
4242        // empty module list and retire the entire running fleet. An editor
4243        // writing via write-new-then-rename, or a half-finished edit, is enough
4244        // to open that window. Refuse instead: a config that cannot be read
4245        // carries no instruction to remove anything.
4246        let Some(config) = loaded else {
4247            return Ok(vec![control_error_frame(
4248                &frame,
4249                "invalid_daemon_config",
4250                format!(
4251                    "daemon config not found at {}; refusing to rescan (an absent config would \
4252                     retire every supervised module)",
4253                    context.config_path.display()
4254                ),
4255            )?]);
4256        };
4257        let (
4258            configured_port,
4259            storage_config,
4260            admission_facts_carrier_module_id,
4261            admission_facts_targets,
4262            scope_authority_owners,
4263            modules,
4264            reserved_capabilities,
4265        ) = (
4266            config.port,
4267            config.storage,
4268            config.admission_facts_carrier_module_id,
4269            config.admission_facts_targets,
4270            config.scope_authority_owners,
4271            config.modules,
4272            config.reserved_capabilities,
4273        );
4274
4275        // Collect the sections rescan cannot apply, so the REPLY carries them.
4276        //
4277        // The warning below has always been correct and has always gone only to
4278        // the journal -- addressed to whoever reads logs, while the person who
4279        // just edited the config is looking at the CLI. Naming each section
4280        // individually rather than setting a flag: "something outside modules
4281        // changed" sends the operator back to diffing their own file, which is
4282        // the work this is meant to save.
4283        let mut restart_required = Vec::new();
4284        for section in RestartRequiredSection::ALL {
4285            let changed = match section {
4286                RestartRequiredSection::Port => configured_port != context.configured_port,
4287                RestartRequiredSection::Storage => storage_config != context.storage_config,
4288                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
4289                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
4290                }
4291                RestartRequiredSection::AdmissionFactsTargets => {
4292                    admission_facts_targets != context.admission_facts_targets
4293                }
4294                RestartRequiredSection::ScopeAuthorityOwners => {
4295                    scope_authority_owners != context.scope_authority_owners
4296                }
4297            };
4298            if changed {
4299                restart_required.push(section.label().to_string());
4300            }
4301        }
4302        if !restart_required.is_empty() {
4303            warn!(
4304                config_path = %context.config_path.display(),
4305                sections = %restart_required.join(", "),
4306                "daemon config changed outside the modules section; restart the daemon to apply those changes"
4307            );
4308        }
4309
4310        for configured in &modules {
4311            if let Err(err) = validate_spec(&configured.module_spec()) {
4312                return Ok(vec![control_error_frame(
4313                    &frame,
4314                    "invalid_daemon_config",
4315                    format!("supervisor rescan rejected daemon config: {err}"),
4316                )?]);
4317            }
4318        }
4319
4320        let configured_capabilities = modules
4321            .iter()
4322            .map(|module| (module.module_id.clone(), module.enabled))
4323            .collect::<Vec<_>>();
4324        let preview_capability_warnings = if preview {
4325            let (_, registrations) = self.runtime_capability_snapshot()?;
4326            let current_modules = self
4327                .supervisor
4328                .list()
4329                .into_iter()
4330                .map(|module| module.module_id().to_string())
4331                .collect::<BTreeSet<_>>();
4332            let resulting_modules = configured_capabilities.clone();
4333            let removed = current_modules
4334                .into_iter()
4335                .filter(|module_id| {
4336                    !resulting_modules
4337                        .iter()
4338                        .any(|(configured_id, _)| configured_id == module_id)
4339                })
4340                .collect::<Vec<_>>();
4341            self.capability_evaluator.preview_removal_warnings(
4342                resulting_modules,
4343                &removed,
4344                &registrations,
4345            )
4346        } else {
4347            Vec::new()
4348        };
4349        let result = match self
4350            .reconcile_supervised_modules(&context.supervisor, modules, preview)
4351            .await
4352        {
4353            Ok(result) => result,
4354            Err(message) => {
4355                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
4356            }
4357        };
4358        if !preview {
4359            self.capability_evaluator
4360                .configure(configured_capabilities, reserved_capabilities);
4361            self.capability_evaluator.wake_deadline_loop();
4362            self.refresh_capability_requirements();
4363        }
4364        let mut result = result;
4365        result.restart_required = restart_required;
4366        result.capability_warnings = preview_capability_warnings;
4367        let response = ClientControlResponse::SupervisorRescan { result };
4368        Ok(vec![control_response_body_frame(
4369            &frame,
4370            &response,
4371            "ClientControlResponse::SupervisorRescan",
4372        )?])
4373    }
4374
4375    async fn handle_supervisor_release_reserved(
4376        &self,
4377        frame: Frame,
4378        module_id: String,
4379    ) -> Result<Vec<Frame>, RouterError> {
4380        let Some(context) = self.rescan.clone() else {
4381            return Ok(vec![control_error_frame(
4382                &frame,
4383                "release_unavailable",
4384                "reserved-id release requires a daemon started with a reloadable config path",
4385            )?]);
4386        };
4387        let operation_lock = self.supervisor.operation_lock();
4388        let _operation_guard = operation_lock.lock().await;
4389        let loaded = match crate::daemon_config::load(&context.config_path) {
4390            Ok(Some(config)) => config,
4391            Ok(None) => {
4392                return Ok(vec![control_error_frame(
4393                    &frame,
4394                    "invalid_daemon_config",
4395                    format!(
4396                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
4397                        context.config_path.display()
4398                    ),
4399                )?])
4400            }
4401            Err(err) => {
4402                return Ok(vec![control_error_frame(
4403                    &frame,
4404                    "invalid_daemon_config",
4405                    format!("unable to verify reserved-id release against daemon config: {err}"),
4406                )?])
4407            }
4408        };
4409        if loaded
4410            .modules
4411            .iter()
4412            .any(|configured| configured.module_id == module_id)
4413        {
4414            return Ok(vec![control_error_frame(
4415                &frame,
4416                "reserved_module_configured",
4417                format!(
4418                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
4419                ),
4420            )?]);
4421        }
4422        if !self.supervisor.release_retained_reserved_gate(&module_id) {
4423            return Ok(vec![control_error_frame(
4424                &frame,
4425                "reserved_gate_not_retained",
4426                format!(
4427                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
4428                ),
4429            )?]);
4430        }
4431
4432        let response = ClientControlResponse::SupervisorAck {
4433            module_id,
4434            applied: true,
4435        };
4436        Ok(vec![control_response_body_frame(
4437            &frame,
4438            &response,
4439            "ClientControlResponse::SupervisorAck",
4440        )?])
4441    }
4442
4443    /// Reconcile the running module set against the configured one.
4444    ///
4445    /// With `preview` set, the diff is computed and returned WITHOUT applying any
4446    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
4447    /// deliberately shares this function with the executing path rather than
4448    /// computing the same diff somewhere else -- two implementations of one
4449    /// decision agree until they do not, and the whole value of a preview is that
4450    /// it describes the operation that will actually run.
4451    async fn reconcile_supervised_modules(
4452        &self,
4453        supervisor: &Supervisor,
4454        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
4455        preview: bool,
4456    ) -> Result<SupervisorRescanResult, String> {
4457        let mut current = BTreeMap::new();
4458        for module in self.supervisor.list() {
4459            let (spec, health) = module.configuration().map_err(|err| {
4460                format!(
4461                    "failed to read configuration for module_id '{}': {err}",
4462                    module.module_id()
4463                )
4464            })?;
4465            let enabled = module
4466                .status()
4467                .map_err(|err| {
4468                    format!(
4469                        "failed to read status for module_id '{}': {err}",
4470                        module.module_id()
4471                    )
4472                })?
4473                .enabled;
4474            current.insert(
4475                module.module_id().to_string(),
4476                (module, spec, health, enabled),
4477            );
4478        }
4479        let configured = configured_modules
4480            .into_iter()
4481            .map(|module| (module.module_id.clone(), module))
4482            .collect::<BTreeMap<_, _>>();
4483
4484        let added = configured
4485            .keys()
4486            .filter(|module_id| !current.contains_key(*module_id))
4487            .cloned()
4488            .collect::<Vec<_>>();
4489        let removed = current
4490            .keys()
4491            .filter(|module_id| !configured.contains_key(*module_id))
4492            .cloned()
4493            .collect::<Vec<_>>();
4494        let mut changed_pending_reload = Vec::new();
4495        let mut configuration_changes = BTreeSet::new();
4496        let mut enabled_changes = BTreeSet::new();
4497        let mut unchanged = 0_u32;
4498
4499        for (module_id, configured_module) in &configured {
4500            let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
4501            else {
4502                continue;
4503            };
4504            let configuration_changed = *current_spec != configured_module.module_spec()
4505                || *current_health != configured_module.health;
4506            let enabled_changed = *current_enabled != configured_module.enabled;
4507            if configuration_changed {
4508                configuration_changes.insert(module_id.clone());
4509                changed_pending_reload.push(module_id.clone());
4510            }
4511            if enabled_changed {
4512                enabled_changes.insert(module_id.clone());
4513            }
4514            if !configuration_changed && !enabled_changed {
4515                unchanged = unchanged.saturating_add(1);
4516            }
4517        }
4518
4519        // Everything above this point is pure computation over two snapshots.
4520        // Everything below MUTATES. The preview returns here so the boundary is a
4521        // single early return rather than a condition repeated at each mutation
4522        // site, where one missed guard would apply part of a change the caller was
4523        // told would not happen.
4524        if preview {
4525            return Ok(SupervisorRescanResult {
4526                added,
4527                removed,
4528                changed_pending_reload,
4529                enabled_changes: enabled_changes.iter().cloned().collect(),
4530                unchanged,
4531                preview: true,
4532                // Filled by the caller on both paths, so the preview reports
4533                // restart-required sections identically to an executed rescan --
4534                // the preview is where an operator is most likely to be looking.
4535                restart_required: Vec::new(),
4536                capability_warnings: Vec::new(),
4537            });
4538        }
4539
4540        for module_id in &removed {
4541            let module = &current
4542                .get(module_id)
4543                .expect("removed module came from current supervisor state")
4544                .0;
4545            module.retire().await.map_err(|err| {
4546                format!("failed to retire module_id '{module_id}' during rescan: {err}")
4547            })?;
4548            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
4549            //
4550            // `handle_route_open` resolves an absent module in three steps:
4551            // registry, then supervisor status, then tombstone. Retiring first
4552            // opens a window where ALL THREE ARE ABSENT -- the registry entry
4553            // went with the teardown above, the supervisor entry went with
4554            // `retire`, and the tombstone does not exist yet -- so a route.open
4555            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
4556            // it") for a module that was deliberately removed and whose caller
4557            // should get `module_removed` (TERMINAL, carrying a removal age).
4558            //
4559            // Writing the tombstone first closes it: during the window the
4560            // supervisor entry still answers, so the caller gets
4561            // `target_unavailable` -- retryable, and TRUE, because the module
4562            // is mid-teardown. After both statements it is `module_removed`.
4563            // No instant remains where a removed module reads as one that
4564            // never existed.
4565            //
4566            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
4567            // the absence of a test beside a fix invites deletion: these are
4568            // two sync statements with no await between them, so reaching the
4569            // window needs a second worker thread to land exactly between them
4570            // and there is no hook to force it. MEASURED: the 25 daemon_config
4571            // tests pass identically with the old order and the new one, so
4572            // the existing suite cannot see this and a green run is not
4573            // evidence either way. What the suite does hold is the
4574            // post-condition -- a removed module answers `module_removed` --
4575            // which this preserves.
4576            //
4577            // Found by an Athena panel reading the shipped tree against a
4578            // design note (2026-09-19), as the one concrete instance of that
4579            // note's class that survived contact with source. Direction is
4580            // benign: retryable where terminal was intended, never the reverse.
4581            self.supervisor.record_rescan_removal(module_id);
4582            self.supervisor.retire(module_id);
4583            self.route_outages.forget(module_id);
4584        }
4585
4586        for module_id in configured.keys() {
4587            let Some((module, _, _, _)) = current.get(module_id) else {
4588                continue;
4589            };
4590            let configured_module = configured
4591                .get(module_id)
4592                .expect("configured module id came from configured map");
4593            if configuration_changes.contains(module_id) {
4594                module
4595                    .update_configuration(
4596                        configured_module.module_spec(),
4597                        configured_module.health,
4598                        configured_module.drain_timeout_ms,
4599                    )
4600                    .await
4601                    .map_err(|err| {
4602                        format!(
4603                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
4604                        )
4605                    })?;
4606            }
4607            if enabled_changes.contains(module_id) {
4608                // A rescan that starts or stops a module applies an operator's
4609                // edit to the config, so the resulting outage was asked for.
4610                self.route_outages.mark_operator_action(module_id);
4611                module
4612                    .set_enabled(configured_module.enabled)
4613                    .await
4614                    .map_err(|err| {
4615                        self.route_outages.operator_action_ended_unrefused(module_id);
4616                        format!(
4617                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
4618                            configured_module.enabled
4619                        )
4620                    })?;
4621            }
4622        }
4623
4624        for module_id in &added {
4625            let configured_module = configured
4626                .get(module_id)
4627                .expect("added module id came from configured map");
4628            supervisor
4629                .supervise_configured_with_health(
4630                    configured_module.module_spec(),
4631                    configured_module.enabled,
4632                    configured_module.health,
4633                    configured_module.drain_timeout_ms,
4634                    configured_module.restart,
4635                )
4636                .map_err(|err| {
4637                    format!("failed to add module_id '{module_id}' during rescan: {err}")
4638                })?;
4639        }
4640
4641        Ok(SupervisorRescanResult {
4642            added,
4643            removed,
4644            changed_pending_reload,
4645            enabled_changes: enabled_changes.iter().cloned().collect(),
4646            unchanged,
4647            preview: false,
4648            // Filled by the caller, which is the only layer that can see the
4649            // previous config to diff against.
4650            restart_required: Vec::new(),
4651            capability_warnings: Vec::new(),
4652        })
4653    }
4654
4655    async fn handle_supervisor_set_enabled(
4656        &self,
4657        frame: Frame,
4658        module_id: String,
4659        enabled: bool,
4660    ) -> Result<Vec<Frame>, RouterError> {
4661        let operation_lock = self.supervisor.operation_lock();
4662        let _operation_guard = operation_lock.lock().await;
4663        let Some(module) = self.supervisor.get(&module_id) else {
4664            return Ok(vec![control_error_frame(
4665                &frame,
4666                "unknown_module",
4667                format!("module_id '{module_id}' is not supervised"),
4668            )?]);
4669        };
4670
4671        // Enabling counts as well as disabling: a module an operator starts
4672        // is refused until it registers, and that wait was asked for.
4673        self.route_outages.mark_operator_action(&module_id);
4674        let applied = match module.set_enabled(enabled).await {
4675            Ok(applied) => applied,
4676            Err(err) => {
4677                self.route_outages
4678                    .operator_action_ended_unrefused(&module_id);
4679                return Ok(vec![control_error_frame(
4680                    &frame,
4681                    "target_unavailable",
4682                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
4683                )?]);
4684            }
4685        };
4686        if !applied {
4687            // Already in the requested state: nothing was made unavailable,
4688            // so the mark must not outlive this request.
4689            self.route_outages
4690                .operator_action_ended_unrefused(&module_id);
4691        }
4692
4693        self.capability_evaluator.wake_deadline_loop();
4694        self.refresh_capability_requirements();
4695        let response = ClientControlResponse::SupervisorAck { module_id, applied };
4696        Ok(vec![control_response_body_frame(
4697            &frame,
4698            &response,
4699            "ClientControlResponse::SupervisorAck",
4700        )?])
4701    }
4702
4703    async fn handle_supervisor_health_probe(
4704        &self,
4705        frame: Frame,
4706        module_id: String,
4707    ) -> Result<Vec<Frame>, RouterError> {
4708        self.refresh_capability_requirements();
4709        let Some(registration) = self
4710            .registry
4711            .get_module(&module_id)
4712            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4713        else {
4714            return Ok(vec![control_error_frame(
4715                &frame,
4716                "unknown_module",
4717                format!("module_id '{module_id}' is not registered"),
4718            )?]);
4719        };
4720
4721        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
4722        // named for it. Making `module_registration_grants_op` return false
4723        // unconditionally reddens five tests, and every one is named for something
4724        // else -- capability relay, probe/bind demultiplexing, supervision-only
4725        // probing. They exercise a successful advertisement check on the way to their
4726        // own subject.
4727        //
4728        // Real protection, fragile in a specific way: narrowing any of those tests to
4729        // focus on its stated subject would silently remove coverage nobody knows
4730        // they are carrying. Recorded here rather than as a sixth test, because the
4731        // useful fact is WHICH tests hold the guard up -- a new test would add
4732        // coverage without telling the next person what the existing ones quietly do.
4733        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
4734        {
4735            return Ok(vec![control_error_frame(
4736                &frame,
4737                "health_not_advertised",
4738                format!("module_id '{module_id}' did not advertise health.check"),
4739            )?]);
4740        }
4741
4742        let deadline = Instant::now() + self.health_probe_timeout;
4743        let pending = match self.forwarding.begin_module_control_rpc_for(
4744            &module_id,
4745            MODULE_CONTROL_OP_HEALTH_CHECK,
4746            deadline,
4747        ) {
4748            Ok(pending) => pending,
4749            Err(err) => {
4750                return Ok(vec![control_error_frame(
4751                    &frame,
4752                    forwarding_error_code(&err),
4753                    err.to_string(),
4754                )?])
4755            }
4756        };
4757
4758        let PendingModuleControlRpc {
4759            endpoint,
4760            module_sink,
4761            negotiated_ver,
4762            corr: probe_corr,
4763            receiver,
4764        } = pending;
4765        let mut guard =
4766            ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
4767        let probe_body =
4768            serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
4769                RouterError::backend(
4770                    0,
4771                    frame.header.corr,
4772                    format!("failed to encode health.check request: {err}"),
4773                )
4774            })?;
4775        let probe_frame = Frame::build_with_version(
4776            negotiated_ver,
4777            FrameType::Request,
4778            control_flags(),
4779            0,
4780            0,
4781            probe_corr,
4782            probe_body,
4783        )
4784        .map_err(RouterError::FrameBuild)?;
4785
4786        if let Err(err) = module_sink.send(probe_frame).await {
4787            return Ok(vec![control_error_frame(
4788                &frame,
4789                "target_unavailable",
4790                err.to_string(),
4791            )?]);
4792        }
4793
4794        match timeout_at(deadline, receiver).await {
4795            Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
4796                guard.disarm();
4797                let Some(report) = response.health_report() else {
4798                    return Ok(vec![control_error_frame(
4799                        &frame,
4800                        "invalid_control_body",
4801                        "health.check RPC returned a non-health response",
4802                    )?]);
4803                };
4804                // Metrics go out whole here. The supervisor's cached snapshot
4805                // caps this blob (see truncate_health_metrics), and this path
4806                // exists precisely to answer without that cap -- so applying it
4807                // here would leave no way to see what the cached view drops.
4808                let HealthReport {
4809                    status,
4810                    detail,
4811                    metrics,
4812                } = report;
4813                let capability_detail = self
4814                    .capability_evaluator
4815                    .required_problem_detail(&module_id);
4816                let response = ClientControlResponse::SupervisorHealthProbe {
4817                    module_id,
4818                    status,
4819                    detail: append_capability_problem_detail(detail, capability_detail),
4820                    metrics,
4821                };
4822                Ok(vec![control_response_body_frame(
4823                    &frame,
4824                    &response,
4825                    "ClientControlResponse::SupervisorHealthProbe",
4826                )?])
4827            }
4828            Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
4829                guard.disarm();
4830                Ok(vec![control_error_body_frame(&frame, body)?])
4831            }
4832            Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
4833                guard.disarm();
4834                Ok(vec![control_error_frame(
4835                    &frame,
4836                    "target_unavailable",
4837                    message,
4838                )?])
4839            }
4840            Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
4841                guard.disarm();
4842                Ok(vec![control_error_frame(
4843                    &frame,
4844                    "invalid_control_body",
4845                    message,
4846                )?])
4847            }
4848            Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
4849                guard.disarm();
4850                Ok(vec![control_error_frame(
4851                    &frame,
4852                    "invalid_control_body",
4853                    format!("expected module-control op '{expected}', got '{actual}'"),
4854                )?])
4855            }
4856            Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
4857                guard.disarm();
4858                Ok(vec![control_error_frame(
4859                    &frame,
4860                    "module_timeout",
4861                    format!(
4862                        "module_id '{module_id}' answered health.check after {:?}",
4863                        self.health_probe_timeout
4864                    ),
4865                )?])
4866            }
4867            Ok(Err(_)) => Ok(vec![control_error_frame(
4868                &frame,
4869                "target_unavailable",
4870                "health.check waiter was canceled before the module responded",
4871            )?]),
4872            Err(_) => Ok(vec![control_error_frame(
4873                &frame,
4874                "module_timeout",
4875                format!(
4876                    "module_id '{module_id}' did not answer health.check within {:?}",
4877                    self.health_probe_timeout
4878                ),
4879            )?]),
4880        }
4881    }
4882
4883    fn supervisor_status(
4884        &self,
4885        module_id: &str,
4886        corr: u64,
4887    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
4888        self.supervisor
4889            .get(module_id)
4890            .map(|module| {
4891                let warming = module.is_warming_for_control("status").map_err(|err| {
4892                    RouterError::backend(
4893                        0,
4894                        corr,
4895                        format!(
4896                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
4897                        ),
4898                    )
4899                })?;
4900                module.status_for_control("status").map_err(|err| {
4901                    RouterError::backend(
4902                        0,
4903                        corr,
4904                        format!(
4905                            "failed to read supervisor status for module_id '{module_id}': {err}"
4906                        ),
4907                    )
4908                }).map(|status| (status, warming))
4909            })
4910            .transpose()
4911    }
4912
4913    fn guard_module_control_op(
4914        &self,
4915        frame: &Frame,
4916        module_id: &str,
4917        op: &str,
4918    ) -> Result<Option<Frame>, RouterError> {
4919        if self.module_grants_op(module_id, op, frame.header.corr)? {
4920            return Ok(None);
4921        }
4922
4923        Ok(Some(control_error_frame(
4924            frame,
4925            "op_not_allowed",
4926            format!("module_id '{module_id}' did not grant control op '{op}'"),
4927        )?))
4928    }
4929
4930    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
4931        let Some(registration) = self
4932            .registry
4933            .get_module(module_id)
4934            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
4935        else {
4936            return Ok(false);
4937        };
4938        Ok(module_registration_grants_op(&registration.control_ops, op))
4939    }
4940
4941    fn handle_status_update(
4942        &self,
4943        endpoint: ModuleEndpointId,
4944        frame: Frame,
4945    ) -> Result<Vec<Frame>, RouterError> {
4946        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
4947            Ok(update) => update,
4948            Err(err) => {
4949                // Forward-compat: a newer module may push a channel-0 op this subc
4950                // version doesn't know. The control contract says unknown push ops
4951                // are IGNORED, never answered with an error. Only a malformed body
4952                // for an op we DO know is a real error worth surfacing.
4953                if is_known_module_push_op(&frame.body) {
4954                    return Ok(vec![control_error_frame(
4955                        &frame,
4956                        "invalid_control_body",
4957                        format!("malformed module control push body: {err}"),
4958                    )?]);
4959                }
4960                return Ok(Vec::new());
4961            }
4962        };
4963
4964        match update {
4965            ModuleControlPush::RouteStatus {
4966                route_channel,
4967                route_epoch,
4968                status,
4969            } => {
4970                self.forwarding
4971                    .cache_status(endpoint, route_channel, route_epoch, status)
4972                    .map_err(RouterError::Forwarding)?;
4973            }
4974        }
4975        Ok(Vec::new())
4976    }
4977
4978    fn handle_route_poll(
4979        &self,
4980        ctx: &RouteCtx,
4981        frame: Frame,
4982        route_channel: u16,
4983        route_epoch: u32,
4984        kind: PollKind,
4985    ) -> Result<Vec<Frame>, RouterError> {
4986        let snapshot = self
4987            .forwarding
4988            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
4989            .map_err(RouterError::Forwarding)?;
4990        let response = match (kind, snapshot) {
4991            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
4992                ClientControlResponse::RoutePoll {
4993                    route_channel,
4994                    route_epoch,
4995                    status,
4996                    live: None,
4997                }
4998            }
4999            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5000                route_channel,
5001                route_epoch,
5002                status: None,
5003                live: None,
5004            },
5005            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
5006                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
5007                // is what makes reporting `true` correct rather than a
5008                // confident guess. `process_live` returns None only when the
5009                // module id has no supervisor snapshot at all -- an
5010                // externally-started module the daemon did not spawn -- and
5011                // for those the supervisor has no opinion to offer, ever. It
5012                // is never None for a supervised module in an unknown state:
5013                // a supervised module always has a snapshot, and the answer
5014                // comes from `state == Running && process_alive`.
5015                //
5016                // The route is Bound, so the module completed a HELLO on a
5017                // live connection; "the process this route points at is
5018                // running" is therefore attested by the binding rather than
5019                // assumed. Reporting `false` for an unsupervised module would
5020                // be the actual lie -- it would tell a client its healthy
5021                // route is dead because the daemon does not manage the
5022                // process.
5023                //
5024                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
5025                // module whose liveness is genuinely unknown, e.g. a snapshot
5026                // that has not been populated yet -- THIS DEFAULT BECOMES
5027                // WRONG and must split: unsupervised stays true, unknown
5028                // becomes null so the client can tell the two apart. The
5029                // response field is already `Option<bool>`, so the wire can
5030                // carry that distinction today.
5031                let live = self
5032                    .process_liveness
5033                    .as_ref()
5034                    .and_then(|source| source.process_live(&module_id))
5035                    .unwrap_or(true);
5036                ClientControlResponse::RoutePoll {
5037                    route_channel,
5038                    route_epoch,
5039                    status: None,
5040                    live: Some(live),
5041                }
5042            }
5043            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5044                route_channel,
5045                route_epoch,
5046                status: None,
5047                live: Some(false),
5048            },
5049        };
5050
5051        Ok(vec![control_response_body_frame(
5052            &frame,
5053            &response,
5054            "ClientControlResponse::RoutePoll",
5055        )?])
5056    }
5057
5058    pub(crate) fn observe_module_control_completion(
5059        &self,
5060        completion: ModuleControlRpcCompletion,
5061    ) -> bool {
5062        match completion {
5063            ModuleControlRpcCompletion::Unknown => false,
5064            ModuleControlRpcCompletion::Settled => true,
5065            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
5066                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
5067                info!(
5068                    module_id = %module_id,
5069                    latency_ms,
5070                    "late health.check answer proves the module is alive"
5071                );
5072                match self
5073                    .supervisor
5074                    .record_late_health_answer(&module_id, latency_ms)
5075                {
5076                    Ok(true) => {}
5077                    Ok(false) => debug!(
5078                        module_id = %module_id,
5079                        latency_ms,
5080                        "late health.check answer has no active supervisor snapshot"
5081                    ),
5082                    Err(err) => warn!(
5083                        module_id = %module_id,
5084                        latency_ms,
5085                        error = %err,
5086                        "failed to record late health.check answer"
5087                    ),
5088                }
5089                true
5090            }
5091        }
5092    }
5093
5094    /// Decide whether a failure while settling a relayed `route.bind` belongs to
5095    /// the module connection whose frame is being handled, or to the client that
5096    /// relay was opened for.
5097    ///
5098    /// This runs on the MODULE connection's frame handler, where returning `Err`
5099    /// ends that connection -- and a module connection carries every client's
5100    /// routes to that module, so ending it costs the whole fleet its tools.
5101    /// `ConnectionClosing` carries the id of the connection that is closing, and
5102    /// when that id is a CLIENT's, the condition is entirely about that one
5103    /// client's route.open. A client-scoped condition has no authority over a
5104    /// shared module connection, so it is logged and the single relay is dropped:
5105    /// the client is going away, and `complete_pending_relay` already removed the
5106    /// relay before failing, so there is nothing left to settle. Anything that
5107    /// relay still reserved is released by that client's own connection teardown,
5108    /// which is already under way -- that is what "closing" means.
5109    ///
5110    /// Every other failure is a statement about THIS connection and stays fatal:
5111    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
5112    /// id in `ConnectionClosing` all mean this connection cannot keep serving
5113    /// frames correctly.
5114    fn refuse_to_end_module_connection_for_a_client(
5115        &self,
5116        module_connection_id: ConnectionId,
5117        corr: u64,
5118        err: ForwardingError,
5119    ) -> Result<(), RouterError> {
5120        if let ForwardingError::ConnectionClosing { connection_id } = err {
5121            if connection_id != module_connection_id {
5122                warn!(
5123                    module_connection_id = module_connection_id.get(),
5124                    client_connection_id = connection_id.get(),
5125                    corr,
5126                    "dropping a route.bind response for a closing client; the module connection keeps serving"
5127                );
5128                return Ok(());
5129            }
5130        }
5131        Err(RouterError::Forwarding(err))
5132    }
5133
5134    fn handle_module_relay_response(
5135        &self,
5136        connection_id: ConnectionId,
5137        frame: Frame,
5138    ) -> Result<Vec<Frame>, RouterError> {
5139        let mut secondary_error = None;
5140        let outcome = match frame.header.ty {
5141            FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
5142                Ok(probe) if probe.op == "route.bind" => {
5143                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
5144                        Ok(ModuleControlResponse::RouteBindAck {}) => {
5145                            RouteBindRelayOutcome::Accepted
5146                        }
5147                        Ok(other) => {
5148                            let message =
5149                                format!("route.bind response carried unexpected body: {other:?}");
5150                            secondary_error = Some(control_error_frame(
5151                                &frame,
5152                                "invalid_control_body",
5153                                message.clone(),
5154                            )?);
5155                            RouteBindRelayOutcome::ModuleGone(message)
5156                        }
5157                        Err(err) => {
5158                            let message = format!("malformed route.bind response body: {err}");
5159                            secondary_error = Some(control_error_frame(
5160                                &frame,
5161                                "invalid_control_body",
5162                                message.clone(),
5163                            )?);
5164                            RouteBindRelayOutcome::ModuleGone(message)
5165                        }
5166                    }
5167                }
5168                Ok(probe) => {
5169                    let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
5170                    {
5171                        Ok(response) => ModuleControlRpcOutcome::Response(response),
5172                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5173                            "malformed {} response body: {err}",
5174                            probe.op
5175                        )),
5176                    };
5177                    let completion = self
5178                        .forwarding
5179                        .complete_module_control_rpc(
5180                            connection_id,
5181                            frame.header.corr,
5182                            Some(&probe.op),
5183                            outcome,
5184                        )
5185                        .map_err(RouterError::Forwarding)?;
5186                    if !self.observe_module_control_completion(completion) {
5187                        debug!(
5188                            connection_id = connection_id.get(),
5189                            corr = frame.header.corr,
5190                            op = %probe.op,
5191                            "dropping late or unknown module-control RPC response"
5192                        );
5193                    }
5194                    return Ok(Vec::new());
5195                }
5196                Err(err) => {
5197                    if let Some(expected_op) = self
5198                        .forwarding
5199                        .pending_module_control_op(connection_id, frame.header.corr)
5200                        .map_err(RouterError::Forwarding)?
5201                    {
5202                        let completion = self
5203                            .forwarding
5204                            .complete_module_control_rpc(
5205                                connection_id,
5206                                frame.header.corr,
5207                                None,
5208                                ModuleControlRpcOutcome::MalformedResponse(format!(
5209                                    "malformed {expected_op} response body: {err}"
5210                                )),
5211                            )
5212                            .map_err(RouterError::Forwarding)?;
5213                        if !self.observe_module_control_completion(completion) {
5214                            debug!(
5215                                connection_id = connection_id.get(),
5216                                corr = frame.header.corr,
5217                                "dropping late malformed module-control RPC response"
5218                            );
5219                        }
5220                        return Ok(Vec::new());
5221                    }
5222                    let message = format!("malformed route.bind response body: {err}");
5223                    secondary_error = Some(control_error_frame(
5224                        &frame,
5225                        "invalid_control_body",
5226                        message.clone(),
5227                    )?);
5228                    RouteBindRelayOutcome::ModuleGone(message)
5229                }
5230            },
5231            FrameType::Error => {
5232                if self
5233                    .forwarding
5234                    .pending_module_control_op(connection_id, frame.header.corr)
5235                    .map_err(RouterError::Forwarding)?
5236                    .is_some()
5237                {
5238                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
5239                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
5240                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5241                            "malformed module-control ERROR body: {err}"
5242                        )),
5243                    };
5244                    let completion = self
5245                        .forwarding
5246                        .complete_module_control_rpc(
5247                            connection_id,
5248                            frame.header.corr,
5249                            None,
5250                            outcome,
5251                        )
5252                        .map_err(RouterError::Forwarding)?;
5253                    if !self.observe_module_control_completion(completion) {
5254                        debug!(
5255                            connection_id = connection_id.get(),
5256                            corr = frame.header.corr,
5257                            "dropping late or unknown module-control RPC error"
5258                        );
5259                    }
5260                    return Ok(Vec::new());
5261                }
5262                match serde_json::from_slice::<ErrorBody>(&frame.body) {
5263                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
5264                    Err(err) => {
5265                        let message = format!("malformed route.bind ERROR body: {err}");
5266                        secondary_error = Some(control_error_frame(
5267                            &frame,
5268                            "invalid_control_body",
5269                            message.clone(),
5270                        )?);
5271                        RouteBindRelayOutcome::ModuleGone(message)
5272                    }
5273                }
5274            }
5275            ty => {
5276                return Ok(vec![control_error_frame(
5277                    &frame,
5278                    "unsupported_control_frame",
5279                    format!("unsupported module channel-0 frame {ty:?}"),
5280                )?])
5281            }
5282        };
5283
5284        let settled =
5285            self.forwarding
5286                .complete_pending_relay(connection_id, frame.header.corr, outcome);
5287        let completion = match settled {
5288            Ok(completion) => completion,
5289            Err(err) => {
5290                self.refuse_to_end_module_connection_for_a_client(
5291                    connection_id,
5292                    frame.header.corr,
5293                    err,
5294                )?;
5295                return Ok(secondary_error.into_iter().collect());
5296            }
5297        };
5298        if let Some(target) = completion.abandoned.as_ref() {
5299            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
5300        }
5301        if !completion.settled {
5302            debug!(
5303                connection_id = connection_id.get(),
5304                corr = frame.header.corr,
5305                frame_type = ?frame.header.ty,
5306                "dropping late or unknown route.bind relay response"
5307            );
5308        }
5309        Ok(secondary_error.into_iter().collect())
5310    }
5311
5312    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
5313        debug!(connection_id = connection_id.get(), "handling GOODBYE");
5314        let registrations = self
5315            .deregister_connection(connection_id)
5316            .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
5317        let released_routes = self
5318            .forwarding
5319            .cleanup_connection(connection_id)
5320            .map_err(RouterError::Forwarding)?;
5321        self.emit_route_goodbyes(released_routes);
5322        // Notify only after forwarding teardown completes (see cleanup_connection).
5323        if !registrations.is_empty() {
5324            crate::supervise::notify_registration_release();
5325        }
5326        Ok(Vec::new())
5327    }
5328}
5329
5330impl Default for ControlHandler {
5331    fn default() -> Self {
5332        Self::new(Arc::new(Registry::default()))
5333    }
5334}
5335
5336impl crate::supervise::SwapPromotionObserver for ControlHandler {
5337    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
5338        self.apply_registration_capabilities(registration);
5339    }
5340}
5341
5342fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
5343    CapabilityRequirementStatus {
5344        consumer: status.consumer,
5345        capability: status.capability,
5346        need: match status.need {
5347            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
5348            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
5349        },
5350        verdict: status.verdict.as_str().to_string(),
5351        episode_seq: status.episode_seq,
5352        config_satisfiable: status.config_satisfiable,
5353        runtime_available: status.runtime_available,
5354        detail: status.detail,
5355    }
5356}
5357
5358fn append_capability_problem_detail(
5359    detail: Option<String>,
5360    capability_detail: Option<String>,
5361) -> Option<String> {
5362    match (detail, capability_detail) {
5363        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
5364        (Some(detail), None) => Some(detail),
5365        (None, Some(capability_detail)) => Some(capability_detail),
5366        (None, None) => None,
5367    }
5368}
5369
5370fn subc_ops() -> Vec<String> {
5371    SUBC_CONTROL_OPS
5372        .iter()
5373        .map(|op| (*op).to_string())
5374        .collect()
5375}
5376
5377fn module_subc_ops() -> Vec<String> {
5378    SUBC_CONTROL_OPS
5379        .iter()
5380        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
5381        .map(|op| (*op).to_string())
5382        .collect()
5383}
5384
5385#[cfg(test)]
5386fn module_baseline_control_ops() -> Vec<String> {
5387    MODULE_BASELINE_CONTROL_OPS
5388        .iter()
5389        .map(|op| (*op).to_string())
5390        .collect()
5391}
5392
5393fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
5394    let mut seen = HashSet::new();
5395    let mut effective = Vec::new();
5396    for op in MODULE_BASELINE_CONTROL_OPS {
5397        if seen.insert((*op).to_string()) {
5398            effective.push((*op).to_string());
5399        }
5400    }
5401    for op in declared.unwrap_or_default() {
5402        if seen.insert(op.clone()) {
5403            effective.push(op);
5404        }
5405    }
5406    effective
5407}
5408
5409fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
5410    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
5411}
5412
5413fn target_module_id(target: &RouteTarget) -> &str {
5414    match target {
5415        RouteTarget::ToolProvider { module_id }
5416        | RouteTarget::ManagementSurface { module_id }
5417        | RouteTarget::InternalService { module_id, .. } => module_id,
5418    }
5419}
5420
5421fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
5422    roles.iter().any(|role| match (target, role) {
5423        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
5424        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
5425        (
5426            RouteTarget::InternalService { service_id, .. },
5427            ProviderRole::InternalService {
5428                service_id: provided,
5429                ..
5430            },
5431        ) => service_id == provided,
5432        _ => false,
5433    })
5434}
5435
5436fn is_routable_role(role: &ProviderRole) -> bool {
5437    matches!(
5438        role,
5439        ProviderRole::ToolProvider { .. }
5440            | ProviderRole::ManagementSurface { .. }
5441            | ProviderRole::InternalService { .. }
5442    )
5443}
5444
5445#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5446enum ControlRequestBodyError {
5447    UnknownOp,
5448    InvalidBody,
5449}
5450
5451#[derive(Debug, Deserialize)]
5452struct ControlOpProbe {
5453    op: String,
5454}
5455
5456/// Channel-0 push ops this subc version understands. A push whose `op` is not in
5457/// this set is treated as a forward-compat unknown and ignored rather than errored.
5458const MODULE_PUSH_OPS: &[&str] = &["route.status"];
5459
5460fn is_known_module_push_op(body: &[u8]) -> bool {
5461    serde_json::from_slice::<ControlOpProbe>(body)
5462        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
5463        .unwrap_or(false)
5464}
5465
5466fn is_known_module_request_op(body: &[u8]) -> bool {
5467    serde_json::from_slice::<ControlOpProbe>(body)
5468        .map(|probe| is_module_to_subc_op(&probe.op))
5469        .unwrap_or(false)
5470}
5471
5472fn is_module_to_subc_op(op: &str) -> bool {
5473    MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
5474}
5475
5476fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
5477    debug!(
5478        op = %op,
5479        connection_id = connection_id.get(),
5480        corr,
5481        "control dispatch"
5482    );
5483}
5484
5485fn log_slow_control_dispatch(
5486    dispatch_started_at: Option<StdInstant>,
5487    op: &'static str,
5488    connection_id: ConnectionId,
5489    corr: u64,
5490) {
5491    let Some(dispatch_started_at) = dispatch_started_at else {
5492        return;
5493    };
5494    let elapsed = dispatch_started_at.elapsed();
5495    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
5496        warn!(
5497            op = %op,
5498            connection_id = connection_id.get(),
5499            corr,
5500            elapsed_ms = elapsed.as_millis() as u64,
5501            "slow control dispatch"
5502        );
5503    }
5504}
5505
5506fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
5507    match request {
5508        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
5509        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
5510        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
5511        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
5512        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
5513        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
5514        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
5515        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
5516        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
5517        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
5518        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
5519        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
5520        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
5521        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
5522        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
5523        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
5524        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
5525        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
5526        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
5527    }
5528}
5529
5530fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
5531    match request {
5532        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
5533        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
5534        ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
5535        ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
5536    }
5537}
5538
5539fn parse_client_control_request(
5540    body: &[u8],
5541) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
5542    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
5543        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5544            Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5545                ControlRequestBodyError::InvalidBody
5546            }
5547            Ok(_) => ControlRequestBodyError::UnknownOp,
5548            Err(_) => ControlRequestBodyError::InvalidBody,
5549        };
5550        (err, classification)
5551    })
5552}
5553
5554fn parse_module_control_request_from_module(
5555    body: &[u8],
5556) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
5557    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
5558        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5559            Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
5560            Ok(_) => ControlRequestBodyError::UnknownOp,
5561            Err(_) => ControlRequestBodyError::InvalidBody,
5562        };
5563        (err, classification)
5564    })
5565}
5566
5567#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
5568enum ProviderRoleKind {
5569    ToolProvider,
5570    PipelineStage,
5571    ManagementSurface,
5572    InternalService,
5573}
5574
5575fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
5576    match role {
5577        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
5578        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
5579        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
5580        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
5581    }
5582}
5583
5584fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
5585    roles.iter().map(provider_role_kind).collect()
5586}
5587
5588/// Most refused scope records named individually in the log per sync; the
5589/// `refused` count on the accepted line is always complete.
5590const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
5591
5592/// Per-outcome counts of one accepted `scope.sync`, for its log line.
5593#[derive(Debug, Default, PartialEq, Eq)]
5594struct ScopeOutcomeCounts {
5595    created: usize,
5596    replaced: usize,
5597    updated: usize,
5598    unchanged: usize,
5599    refused: usize,
5600}
5601
5602impl ScopeOutcomeCounts {
5603    fn of(results: &[ScopeRecordResult]) -> Self {
5604        let mut counts = Self::default();
5605        for result in results {
5606            let slot = match result.outcome {
5607                ScopeRecordOutcome::Created => &mut counts.created,
5608                ScopeRecordOutcome::Replaced => &mut counts.replaced,
5609                ScopeRecordOutcome::Updated => &mut counts.updated,
5610                ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
5611                ScopeRecordOutcome::Refused => &mut counts.refused,
5612            };
5613            *slot += 1;
5614        }
5615        counts
5616    }
5617}
5618
5619#[cfg(test)]
5620mod scope_outcome_count_tests {
5621    use super::*;
5622
5623    fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
5624        ScopeRecordResult {
5625            scope_ref: "r".to_string(),
5626            scope_epoch: 1,
5627            outcome,
5628            code: None,
5629            message: None,
5630            version: None,
5631            parent_state: None,
5632        }
5633    }
5634
5635    /// Each outcome lands in its own count, so a refused record can never be
5636    /// hidden inside the total the log already printed.
5637    #[test]
5638    fn every_outcome_is_counted_in_its_own_field() {
5639        let results = [
5640            result(ScopeRecordOutcome::Created),
5641            result(ScopeRecordOutcome::Created),
5642            result(ScopeRecordOutcome::Replaced),
5643            result(ScopeRecordOutcome::Updated),
5644            result(ScopeRecordOutcome::Unchanged),
5645            result(ScopeRecordOutcome::Refused),
5646            result(ScopeRecordOutcome::Refused),
5647            result(ScopeRecordOutcome::Refused),
5648        ];
5649        assert_eq!(
5650            ScopeOutcomeCounts::of(&results),
5651            ScopeOutcomeCounts {
5652                created: 2,
5653                replaced: 1,
5654                updated: 1,
5655                unchanged: 1,
5656                refused: 3,
5657            }
5658        );
5659    }
5660}
5661
5662/// Return whether a catalog change can create a newly violating live route.
5663/// Removing an attested claim is intentionally excluded: it makes fewer routes
5664/// forbidden and therefore must leave the existing route census untouched.
5665fn capability_census_trigger(
5666    old: Option<&CapabilityDeclarations>,
5667    new: Option<&CapabilityDeclarations>,
5668) -> bool {
5669    let old_provides = old
5670        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
5671        .unwrap_or_default();
5672    let old_denies = old
5673        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
5674        .unwrap_or_default();
5675    let new = new.cloned().unwrap_or(CapabilityDeclarations {
5676        provides: Vec::new(),
5677        requires: Vec::new(),
5678        must_never_reach: Vec::new(),
5679    });
5680
5681    new.provides
5682        .iter()
5683        .any(|capability| !old_provides.contains(capability))
5684        || new
5685            .must_never_reach
5686            .iter()
5687            .any(|capability| !old_denies.contains(capability))
5688}
5689
5690/// Find the first capability an attested opener denies that an attested target
5691/// claims. Both manifests are live registry records, never cached or client data.
5692fn denied_capability<'a>(
5693    opening_manifest: &'a ModuleManifest,
5694    target_manifest: &ModuleManifest,
5695) -> Option<&'a str> {
5696    let opening_capabilities = opening_manifest.capabilities.as_ref()?;
5697    let target_capabilities = target_manifest.capabilities.as_ref()?;
5698    opening_capabilities
5699        .must_never_reach
5700        .iter()
5701        .find(|denied| {
5702            target_capabilities
5703                .provides
5704                .iter()
5705                .any(|provided| provided == *denied)
5706        })
5707        .map(String::as_str)
5708}
5709
5710fn catalog_update_frozen_field_message(
5711    registered: &ModuleManifest,
5712    provides: &[ProviderRole],
5713) -> Option<String> {
5714    let old_has_provides = !registered.provides.is_empty();
5715    let new_has_provides = !provides.is_empty();
5716    if old_has_provides != new_has_provides {
5717        return Some(format!(
5718            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
5719            registered.module_id
5720        ));
5721    }
5722
5723    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
5724        return Some(format!(
5725            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
5726            registered.module_id
5727        ));
5728    }
5729
5730    let registered_concurrency = manifest_concurrency(registered);
5731    let mut candidate = registered.clone();
5732    candidate.provides = provides.to_vec();
5733    let candidate_concurrency = manifest_concurrency(&candidate);
5734    if candidate_concurrency != registered_concurrency {
5735        return Some(format!(
5736            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
5737            registered.module_id, registered_concurrency, candidate_concurrency
5738        ));
5739    }
5740
5741    // control_ops live beside the manifest in the HELLO body, not inside
5742    // ModuleManifest, so a provides-only catalog.update cannot change them.
5743    None
5744}
5745
5746fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
5747    manifest.provides.iter().any(is_routable_role)
5748}
5749
5750/// Returns the routable-provider concurrency subc should enforce for this manifest.
5751///
5752/// ToolProvider and ManagementSurface store their delivery concurrency directly.
5753/// InternalService has no role-specific concurrency field, so it retains the
5754/// existing ModuleManaged default for backward compatibility.
5755fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
5756    manifest
5757        .provides
5758        .iter()
5759        .find_map(|provider| match provider {
5760            ProviderRole::ToolProvider { concurrency, .. }
5761            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
5762            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
5763        })
5764        .unwrap_or(Concurrency::ModuleManaged)
5765}
5766
5767/// True when the manifest carries a ManagementSurface role whose concurrency
5768/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
5769/// bytes because the typed manifest deliberately erases that distinction: the
5770/// default exists for wire compatibility, and this probe exists so the default
5771/// stays observable. Any parse irregularity returns false -- the caller only
5772/// logs, and a malformed body already failed registration upstream.
5773fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
5774    let has_management_surface = manifest
5775        .provides
5776        .iter()
5777        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
5778    if !has_management_surface {
5779        return false;
5780    }
5781    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
5782        return false;
5783    };
5784    let Some(provides) = raw
5785        .get("manifest")
5786        .and_then(|manifest| manifest.get("provides"))
5787        .and_then(serde_json::Value::as_array)
5788    else {
5789        return false;
5790    };
5791    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
5792    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
5793    // against the management_surface_manifest_without_concurrency golden, not
5794    // recalled (the externally-tagged guess was this function's first bug).
5795    provides.iter().any(|role| {
5796        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
5797            && role.get("concurrency").is_none()
5798    })
5799}
5800
5801fn negotiate_version(peer_version: u8) -> Result<u8, String> {
5802    if peer_version != PROTOCOL_VERSION {
5803        return Err(format!(
5804            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
5805        ));
5806    }
5807    Ok(PROTOCOL_VERSION)
5808}
5809
5810fn pong(frame: &Frame) -> Result<Frame, RouterError> {
5811    Frame::build_with_version(
5812        response_version(frame),
5813        FrameType::Pong,
5814        frame.header.flags,
5815        0,
5816        0,
5817        frame.header.corr,
5818        Vec::new(),
5819    )
5820    .map_err(RouterError::FrameBuild)
5821}
5822
5823fn control_error_frame(
5824    frame: &Frame,
5825    code: &'static str,
5826    message: impl Into<String>,
5827) -> Result<Frame, RouterError> {
5828    control_error_body_frame(
5829        frame,
5830        ErrorBody {
5831            code: code.to_string(),
5832            message: message.into(),
5833            detail: None,
5834        },
5835    )
5836}
5837
5838fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
5839    let body = serde_json::to_vec(&error).map_err(|err| {
5840        RouterError::backend(
5841            0,
5842            frame.header.corr,
5843            format!("failed to encode control ERROR: {err}"),
5844        )
5845    })?;
5846
5847    Frame::build_with_version(
5848        response_version(frame),
5849        FrameType::Error,
5850        control_flags(),
5851        0,
5852        0,
5853        frame.header.corr,
5854        body,
5855    )
5856    .map_err(RouterError::FrameBuild)
5857}
5858
5859fn control_response_body_frame<T: Serialize>(
5860    frame: &Frame,
5861    reply: &T,
5862    label: &'static str,
5863) -> Result<Frame, RouterError> {
5864    let body = serde_json::to_vec(reply).map_err(|err| {
5865        RouterError::backend(
5866            0,
5867            frame.header.corr,
5868            format!("failed to encode {label}: {err}"),
5869        )
5870    })?;
5871
5872    Frame::build_with_version(
5873        response_version(frame),
5874        FrameType::Response,
5875        control_flags(),
5876        0,
5877        0,
5878        frame.header.corr,
5879        body,
5880    )
5881    .map_err(RouterError::FrameBuild)
5882}
5883
5884/// Map a forwarding failure to the wire code a client sees.
5885///
5886/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
5887/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
5888/// chosen here decides whether a caller retries or gives up.
5889///
5890/// That makes attribution the load-bearing property, not merely having a code. A
5891/// permanent fault published as a retryable one produces a fleet-wide retry storm
5892/// against something that can never recover; a transient fault published as
5893/// permanent gives up on work that would have succeeded. Both look correct in a
5894/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
5895/// the mapping per variant rather than merely asserting that some code exists.
5896///
5897/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
5898/// two codes on the same side of the boundary passes it. Measured rather than
5899/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
5900/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
5901/// a test named for something else that happens to assert the string.
5902///
5903/// That accidental coverage is deliberately left alone rather than promoted to a
5904/// named test, because it guards a property this function does not promise.
5905/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
5906/// specific code within a class, so identity is free to change and only the
5907/// partition is a contract. Splitting it out would assert a guarantee nothing
5908/// depends on — and a suite that promises more than the code does is the harder
5909/// thing to correct later, because the next reader cannot tell which assertions
5910/// are load-bearing.
5911///
5912/// Pin identity here the moment a consumer branches on a specific code.
5913fn forwarding_error_code(err: &ForwardingError) -> &'static str {
5914    match err {
5915        ForwardingError::NoModuleConnection => "target_unavailable",
5916        ForwardingError::ModuleReloading { .. } => "module_reloading",
5917        ForwardingError::ClientRouteChannelExhausted { .. }
5918        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
5919        ForwardingError::StaleModuleEndpoint
5920        | ForwardingError::UnknownReservation { .. }
5921        | ForwardingError::ConnectionClosing { .. }
5922        | ForwardingError::ClientEgressClosed { .. }
5923        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
5924        // Only a swap candidate's registration can produce this, and it means
5925        // exactly what a second active HELLO for a live id means.
5926        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
5927        ForwardingError::RelayCorrelationExhausted
5928        | ForwardingError::RouteOpenBuild(_)
5929        | ForwardingError::Poisoned => "forwarding_error",
5930    }
5931}
5932
5933fn response_version(frame: &Frame) -> u8 {
5934    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
5935        frame.header.ver
5936    } else {
5937        PROTOCOL_VERSION
5938    }
5939}
5940
5941fn control_flags() -> Flags {
5942    Flags::new(false, Priority::Passive, false)
5943}
5944
5945/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
5946/// channel. The target is the module (a client never saw the route), so this
5947/// takes the module path: delivered late rather than dropped when the module's
5948/// queue is momentarily full, and never closing its connection.
5949fn send_goodbye_target_best_effort(
5950    counters: &DaemonCounters,
5951    target: &GoodbyeTarget,
5952    context: &'static str,
5953) {
5954    let Ok(frame) = Frame::build_with_version(
5955        target.negotiated_ver,
5956        FrameType::Goodbye,
5957        control_flags(),
5958        target.channel,
5959        target.epoch,
5960        0,
5961        Vec::new(),
5962    ) else {
5963        return;
5964    };
5965    crate::forwarding::send_module_route_goodbye(
5966        counters,
5967        &target.sink,
5968        frame,
5969        target.module_id.as_deref(),
5970        context,
5971    );
5972}
5973
5974pub(crate) fn send_route_control_pushes(
5975    forwarding: &ForwardingTable,
5976    routes: Vec<EndpointRoute>,
5977    push: ClientControlPush,
5978) {
5979    let body = match serde_json::to_vec(&push) {
5980        Ok(body) => body,
5981        Err(err) => {
5982            warn!(error = %err, "failed to serialize route lifecycle control PUSH");
5983            return;
5984        }
5985    };
5986    let mut targets = Vec::new();
5987    for route in routes {
5988        let target = route.goodbye_target;
5989        if let Some(existing) = targets
5990            .iter()
5991            .find(|existing: &&GoodbyeTarget| existing.connection_id == target.connection_id)
5992        {
5993            debug_assert_eq!(
5994                existing.negotiated_ver, target.negotiated_ver,
5995                "one connection cannot negotiate multiple frame versions"
5996            );
5997            continue;
5998        }
5999        targets.push(target);
6000    }
6001    for target in targets {
6002        let frame = match Frame::build_with_version(
6003            target.negotiated_ver,
6004            FrameType::Push,
6005            control_flags(),
6006            0,
6007            0,
6008            0,
6009            body.clone(),
6010        ) {
6011            Ok(frame) => frame,
6012            Err(err) => {
6013                warn!(
6014                    route_channel = target.channel,
6015                    error = %err,
6016                    "failed to build route lifecycle control PUSH frame"
6017                );
6018                continue;
6019            }
6020        };
6021        if let Err(err) = target.sink.try_send(frame) {
6022            if target.close_on_delivery_failure() {
6023                warn!(
6024                    target_connection_id = target.connection_id.get(),
6025                    route_channel = target.channel,
6026                    error = %err,
6027                    "route lifecycle control PUSH was not delivered to client; closing target connection"
6028                );
6029                let _ = forwarding.escalate_client_delivery_failure(
6030                    target.connection_id,
6031                    target.channel,
6032                    target.epoch,
6033                    CloseReason::new(
6034                        "route_lifecycle_push_delivery_failed",
6035                        format!(
6036                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
6037                            target.channel
6038                        ),
6039                    ),
6040                    crate::forwarding::UndeliveredFrame {
6041                        module_id: target.module_id.as_deref(),
6042                        sink: &target.sink,
6043                    },
6044                );
6045            }
6046        }
6047    }
6048}
6049
6050#[cfg(test)]
6051mod tests {
6052    use std::{
6053        collections::BTreeMap,
6054        fmt,
6055        path::PathBuf,
6056        sync::{Arc, Mutex},
6057        time::Duration,
6058    };
6059    use subc_test_support::TestTempDir;
6060
6061    use serde_json::{json, Value};
6062    use subc_protocol::{
6063        manifest::{
6064            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
6065            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
6066        },
6067        session::HealthStatus,
6068        FrameType,
6069    };
6070
6071    use super::*;
6072    use crate::{
6073        forwarding::{DataRoute, DataRouteState},
6074        registry::ChannelState,
6075        router::FrameSink,
6076        stderr_tail::DEFAULT_MAX_LINE_BYTES,
6077        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
6078        RouteCtx, Router,
6079    };
6080    use tokio::{
6081        sync::mpsc,
6082        time::{sleep, Instant},
6083    };
6084    use tracing::{
6085        field::{Field, Visit},
6086        Event, Subscriber,
6087    };
6088    use tracing_subscriber::{layer::Context, prelude::*, Layer};
6089
6090    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
6091    ///
6092    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
6093    /// is only populated for `tests/*.rs` integration test binaries -- this file
6094    /// compiles as part of the library target, which gets neither. This test's
6095    /// own executable path is `<target-dir>/<profile>/deps/subc_core-<hash>`,
6096    /// and the sibling binary lives two directories up at
6097    /// `<target-dir>/<profile>/fake-aft-stub`.
6098    ///
6099    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
6100    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
6101    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
6102    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
6103    /// `NotFound`, which reads as a broken test rather than an unbuilt
6104    /// dependency -- so state the cause and the remedy instead. Deliberately a
6105    /// panic and not a silent skip: a test that quietly passes when it could not
6106    /// run is worse than one that fails, because it reports health it never
6107    /// verified.
6108    fn fake_aft_stub_path() -> PathBuf {
6109        let mut path = std::env::current_exe().expect("current_exe available in tests");
6110        path.pop(); // .../deps/
6111        path.pop(); // .../<profile>/
6112        path.push(if cfg!(windows) {
6113            "fake-aft-stub.exe"
6114        } else {
6115            "fake-aft-stub"
6116        });
6117        assert!(
6118            path.exists(),
6119            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
6120             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
6121            path.display()
6122        );
6123        path
6124    }
6125
6126    /// Whether clients retry `code` in place: the predicate itself, never a copy
6127    /// of its set. A copied list breaks silently when a code is added to or
6128    /// removed from the real one, and a stale copy here would let exactly the
6129    /// failure this test exists to catch pass.
6130    fn client_retries(code: &str) -> bool {
6131        subc_protocol::error_codes::is_retryable_route_open(code)
6132    }
6133
6134    /// A code is not a label — clients branch on it, so publishing the wrong KIND
6135    /// of failure is worse than publishing none. A permanent fault dressed as
6136    /// retryable makes every client in the fleet retry forever against something
6137    /// that cannot recover; a transient fault dressed as permanent abandons work
6138    /// that would have succeeded.
6139    ///
6140    /// Asserting "a code exists" cannot catch either, because the string is free
6141    /// to say anything. This enumerates every variant and pins which side of the
6142    /// retry boundary it lands on, so a new variant must be classified here
6143    /// deliberately rather than inheriting whichever arm it was appended to.
6144    #[test]
6145    fn retryability_of_forwarding_codes_matches_the_failure() {
6146        // Transient by nature: the target is booting, reloading, or its endpoint
6147        // was swapped mid-flight. Retrying is how these resolve.
6148        let transient = [
6149            ForwardingError::NoModuleConnection,
6150            ForwardingError::ModuleReloading {
6151                module_id: "m".into(),
6152            },
6153            ForwardingError::StaleModuleEndpoint,
6154            ForwardingError::UnknownReservation {
6155                client_channel: 1,
6156                module_channel: 1,
6157            },
6158            ForwardingError::ConnectionClosing {
6159                connection_id: ConnectionId::new(1),
6160            },
6161            ForwardingError::ClientEgressClosed {
6162                connection_id: ConnectionId::new(1),
6163            },
6164            ForwardingError::ModuleEgressUnavailable {
6165                connection_id: ConnectionId::new(1),
6166            },
6167        ];
6168        for err in transient {
6169            let code = forwarding_error_code(&err);
6170            assert!(
6171                client_retries(code),
6172                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
6173            );
6174        }
6175
6176        // Not fixed by retrying. Channel and correlation exhaustion need the
6177        // caller to close routes, and a poisoned lock is a daemon that cannot
6178        // recover at all — the worst thing to advertise as retryable, since every
6179        // client would storm a daemon that will never answer.
6180        let permanent = [
6181            ForwardingError::ClientRouteChannelExhausted {
6182                connection_id: ConnectionId::new(1),
6183            },
6184            ForwardingError::ModuleRouteChannelExhausted {
6185                endpoint: ModuleEndpointId {
6186                    connection_id: ConnectionId::new(1),
6187                    generation: 1,
6188                },
6189            },
6190            ForwardingError::RelayCorrelationExhausted,
6191            ForwardingError::RouteOpenBuild("x".into()),
6192            ForwardingError::Poisoned,
6193        ];
6194        for err in permanent {
6195            let code = forwarding_error_code(&err);
6196            assert!(
6197                !client_retries(code),
6198                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
6199            );
6200        }
6201    }
6202
6203    /// The principal is the daemon's answer to "who is calling", and modules
6204    /// branch on it: aft gates bash on it, cerebellum gates browser control,
6205    /// plexus gates connector invocation. So a stamp is an authorization input in
6206    /// another process, not a label — and both possible answers SUCCEED, which is
6207    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
6208    /// first-party capability to something that never proved it; a supervised one
6209    /// stamped `Direct` silently strips a module of capability it is entitled to.
6210    ///
6211    /// Neither shows up in a test that only checks the bind succeeded. Before this
6212    /// test the only coverage was accidental —
6213    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
6214    /// stamped principal on its way past, so narrowing that wire-shape test to its
6215    /// stated subject would have deleted the last assertion on this value. It
6216    /// still asserts the stamp, which is now redundancy rather than the only
6217    /// guard: both fail under the same mutation, and this one names the reason.
6218    /// SCOPE: this handler's supervisor has spawned nothing, so
6219    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
6220    /// is unreachable here. Both assertions below are refusals, and a mutant that
6221    /// refuses everything would satisfy them.
6222    ///
6223    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
6224    /// spawns a supervised consumer, reads its live nonce, and asserts the module
6225    /// observed `principal.kind == "reserved"` carrying that module_id — verified
6226    /// at source rather than assumed, since a citation is a claim about another
6227    /// file and ages like one. Recorded because a harness that structurally
6228    /// cannot reach an arm reports "none" for that arm identically to one that
6229    /// covers it and found nothing.
6230    #[tokio::test]
6231    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
6232        let handler = ControlHandler::default();
6233        let frame =
6234            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
6235
6236        // Absent consumer_identity is the ordinary case: a human at a terminal, or
6237        // any process holding the connection file. Nothing was proved, so nothing
6238        // may be granted beyond the unattested floor.
6239        let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
6240        assert_eq!(
6241            stamped,
6242            Principal::Direct,
6243            "a caller that proved nothing must not be stamped as a supervised module"
6244        );
6245
6246        // A claimed module_id with a nonce no supervised child was given is a
6247        // forgery attempt, not a weaker caller: it must be REFUSED rather than
6248        // quietly demoted to Direct, or an impersonation attempt looks identical
6249        // to an ordinary unattested connection.
6250        let forged = handler
6251            .route_open_principal(
6252                &frame,
6253                Some(ConsumerIdentity {
6254                    module_id: "aft".to_string(),
6255                    launch_nonce: "not-a-real-nonce".to_string(),
6256                }),
6257            )
6258            .unwrap();
6259        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
6260        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
6261    }
6262
6263    /// The test above hands `route_open_principal` an identity it built itself,
6264    /// which proves the stamping rule and nothing about where the identity comes
6265    /// from. The real producer is a wire body, and the two are joined by a serde
6266    /// field name that nothing else asserts.
6267    ///
6268    /// That join fails quietly in one specific way: an unrecognised key is simply
6269    /// absent after parsing, so a renamed or misspelled `consumer_identity`
6270    /// yields `None` and every supervised module silently drops to `Direct`.
6271    /// Capability-wise that is the safe direction, but it surfaces far from its
6272    /// cause — as a module mysteriously refused bash — and it would pass every
6273    /// test that builds its own input.
6274    ///
6275    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
6276    /// would break every client the moment the daemon gains a field, trading a
6277    /// quiet demotion for a hard refusal on additive change. Asserting the join
6278    /// instead means a rename breaks a test here rather than the fleet.
6279    #[test]
6280    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
6281        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
6282        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
6283        let ClientControlRequest::RouteOpen {
6284            consumer_identity, ..
6285        } = parsed
6286        else {
6287            panic!("route.open body must parse as RouteOpen");
6288        };
6289        assert_eq!(
6290            consumer_identity,
6291            Some(ConsumerIdentity {
6292                module_id: "aft".to_string(),
6293                launch_nonce: "n".to_string(),
6294            }),
6295            "the wire field name must reach the value route_open_principal reads"
6296        );
6297    }
6298
6299    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
6300        ModuleManifest::builder(module_id, "0.1.0")
6301            .protocol_ver(protocol_ver)
6302            .provides(vec![ProviderRole::ToolProvider {
6303                tools: vec![Tool {
6304                    name: "read".to_string(),
6305                    description: None,
6306                    execution_mode: ExecutionMode::Pure,
6307                    schema: json!({"type": "object"}),
6308                }],
6309                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
6310                concurrency: Concurrency::ModuleManaged,
6311                emits_push: true,
6312                sub_supervises: true,
6313            }])
6314            .build()
6315    }
6316
6317    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
6318        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
6319    }
6320
6321    fn hello_frame_with_control_ops(
6322        module_id: &str,
6323        protocol_ver: u8,
6324        corr: u64,
6325        control_ops: Option<Vec<String>>,
6326    ) -> Frame {
6327        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
6328    }
6329
6330    fn hello_frame_with_nonce(
6331        module_id: &str,
6332        protocol_ver: u8,
6333        corr: u64,
6334        launch_nonce: Option<&str>,
6335    ) -> Frame {
6336        hello_frame_full(
6337            module_id,
6338            protocol_ver,
6339            corr,
6340            None,
6341            launch_nonce.map(ToOwned::to_owned),
6342        )
6343    }
6344
6345    fn hello_frame_full(
6346        module_id: &str,
6347        protocol_ver: u8,
6348        corr: u64,
6349        control_ops: Option<Vec<String>>,
6350        launch_nonce: Option<String>,
6351    ) -> Frame {
6352        let body = serde_json::to_vec(&ModuleHelloBody {
6353            manifest: manifest(module_id, protocol_ver),
6354            protocol_ver,
6355            control_ops,
6356            launch_nonce,
6357        })
6358        .unwrap();
6359        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6360    }
6361
6362    fn non_routable_hello_frame_with_control_ops(
6363        module_id: &str,
6364        corr: u64,
6365        control_ops: Option<Vec<String>>,
6366    ) -> Frame {
6367        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
6368        manifest.provides.clear();
6369        let body = serde_json::to_vec(&ModuleHelloBody {
6370            manifest,
6371            protocol_ver: PROTOCOL_VERSION,
6372            control_ops,
6373            launch_nonce: None,
6374        })
6375        .unwrap();
6376        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6377    }
6378
6379    fn capability_grammar_hello_frame(
6380        capabilities: Value,
6381        runtime_computed: Option<Value>,
6382        corr: u64,
6383    ) -> Frame {
6384        let mut body = serde_json::to_value(ModuleHelloBody {
6385            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
6386            protocol_ver: PROTOCOL_VERSION,
6387            control_ops: None,
6388            launch_nonce: None,
6389        })
6390        .expect("HELLO body serializes");
6391        body["manifest"]["capabilities"] = capabilities;
6392        if let Some(runtime_computed) = runtime_computed {
6393            body["runtime_computed"] = runtime_computed;
6394        }
6395        Frame::build(
6396            FrameType::Hello,
6397            control_flags(),
6398            0,
6399            0,
6400            corr,
6401            serde_json::to_vec(&body).expect("HELLO body reserializes"),
6402        )
6403        .expect("HELLO frame builds")
6404    }
6405
6406    fn channel_request(channel: u16, corr: u64) -> Frame {
6407        Frame::build(
6408            FrameType::Request,
6409            Flags::new(true, Priority::Interactive, false),
6410            channel,
6411            0,
6412            corr,
6413            b"opaque".to_vec(),
6414        )
6415        .unwrap()
6416    }
6417
6418    fn route_ctx(
6419        connection_id: ConnectionId,
6420    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
6421        let (tx, rx) = mpsc::channel(8);
6422        (
6423            RouteCtx {
6424                connection_id,
6425                egress: FrameSink::new(tx),
6426            },
6427            rx,
6428        )
6429    }
6430
6431    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
6432        serde_json::from_slice(&frame.body).unwrap()
6433    }
6434
6435    /// Register a module over a connection that has a sink and return the
6436    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
6437    /// module's own sink rather than returning it as a reply, so the ack is
6438    /// taken off `rx` here and whatever the test reads next is what followed it.
6439    async fn hello_via_sink(
6440        handler: &ControlHandler,
6441        ctx: &RouteCtx,
6442        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
6443        hello: Frame,
6444    ) -> Frame {
6445        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
6446        assert!(
6447            replies.is_empty(),
6448            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
6449        );
6450        let ack = rx
6451            .try_recv()
6452            .expect("HELLO_ACK is queued on the module sink")
6453            .frame;
6454        assert_eq!(ack.header.ty, FrameType::HelloAck);
6455        ack
6456    }
6457
6458    fn parse_error(frame: &Frame) -> Value {
6459        serde_json::from_slice(&frame.body).unwrap()
6460    }
6461
6462    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
6463        serde_json::from_slice(&frame.body).unwrap()
6464    }
6465
6466    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
6467        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
6468            route_channel,
6469            route_epoch: 0,
6470            kind,
6471        })
6472        .unwrap();
6473        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6474    }
6475
6476    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
6477        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
6478            module_id: module_id.to_string(),
6479        })
6480        .unwrap();
6481        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6482    }
6483
6484    fn route_open_frame(corr: u64, module_id: &str, project_root: TestTempDir) -> Frame {
6485        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
6486    }
6487
6488    fn route_open_frame_with_consumer_capabilities(
6489        corr: u64,
6490        module_id: &str,
6491        project_root: TestTempDir,
6492        consumer_capabilities: Option<Vec<String>>,
6493    ) -> Frame {
6494        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6495            target: RouteTarget::ToolProvider {
6496                module_id: module_id.to_string(),
6497            },
6498            identity: BindIdentity::new(
6499                project_root.path().to_path_buf(),
6500                "unit".to_string(),
6501                "session".to_string(),
6502            ),
6503            consumer_identity: None,
6504            consumer_capabilities,
6505            role_versions: None,
6506            admission_facts: None,
6507            scope: None,
6508        })
6509        .unwrap();
6510        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6511    }
6512
6513    fn route_open_frame_with_role_versions(
6514        corr: u64,
6515        module_id: &str,
6516        project_root: TestTempDir,
6517        role_versions: Option<BTreeMap<String, String>>,
6518    ) -> Frame {
6519        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6520            target: RouteTarget::ToolProvider {
6521                module_id: module_id.to_string(),
6522            },
6523            identity: BindIdentity::new(
6524                project_root.path().to_path_buf(),
6525                "unit".to_string(),
6526                format!("session-{corr}"),
6527            ),
6528            consumer_identity: None,
6529            consumer_capabilities: None,
6530            role_versions,
6531            admission_facts: None,
6532            scope: None,
6533        })
6534        .unwrap();
6535        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6536    }
6537
6538    fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
6539        entries
6540            .iter()
6541            .map(|(role, version)| (role.to_string(), version.to_string()))
6542            .collect()
6543    }
6544
6545    fn route_open_frame_with_admission_facts(
6546        corr: u64,
6547        module_id: &str,
6548        project_root: TestTempDir,
6549        consumer_identity: Option<subc_control::ConsumerIdentity>,
6550        facts: Option<Value>,
6551    ) -> Frame {
6552        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6553            target: RouteTarget::ToolProvider {
6554                module_id: module_id.to_string(),
6555            },
6556            identity: BindIdentity::new(
6557                project_root.path().to_path_buf(),
6558                "unit".to_string(),
6559                format!("session-{corr}"),
6560            ),
6561            consumer_identity,
6562            consumer_capabilities: None,
6563            role_versions: None,
6564            admission_facts: facts,
6565            scope: None,
6566        })
6567        .unwrap();
6568        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6569    }
6570
6571    #[derive(Clone, Default)]
6572    struct EventCapture {
6573        events: Arc<Mutex<Vec<CapturedEvent>>>,
6574    }
6575
6576    #[derive(Clone, Debug)]
6577    struct CapturedEvent {
6578        target: String,
6579        level: tracing::Level,
6580        fields: BTreeMap<String, String>,
6581    }
6582
6583    impl EventCapture {
6584        fn events(&self) -> Vec<CapturedEvent> {
6585            self.events.lock().unwrap().clone()
6586        }
6587    }
6588
6589    impl<S> Layer<S> for EventCapture
6590    where
6591        S: Subscriber,
6592    {
6593        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
6594            let mut visitor = EventFieldVisitor::default();
6595            event.record(&mut visitor);
6596            self.events.lock().unwrap().push(CapturedEvent {
6597                target: event.metadata().target().to_string(),
6598                level: *event.metadata().level(),
6599                fields: visitor.fields,
6600            });
6601        }
6602    }
6603
6604    #[derive(Default)]
6605    struct EventFieldVisitor {
6606        fields: BTreeMap<String, String>,
6607    }
6608
6609    impl Visit for EventFieldVisitor {
6610        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
6611            self.fields
6612                .insert(field.name().to_string(), format!("{value:?}"));
6613        }
6614    }
6615
6616    fn health_response(corr: u64, status: HealthStatus) -> Frame {
6617        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
6618            status,
6619            detail: Some("warming".to_string()),
6620            metrics: Some(json!({"queue_depth": 3})),
6621        })
6622        .unwrap();
6623        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6624    }
6625
6626    fn route_bind_ack(corr: u64) -> Frame {
6627        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
6628        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6629    }
6630
6631    fn unique_project_root(label: &str) -> TestTempDir {
6632        TestTempDir::new(label)
6633    }
6634
6635    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
6636        match parse_route_poll(frame) {
6637            ClientControlResponse::RoutePoll {
6638                status: None,
6639                live: Some(live),
6640                ..
6641            } => assert_eq!(live, expected_live),
6642            other => panic!("unexpected route.poll response: {other:?}"),
6643        }
6644    }
6645
6646    fn bind_liveness_route(
6647        registry: &Registry,
6648        forwarding: &ForwardingTable,
6649        module_id: &str,
6650    ) -> (RouteCtx, u16, u32) {
6651        let module_connection = ConnectionId::new(101);
6652        let client_connection = ConnectionId::new(202);
6653        let registration = registry
6654            .register_with_control_ops(
6655                manifest(module_id, PROTOCOL_VERSION),
6656                PROTOCOL_VERSION,
6657                module_connection,
6658                module_baseline_control_ops(),
6659            )
6660            .unwrap();
6661        let (module_tx, _module_rx) = mpsc::channel(8);
6662        let endpoint = forwarding
6663            .register_module_connection(
6664                module_connection,
6665                module_id.to_string(),
6666                PROTOCOL_VERSION,
6667                manifest_concurrency(&registration.manifest),
6668                FrameSink::new(module_tx),
6669            )
6670            .unwrap();
6671        let (client_ctx, _client_rx) = route_ctx(client_connection);
6672        let pending = forwarding
6673            .begin_route_bind_relay_for_test(
6674                client_connection,
6675                client_ctx.egress.clone(),
6676                1,
6677                module_id,
6678            )
6679            .unwrap();
6680        assert_eq!(pending.endpoint, endpoint);
6681        let route_channel = pending.client_channel;
6682        let route_epoch = pending.client_epoch;
6683        forwarding
6684            .complete_pending_relay(
6685                module_connection,
6686                pending.corr,
6687                RouteBindRelayOutcome::Accepted,
6688            )
6689            .unwrap();
6690        (client_ctx, route_channel, route_epoch)
6691    }
6692
6693    struct FakeProcessLiveness {
6694        live: Option<bool>,
6695    }
6696
6697    impl ModuleProcessLiveness for FakeProcessLiveness {
6698        fn process_live(&self, _module_id: &str) -> Option<bool> {
6699            self.live
6700        }
6701    }
6702
6703    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6704    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
6705    {
6706        let registry = Arc::new(Registry::default());
6707        let supervisor_handle = SupervisorHandle::new();
6708        let supervisor = Supervisor::new(
6709            Arc::clone(&registry),
6710            RestartPolicy::new(1, Duration::from_millis(10)),
6711        )
6712        .with_handle(supervisor_handle.clone());
6713        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
6714        let module = supervisor
6715            .spawn(ModuleSpec {
6716                launch_nonce_env: true,
6717                module_id: "stderr-tail-wire".to_string(),
6718                program: fake_aft_stub_path(),
6719                args: Vec::new(),
6720                env: vec![
6721                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
6722                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
6723                ],
6724                reserved: false,
6725                reserved_prefixes: Vec::new(),
6726                protocol: ModuleProtocol::Subc,
6727                overlap: Default::default(),
6728            })
6729            .unwrap();
6730
6731        let deadline = Instant::now() + Duration::from_secs(5);
6732        loop {
6733            let tail = module.stderr_tail(None, None);
6734            if tail
6735                .entries
6736                .iter()
6737                .any(|entry| matches!(entry, TailEntry::ProcessStart))
6738                && tail.entries.iter().any(|entry| {
6739                    matches!(
6740                        entry,
6741                        TailEntry::Line {
6742                            truncated: true,
6743                            ..
6744                        }
6745                    )
6746                })
6747            {
6748                break;
6749            }
6750            assert!(
6751                Instant::now() < deadline,
6752                "module did not produce a truncated line and restart boundary: {tail:?}"
6753            );
6754            sleep(Duration::from_millis(10)).await;
6755        }
6756
6757        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
6758        let request = ClientControlRequest::SupervisorStderrTail {
6759            module_id: "stderr-tail-wire".to_string(),
6760            max_lines: None,
6761            max_bytes: None,
6762        };
6763        let frame = Frame::build(
6764            FrameType::Request,
6765            control_flags(),
6766            0,
6767            0,
6768            1,
6769            serde_json::to_vec(&request).unwrap(),
6770        )
6771        .unwrap();
6772        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6773        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6774        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
6775            serde_json::from_slice(&responses[0].body).unwrap()
6776        else {
6777            panic!("expected supervisor.stderr_tail response");
6778        };
6779
6780        assert!(
6781            tail.entries
6782                .iter()
6783                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
6784            "the control response lost the restart boundary"
6785        );
6786        let Some(StderrTailEntry::Line {
6787            text,
6788            truncated,
6789            at_ms,
6790        }) = tail.entries.iter().find(|entry| {
6791            matches!(
6792                entry,
6793                StderrTailEntry::Line {
6794                    truncated: true,
6795                    ..
6796                }
6797            )
6798        })
6799        else {
6800            panic!("the control response lost the truncated line");
6801        };
6802        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
6803        assert!(*truncated);
6804        assert!(
6805            at_ms.is_some(),
6806            "the control response lost the line's capture time"
6807        );
6808    }
6809
6810    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
6811    /// read done on the worker thread would stall every other task until it
6812    /// finished; the read must run off the worker so this test's own task keeps
6813    /// running while the read is paused.
6814    #[tokio::test(flavor = "current_thread")]
6815    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
6816        let dir = TestTempDir::new("terminals-off-worker");
6817        let journal_path = dir.join("terminals.jsonl");
6818        let registry = Arc::new(Registry::default());
6819        let supervisor_handle = SupervisorHandle::new();
6820        let supervisor =
6821            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
6822                .with_handle(supervisor_handle.clone())
6823                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
6824        let module = supervisor
6825            .spawn(ModuleSpec {
6826                launch_nonce_env: true,
6827                module_id: "terminal-off-worker".to_string(),
6828                program: fake_aft_stub_path(),
6829                args: Vec::new(),
6830                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6831                reserved: false,
6832                reserved_prefixes: Vec::new(),
6833                protocol: ModuleProtocol::Subc,
6834                overlap: Default::default(),
6835            })
6836            .unwrap();
6837        let deadline = Instant::now() + Duration::from_secs(5);
6838        while module.terminal_history().entries.len() != 2 {
6839            assert!(Instant::now() < deadline, "module did not record two exits");
6840            sleep(Duration::from_millis(10)).await;
6841        }
6842
6843        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
6844        let handler =
6845            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
6846        let frame = Frame::build(
6847            FrameType::Request,
6848            control_flags(),
6849            0,
6850            0,
6851            1,
6852            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
6853                module_id: "terminal-off-worker".to_string(),
6854            })
6855            .unwrap(),
6856        )
6857        .unwrap();
6858        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6859        let spawned_at = std::time::Instant::now();
6860        let read = tokio::spawn({
6861            let handler = Arc::clone(&handler);
6862            async move { handler.handle_control_frame(&ctx, frame).await }
6863        });
6864        // Waiting for the pause from a blocking thread keeps this task pending,
6865        // so the runtime's single worker is free to run the read task.
6866        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
6867            .await
6868            .unwrap()
6869            .expect("the history read reached its pause");
6870        let elapsed = spawned_at.elapsed();
6871        assert!(
6872            elapsed < Duration::from_secs(2) && !read.is_finished(),
6873            "this task could not run while the history read was paused \
6874             (resumed after {elapsed:?}, read finished: {})",
6875            read.is_finished()
6876        );
6877
6878        drop(release);
6879        let responses = read.await.unwrap().unwrap();
6880        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6881        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
6882            panic!("expected supervisor.terminals response");
6883        };
6884        assert_eq!(terminals.entries.len(), 2);
6885        assert_eq!(terminals.journal_skipped_lines, 0);
6886        assert_eq!(terminals.journal_read_errors, 0);
6887    }
6888
6889    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6890    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
6891        let registry = Arc::new(Registry::default());
6892        let supervisor_handle = SupervisorHandle::new();
6893        let supervisor =
6894            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
6895                .with_handle(supervisor_handle.clone());
6896        let module = supervisor
6897            .spawn(ModuleSpec {
6898                launch_nonce_env: true,
6899                module_id: "terminal-golden".to_string(),
6900                program: fake_aft_stub_path(),
6901                args: Vec::new(),
6902                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6903                reserved: false,
6904                reserved_prefixes: Vec::new(),
6905                protocol: ModuleProtocol::Subc,
6906                overlap: Default::default(),
6907            })
6908            .unwrap();
6909
6910        let deadline = Instant::now() + Duration::from_secs(5);
6911        while module.terminal_history().entries.len() != 2 {
6912            assert!(
6913                Instant::now() < deadline,
6914                "module did not retain two terminal exits: {:?}",
6915                module.terminal_history()
6916            );
6917            sleep(Duration::from_millis(10)).await;
6918        }
6919
6920        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
6921        let request = ClientControlRequest::SupervisorTerminals {
6922            module_id: "terminal-golden".to_string(),
6923        };
6924        let frame = Frame::build(
6925            FrameType::Request,
6926            control_flags(),
6927            0,
6928            0,
6929            1,
6930            serde_json::to_vec(&request).unwrap(),
6931        )
6932        .unwrap();
6933        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6934        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6935        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6936        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
6937            panic!("expected supervisor.terminals response");
6938        };
6939        assert_eq!(terminals.entries.len(), 2);
6940        assert_eq!(terminals.dropped, 0);
6941
6942        let mut rendered = serde_json::to_value(response).unwrap();
6943        // Wall-clock fields are the observation contract, but not stable fixture
6944        // bytes; normalize only them after the real handler has shaped the response.
6945        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
6946        for (index, entry) in rendered["entries"]
6947            .as_array_mut()
6948            .expect("terminal response entries array")
6949            .iter_mut()
6950            .enumerate()
6951        {
6952            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
6953        }
6954
6955        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
6956            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
6957        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
6958        if std::env::var_os("UPDATE_GOLDEN").is_some() {
6959            std::fs::write(&golden_path, &serialized).unwrap();
6960        }
6961        let expected: Value =
6962            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
6963        assert_eq!(rendered, expected);
6964    }
6965
6966    #[test]
6967    fn hello_registers_manifest_and_returns_ack() {
6968        let registry = Arc::new(Registry::default());
6969        let handler = ControlHandler::new(Arc::clone(&registry));
6970        let conn = ConnectionId::new(1);
6971
6972        let responses = handler
6973            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
6974            .unwrap();
6975
6976        assert_eq!(responses.len(), 1);
6977        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6978        assert_eq!(responses[0].header.channel, 0);
6979        assert_eq!(responses[0].header.corr, 7);
6980        let ack = parse_ack(&responses[0]);
6981        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
6982        assert!(ack
6983            .subc_capabilities
6984            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
6985        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
6986        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
6987        assert!(ack
6988            .subc_ops
6989            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
6990        assert!(ack
6991            .subc_ops
6992            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
6993
6994        let registration = registry.get_module("aft").unwrap().unwrap();
6995        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
6996        assert_eq!(registration.state, ChannelState::Active);
6997        assert_eq!(registration.connection_id, conn);
6998        assert_eq!(registration.control_ops, module_baseline_control_ops());
6999    }
7000
7001    #[test]
7002    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
7003        let invalid_identifiers = [
7004            ("case_change", "credentials-Provider/v1"),
7005            ("leading_zero", "credentials-provider/v01"),
7006            ("trailing_hyphen", "credentials-provider-/v1"),
7007            ("consecutive_hyphens", "credentials--provider/v1"),
7008            ("uppercase", "Credentials-provider/v1"),
7009            ("missing_v", "credentials-provider/1"),
7010            ("whitespace", "credentials provider/v1"),
7011            ("zero_version", "credentials-provider/v0"),
7012            ("out_of_range_version", "credentials-provider/v4294967296"),
7013            (
7014                "overlength_name",
7015                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
7016            ),
7017        ];
7018        let mut cases = invalid_identifiers
7019            .into_iter()
7020            .map(|(name, identifier)| {
7021                (
7022                    format!("identifier_{name}"),
7023                    "capabilities.provides[0]".to_string(),
7024                    identifier.to_string(),
7025                    json!({ "provides": [identifier] }),
7026                    None,
7027                )
7028            })
7029            .collect::<Vec<_>>();
7030        cases.extend([
7031            (
7032                "unknown_need".to_string(),
7033                "capabilities.requires[0].need".to_string(),
7034                "deferred".to_string(),
7035                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
7036                None,
7037            ),
7038            (
7039                "duplicate_provides".to_string(),
7040                "capabilities.provides[1]".to_string(),
7041                "credentials-provider/v1".to_string(),
7042                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
7043                None,
7044            ),
7045            (
7046                "duplicate_must_never_reach".to_string(),
7047                "capabilities.must_never_reach[1]".to_string(),
7048                "credentials-provider/v1".to_string(),
7049                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
7050                None,
7051            ),
7052            (
7053                "duplicate_requires_same_need".to_string(),
7054                "capabilities.requires[1]".to_string(),
7055                "credentials-provider/v1".to_string(),
7056                json!({ "requires": [
7057                    { "capability": "credentials-provider/v1", "need": "required" },
7058                    { "capability": "credentials-provider/v1", "need": "required" }
7059                ] }),
7060                None,
7061            ),
7062            (
7063                "duplicate_requires_conflicting_need".to_string(),
7064                "capabilities.requires[1]".to_string(),
7065                "credentials-provider/v1".to_string(),
7066                json!({ "requires": [
7067                    { "capability": "credentials-provider/v1", "need": "required" },
7068                    { "capability": "credentials-provider/v1", "need": "optional" }
7069                ] }),
7070                None,
7071            ),
7072            (
7073                "capabilities_root_pointer".to_string(),
7074                "runtime_computed[0]".to_string(),
7075                "/capabilities".to_string(),
7076                json!({}),
7077                Some(json!(["/capabilities"])),
7078            ),
7079            (
7080                "capabilities_descendant_pointer".to_string(),
7081                "runtime_computed[0]".to_string(),
7082                "/capabilities/provides".to_string(),
7083                json!({}),
7084                Some(json!(["/capabilities/provides"])),
7085            ),
7086            (
7087                "malformed_pointer_without_leading_slash".to_string(),
7088                "runtime_computed[0]".to_string(),
7089                "capabilities".to_string(),
7090                json!({}),
7091                Some(json!(["capabilities"])),
7092            ),
7093            (
7094                "malformed_pointer_escape".to_string(),
7095                "runtime_computed[0]".to_string(),
7096                "/roles/~2/tools".to_string(),
7097                json!({}),
7098                Some(json!(["/roles/~2/tools"])),
7099            ),
7100            (
7101                "unknown_capabilities_field".to_string(),
7102                "capabilities.future".to_string(),
7103                "<array>".to_string(),
7104                json!({ "future": [] }),
7105                None,
7106            ),
7107        ]);
7108
7109        for (index, (name, field, value, capabilities, runtime_computed)) in
7110            cases.into_iter().enumerate()
7111        {
7112            let registry = Arc::new(Registry::default());
7113            let handler = ControlHandler::new(Arc::clone(&registry));
7114            let response = handler
7115                .handle_control(
7116                    ConnectionId::new((index + 1) as u64),
7117                    capability_grammar_hello_frame(
7118                        capabilities,
7119                        runtime_computed,
7120                        index as u64 + 1,
7121                    ),
7122                )
7123                .expect("invalid HELLO returns a refusal");
7124
7125            assert_eq!(response.len(), 1, "{name} must emit one refusal");
7126            let error = parse_error(&response[0]);
7127            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
7128            let message = error["message"]
7129                .as_str()
7130                .expect("error message is a string");
7131            assert!(
7132                message.contains(&field),
7133                "{name}: field missing from {message}"
7134            );
7135            assert!(
7136                message.contains(&value),
7137                "{name}: value missing from {message}"
7138            );
7139            assert_eq!(
7140                registry
7141                    .active_registration_count()
7142                    .expect("registry reads"),
7143                0,
7144                "{name}: refused HELLO must not create a catalog entry"
7145            );
7146        }
7147    }
7148
7149    #[test]
7150    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
7151        let registry = Arc::new(Registry::default());
7152        let handler = ControlHandler::new(Arc::clone(&registry));
7153        let capabilities = json!({
7154            "provides": ["credentials-provider/v1"],
7155            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
7156            "must_never_reach": ["federation-transport/v1"]
7157        });
7158        let response = handler
7159            .handle_control(
7160                ConnectionId::new(99),
7161                capability_grammar_hello_frame(
7162                    capabilities.clone(),
7163                    Some(json!(["/roles/0/tools"])),
7164                    99,
7165                ),
7166            )
7167            .expect("valid HELLO registers");
7168        assert_eq!(response[0].header.ty, FrameType::HelloAck);
7169
7170        let request = Frame::build(
7171            FrameType::Request,
7172            control_flags(),
7173            0,
7174            0,
7175            100,
7176            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7177                .expect("catalog request serializes"),
7178        )
7179        .expect("catalog request frame builds");
7180        let response = handler
7181            .handle_catalog_list(request, None)
7182            .expect("catalog list succeeds");
7183        let ClientControlResponse::CatalogList { modules, .. } =
7184            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
7185        else {
7186            panic!("catalog request must return catalog.list");
7187        };
7188        assert_eq!(modules.len(), 1);
7189        assert_eq!(
7190            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
7191            capabilities
7192        );
7193    }
7194
7195    #[test]
7196    fn catalog_list_mirrors_management_operation_description() {
7197        let registry = Arc::new(Registry::default());
7198        let handler = ControlHandler::new(Arc::clone(&registry));
7199        let description = "List managed records and return their identifiers and metadata.";
7200        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
7201        manifest.provides = vec![ProviderRole::ManagementSurface {
7202            operations: vec![ManagementOperation {
7203                name: "records.list".to_string(),
7204                kind: ManagementOperationKind::Query,
7205                description: Some(description.to_string()),
7206            }],
7207            config_schema: json!({"type": "object"}),
7208            observability: vec![ObservabilitySurface {
7209                name: "records.stats".to_string(),
7210                kind: ObservabilityKind::Snapshot,
7211            }],
7212            identity_scope: vec![IdentityScope::Project],
7213            concurrency: Concurrency::ModuleManaged,
7214        }];
7215        registry
7216            .register_with_control_ops(
7217                manifest,
7218                PROTOCOL_VERSION,
7219                ConnectionId::new(99),
7220                Vec::new(),
7221            )
7222            .expect("described management manifest registers");
7223
7224        let request = Frame::build(
7225            FrameType::Request,
7226            control_flags(),
7227            0,
7228            0,
7229            100,
7230            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7231                .expect("catalog request serializes"),
7232        )
7233        .expect("catalog request frame builds");
7234        let response = handler
7235            .handle_catalog_list(request, None)
7236            .expect("catalog list succeeds");
7237        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7238        assert_eq!(
7239            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
7240            "catalog.list must preserve the declared operation description verbatim"
7241        );
7242    }
7243
7244    #[test]
7245    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
7246        let registry = Arc::new(Registry::default());
7247        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
7248            [("vault".to_string(), true), ("squatter".to_string(), true)],
7249            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
7250        );
7251        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
7252        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7253            provides: vec!["credentials-provider/v1".to_string()],
7254            requires: Vec::new(),
7255            must_never_reach: Vec::new(),
7256        });
7257        let frame = Frame::build(
7258            FrameType::Hello,
7259            control_flags(),
7260            0,
7261            0,
7262            77,
7263            serde_json::to_vec(&ModuleHelloBody {
7264                manifest: squatter,
7265                protocol_ver: PROTOCOL_VERSION,
7266                control_ops: None,
7267                launch_nonce: None,
7268            })
7269            .expect("HELLO serializes"),
7270        )
7271        .expect("HELLO frame builds");
7272        let response = handler
7273            .handle_control(ConnectionId::new(77), frame)
7274            .expect("reserved claim receives a typed refusal");
7275        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
7276        assert_eq!(
7277            registry
7278                .active_registration_count()
7279                .expect("registry reads"),
7280            0,
7281            "a reserved capability refusal must not leave a catalog entry"
7282        );
7283    }
7284
7285    #[test]
7286    fn server_describe_surfaces_required_capability_verdict_fields() {
7287        let registry = Arc::new(Registry::default());
7288        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
7289            [
7290                ("consumer".to_string(), true),
7291                ("provider".to_string(), false),
7292            ],
7293            BTreeMap::new(),
7294        );
7295        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
7296        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7297            provides: Vec::new(),
7298            requires: vec![subc_protocol::manifest::CapabilityRequirement {
7299                capability: "credentials-provider/v1".to_string(),
7300                need: subc_protocol::manifest::CapabilityNeed::Required,
7301            }],
7302            must_never_reach: Vec::new(),
7303        });
7304        let hello = Frame::build(
7305            FrameType::Hello,
7306            control_flags(),
7307            0,
7308            0,
7309            78,
7310            serde_json::to_vec(&ModuleHelloBody {
7311                manifest: consumer,
7312                protocol_ver: PROTOCOL_VERSION,
7313                control_ops: None,
7314                launch_nonce: None,
7315            })
7316            .expect("HELLO serializes"),
7317        )
7318        .expect("HELLO frame builds");
7319        handler
7320            .handle_control(ConnectionId::new(78), hello)
7321            .expect("consumer registers");
7322        let describe = Frame::build(
7323            FrameType::Request,
7324            control_flags(),
7325            0,
7326            0,
7327            79,
7328            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
7329                .expect("request serializes"),
7330        )
7331        .expect("describe frame builds");
7332        let response = handler
7333            .handle_server_describe(describe)
7334            .expect("server.describe succeeds");
7335        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
7336        let requirement = &rendered["capability_requirements"][0];
7337        assert_eq!(requirement["consumer"], "consumer");
7338        assert_eq!(requirement["verdict"], "never_provided");
7339        assert_eq!(requirement["episode_seq"], 1);
7340        assert_eq!(requirement["config_satisfiable"], false);
7341        assert_eq!(requirement["runtime_available"], false);
7342        assert!(requirement["detail"]
7343            .as_str()
7344            .expect("detail string")
7345            .contains("credentials-provider/v1"));
7346    }
7347
7348    #[test]
7349    fn catalog_list_omits_capabilities_for_legacy_manifest() {
7350        let registry = Arc::new(Registry::default());
7351        let handler = ControlHandler::new(Arc::clone(&registry));
7352        let hello = handler
7353            .handle_control(
7354                ConnectionId::new(101),
7355                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
7356            )
7357            .expect("legacy HELLO registers");
7358        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
7359
7360        let request = Frame::build(
7361            FrameType::Request,
7362            control_flags(),
7363            0,
7364            0,
7365            102,
7366            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7367                .expect("catalog request serializes"),
7368        )
7369        .expect("catalog request frame builds");
7370        let response = handler
7371            .handle_catalog_list(request, None)
7372            .expect("catalog list succeeds");
7373        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7374        assert!(
7375            body["modules"][0].get("capabilities").is_none(),
7376            "legacy manifest must retain an absent capabilities field on catalog.list"
7377        );
7378    }
7379
7380    #[test]
7381    fn hello_ack_omits_storage_when_no_storage_config() {
7382        let registry = Arc::new(Registry::default());
7383        let handler = ControlHandler::new(Arc::clone(&registry));
7384        let responses = handler
7385            .handle_control(
7386                ConnectionId::new(1),
7387                hello_frame("aft", PROTOCOL_VERSION, 7),
7388            )
7389            .unwrap();
7390        let ack = parse_ack(&responses[0]);
7391        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
7392        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
7393    }
7394
7395    #[tokio::test]
7396    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
7397        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
7398        let registry = Arc::new(Registry::default());
7399        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
7400        let responses = handler
7401            .handle_control(
7402                ConnectionId::new(1),
7403                hello_frame("aft", PROTOCOL_VERSION, 7),
7404            )
7405            .unwrap();
7406        let ack = parse_ack(&responses[0]);
7407        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
7408
7409        let described = handler
7410            .handle_control_frame(
7411                &route_ctx(ConnectionId::new(2)).0,
7412                Frame::build(
7413                    FrameType::Request,
7414                    control_flags(),
7415                    0,
7416                    0,
7417                    9,
7418                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
7419                )
7420                .unwrap(),
7421            )
7422            .await
7423            .unwrap();
7424        let ClientControlResponse::ServerDescribe { machine_id, .. } =
7425            serde_json::from_slice(&described[0].body).unwrap()
7426        else {
7427            panic!("server.describe answered with another shape");
7428        };
7429        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
7430    }
7431
7432    #[test]
7433    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
7434        // With a central sqlite storage policy, each registering module gets its
7435        // own resolved descriptor in HELLO_ACK, keyed by its module id.
7436        let registry = Arc::new(Registry::default());
7437        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
7438            crate::daemon_config::StorageConfig::Sqlite {
7439                data_home: std::path::PathBuf::from("/data"),
7440            },
7441        ));
7442
7443        let responses = handler
7444            .handle_control(
7445                ConnectionId::new(1),
7446                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
7447            )
7448            .unwrap();
7449        let ack = parse_ack(&responses[0]);
7450        assert_eq!(
7451            ack.storage,
7452            Some(serde_json::json!({
7453                "module_id": "alfonso-routing",
7454                "storage_namespace": "default",
7455                "isolation": { "kind": "module" },
7456                "backend": {
7457                    "backend": "sqlite",
7458                    "path": "/data/cortexkit/alfonso-routing/store.db"
7459                }
7460            })),
7461            "the delivered descriptor is the module's own sqlite store path"
7462        );
7463    }
7464
7465    #[test]
7466    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
7467        let registry = Arc::new(Registry::default());
7468        let handler = ControlHandler::new(Arc::clone(&registry));
7469        let conn = ConnectionId::new(1);
7470        let responses = handler
7471            .handle_control(
7472                conn,
7473                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
7474            )
7475            .unwrap();
7476        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
7477        let registration = registry.get_module("aft").unwrap().unwrap();
7478        assert_eq!(registration.control_ops, module_baseline_control_ops());
7479
7480        let frame =
7481            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
7482        assert!(handler
7483            .guard_module_control_op(&frame, "aft", "route.bind")
7484            .unwrap()
7485            .is_none());
7486        let error = handler
7487            .guard_module_control_op(&frame, "aft", "test.synthetic")
7488            .unwrap()
7489            .expect("synthetic ungranted op should be rejected");
7490        assert_eq!(error.header.ty, FrameType::Error);
7491        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
7492    }
7493
7494    #[test]
7495    fn hello_control_ops_some_adds_optional_grants() {
7496        let registry = Arc::new(Registry::default());
7497        let handler = ControlHandler::new(Arc::clone(&registry));
7498        handler
7499            .handle_control(
7500                ConnectionId::new(1),
7501                hello_frame_with_control_ops(
7502                    "aft",
7503                    PROTOCOL_VERSION,
7504                    7,
7505                    Some(vec![
7506                        "future.synthetic".to_string(),
7507                        "route.bind".to_string(),
7508                    ]),
7509                ),
7510            )
7511            .unwrap();
7512        let registration = registry.get_module("aft").unwrap().unwrap();
7513        assert_eq!(
7514            registration.control_ops,
7515            vec![
7516                "route.bind".to_string(),
7517                "route.status".to_string(),
7518                "future.synthetic".to_string(),
7519            ]
7520        );
7521        let frame =
7522            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
7523        assert!(handler
7524            .guard_module_control_op(&frame, "aft", "future.synthetic")
7525            .unwrap()
7526            .is_none());
7527    }
7528
7529    #[tokio::test]
7530    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
7531        let registry = Arc::new(Registry::default());
7532        let forwarding = Arc::new(ForwardingTable::default());
7533        let handler =
7534            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
7535        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
7536        hello_via_sink(
7537            &handler,
7538            &module_ctx,
7539            &mut module_rx,
7540            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
7541        )
7542        .await;
7543
7544        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
7545        let responses = handler
7546            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
7547            .await
7548            .unwrap();
7549        assert_eq!(responses.len(), 1);
7550        assert_eq!(responses[0].header.ty, FrameType::Error);
7551        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
7552        assert!(module_rx.try_recv().is_err());
7553    }
7554
7555    #[tokio::test]
7556    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
7557        let registry = Arc::new(Registry::default());
7558        let forwarding = Arc::new(ForwardingTable::default());
7559        let handler =
7560            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
7561        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
7562        hello_via_sink(
7563            &handler,
7564            &module_ctx,
7565            &mut module_rx,
7566            hello_frame_with_control_ops(
7567                "aft",
7568                PROTOCOL_VERSION,
7569                7,
7570                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
7571            ),
7572        )
7573        .await;
7574
7575        let project_root = unique_project_root("demux");
7576        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
7577        let route_handler = handler.clone();
7578        let route_task = tokio::spawn(async move {
7579            route_handler
7580                .handle_control_frame(
7581                    &route_client_ctx,
7582                    route_open_frame(100, "aft", project_root),
7583                )
7584                .await
7585                .unwrap()
7586        });
7587        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7588            .await
7589            .unwrap()
7590            .unwrap();
7591        assert!(matches!(
7592            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
7593            ModuleControlRequest::RouteBind { .. }
7594        ));
7595
7596        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
7597        let health_handler = handler.clone();
7598        let health_task = tokio::spawn(async move {
7599            health_handler
7600                .handle_control_frame(
7601                    &health_client_ctx,
7602                    supervisor_health_probe_frame(101, "aft"),
7603                )
7604                .await
7605                .unwrap()
7606        });
7607        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7608            .await
7609            .unwrap()
7610            .unwrap();
7611        assert_eq!(
7612            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
7613            ModuleControlRequest::HealthCheck {}
7614        );
7615
7616        handler
7617            .handle_control_frame(
7618                &module_ctx,
7619                health_response(health_frame.header.corr, HealthStatus::Degraded),
7620            )
7621            .await
7622            .unwrap();
7623        let health_response = health_task.await.unwrap();
7624        assert_eq!(health_response.len(), 1);
7625        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
7626            ClientControlResponse::SupervisorHealthProbe {
7627                module_id,
7628                status,
7629                detail,
7630                metrics,
7631            } => {
7632                assert_eq!(module_id, "aft");
7633                assert_eq!(status, HealthStatus::Degraded);
7634                assert_eq!(detail.as_deref(), Some("warming"));
7635                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
7636            }
7637            other => panic!("unexpected health response: {other:?}"),
7638        }
7639
7640        handler
7641            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
7642            .await
7643            .unwrap();
7644        let route_response = route_task.await.unwrap();
7645        assert!(route_response.is_empty());
7646        let published = route_client_rx.recv().await.unwrap();
7647        assert!(matches!(
7648            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
7649            ClientControlResponse::RouteOpen { .. }
7650        ));
7651    }
7652
7653    /// Start one `route.open` on `client_connection` and return its still-running
7654    /// handler task together with the `route.bind` the module received for it.
7655    /// The handler blocks until the module answers, so it has to run as a task
7656    /// while the test drives the module side.
7657    async fn relay_route_open(
7658        handler: &ControlHandler,
7659        client_connection: ConnectionId,
7660        client_egress: &FrameSink,
7661        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7662        corr: u64,
7663        module_id: &str,
7664        project_root_label: &str,
7665    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
7666        let ctx = RouteCtx {
7667            connection_id: client_connection,
7668            egress: client_egress.clone(),
7669        };
7670        let handler = handler.clone();
7671        let project_root = unique_project_root(project_root_label);
7672        let module_id = module_id.to_string();
7673        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
7674        let task = tokio::spawn(async move {
7675            let _guard = tracing::dispatcher::set_default(&dispatch);
7676            handler
7677                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
7678                .await
7679                .unwrap()
7680        });
7681        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
7682            .await
7683            .expect("module receives the relayed route.bind")
7684            .expect("module egress is open");
7685        (task, bind.frame)
7686    }
7687
7688    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
7689        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
7690            ModuleControlRequest::RouteBind {
7691                route_channel,
7692                epoch,
7693                ..
7694            } => (route_channel, epoch),
7695            other => panic!("expected a route.bind request, got {other:?}"),
7696        }
7697    }
7698
7699    fn published_route(frame: &Frame) -> (u16, u32) {
7700        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
7701            ClientControlResponse::RouteOpen {
7702                route_channel,
7703                route_epoch,
7704            } => (route_channel, route_epoch),
7705            other => panic!("expected a route.open response, got {other:?}"),
7706        }
7707    }
7708
7709    /// Reproduction of a production outage. A client had `route.open`s in
7710    /// flight to a module and was already marked closing -- its egress had refused a
7711    /// module frame, so the daemon asked its connection to end -- while its sink
7712    /// was still open. When the module acked those binds, the daemon refused to
7713    /// commit a route for a closing client, and that refusal was returned from
7714    /// the MODULE connection's frame handler, where a router error that has no
7715    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
7716    /// the supervisor correctly did not respawn a clean exit, and every seat lost
7717    /// its tools for hours -- one client's teardown took down a connection
7718    /// carrying ~170 other routes.
7719    ///
7720    /// The window is opened here by calling the production path that opens it
7721    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
7722    /// state that matters is "in `closing_connections`, sink still open, relay
7723    /// still pending", and it lasts only from the close request until the
7724    /// connection loop reacts to it; a socket-level test can flood a client into
7725    /// that escalation but cannot pin the module's ack inside the window. Closing
7726    /// the socket instead takes the other path entirely -- connection teardown
7727    /// removes the pending relay under the same lock, so the ack finds nothing.
7728    #[tokio::test]
7729    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
7730        let registry = Arc::new(Registry::default());
7731        let forwarding = Arc::new(ForwardingTable::default());
7732        let handler =
7733            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
7734
7735        let module_connection = ConnectionId::new(30);
7736        let (module_ctx, mut module_rx) = route_ctx(module_connection);
7737        hello_via_sink(
7738            &handler,
7739            &module_ctx,
7740            &mut module_rx,
7741            hello_frame("aft", PROTOCOL_VERSION, 7),
7742        )
7743        .await;
7744
7745        let dying_client = ConnectionId::new(31);
7746        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
7747
7748        // A published route on the dying client. The escalation below only marks
7749        // a connection closing for a route it has already published.
7750        let (first_task, first_bind) = relay_route_open(
7751            &handler,
7752            dying_client,
7753            &dying_ctx.egress,
7754            &mut module_rx,
7755            100,
7756            "aft",
7757            "closing-first",
7758        )
7759        .await;
7760        handler
7761            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
7762            .await
7763            .unwrap();
7764        assert!(first_task.await.unwrap().is_empty());
7765        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
7766
7767        // A second route.open from the same client, relayed and awaiting its ack.
7768        let (second_task, second_bind) = relay_route_open(
7769            &handler,
7770            dying_client,
7771            &dying_ctx.egress,
7772            &mut module_rx,
7773            101,
7774            "aft",
7775            "closing-second",
7776        )
7777        .await;
7778        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
7779
7780        // The window: the client is closing, its sink is still open, and its
7781        // second bind is still pending.
7782        assert!(forwarding
7783            .escalate_client_delivery_failure(
7784                dying_client,
7785                first_channel,
7786                first_epoch,
7787                CloseReason::new(
7788                    "module_to_client_delivery_failed",
7789                    "client egress refused a module frame",
7790                ),
7791                crate::forwarding::UndeliveredFrame {
7792                    module_id: None,
7793                    sink: &dying_ctx.egress,
7794                },
7795            )
7796            .unwrap());
7797        assert!(!dying_ctx.egress.is_closed());
7798
7799        // The frame that used to end the module connection.
7800        let ack = handler
7801            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
7802            .await;
7803        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
7804        if module_loop_error.is_some() {
7805            // What the server's connection loop does with a router error that has
7806            // no ERROR-frame translation: end the connection, which releases the
7807            // module's registration and every route on it.
7808            handler.cleanup_connection(module_connection).unwrap();
7809        }
7810        // Read the module's next frame before opening the co-tenant's route, so
7811        // the GOODBYE assertion below is about THIS ack and not about later
7812        // traffic. `None` means the module was told nothing.
7813        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7814            .await
7815            .ok()
7816            .flatten();
7817
7818        // 1. The module connection is still registered.
7819        assert!(
7820            registry
7821                .get_module_by_connection(module_connection)
7822                .unwrap()
7823                .is_some(),
7824            "one client's closing connection ended the shared module connection: \
7825             {module_loop_error:?}"
7826        );
7827        // ...and still serving: another client can open and use a route on it.
7828        let cotenant = ConnectionId::new(32);
7829        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
7830        let (cotenant_task, cotenant_bind) = relay_route_open(
7831            &handler,
7832            cotenant,
7833            &cotenant_ctx.egress,
7834            &mut module_rx,
7835            102,
7836            "aft",
7837            "closing-cotenant",
7838        )
7839        .await;
7840        handler
7841            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
7842            .await
7843            .unwrap();
7844        assert!(cotenant_task.await.unwrap().is_empty());
7845        let (cotenant_channel, cotenant_epoch) =
7846            published_route(&cotenant_rx.recv().await.unwrap());
7847        assert!(matches!(
7848            forwarding
7849                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
7850                .unwrap(),
7851            DataRoute::Client(DataRouteState::Bound(_))
7852        ));
7853
7854        // 2. The module was told to drop the binding it created for the route
7855        //    that will never be published.
7856        let goodbye = post_ack_module_frame
7857            .expect("module receives a GOODBYE for the abandoned route channel");
7858        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
7859        assert_eq!(goodbye.header.channel, abandoned_channel);
7860        assert_eq!(goodbye.header.epoch, abandoned_epoch);
7861
7862        // 3. The dying client received nothing: no route was ever published to
7863        //    it. Its route.open is answered as unavailable, which the connection
7864        //    loop would write to a socket that is already going away.
7865        assert!(dying_rx.try_recv().is_err());
7866        let second_response = second_task.await.unwrap();
7867        assert_eq!(second_response.len(), 1);
7868        assert_eq!(
7869            parse_error(&second_response[0])["code"],
7870            "target_unavailable"
7871        );
7872    }
7873
7874    /// The fence at the module-loop boundary, stated as its own contract: which
7875    /// forwarding failures are allowed to end the module connection that is being
7876    /// served. A `ConnectionClosing` naming some client is about that client, and
7877    /// a module connection is shared; the same error naming the module's own
7878    /// connection is about this connection and must stay fatal, as must failures
7879    /// that are about the forwarding table itself.
7880    #[test]
7881    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
7882        let handler = ControlHandler::default();
7883        let module_connection = ConnectionId::new(30);
7884        let client_connection = ConnectionId::new(31);
7885
7886        handler
7887            .refuse_to_end_module_connection_for_a_client(
7888                module_connection,
7889                77,
7890                ForwardingError::ConnectionClosing {
7891                    connection_id: client_connection,
7892                },
7893            )
7894            .expect("a closing client must never end the module connection");
7895
7896        assert!(matches!(
7897            handler.refuse_to_end_module_connection_for_a_client(
7898                module_connection,
7899                78,
7900                ForwardingError::ConnectionClosing {
7901                    connection_id: module_connection,
7902                },
7903            ),
7904            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
7905                connection_id
7906            })) if connection_id == module_connection
7907        ));
7908        assert!(matches!(
7909            handler.refuse_to_end_module_connection_for_a_client(
7910                module_connection,
7911                79,
7912                ForwardingError::Poisoned,
7913            ),
7914            Err(RouterError::Forwarding(ForwardingError::Poisoned))
7915        ));
7916        assert!(matches!(
7917            handler.refuse_to_end_module_connection_for_a_client(
7918                module_connection,
7919                80,
7920                ForwardingError::StaleModuleEndpoint,
7921            ),
7922            Err(RouterError::Forwarding(
7923                ForwardingError::StaleModuleEndpoint
7924            ))
7925        ));
7926    }
7927
7928    /// The spawn-attestation guard is what stops a connected module from claiming
7929    /// another module's identity and being stamped `Reserved` for it. Every other
7930    /// test that supplies a consumer_identity supplies a CORRECT one, because a
7931    /// correct one is what the rest of the flow needs -- so the guard's rejection
7932    /// branch was never the subject of an assertion, only its acceptance branch.
7933    ///
7934    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
7935    /// whole subc-core library suite green; only the forwarding integration tests
7936    /// notice, and they notice for unrelated reasons. This test exists so the
7937    /// refusal itself is asserted where the guard lives: it fails if the identity
7938    /// check stops refusing, which is the direction that matters, since a guard
7939    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
7940    #[tokio::test]
7941    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
7942        let registry = Arc::new(Registry::default());
7943        let forwarding = Arc::new(ForwardingTable::default());
7944        let supervisor = SupervisorHandle::new();
7945        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7946        let handler =
7947            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
7948                .with_supervisor(supervisor);
7949
7950        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
7951        hello_via_sink(
7952            &handler,
7953            &target_ctx,
7954            &mut target_rx,
7955            hello_frame("target", PROTOCOL_VERSION, 1),
7956        )
7957        .await;
7958
7959        // A real supervised module id presenting the wrong nonce. This is the
7960        // impersonation case: the attacker knows a privileged module_id, which is
7961        // public, and guesses at the nonce, which is not.
7962        let wrong_nonce = handler
7963            .handle_control_frame(
7964                &route_ctx(ConnectionId::new(91)).0,
7965                route_open_frame_with_admission_facts(
7966                    20,
7967                    "target",
7968                    unique_project_root("admission-facts"),
7969                    Some(subc_control::ConsumerIdentity {
7970                        module_id: "fed".to_string(),
7971                        launch_nonce: "not-the-real-nonce".to_string(),
7972                    }),
7973                    None,
7974                ),
7975            )
7976            .await
7977            .unwrap();
7978        assert_eq!(
7979            parse_error(&wrong_nonce[0])["code"],
7980            "bad_consumer_identity",
7981            "a mismatched launch nonce must be refused, not stamped Reserved"
7982        );
7983
7984        // A module id the supervisor never spawned at all, so no nonce exists to
7985        // compare against. An implementation that treats "no record" as "nothing
7986        // to check" fails open here while passing the case above.
7987        let never_spawned = handler
7988            .handle_control_frame(
7989                &route_ctx(ConnectionId::new(92)).0,
7990                route_open_frame_with_admission_facts(
7991                    21,
7992                    "target",
7993                    unique_project_root("admission-facts"),
7994                    Some(subc_control::ConsumerIdentity {
7995                        module_id: "never-spawned".to_string(),
7996                        launch_nonce: "any-nonce".to_string(),
7997                    }),
7998                    None,
7999                ),
8000            )
8001            .await
8002            .unwrap();
8003        assert_eq!(
8004            parse_error(&never_spawned[0])["code"],
8005            "bad_consumer_identity",
8006            "an unspawned module_id must be refused rather than accepted for lack of a record"
8007        );
8008    }
8009
8010    /// The refusal test above proves the guard says NO. Nothing proved it can say
8011    /// YES, and the difference is not academic: replacing the whole authorization
8012    /// with `false` -- admitting no consumer identity at all, revoking Reserved
8013    /// standing for every supervised module in the fleet -- leaves 110 of the 111
8014    /// library tests GREEN. The one that notices does so by HANGING, because it
8015    /// waits for a bind that can no longer happen.
8016    ///
8017    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
8018    /// or flaky test, invites a RETRY rather than an investigation, and the retry
8019    /// hangs too and gets blamed on the runner. So a total revocation of the
8020    /// daemon's trust grant would have shipped behind a symptom nobody attributes
8021    /// to code.
8022    ///
8023    /// The bias is structural rather than accidental. A REFUSAL looks like a
8024    /// failure someone writes a test for; a GRANT looks like the happy path. Every
8025    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
8026    /// suite for that reason, and this one is the purest case in the daemon.
8027    ///
8028    /// This test asserts the EFFECT rather than the absence of an error: the module
8029    /// receives a RouteBind and it carries `Reserved` naming the attested module.
8030    /// A guard that admitted nobody would produce no bind at all; one that admitted
8031    /// everybody would stamp the wrong principal, which the refusal test catches.
8032    #[tokio::test]
8033    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
8034        let registry = Arc::new(Registry::default());
8035        let forwarding = Arc::new(ForwardingTable::default());
8036        let supervisor = SupervisorHandle::new();
8037        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8038        let handler =
8039            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8040                .with_supervisor(supervisor);
8041
8042        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
8043        hello_via_sink(
8044            &handler,
8045            &target_ctx,
8046            &mut target_rx,
8047            hello_frame("target", PROTOCOL_VERSION, 1),
8048        )
8049        .await;
8050
8051        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
8052        let route_handler = handler.clone();
8053        let route_task = tokio::spawn(async move {
8054            route_handler
8055                .handle_control_frame(
8056                    &client_ctx,
8057                    route_open_frame_with_admission_facts(
8058                        30,
8059                        "target",
8060                        unique_project_root("admission-facts"),
8061                        Some(subc_control::ConsumerIdentity {
8062                            module_id: "fed".to_string(),
8063                            launch_nonce: "fed-nonce".to_string(),
8064                        }),
8065                        None,
8066                    ),
8067                )
8068                .await
8069                .unwrap()
8070        });
8071
8072        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
8073        // the very mutation it exists to catch -- a guard that admits nobody -- no
8074        // bind is ever sent, so it HUNG rather than failing. That reproduces the
8075        // exact defect being fixed: a total revocation detected only as a stalled
8076        // suite, which reads as flakiness and invites a retry. An acceptance test
8077        // that waits for an effect must bound the wait, or a red becomes a hang.
8078        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8079            .await
8080            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
8081            .expect("module control channel closed before route.bind");
8082        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8083        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
8084            panic!("expected route.bind")
8085        };
8086        assert_eq!(
8087            principal,
8088            Some(Principal::Reserved {
8089                module_id: "fed".to_string()
8090            }),
8091            "a correctly attested consumer must be stamped Reserved for its own id"
8092        );
8093
8094        handler
8095            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8096            .await
8097            .unwrap();
8098        assert!(route_task.await.unwrap().is_empty());
8099        assert!(
8100            matches!(
8101                serde_json::from_slice::<ClientControlResponse>(
8102                    &client_rx.recv().await.unwrap().body
8103                )
8104                .unwrap(),
8105                ClientControlResponse::RouteOpen { .. }
8106            ),
8107            "the route must actually open, not merely avoid an error"
8108        );
8109    }
8110
8111    #[tokio::test(start_paused = true)]
8112    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
8113        let registry = Arc::new(Registry::default());
8114        let forwarding = Arc::new(ForwardingTable::default());
8115        let supervisor = SupervisorHandle::new();
8116        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8117        let handler =
8118            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8119                .with_supervisor(supervisor);
8120
8121        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
8122        hello_via_sink(
8123            &handler,
8124            &target_ctx,
8125            &mut target_rx,
8126            hello_frame("target", PROTOCOL_VERSION, 1),
8127        )
8128        .await;
8129
8130        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
8131        let direct_handler = handler.clone();
8132        let direct_open = tokio::spawn(async move {
8133            direct_handler
8134                .handle_control_frame(
8135                    &direct_ctx,
8136                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
8137                )
8138                .await
8139                .unwrap()
8140        });
8141        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8142            .await
8143            .expect("no direct route.bind within 5s")
8144            .expect("target control channel closed before direct route.bind");
8145        handler
8146            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
8147            .await
8148            .unwrap();
8149        assert!(direct_open.await.unwrap().is_empty());
8150        let _ = direct_rx.recv().await.unwrap();
8151
8152        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
8153        let reserved_handler = handler.clone();
8154        let reserved_open = tokio::spawn(async move {
8155            reserved_handler
8156                .handle_control_frame(
8157                    &reserved_ctx,
8158                    route_open_frame_with_admission_facts(
8159                        3,
8160                        "target",
8161                        unique_project_root("admission-facts"),
8162                        Some(ConsumerIdentity {
8163                            module_id: "fed".to_string(),
8164                            launch_nonce: "fed-nonce".to_string(),
8165                        }),
8166                        None,
8167                    ),
8168                )
8169                .await
8170                .unwrap()
8171        });
8172        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8173            .await
8174            .expect("no reserved route.bind within 5s")
8175            .expect("target control channel closed before reserved route.bind");
8176        handler
8177            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
8178            .await
8179            .unwrap();
8180        assert!(reserved_open.await.unwrap().is_empty());
8181        let _ = reserved_rx.recv().await.unwrap();
8182
8183        forwarding
8184            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
8185            .unwrap();
8186        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
8187        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
8188            module_id: Some("target".to_string()),
8189        })
8190        .unwrap();
8191        let census_frame =
8192            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
8193        let response = handler
8194            .handle_control_frame(&census_ctx, census_frame)
8195            .await
8196            .unwrap()
8197            .pop()
8198            .unwrap();
8199        let actual: Value = serde_json::from_slice(&response.body).unwrap();
8200        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
8201        assert!(matches!(
8202            decoded,
8203            ClientControlResponse::SupervisorRoutes { .. }
8204        ));
8205        let routes = actual["modules"][0]["routes"].as_array().unwrap();
8206        assert_eq!(routes.len(), 2);
8207        assert!(routes.iter().all(|route| route["draining"] == true));
8208        // The census carries WHY: the reason the drain was begun with, in the
8209        // route.closing vocabulary, on every draining route this drain marked.
8210        assert!(
8211            routes.iter().all(|route| route["drain_reason"] == "reload"),
8212            "draining routes must name the drain's reason: {routes:?}"
8213        );
8214        assert!(routes.iter().any(|route| {
8215            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
8216        }));
8217        assert!(routes.iter().any(|route| {
8218            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
8219        }));
8220
8221        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8222            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
8223        if std::env::var_os("UPDATE_GOLDEN").is_some() {
8224            std::fs::write(
8225                &golden_path,
8226                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
8227            )
8228            .unwrap();
8229        }
8230        let expected: Value =
8231            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
8232        assert_eq!(actual, expected);
8233    }
8234
8235    async fn query_live_roots(
8236        handler: &ControlHandler,
8237        module_ctx: &RouteCtx,
8238    ) -> ModuleControlResponseToModule {
8239        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
8240        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
8241        let response = handler
8242            .handle_control_frame(module_ctx, frame)
8243            .await
8244            .unwrap()
8245            .pop()
8246            .unwrap();
8247        serde_json::from_slice(&response.body).unwrap()
8248    }
8249
8250    #[tokio::test(start_paused = true)]
8251    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
8252        let registry = Arc::new(Registry::default());
8253        let forwarding = Arc::new(ForwardingTable::default());
8254        let handler = ControlHandler::with_forwarding(registry, forwarding);
8255        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
8256        hello_via_sink(
8257            &handler,
8258            &target_ctx,
8259            &mut target_rx,
8260            hello_frame("target", PROTOCOL_VERSION, 1),
8261        )
8262        .await;
8263        let root = unique_project_root("live-roots-known");
8264        let path = ProjectRootId::from_path_allowing_missing(root.path())
8265            .unwrap()
8266            .as_path()
8267            .to_path_buf();
8268        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
8269        let open_handler = handler.clone();
8270        let opened = tokio::spawn(async move {
8271            open_handler
8272                .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
8273                .await
8274                .unwrap()
8275        });
8276        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8277            .await
8278            .unwrap()
8279            .unwrap();
8280        handler
8281            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
8282            .await
8283            .unwrap();
8284        assert!(opened.await.unwrap().is_empty());
8285        let _ = client_rx.recv().await.unwrap();
8286
8287        let root = unique_project_root("live-roots-pending");
8288        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
8289            .unwrap()
8290            .as_path()
8291            .to_path_buf();
8292        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
8293        let open_handler = handler.clone();
8294        let pending = tokio::spawn(async move {
8295            open_handler
8296                .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
8297                .await
8298                .unwrap()
8299        });
8300        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8301            .await
8302            .unwrap()
8303            .unwrap();
8304        let actual = query_live_roots(&handler, &target_ctx).await;
8305        let ModuleControlResponseToModule::LiveRoots {
8306            roots,
8307            unknown_root_bindings,
8308            total_bindings,
8309        } = actual
8310        else {
8311            panic!("expected live roots")
8312        };
8313        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
8314        assert_eq!(unknown_root_bindings, 0);
8315        assert_eq!(
8316            roots.len(),
8317            2,
8318            "root-known arm must retain each canonical root"
8319        );
8320        assert_eq!(
8321            total_bindings,
8322            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8323        );
8324        let counts = roots
8325            .iter()
8326            .map(|root| (root.project_root.clone(), root.bound, root.pending))
8327            .collect::<Vec<_>>();
8328        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
8329        expected.sort_by(|a, b| a.0.cmp(&b.0));
8330        assert_eq!(
8331            counts, expected,
8332            "roots must sort by path and count pending separately"
8333        );
8334        handler
8335            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
8336            .await
8337            .unwrap();
8338        assert!(pending.await.unwrap().is_empty());
8339    }
8340
8341    #[tokio::test(start_paused = true)]
8342    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
8343        let forwarding = Arc::new(ForwardingTable::default());
8344        let handler =
8345            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
8346        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
8347        hello_via_sink(
8348            &handler,
8349            &target_ctx,
8350            &mut target_rx,
8351            hello_frame("target", PROTOCOL_VERSION, 1),
8352        )
8353        .await;
8354        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
8355        let pending = forwarding
8356            .begin_route_bind_relay_for_test(
8357                client_ctx.connection_id,
8358                client_ctx.egress.clone(),
8359                2,
8360                "target",
8361            )
8362            .unwrap();
8363        forwarding
8364            .complete_pending_relay(
8365                target_ctx.connection_id,
8366                pending.corr,
8367                RouteBindRelayOutcome::Accepted,
8368            )
8369            .unwrap();
8370        let actual = query_live_roots(&handler, &target_ctx).await;
8371        let ModuleControlResponseToModule::LiveRoots {
8372            roots,
8373            unknown_root_bindings,
8374            total_bindings,
8375        } = actual
8376        else {
8377            panic!("expected live roots")
8378        };
8379        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
8380        assert_eq!(
8381            unknown_root_bindings, 1,
8382            "unknown-root arm must not read as no bindings"
8383        );
8384        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
8385        assert_eq!(
8386            total_bindings,
8387            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8388        );
8389    }
8390
8391    /// A module reads HELLO_ACK as its first frame and exits on anything else,
8392    /// so the ack has to be on its outbound queue before the module is
8393    /// routable. The connection loop writes a handler's replies only after the
8394    /// handler returns; this test stops in exactly that gap, runs a real
8395    /// route.open from another connection, and only then writes whatever the
8396    /// HELLO handler returned, the way the loop would. If the ack were still a
8397    /// reply, the route.bind request would reach the module first.
8398    #[tokio::test(start_paused = true)]
8399    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
8400        let forwarding = Arc::new(ForwardingTable::default());
8401        let handler =
8402            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
8403        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
8404        let replies = handler
8405            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
8406            .await
8407            .unwrap();
8408        let queued_by_hello = module_rx.len();
8409
8410        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
8411        let open_handler = handler.clone();
8412        let open = tokio::spawn(async move {
8413            open_handler
8414                .handle_control_frame(
8415                    &client_ctx,
8416                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
8417                )
8418                .await
8419                .unwrap()
8420        });
8421        // Let the route.open run until its route.bind is on the module's queue.
8422        let mut spins = 0;
8423        while module_rx.len() == queued_by_hello {
8424            spins += 1;
8425            assert!(spins < 10_000, "route.open never queued a route.bind");
8426            tokio::task::yield_now().await;
8427        }
8428
8429        // Now the connection loop's half: write the HELLO handler's replies.
8430        for reply in replies {
8431            module_ctx.egress.send(reply).await.unwrap();
8432        }
8433
8434        let first = module_rx.recv().await.unwrap().frame;
8435        assert_eq!(
8436            first.header.ty,
8437            FrameType::HelloAck,
8438            "the first frame a registering module reads must be its HELLO_ACK"
8439        );
8440        assert_eq!(first.header.corr, 7);
8441        let second = module_rx.recv().await.unwrap().frame;
8442        assert_eq!(second.header.ty, FrameType::Request);
8443        assert!(
8444            matches!(
8445                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
8446                ModuleControlRequest::RouteBind { .. }
8447            ),
8448            "the route.bind follows the ack"
8449        );
8450        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
8451
8452        handler
8453            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
8454            .await
8455            .unwrap();
8456        assert!(open.await.unwrap().is_empty());
8457        let _ = client_rx.recv().await.unwrap();
8458    }
8459
8460    #[tokio::test(start_paused = true)]
8461    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
8462        let handler = ControlHandler::with_forwarding(
8463            Arc::new(Registry::default()),
8464            Arc::new(ForwardingTable::default()),
8465        );
8466        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
8467        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
8468        hello_via_sink(
8469            &handler,
8470            &first_ctx,
8471            &mut first_rx,
8472            hello_frame("first", PROTOCOL_VERSION, 1),
8473        )
8474        .await;
8475        hello_via_sink(
8476            &handler,
8477            &second_ctx,
8478            &mut second_rx,
8479            hello_frame("second", PROTOCOL_VERSION, 2),
8480        )
8481        .await;
8482        let root = unique_project_root("second-only");
8483        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
8484        let cloned = handler.clone();
8485        let open = tokio::spawn(async move {
8486            cloned
8487                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
8488                .await
8489                .unwrap()
8490        });
8491        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
8492            .await
8493            .unwrap()
8494            .unwrap();
8495        let first = query_live_roots(&handler, &first_ctx).await;
8496        let second = query_live_roots(&handler, &second_ctx).await;
8497        assert!(
8498            matches!(
8499                first,
8500                ModuleControlResponseToModule::LiveRoots {
8501                    total_bindings: 0,
8502                    ..
8503                }
8504            ),
8505            "cross-module scope must not expose another module's roots"
8506        );
8507        assert!(
8508            matches!(
8509                second,
8510                ModuleControlResponseToModule::LiveRoots {
8511                    total_bindings: 1,
8512                    ..
8513                }
8514            ),
8515            "second module must see its pending route"
8516        );
8517        handler
8518            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
8519            .await
8520            .unwrap();
8521        assert!(open.await.unwrap().is_empty());
8522    }
8523
8524    #[tokio::test(start_paused = true)]
8525    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
8526        let handler = ControlHandler::with_forwarding(
8527            Arc::new(Registry::default()),
8528            Arc::new(ForwardingTable::default()),
8529        );
8530        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
8531        hello_via_sink(
8532            &handler,
8533            &target_ctx,
8534            &mut target_rx,
8535            hello_frame("target", PROTOCOL_VERSION, 1),
8536        )
8537        .await;
8538        let actual = query_live_roots(&handler, &target_ctx).await;
8539        let ModuleControlResponseToModule::LiveRoots {
8540            roots,
8541            unknown_root_bindings,
8542            total_bindings,
8543        } = actual
8544        else {
8545            panic!("expected live roots")
8546        };
8547        assert!(roots.is_empty());
8548        assert_eq!(unknown_root_bindings, 0);
8549        assert_eq!(total_bindings, 0);
8550        assert_eq!(
8551            total_bindings,
8552            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8553        );
8554    }
8555
8556    /// Read the vendored fed corpus rather than hand-building a package.
8557    ///
8558    /// A hand-built object encodes what the test author believed the carrier
8559    /// emits. These vectors are what it actually emits, and one of them exists
8560    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
8561    /// ignores additive unknown fields at the traversal emit terminus."
8562    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
8563        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8564            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
8565        let text = std::fs::read_to_string(&path)
8566            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
8567        let vectors: Vec<(String, Value)> = text
8568            .lines()
8569            .filter(|line| !line.trim().is_empty())
8570            .map(|line| {
8571                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
8572                let id = entry["corpus_id"]
8573                    .as_str()
8574                    .expect("every vector carries a corpus_id")
8575                    .to_string();
8576                (id, entry["package"].clone())
8577            })
8578            .collect();
8579        // Pin the count: a corpus that silently shrinks would take its coverage
8580        // with it, and a suite reading N-1 vectors reports the same clean pass
8581        // as one reading N.
8582        assert_eq!(
8583            vectors.len(),
8584            3,
8585            "vendored fed corpus changed size; re-sync from subc-federation"
8586        );
8587
8588        // Pin what makes the corpus DISCRIMINATING, not just present.
8589        //
8590        // The relay test below takes its expected value from the corpus, so the
8591        // corpus supplies the test's power to detect a lossy relay rather than
8592        // its correctness. A relay that dropped unrecognised fields would still
8593        // be caught -- but only by a package carrying fields it does not know.
8594        // Shrink every package to the handful of keys any implementation would
8595        // recognise and the test keeps passing over an input that can no longer
8596        // fail, which is the same clean green as a corpus that shrank away.
8597        //
8598        // So assert the precondition rather than duplicating the packages here:
8599        // at least one vector must carry a field beyond the small common set.
8600        // That is one claim to maintain instead of nine, and it fails loudly if
8601        // a re-sync ever flattens the corpus.
8602        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
8603        let richest = vectors
8604            .iter()
8605            .filter_map(|(_, package)| package.as_object())
8606            .map(|object| {
8607                object
8608                    .keys()
8609                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
8610                    .count()
8611            })
8612            .max()
8613            .unwrap_or(0);
8614        assert!(
8615            richest >= 2,
8616            "vendored corpus no longer carries a package with unmodelled fields, \
8617             so the relay test can no longer distinguish a verbatim relay from a lossy one"
8618        );
8619
8620        vectors
8621    }
8622
8623    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
8624    ///
8625    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
8626    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
8627    /// three-key object, so a relay that quietly dropped fields it did not
8628    /// recognise would satisfy it. These vectors carry nine keys including ones
8629    /// this crate has no type for, so a typed relay fails here and only here.
8630    #[tokio::test]
8631    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
8632        for (corpus_id, package) in fed_admission_facts_vectors() {
8633            let registry = Arc::new(Registry::default());
8634            let forwarding = Arc::new(ForwardingTable::default());
8635            let supervisor = SupervisorHandle::new();
8636            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8637            let handler =
8638                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8639                    .with_supervisor(supervisor)
8640                    .with_admission_facts_config(
8641                        Some("fed".to_string()),
8642                        Some(vec!["target".to_string()]),
8643                    );
8644
8645            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
8646            hello_via_sink(
8647                &handler,
8648                &target_ctx,
8649                &mut target_rx,
8650                hello_frame("target", PROTOCOL_VERSION, 1),
8651            )
8652            .await;
8653
8654            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
8655            let route_handler = handler.clone();
8656            let expected = package.clone();
8657            let route_task = tokio::spawn(async move {
8658                route_handler
8659                    .handle_control_frame(
8660                        &client_ctx,
8661                        route_open_frame_with_admission_facts(
8662                            20,
8663                            "target",
8664                            unique_project_root("admission-facts"),
8665                            Some(subc_control::ConsumerIdentity {
8666                                module_id: "fed".to_string(),
8667                                launch_nonce: "fed-nonce".to_string(),
8668                            }),
8669                            Some(package),
8670                        ),
8671                    )
8672                    .await
8673                    .unwrap()
8674            });
8675
8676            let bind_frame = target_rx.recv().await.unwrap();
8677            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8678            let ModuleControlRequest::RouteBind {
8679                admission_facts, ..
8680            } = bind
8681            else {
8682                panic!("{corpus_id}: expected route.bind")
8683            };
8684            assert_eq!(
8685                admission_facts,
8686                Some(expected),
8687                "{corpus_id}: relay must not add, drop or reshape any field"
8688            );
8689
8690            handler
8691                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8692                .await
8693                .unwrap();
8694            route_task.await.unwrap();
8695        }
8696    }
8697
8698    #[tokio::test]
8699    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
8700        let registry = Arc::new(Registry::default());
8701        let forwarding = Arc::new(ForwardingTable::default());
8702        let supervisor = SupervisorHandle::new();
8703        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8704        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
8705        let handler =
8706            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8707                .with_supervisor(supervisor)
8708                .with_admission_facts_config(
8709                    Some("fed".to_string()),
8710                    Some(vec!["target".to_string()]),
8711                );
8712
8713        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
8714        hello_via_sink(
8715            &handler,
8716            &target_ctx,
8717            &mut target_rx,
8718            hello_frame("target", PROTOCOL_VERSION, 1),
8719        )
8720        .await;
8721        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
8722        hello_via_sink(
8723            &handler,
8724            &other_ctx,
8725            &mut other_rx,
8726            hello_frame("other", PROTOCOL_VERSION, 2),
8727        )
8728        .await;
8729
8730        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
8731        let expected_facts = facts.clone();
8732        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
8733        let route_handler = handler.clone();
8734        let route_task = tokio::spawn(async move {
8735            route_handler
8736                .handle_control_frame(
8737                    &client_ctx,
8738                    route_open_frame_with_admission_facts(
8739                        10,
8740                        "target",
8741                        unique_project_root("admission-facts"),
8742                        Some(subc_control::ConsumerIdentity {
8743                            module_id: "fed".to_string(),
8744                            launch_nonce: "fed-nonce".to_string(),
8745                        }),
8746                        Some(facts.clone()),
8747                    ),
8748                )
8749                .await
8750                .unwrap()
8751        });
8752        let bind_frame = target_rx.recv().await.unwrap();
8753        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8754        let ModuleControlRequest::RouteBind {
8755            admission_facts, ..
8756        } = bind
8757        else {
8758            panic!("expected route.bind")
8759        };
8760        assert_eq!(admission_facts, Some(expected_facts));
8761        handler
8762            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8763            .await
8764            .unwrap();
8765        assert!(route_task.await.unwrap().is_empty());
8766        assert!(matches!(
8767            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
8768                .unwrap(),
8769            ClientControlResponse::RouteOpen { .. }
8770        ));
8771
8772        let direct = handler
8773            .handle_control_frame(
8774                &route_ctx(ConnectionId::new(73)).0,
8775                route_open_frame_with_admission_facts(
8776                    11,
8777                    "target",
8778                    unique_project_root("admission-facts"),
8779                    None,
8780                    Some(json!({"x": 1})),
8781                ),
8782            )
8783            .await
8784            .unwrap();
8785        assert_eq!(
8786            parse_error(&direct[0])["code"],
8787            "admission_facts_not_permitted"
8788        );
8789
8790        let different_reserved = handler
8791            .handle_control_frame(
8792                &route_ctx(ConnectionId::new(77)).0,
8793                route_open_frame_with_admission_facts(
8794                    15,
8795                    "target",
8796                    unique_project_root("admission-facts"),
8797                    Some(subc_control::ConsumerIdentity {
8798                        module_id: "other".to_string(),
8799                        launch_nonce: "other-nonce".to_string(),
8800                    }),
8801                    Some(json!({"x": 1})),
8802                ),
8803            )
8804            .await
8805            .unwrap();
8806        assert_eq!(
8807            parse_error(&different_reserved[0])["code"],
8808            "admission_facts_not_permitted"
8809        );
8810
8811        let other_target = handler
8812            .handle_control_frame(
8813                &route_ctx(ConnectionId::new(74)).0,
8814                route_open_frame_with_admission_facts(
8815                    12,
8816                    "other",
8817                    unique_project_root("admission-facts"),
8818                    Some(subc_control::ConsumerIdentity {
8819                        module_id: "fed".to_string(),
8820                        launch_nonce: "fed-nonce".to_string(),
8821                    }),
8822                    Some(json!({"x": 1})),
8823                ),
8824            )
8825            .await
8826            .unwrap();
8827        assert_eq!(
8828            parse_error(&other_target[0])["code"],
8829            "admission_facts_target_not_allowed"
8830        );
8831
8832        let nonexistent = handler
8833            .handle_control_frame(
8834                &route_ctx(ConnectionId::new(75)).0,
8835                route_open_frame_with_admission_facts(
8836                    13,
8837                    "missing",
8838                    unique_project_root("admission-facts"),
8839                    None,
8840                    Some(json!({"x": 1})),
8841                ),
8842            )
8843            .await
8844            .unwrap();
8845        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
8846
8847        let described = handler
8848            .handle_control_frame(
8849                &route_ctx(ConnectionId::new(76)).0,
8850                Frame::build(
8851                    FrameType::Request,
8852                    control_flags(),
8853                    0,
8854                    0,
8855                    14,
8856                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8857                )
8858                .unwrap(),
8859            )
8860            .await
8861            .unwrap();
8862        let ClientControlResponse::ServerDescribe { capabilities, .. } =
8863            serde_json::from_slice(&described[0].body).unwrap()
8864        else {
8865            panic!("expected server.describe response")
8866        };
8867        assert!(capabilities
8868            .iter()
8869            .any(|cap| cap == "admission_facts_relay_v1"));
8870    }
8871
8872    #[tokio::test]
8873    async fn admission_facts_without_configured_carrier_are_rejected() {
8874        let registry = Arc::new(Registry::default());
8875        let forwarding = Arc::new(ForwardingTable::default());
8876        let handler = ControlHandler::with_forwarding(registry, forwarding);
8877        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
8878        hello_via_sink(
8879            &handler,
8880            &target_ctx,
8881            &mut target_rx,
8882            hello_frame("target", PROTOCOL_VERSION, 1),
8883        )
8884        .await;
8885
8886        let responses = handler
8887            .handle_control_frame(
8888                &route_ctx(ConnectionId::new(79)).0,
8889                route_open_frame_with_admission_facts(
8890                    16,
8891                    "target",
8892                    unique_project_root("admission-facts"),
8893                    None,
8894                    Some(json!({"x": 1})),
8895                ),
8896            )
8897            .await
8898            .unwrap();
8899        assert_eq!(
8900            parse_error(&responses[0])["code"],
8901            "admission_facts_not_permitted"
8902        );
8903    }
8904
8905    #[tokio::test]
8906    async fn route_open_relays_consumer_capabilities_verbatim() {
8907        let registry = Arc::new(Registry::default());
8908        let forwarding = Arc::new(ForwardingTable::default());
8909        let handler =
8910            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
8911        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
8912        hello_via_sink(
8913            &handler,
8914            &module_ctx,
8915            &mut module_rx,
8916            hello_frame("aft", PROTOCOL_VERSION, 7),
8917        )
8918        .await;
8919
8920        let expected = vec!["elicitation".to_string(), "roots".to_string()];
8921        let expected_for_request = expected.clone();
8922        let project_root = unique_project_root("consumer-capabilities-present");
8923        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
8924        let route_handler = handler.clone();
8925        let route_task = tokio::spawn(async move {
8926            route_handler
8927                .handle_control_frame(
8928                    &client_ctx,
8929                    route_open_frame_with_consumer_capabilities(
8930                        401,
8931                        "aft",
8932                        project_root,
8933                        Some(expected_for_request),
8934                    ),
8935                )
8936                .await
8937                .unwrap()
8938        });
8939        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8940            .await
8941            .unwrap()
8942            .unwrap();
8943        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8944        let ModuleControlRequest::RouteBind {
8945            consumer_capabilities,
8946            ..
8947        } = bind
8948        else {
8949            panic!("expected route.bind request, got {bind:?}");
8950        };
8951        assert_eq!(consumer_capabilities, Some(expected.clone()));
8952
8953        handler
8954            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8955            .await
8956            .unwrap();
8957        let route_response = route_task.await.unwrap();
8958        assert!(route_response.is_empty());
8959        let published = client_rx.recv().await.unwrap();
8960        assert!(matches!(
8961            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8962            ClientControlResponse::RouteOpen { .. }
8963        ));
8964    }
8965
8966    #[tokio::test]
8967    async fn route_open_without_consumer_capabilities_relays_none() {
8968        let registry = Arc::new(Registry::default());
8969        let forwarding = Arc::new(ForwardingTable::default());
8970        let handler =
8971            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
8972        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
8973        hello_via_sink(
8974            &handler,
8975            &module_ctx,
8976            &mut module_rx,
8977            hello_frame("aft", PROTOCOL_VERSION, 7),
8978        )
8979        .await;
8980
8981        let project_root = unique_project_root("consumer-capabilities-absent");
8982        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
8983        let route_handler = handler.clone();
8984        let route_task = tokio::spawn(async move {
8985            route_handler
8986                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
8987                .await
8988                .unwrap()
8989        });
8990        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8991            .await
8992            .unwrap()
8993            .unwrap();
8994        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8995        let ModuleControlRequest::RouteBind {
8996            consumer_capabilities,
8997            ..
8998        } = bind
8999        else {
9000            panic!("expected route.bind request, got {bind:?}");
9001        };
9002        assert_eq!(consumer_capabilities, None);
9003
9004        handler
9005            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9006            .await
9007            .unwrap();
9008        let route_response = route_task.await.unwrap();
9009        assert!(route_response.is_empty());
9010        let published = client_rx.recv().await.unwrap();
9011        assert!(matches!(
9012            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9013            ClientControlResponse::RouteOpen { .. }
9014        ));
9015    }
9016
9017    /// Opens a route to a freshly registered `aft` with `sent` as the
9018    /// route.open's role_versions, acks the bind, and returns the role_versions
9019    /// the module's bind carried.
9020    async fn bind_role_versions_for(
9021        sent: Option<BTreeMap<String, String>>,
9022        connection: u64,
9023    ) -> Option<BTreeMap<String, String>> {
9024        let registry = Arc::new(Registry::default());
9025        let forwarding = Arc::new(ForwardingTable::default());
9026        let handler =
9027            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9028        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
9029        hello_via_sink(
9030            &handler,
9031            &module_ctx,
9032            &mut module_rx,
9033            hello_frame("aft", PROTOCOL_VERSION, 7),
9034        )
9035        .await;
9036        let project_root = unique_project_root("role-versions");
9037        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
9038        let route_handler = handler.clone();
9039        let route_task = tokio::spawn(async move {
9040            route_handler
9041                .handle_control_frame(
9042                    &client_ctx,
9043                    route_open_frame_with_role_versions(403, "aft", project_root, sent),
9044                )
9045                .await
9046                .unwrap()
9047        });
9048        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9049            .await
9050            .expect("a well-formed route.open reaches the module as a bind")
9051            .unwrap();
9052        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9053        let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
9054            panic!("expected route.bind request, got {bind:?}");
9055        };
9056        handler
9057            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9058            .await
9059            .unwrap();
9060        assert!(route_task.await.unwrap().is_empty());
9061        let published = client_rx.recv().await.unwrap();
9062        assert!(matches!(
9063            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9064            ClientControlResponse::RouteOpen { .. }
9065        ));
9066        role_versions
9067    }
9068
9069    #[tokio::test]
9070    async fn route_open_relays_role_versions_verbatim() {
9071        let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
9072        assert_eq!(
9073            bind_role_versions_for(Some(sent.clone()), 141).await,
9074            Some(sent)
9075        );
9076    }
9077
9078    /// An empty map declares nothing, so the provider sees no field rather
9079    /// than an empty object it would have to treat as a second "none".
9080    #[tokio::test]
9081    async fn route_open_with_empty_or_absent_role_versions_relays_none() {
9082        assert_eq!(bind_role_versions_for(None, 143).await, None);
9083        assert_eq!(
9084            bind_role_versions_for(Some(BTreeMap::new()), 145).await,
9085            None
9086        );
9087    }
9088
9089    #[tokio::test]
9090    async fn route_open_refuses_malformed_role_versions_before_any_bind() {
9091        let registry = Arc::new(Registry::default());
9092        let forwarding = Arc::new(ForwardingTable::default());
9093        let handler =
9094            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9095        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
9096        hello_via_sink(
9097            &handler,
9098            &module_ctx,
9099            &mut module_rx,
9100            hello_frame("aft", PROTOCOL_VERSION, 7),
9101        )
9102        .await;
9103
9104        let nine: BTreeMap<String, String> = (0..9)
9105            .map(|index| (format!("role-{index}"), "v1".to_string()))
9106            .collect();
9107        for (label, malformed) in [
9108            (
9109                "invalid role name",
9110                role_versions(&[("Tool_Provider", "v1")]),
9111            ),
9112            ("invalid version", role_versions(&[("tool-provider", "v0")])),
9113            ("nine entries", nine),
9114        ] {
9115            // A refused open answers at once; one that reached the module
9116            // would wait for its bind ack and trip this timeout.
9117            let responses = tokio::time::timeout(
9118                Duration::from_secs(1),
9119                handler.handle_control_frame(
9120                    &route_ctx(ConnectionId::new(148)).0,
9121                    route_open_frame_with_role_versions(
9122                        404,
9123                        "aft",
9124                        unique_project_root("role-versions-malformed"),
9125                        Some(malformed),
9126                    ),
9127                ),
9128            )
9129            .await
9130            .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
9131            .unwrap();
9132            assert_eq!(responses.len(), 1, "{label}");
9133            assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
9134            let error = parse_error(&responses[0]);
9135            assert_eq!(error["code"], "invalid_request", "{label}: {error}");
9136            assert_eq!(
9137                error["detail"]["field"], "role_versions",
9138                "{label}: {error}"
9139            );
9140            assert!(
9141                !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
9142                "{label}: a malformed declaration is terminal"
9143            );
9144            assert!(
9145                module_rx.try_recv().is_err(),
9146                "{label}: the module must never see a bind"
9147            );
9148        }
9149    }
9150
9151    /// `route-role-versions/v1` is in HELLO_ACK and `server.describe`, so a
9152    /// consumer can tell this daemon forwards the field from one that would
9153    /// drop it.
9154    #[tokio::test]
9155    async fn route_role_versions_capability_is_advertised() {
9156        let handler = ControlHandler::new(Arc::new(Registry::default()));
9157        let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
9158        let ack = hello_via_sink(
9159            &handler,
9160            &ctx,
9161            &mut rx,
9162            hello_frame("m", PROTOCOL_VERSION, 1),
9163        )
9164        .await;
9165        let ack = parse_ack(&ack);
9166        assert!(
9167            ack.subc_capabilities
9168                .iter()
9169                .any(|c| c == "route-role-versions/v1"),
9170            "{:?}",
9171            ack.subc_capabilities
9172        );
9173
9174        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
9175        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
9176        let reply = handler
9177            .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
9178            .await
9179            .unwrap()
9180            .pop()
9181            .unwrap();
9182        let ClientControlResponse::ServerDescribe { capabilities, .. } =
9183            serde_json::from_slice(&reply.body).unwrap()
9184        else {
9185            panic!("not a server.describe reply");
9186        };
9187        assert!(
9188            capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
9189            "{capabilities:?}"
9190        );
9191    }
9192
9193    #[tokio::test]
9194    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
9195        let registry = Arc::new(Registry::default());
9196        let forwarding = Arc::new(ForwardingTable::default());
9197        let handler =
9198            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9199                .with_health_probe_timeout(Duration::from_secs(5));
9200        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
9201        hello_via_sink(
9202            &handler,
9203            &module_ctx,
9204            &mut module_rx,
9205            non_routable_hello_frame_with_control_ops(
9206                "mcp",
9207                300,
9208                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9209            ),
9210        )
9211        .await;
9212        assert!(registry
9213            .get_module("mcp")
9214            .unwrap()
9215            .unwrap()
9216            .manifest
9217            .provides
9218            .is_empty());
9219
9220        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
9221        let route_response = handler
9222            .handle_control_frame(
9223                &route_client_ctx,
9224                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
9225            )
9226            .await
9227            .unwrap();
9228        assert_eq!(route_response[0].header.ty, FrameType::Error);
9229        assert_eq!(
9230            parse_error(&route_response[0])["code"],
9231            "target_unavailable"
9232        );
9233        assert!(parse_error(&route_response[0])["message"]
9234            .as_str()
9235            .unwrap()
9236            .contains("does not provide the requested target"));
9237        assert!(module_rx.try_recv().is_err());
9238
9239        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
9240        let health_handler = handler.clone();
9241        let health_task = tokio::spawn(async move {
9242            health_handler
9243                .handle_control_frame(
9244                    &health_client_ctx,
9245                    supervisor_health_probe_frame(302, "mcp"),
9246                )
9247                .await
9248                .unwrap()
9249        });
9250        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9251            .await
9252            .unwrap()
9253            .unwrap();
9254        assert_eq!(
9255            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
9256            ModuleControlRequest::HealthCheck {}
9257        );
9258        handler
9259            .handle_control_frame(
9260                &module_ctx,
9261                health_response(health_frame.header.corr, HealthStatus::Ok),
9262            )
9263            .await
9264            .unwrap();
9265        let health_response = health_task.await.unwrap();
9266        assert_eq!(health_response[0].header.ty, FrameType::Response);
9267        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
9268            ClientControlResponse::SupervisorHealthProbe {
9269                module_id, status, ..
9270            } => {
9271                assert_eq!(module_id, "mcp");
9272                assert_eq!(status, HealthStatus::Ok);
9273            }
9274            other => panic!("unexpected health response: {other:?}"),
9275        }
9276
9277        // Exercise the forwarding cleanup path directly while leaving the registry
9278        // advertisement in place. If cleanup leaves a stale control sink behind,
9279        // the next probe will enqueue onto it and wait for the long probe timeout
9280        // instead of returning an immediate no-connection error.
9281        forwarding
9282            .cleanup_connection(module_ctx.connection_id)
9283            .unwrap();
9284        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
9285        let cleanup_response = tokio::time::timeout(
9286            Duration::from_millis(200),
9287            handler.handle_control_frame(
9288                &cleanup_probe_ctx,
9289                supervisor_health_probe_frame(303, "mcp"),
9290            ),
9291        )
9292        .await
9293        .expect("probe should fail immediately when the control lane is gone")
9294        .unwrap();
9295        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
9296        assert_eq!(
9297            parse_error(&cleanup_response[0])["code"],
9298            "target_unavailable"
9299        );
9300        assert!(parse_error(&cleanup_response[0])["message"]
9301            .as_str()
9302            .unwrap()
9303            .contains("no module connection"));
9304
9305        handler
9306            .cleanup_connection(module_ctx.connection_id)
9307            .unwrap();
9308    }
9309
9310    #[tokio::test]
9311    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
9312        let registry = Arc::new(Registry::default());
9313        let supervisor_handle = SupervisorHandle::new();
9314        let supervisor =
9315            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
9316                .with_handle(supervisor_handle.clone())
9317                .with_connection_file_path(
9318                    std::env::temp_dir()
9319                        .join(format!("subc-route-open-warming-{}", std::process::id())),
9320                );
9321        let module = supervisor
9322            .supervise_configured(
9323                ModuleSpec {
9324                    launch_nonce_env: true,
9325                    module_id: "warming".to_string(),
9326                    program: fake_aft_stub_path(),
9327                    args: Vec::new(),
9328                    env: Vec::new(),
9329                    reserved: false,
9330                    reserved_prefixes: Vec::new(),
9331                    protocol: ModuleProtocol::Subc,
9332                    overlap: Default::default(),
9333                },
9334                true,
9335            )
9336            .unwrap();
9337        assert_eq!(module.state().unwrap(), ModuleState::Running);
9338
9339        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9340        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
9341        let response = handler
9342            .handle_control_frame(
9343                &ctx,
9344                route_open_frame(304, "warming", unique_project_root("warming")),
9345            )
9346            .await
9347            .unwrap();
9348        module.stop().await.unwrap();
9349
9350        assert_eq!(response[0].header.ty, FrameType::Error);
9351        let error = parse_error(&response[0]);
9352        assert_eq!(error["code"], "module_warming");
9353        assert!(error["message"]
9354            .as_str()
9355            .unwrap()
9356            .contains("state=running, enabled=true, live=false"));
9357    }
9358
9359    #[test]
9360    fn route_open_connection_cap_logs_admission_reason_and_capacity() {
9361        let handler = ControlHandler::new(Arc::new(Registry::default()));
9362        let capture = EventCapture::default();
9363        let _subscriber =
9364            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9365        let (ctx, _rx) = route_ctx(ConnectionId::new(96));
9366        let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
9367        let pending = (0..limit).collect::<Vec<_>>();
9368        let response = handler
9369            .route_open_capacity_refusal(
9370                &ctx,
9371                &route_open_frame(396, "busy", unique_project_root("connection-cap")),
9372                "busy",
9373                pending.len(),
9374                limit,
9375            )
9376            .unwrap();
9377        assert_eq!(parse_error(&response)["code"], "target_unavailable");
9378        let event = capture
9379            .events()
9380            .into_iter()
9381            .find(|event| {
9382                event.target == "control"
9383                    && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
9384            })
9385            .expect("connection admission refusal event");
9386        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
9387        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
9388    }
9389
9390    #[test]
9391    fn route_open_target_cap_logs_admission_reason_and_capacity() {
9392        let handler = ControlHandler::new(Arc::new(Registry::default()));
9393        let capture = EventCapture::default();
9394        let _subscriber =
9395            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9396        let (ctx, _rx) = route_ctx(ConnectionId::new(97));
9397        let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
9398        let guards = (0..limit)
9399            .map(|_| {
9400                handler
9401                    .route_bind_concurrency
9402                    .try_admit("busy", limit)
9403                    .unwrap()
9404            })
9405            .collect::<Vec<_>>();
9406        let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
9407            Err(in_flight) => in_flight,
9408            Ok(_) => panic!("target cap must refuse after {limit} admissions"),
9409        };
9410        let response = handler
9411            .route_open_target_capacity_refusal(
9412                &ctx,
9413                &route_open_frame(397, "busy", unique_project_root("target-cap")),
9414                "busy",
9415                in_flight,
9416            )
9417            .unwrap();
9418        assert_eq!(parse_error(&response)["code"], "target_unavailable");
9419        let event = capture
9420            .events()
9421            .into_iter()
9422            .find(|event| {
9423                event.target == "control"
9424                    && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
9425            })
9426            .expect("target admission refusal event");
9427        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
9428        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
9429        drop(guards);
9430    }
9431
9432    /// One wire code has several senders, so the refusal line names the check
9433    /// that refused. This drives the shared refusal path for ordinary refusals
9434    /// with an unregistered
9435    /// target and requires the branch label on the event.
9436    #[tokio::test]
9437    async fn route_open_refusal_names_the_check_that_refused() {
9438        let handler = ControlHandler::new(Arc::new(Registry::default()));
9439        let capture = EventCapture::default();
9440        let _subscriber =
9441            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9442        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
9443        let response = handler
9444            .handle_control_frame(
9445                &ctx,
9446                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
9447            )
9448            .await
9449            .unwrap();
9450
9451        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9452        let event = capture
9453            .events()
9454            .into_iter()
9455            .find(|event| {
9456                event.target == "control"
9457                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
9458            })
9459            .expect("route.open refusal event");
9460        assert_eq!(
9461            event.fields.get("reason"),
9462            Some(&"\"not_registered\"".to_string())
9463        );
9464    }
9465
9466    #[tokio::test]
9467    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
9468        let registry = Arc::new(Registry::default());
9469        let supervisor_handle = SupervisorHandle::new();
9470        let supervisor =
9471            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
9472                .with_handle(supervisor_handle.clone())
9473                .with_connection_file_path(std::env::temp_dir().join(format!(
9474                    "subc-route-open-refusal-info-{}",
9475                    std::process::id()
9476                )));
9477        let module = supervisor
9478            .supervise_configured(
9479                ModuleSpec {
9480                    launch_nonce_env: true,
9481                    module_id: "warming".to_string(),
9482                    program: fake_aft_stub_path(),
9483                    args: Vec::new(),
9484                    env: Vec::new(),
9485                    reserved: false,
9486                    reserved_prefixes: Vec::new(),
9487                    protocol: ModuleProtocol::Subc,
9488                    overlap: Default::default(),
9489                },
9490                true,
9491            )
9492            .unwrap();
9493        assert_eq!(module.state().unwrap(), ModuleState::Running);
9494
9495        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9496        assert!(handler
9497            .counters()
9498            .snapshot()
9499            .get("route_open_refused_by_code")
9500            .is_none());
9501        let capture = EventCapture::default();
9502        let _subscriber =
9503            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9504        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
9505        let response = handler
9506            .handle_control_frame(
9507                &ctx,
9508                route_open_frame(394, "warming", unique_project_root("refusal-info")),
9509            )
9510            .await
9511            .unwrap();
9512        module.stop().await.unwrap();
9513
9514        assert_eq!(parse_error(&response[0])["code"], "module_warming");
9515        let event = capture
9516            .events()
9517            .into_iter()
9518            .find(|event| {
9519                event.target == "control"
9520                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
9521            })
9522            .expect("route.open refusal event");
9523        assert_eq!(
9524            event.fields.get("module_id"),
9525            Some(&"\"warming\"".to_string())
9526        );
9527        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
9528        assert_eq!(
9529            event.fields.get("reason"),
9530            Some(&"\"supervised_not_registered\"".to_string())
9531        );
9532        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
9533        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
9534        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
9535        assert_eq!(
9536            handler.counters().snapshot()["route_open_refused_by_code"],
9537            json!({ "module_warming": 1 })
9538        );
9539    }
9540
9541    const OUTAGE_START: &str = "route.open refusing module: not serving";
9542    const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
9543
9544    fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
9545        capture
9546            .events()
9547            .into_iter()
9548            .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
9549            .collect()
9550    }
9551
9552    fn supervise_stub(
9553        registry: &Arc<Registry>,
9554        module_id: &str,
9555        enabled: bool,
9556    ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
9557        let supervisor_handle = SupervisorHandle::new();
9558        let supervisor =
9559            Supervisor::new(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
9560                .with_handle(supervisor_handle.clone())
9561                .with_connection_file_path(std::env::temp_dir().join(format!(
9562                    "subc-route-outage-{module_id}-{}",
9563                    std::process::id()
9564                )));
9565        let module = supervisor
9566            .supervise_configured(
9567                ModuleSpec {
9568                    launch_nonce_env: true,
9569                    module_id: module_id.to_string(),
9570                    program: fake_aft_stub_path(),
9571                    args: Vec::new(),
9572                    env: Vec::new(),
9573                    reserved: false,
9574                    reserved_prefixes: Vec::new(),
9575                    protocol: ModuleProtocol::Subc,
9576                    overlap: Default::default(),
9577                },
9578                enabled,
9579            )
9580            .unwrap();
9581        (supervisor_handle, module)
9582    }
9583
9584    fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
9585        let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
9586            module_id: module_id.to_string(),
9587            drain_timeout_ms: Some(50),
9588        })
9589        .unwrap();
9590        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9591    }
9592
9593    /// Two handlers built over one forwarding table must share one outage
9594    /// tracker; separate trackers would each log their own opening line for
9595    /// the same outage.
9596    #[test]
9597    fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
9598        let registry = Arc::new(Registry::default());
9599        let forwarding = Arc::new(ForwardingTable::default());
9600        let first = ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9601        let second = ControlHandler::with_forwarding(registry, forwarding);
9602        assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
9603    }
9604
9605    /// A client can name any module id it likes. Refusing an unknown one,
9606    /// however often, must not create outage state or outage lines, or the
9607    /// tracker would be a memory sink any client could fill.
9608    #[tokio::test(flavor = "current_thread")]
9609    async fn route_open_unknown_module_refusals_add_no_outage_state() {
9610        let handler = ControlHandler::new(Arc::new(Registry::default()));
9611        let capture = EventCapture::default();
9612        let _subscriber =
9613            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9614        let (ctx, _rx) = route_ctx(ConnectionId::new(90));
9615        for corr in 0..8 {
9616            let response = handler
9617                .handle_control_frame(
9618                    &ctx,
9619                    route_open_frame(
9620                        380 + corr,
9621                        &format!("nobody-{corr}"),
9622                        unique_project_root("outage-unknown"),
9623                    ),
9624                )
9625                .await
9626                .unwrap();
9627            assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9628        }
9629
9630        assert_eq!(handler.route_outages.tracked_module_count(), 0);
9631        assert!(outage_lines(&capture, OUTAGE_START).is_empty());
9632        assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
9633    }
9634
9635    /// Drives the refusal path end to end: a supervised module that served
9636    /// before and stopped being registered with no instruction to stop is a
9637    /// WARN, and the same module refused after an operator `supervisor.restart`
9638    /// is an INFO.
9639    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9640    async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
9641        let registry = Arc::new(Registry::default());
9642        let (supervisor_handle, module) = supervise_stub(&registry, "outage-restart", true);
9643        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9644        let capture = EventCapture::default();
9645        let _subscriber =
9646            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9647        let (ctx, _rx) = route_ctx(ConnectionId::new(91));
9648        // The stub never registers, so pretend it served once: otherwise every
9649        // refusal would fall in its startup window.
9650        handler.route_outages.record_accepted("outage-restart");
9651
9652        let response = handler
9653            .handle_control_frame(
9654                &ctx,
9655                route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
9656            )
9657            .await
9658            .unwrap();
9659        assert_eq!(response[0].header.ty, FrameType::Error);
9660        let starts = outage_lines(&capture, OUTAGE_START);
9661        assert_eq!(starts.len(), 1, "{starts:?}");
9662        assert_eq!(starts[0].level, tracing::Level::WARN);
9663        assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
9664        assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
9665        assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
9666        handler.route_outages.record_accepted("outage-restart");
9667        assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
9668
9669        let restart = handler
9670            .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
9671            .await
9672            .unwrap();
9673        assert_eq!(
9674            restart[0].header.ty,
9675            FrameType::Response,
9676            "{:?}",
9677            parse_error(&restart[0])
9678        );
9679        handler
9680            .handle_control_frame(
9681                &ctx,
9682                route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
9683            )
9684            .await
9685            .unwrap();
9686        module.stop().await.unwrap();
9687
9688        let starts = outage_lines(&capture, OUTAGE_START);
9689        assert_eq!(starts.len(), 2, "{starts:?}");
9690        assert_eq!(starts[1].level, tracing::Level::INFO);
9691        assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
9692    }
9693
9694    /// A restart refused before it touched the module (here: the module is
9695    /// disabled) must clear its operator mark, so the next real outage is
9696    /// still reported as a warning.
9697    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9698    async fn failed_operator_restart_leaves_no_operator_mark() {
9699        let registry = Arc::new(Registry::default());
9700        let (supervisor_handle, _module) = supervise_stub(&registry, "outage-disabled", false);
9701        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9702        let capture = EventCapture::default();
9703        let _subscriber =
9704            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9705        let (ctx, _rx) = route_ctx(ConnectionId::new(92));
9706        handler.route_outages.record_accepted("outage-disabled");
9707
9708        let restart = handler
9709            .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
9710            .await
9711            .unwrap();
9712        assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
9713        assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
9714
9715        handler
9716            .handle_control_frame(
9717                &ctx,
9718                route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
9719            )
9720            .await
9721            .unwrap();
9722        let starts = outage_lines(&capture, OUTAGE_START);
9723        assert_eq!(starts.len(), 1, "{starts:?}");
9724        assert_eq!(starts[0].level, tracing::Level::WARN);
9725    }
9726
9727    #[tokio::test(flavor = "current_thread")]
9728    async fn route_open_unknown_module_escapes_target_module_id() {
9729        let handler = ControlHandler::new(Arc::new(Registry::default()));
9730        let capture = EventCapture::default();
9731        let _subscriber =
9732            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9733        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
9734        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
9735        let response = handler
9736            .handle_control_frame(
9737                &ctx,
9738                route_open_frame(
9739                    395,
9740                    hostile_module_id,
9741                    unique_project_root("hostile-target-module-id"),
9742                ),
9743            )
9744            .await
9745            .unwrap();
9746
9747        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9748        let event = capture
9749            .events()
9750            .into_iter()
9751            .find(|event| {
9752                event.target == "control"
9753                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
9754            })
9755            .expect("route.open unknown-module refusal event");
9756        let logged = event.fields.get("module_id").expect("module_id field");
9757        assert!(!logged.bytes().any(|byte| byte < 0x20));
9758        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
9759    }
9760
9761    #[tokio::test(flavor = "current_thread")]
9762    async fn route_open_module_rejection_uses_daemon_counter_key() {
9763        let registry = Arc::new(Registry::default());
9764        let forwarding = Arc::new(ForwardingTable::default());
9765        let handler =
9766            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9767        let module_connection = ConnectionId::new(95);
9768        let (module_ctx, mut module_rx) = route_ctx(module_connection);
9769        hello_via_sink(
9770            &handler,
9771            &module_ctx,
9772            &mut module_rx,
9773            hello_frame("aft", PROTOCOL_VERSION, 395),
9774        )
9775        .await;
9776
9777        let client_connection = ConnectionId::new(96);
9778        let (client_ctx, _client_rx) = route_ctx(client_connection);
9779        let capture = EventCapture::default();
9780        let _subscriber =
9781            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9782        let (route_task, bind) = relay_route_open(
9783            &handler,
9784            client_connection,
9785            &client_ctx.egress,
9786            &mut module_rx,
9787            396,
9788            "aft",
9789            "hostile-module-code",
9790        )
9791        .await;
9792        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
9793        let rejection = Frame::build(
9794            FrameType::Error,
9795            control_flags(),
9796            0,
9797            0,
9798            bind.header.corr,
9799            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
9800        )
9801        .unwrap();
9802        handler
9803            .handle_control_frame(&module_ctx, rejection)
9804            .await
9805            .unwrap();
9806
9807        let response = route_task.await.unwrap();
9808        assert_eq!(parse_error(&response[0])["code"], hostile_code);
9809        let counters = handler.counters().snapshot();
9810        assert_eq!(
9811            counters["route_open_refused_by_code"],
9812            json!({ "module_rejected": 1 })
9813        );
9814        assert!(counters["route_open_refused_by_code"]
9815            .get(hostile_code)
9816            .is_none());
9817
9818        let event = capture
9819            .events()
9820            .into_iter()
9821            .find(|event| {
9822                event.target == "control"
9823                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
9824            })
9825            .expect("route.open module-rejection refusal event");
9826        let logged = event.fields.get("module_code").expect("module_code field");
9827        assert!(!logged.bytes().any(|byte| byte < 0x20));
9828        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
9829    }
9830
9831    #[tokio::test]
9832    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
9833        let registry = Arc::new(Registry::default());
9834        let supervisor_handle = SupervisorHandle::new();
9835        let missing_program = std::env::temp_dir().join(format!(
9836            "subc-route-open-missing-program-{}",
9837            std::process::id()
9838        ));
9839        let supervisor =
9840            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
9841                .with_handle(supervisor_handle.clone());
9842        let module = supervisor
9843            .supervise_configured(
9844                ModuleSpec {
9845                    launch_nonce_env: true,
9846                    module_id: "failed".to_string(),
9847                    program: missing_program,
9848                    args: Vec::new(),
9849                    env: Vec::new(),
9850                    reserved: false,
9851                    reserved_prefixes: Vec::new(),
9852                    protocol: ModuleProtocol::Subc,
9853                    overlap: Default::default(),
9854                },
9855                true,
9856            )
9857            .unwrap();
9858        assert_eq!(module.state().unwrap(), ModuleState::Failed);
9859
9860        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9861        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
9862        let response = handler
9863            .handle_control_frame(
9864                &ctx,
9865                route_open_frame(305, "failed", unique_project_root("failed")),
9866            )
9867            .await
9868            .unwrap();
9869
9870        assert_eq!(response[0].header.ty, FrameType::Error);
9871        let error = parse_error(&response[0]);
9872        assert_eq!(error["code"], "target_unavailable");
9873        assert!(error["message"]
9874            .as_str()
9875            .unwrap()
9876            .contains("state=failed, enabled=true, live=false"));
9877    }
9878
9879    #[tokio::test]
9880    async fn route_open_role_mismatch_remains_target_unavailable() {
9881        let registry = Arc::new(Registry::default());
9882        let handler = ControlHandler::new(Arc::clone(&registry));
9883        handler
9884            .handle_control(
9885                ConnectionId::new(41),
9886                non_routable_hello_frame_with_control_ops("health-only", 306, None),
9887            )
9888            .unwrap();
9889
9890        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
9891        let response = handler
9892            .handle_control_frame(
9893                &ctx,
9894                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
9895            )
9896            .await
9897            .unwrap();
9898
9899        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9900        assert!(parse_error(&response[0])["message"]
9901            .as_str()
9902            .unwrap()
9903            .contains("does not provide the requested target"));
9904    }
9905
9906    #[tokio::test]
9907    async fn route_open_inactive_registration_remains_target_unavailable() {
9908        let registry = Arc::new(Registry::default());
9909        let handler = ControlHandler::new(Arc::clone(&registry));
9910        handler
9911            .handle_control(
9912                ConnectionId::new(43),
9913                hello_frame("inactive", PROTOCOL_VERSION, 308),
9914            )
9915            .unwrap();
9916        assert!(registry
9917            .set_module_state_for_test("inactive", ChannelState::Closed)
9918            .unwrap());
9919
9920        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
9921        let response = handler
9922            .handle_control_frame(
9923                &ctx,
9924                route_open_frame(309, "inactive", unique_project_root("inactive")),
9925            )
9926            .await
9927            .unwrap();
9928
9929        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9930        assert!(parse_error(&response[0])["message"]
9931            .as_str()
9932            .unwrap()
9933            .contains("is not active"));
9934    }
9935
9936    #[tokio::test]
9937    async fn late_health_reply_is_recorded_through_the_module_response_path() {
9938        let registry = Arc::new(Registry::default());
9939        let forwarding = Arc::new(ForwardingTable::default());
9940        let supervisor_handle = SupervisorHandle::new();
9941        let supervisor = Supervisor::new(Arc::clone(&registry), crate::RestartPolicy::default())
9942            .with_forwarding(Arc::clone(&forwarding))
9943            .with_handle(supervisor_handle.clone());
9944        let module = supervisor
9945            .supervise_configured(
9946                crate::ModuleSpec {
9947                    launch_nonce_env: true,
9948                    module_id: "late-health-response".to_string(),
9949                    program: PathBuf::from("disabled-module"),
9950                    args: Vec::new(),
9951                    env: Vec::new(),
9952                    reserved: false,
9953                    reserved_prefixes: Vec::new(),
9954                    protocol: ModuleProtocol::Subc,
9955                    overlap: Default::default(),
9956                },
9957                false,
9958            )
9959            .unwrap();
9960        let handler =
9961            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9962                .with_supervisor(supervisor_handle);
9963        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
9964        handler
9965            .handle_control_frame(
9966                &module_ctx,
9967                hello_frame_with_control_ops(
9968                    "late-health-response",
9969                    PROTOCOL_VERSION,
9970                    7,
9971                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9972                ),
9973            )
9974            .await
9975            .unwrap();
9976        let probe_started_at = Instant::now() - Duration::from_millis(80);
9977        let pending = forwarding
9978            .begin_health_probe_rpc_for(
9979                "late-health-response",
9980                MODULE_CONTROL_OP_HEALTH_CHECK,
9981                probe_started_at,
9982                Instant::now() - Duration::from_millis(1),
9983            )
9984            .unwrap();
9985        assert!(forwarding
9986            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
9987            .unwrap());
9988
9989        let responses = handler
9990            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
9991            .await
9992            .unwrap();
9993
9994        assert!(responses.is_empty());
9995        let health = module.status().unwrap().health;
9996        assert_eq!(health.late_answer_count, 1);
9997        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
9998    }
9999
10000    #[tokio::test]
10001    async fn health_probe_timeout_and_module_death_are_typed() {
10002        let registry = Arc::new(Registry::default());
10003        let forwarding = Arc::new(ForwardingTable::default());
10004        let handler =
10005            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10006                .with_health_probe_timeout(Duration::from_millis(50));
10007        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
10008        hello_via_sink(
10009            &handler,
10010            &module_ctx,
10011            &mut module_rx,
10012            hello_frame_with_control_ops(
10013                "aft",
10014                PROTOCOL_VERSION,
10015                7,
10016                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10017            ),
10018        )
10019        .await;
10020
10021        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10022        let responses = handler
10023            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
10024            .await
10025            .unwrap();
10026        assert_eq!(responses[0].header.ty, FrameType::Error);
10027        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
10028        let _ = module_rx.try_recv();
10029
10030        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
10031        let health_handler = handler.clone();
10032        let death_task = tokio::spawn(async move {
10033            health_handler
10034                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
10035                .await
10036                .unwrap()
10037        });
10038        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10039            .await
10040            .unwrap()
10041            .unwrap();
10042        handler
10043            .cleanup_connection(module_ctx.connection_id)
10044            .unwrap();
10045        let responses = death_task.await.unwrap();
10046        assert_eq!(responses[0].header.ty, FrameType::Error);
10047        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
10048    }
10049
10050    #[test]
10051    fn hello_requires_exact_protocol_version() {
10052        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
10053            let registry = Arc::new(Registry::default());
10054            let handler = ControlHandler::new(Arc::clone(&registry));
10055            let responses = handler
10056                .handle_control(
10057                    ConnectionId::new(connection),
10058                    hello_frame("aft", offered, 9),
10059                )
10060                .unwrap();
10061
10062            assert_eq!(responses.len(), 1);
10063            assert_eq!(responses[0].header.ty, FrameType::Error);
10064            let error = parse_error(&responses[0]);
10065            assert_eq!(error["code"], "version_unsupported");
10066            assert!(registry.get_module("aft").unwrap().is_none());
10067            assert_eq!(registry.active_registration_count().unwrap(), 0);
10068        }
10069    }
10070
10071    #[test]
10072    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
10073        let registry = Arc::new(Registry::default());
10074        let forwarding = Arc::new(ForwardingTable::default());
10075        let handler =
10076            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10077        let module_connection = ConnectionId::new(301);
10078        let registration = registry
10079            .register_with_control_ops(
10080                manifest("aft-push", PROTOCOL_VERSION),
10081                PROTOCOL_VERSION,
10082                module_connection,
10083                module_baseline_control_ops(),
10084            )
10085            .unwrap();
10086        let (module_tx, _module_rx) = mpsc::channel(8);
10087        let endpoint = forwarding
10088            .register_module_connection(
10089                module_connection,
10090                "aft-push".to_string(),
10091                PROTOCOL_VERSION,
10092                manifest_concurrency(&registration.manifest),
10093                FrameSink::new(module_tx),
10094            )
10095            .unwrap();
10096
10097        // A push op this version does not know is ignored (forward-compat), not errored.
10098        let unknown = Frame::build(
10099            FrameType::Push,
10100            control_flags(),
10101            0,
10102            0,
10103            5,
10104            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
10105        )
10106        .unwrap();
10107        let out = handler.handle_status_update(endpoint, unknown).unwrap();
10108        assert!(
10109            out.is_empty(),
10110            "unknown push op must be ignored, got {out:?}"
10111        );
10112
10113        // A malformed body for a KNOWN op is a real error worth surfacing.
10114        let malformed = Frame::build(
10115            FrameType::Push,
10116            control_flags(),
10117            0,
10118            0,
10119            6,
10120            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
10121        )
10122        .unwrap();
10123        let out = handler.handle_status_update(endpoint, malformed).unwrap();
10124        assert_eq!(out.len(), 1);
10125        assert_eq!(out[0].header.ty, FrameType::Error);
10126        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
10127    }
10128
10129    #[test]
10130    fn hello_rejected_when_connection_already_owns_client_routes() {
10131        let registry = Arc::new(Registry::default());
10132        let forwarding = Arc::new(ForwardingTable::default());
10133        let handler =
10134            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10135        // Commits a client route on connection 202 (bound to a module on conn 101).
10136        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
10137        let client_connection = ConnectionId::new(202);
10138
10139        // That same connection now tries to register as a module: rejected, so one
10140        // connection never holds both client-route and module-endpoint state.
10141        let responses = handler
10142            .handle_control(
10143                client_connection,
10144                hello_frame("aft-second", PROTOCOL_VERSION, 9),
10145            )
10146            .unwrap();
10147        assert_eq!(responses[0].header.ty, FrameType::Error);
10148        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
10149        assert!(registry.get_module("aft-second").unwrap().is_none());
10150    }
10151
10152    #[test]
10153    fn reserved_module_hello_requires_matching_launch_nonce() {
10154        let registry = Arc::new(Registry::default());
10155        let supervisor = SupervisorHandle::new();
10156        // The supervisor recorded the nonce it injected when it spawned the reserved
10157        // module; the HELLO verifier checks against the same shared handle.
10158        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
10159        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
10160
10161        // A HELLO with NO nonce is rejected.
10162        let no_nonce = handler
10163            .handle_control(
10164                ConnectionId::new(1),
10165                hello_frame("vault", PROTOCOL_VERSION, 1),
10166            )
10167            .unwrap();
10168        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
10169        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
10170        assert!(registry.get_module("vault").unwrap().is_none());
10171
10172        // A HELLO with the WRONG nonce is rejected.
10173        let wrong = handler
10174            .handle_control(
10175                ConnectionId::new(2),
10176                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
10177            )
10178            .unwrap();
10179        assert_eq!(wrong[0].header.ty, FrameType::Error);
10180        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
10181        assert!(registry.get_module("vault").unwrap().is_none());
10182
10183        // A HELLO with the CORRECT nonce registers.
10184        let ok = handler
10185            .handle_control(
10186                ConnectionId::new(3),
10187                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
10188            )
10189            .unwrap();
10190        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
10191        assert!(registry.get_module("vault").unwrap().is_some());
10192    }
10193
10194    #[test]
10195    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
10196        let registry = Arc::new(Registry::default());
10197        let supervisor = SupervisorHandle::new();
10198        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
10199        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
10200        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
10201
10202        let squat = handler
10203            .handle_control(
10204                ConnectionId::new(1),
10205                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
10206            )
10207            .unwrap();
10208        assert_eq!(squat[0].header.ty, FrameType::Error);
10209        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
10210        assert!(parse_error(&squat[0])["message"]
10211            .as_str()
10212            .unwrap()
10213            .contains("fed:"));
10214
10215        let accepted_peer = handler
10216            .handle_control(
10217                ConnectionId::new(2),
10218                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
10219            )
10220            .unwrap();
10221        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
10222
10223        let accepted_short = handler
10224            .handle_control(
10225                ConnectionId::new(3),
10226                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
10227            )
10228            .unwrap();
10229        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
10230
10231        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
10232            let response = handler
10233                .handle_control(
10234                    ConnectionId::new(conn),
10235                    hello_frame(module_id, PROTOCOL_VERSION, conn),
10236                )
10237                .unwrap();
10238            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
10239        }
10240    }
10241
10242    #[test]
10243    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
10244        let registry = Arc::new(Registry::default());
10245        let supervisor = SupervisorHandle::new();
10246        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
10247        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
10248        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
10249        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
10250
10251        let owner_nonce = handler
10252            .handle_control(
10253                ConnectionId::new(1),
10254                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
10255            )
10256            .unwrap();
10257        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
10258        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
10259        assert!(registry.get_module("fed:special").unwrap().is_none());
10260
10261        let exact_nonce = handler
10262            .handle_control(
10263                ConnectionId::new(2),
10264                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
10265            )
10266            .unwrap();
10267        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
10268        assert!(registry.get_module("fed:special").unwrap().is_some());
10269    }
10270
10271    #[test]
10272    fn non_reserved_module_ignores_launch_nonce() {
10273        let registry = Arc::new(Registry::default());
10274        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
10275        // registration succeeds whether a spawned process echoes a nonce or not.
10276        let handler = ControlHandler::new(Arc::clone(&registry));
10277        let no_nonce = handler
10278            .handle_control(
10279                ConnectionId::new(1),
10280                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
10281            )
10282            .unwrap();
10283        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
10284        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
10285
10286        let echoed_nonce = handler
10287            .handle_control(
10288                ConnectionId::new(2),
10289                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
10290            )
10291            .unwrap();
10292        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
10293        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
10294    }
10295
10296    #[test]
10297    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
10298        let handler = ControlHandler::default();
10299        let conn = ConnectionId::new(1);
10300        let malformed = Frame::build(
10301            FrameType::Hello,
10302            control_flags(),
10303            0,
10304            0,
10305            3,
10306            b"{not json".to_vec(),
10307        )
10308        .unwrap();
10309
10310        let error = handler.handle_control(conn, malformed).unwrap();
10311        assert_eq!(error[0].header.ty, FrameType::Error);
10312        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
10313
10314        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
10315        let pong = handler.handle_control(conn, ping).unwrap();
10316        assert_eq!(pong[0].header.ty, FrameType::Pong);
10317        assert_eq!(pong[0].header.corr, 4);
10318    }
10319
10320    #[test]
10321    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
10322        let registry = Arc::new(Registry::default());
10323        let handler = ControlHandler::new(Arc::clone(&registry));
10324
10325        handler
10326            .handle_control(
10327                ConnectionId::new(1),
10328                hello_frame("aft", PROTOCOL_VERSION, 1),
10329            )
10330            .unwrap();
10331        let duplicate = handler
10332            .handle_control(
10333                ConnectionId::new(2),
10334                hello_frame("aft", PROTOCOL_VERSION, 2),
10335            )
10336            .unwrap();
10337
10338        assert_eq!(duplicate[0].header.ty, FrameType::Error);
10339        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10340        let registration = registry.get_module("aft").unwrap().unwrap();
10341        assert_eq!(registration.connection_id, ConnectionId::new(1));
10342    }
10343
10344    #[test]
10345    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
10346        let registry = Arc::new(Registry::default());
10347        let forwarding = Arc::new(ForwardingTable::default());
10348        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
10349        let handler =
10350            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10351                .with_process_liveness(process_liveness);
10352        let (ctx, route_channel, route_epoch) =
10353            bind_liveness_route(&registry, &forwarding, "aft-dead");
10354        let responses = handler
10355            .handle_route_poll(
10356                &ctx,
10357                route_poll_frame(41, PollKind::Liveness, route_channel),
10358                route_channel,
10359                route_epoch,
10360                PollKind::Liveness,
10361            )
10362            .unwrap();
10363
10364        assert_eq!(responses.len(), 1);
10365        assert_eq!(responses[0].header.ty, FrameType::Response);
10366        assert_route_poll_liveness(&responses[0], false);
10367    }
10368
10369    #[test]
10370    fn liveness_poll_without_process_source_uses_bound_route() {
10371        let registry = Arc::new(Registry::default());
10372        let forwarding = Arc::new(ForwardingTable::default());
10373        let handler =
10374            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10375        let (ctx, route_channel, route_epoch) =
10376            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
10377        let responses = handler
10378            .handle_route_poll(
10379                &ctx,
10380                route_poll_frame(42, PollKind::Liveness, route_channel),
10381                route_channel,
10382                route_epoch,
10383                PollKind::Liveness,
10384            )
10385            .unwrap();
10386
10387        assert_route_poll_liveness(&responses[0], true);
10388    }
10389
10390    #[test]
10391    fn liveness_poll_untracked_process_source_uses_bound_route() {
10392        let registry = Arc::new(Registry::default());
10393        let forwarding = Arc::new(ForwardingTable::default());
10394        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
10395        let handler =
10396            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10397                .with_process_liveness(process_liveness);
10398        let (ctx, route_channel, route_epoch) =
10399            bind_liveness_route(&registry, &forwarding, "aft-untracked");
10400        let responses = handler
10401            .handle_route_poll(
10402                &ctx,
10403                route_poll_frame(43, PollKind::Liveness, route_channel),
10404                route_channel,
10405                route_epoch,
10406                PollKind::Liveness,
10407            )
10408            .unwrap();
10409
10410        assert_route_poll_liveness(&responses[0], true);
10411    }
10412
10413    #[tokio::test]
10414    async fn unknown_op_returns_unknown_control_op() {
10415        let handler = ControlHandler::default();
10416        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
10417        let request = Frame::build(
10418            FrameType::Request,
10419            control_flags(),
10420            0,
10421            0,
10422            55,
10423            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
10424        )
10425        .unwrap();
10426
10427        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10428
10429        assert_eq!(response.len(), 1);
10430        assert_eq!(response[0].header.ty, FrameType::Error);
10431        assert_eq!(response[0].header.corr, 55);
10432        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
10433    }
10434
10435    #[tokio::test]
10436    async fn supervisor_provenance_rejects_unknown_exact_module() {
10437        let handler = ControlHandler::default();
10438        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
10439        let request = Frame::build(
10440            FrameType::Request,
10441            control_flags(),
10442            0,
10443            0,
10444            57,
10445            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
10446        )
10447        .unwrap();
10448
10449        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10450
10451        assert_eq!(response.len(), 1);
10452        assert_eq!(response[0].header.ty, FrameType::Error);
10453        assert_eq!(response[0].header.corr, 57);
10454        let error = parse_error(&response[0]);
10455        assert_eq!(error["code"], "unknown_module");
10456        assert_eq!(error["message"], "module_id 'missing' is not supervised");
10457    }
10458
10459    #[test]
10460    fn provenance_probe_override_keeps_handler_tests_deterministic() {
10461        let expected = subc_control::RunningImageAgreement::Unavailable {
10462            reason: subc_control::RunningImageUnavailableReason::HashFailed,
10463        };
10464        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
10465        assert_eq!(handler.provenance_probe_override, Some(expected));
10466    }
10467
10468    #[test]
10469    fn reload_verdict_detects_configured_program_different_from_spawned_path() {
10470        let verdict = reload_verdict(
10471            std::path::Path::new("/bin/new"),
10472            Some(std::path::Path::new("/bin/old")),
10473            subc_control::RunningImageAgreement::Unavailable {
10474                reason: subc_control::RunningImageUnavailableReason::HashFailed,
10475            },
10476        );
10477        assert!(matches!(
10478            verdict.path,
10479            subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
10480                if configured == std::path::Path::new("/bin/new")
10481                    && spawned_from == std::path::Path::new("/bin/old")
10482        ));
10483    }
10484
10485    #[test]
10486    fn reload_verdict_detects_replaced_image_at_same_path() {
10487        let image = subc_control::RunningImageAgreement::Mismatch {
10488            running: subc_control::RunningImageEvidence::LinuxProcSha256 {
10489                digest: "old".into(),
10490            },
10491            disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
10492                digest: "new".into(),
10493            },
10494        };
10495        let verdict = reload_verdict(
10496            std::path::Path::new("/bin/same"),
10497            Some(std::path::Path::new("/bin/same")),
10498            image.clone(),
10499        );
10500        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10501        assert_eq!(verdict.image, image);
10502    }
10503
10504    #[test]
10505    fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
10506        let image = subc_control::RunningImageAgreement::Unavailable {
10507            reason: subc_control::RunningImageUnavailableReason::NotRunning,
10508        };
10509        let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
10510        assert_eq!(
10511            verdict.path,
10512            subc_control::ReloadPathAgreement::Unavailable {
10513                reason: subc_control::ReloadPathUnavailableReason::NotRunning,
10514            }
10515        );
10516        assert_eq!(verdict.image, image);
10517
10518        let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
10519            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
10520        };
10521        let verdict = reload_verdict(
10522            std::path::Path::new("/bin/same"),
10523            Some(std::path::Path::new("/bin/same")),
10524            unconfirmed.clone(),
10525        );
10526        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10527        assert_eq!(verdict.image, unconfirmed);
10528    }
10529
10530    #[test]
10531    fn reload_verdict_preserves_each_image_unavailability_reason() {
10532        use subc_control::RunningImageUnavailableReason as Reason;
10533
10534        for reason in [
10535            Reason::NotRunning,
10536            Reason::UnsupportedPlatform,
10537            Reason::RunningExecutableUnreadable,
10538            Reason::SpawnedPathUnreadable,
10539            Reason::HashFailed,
10540            Reason::ProcessIdentityUnconfirmed,
10541            Reason::Unknown("future_probe_reason".to_string()),
10542        ] {
10543            let image = subc_control::RunningImageAgreement::Unavailable {
10544                reason: reason.clone(),
10545            };
10546            let verdict = reload_verdict(
10547                std::path::Path::new("/bin/same"),
10548                Some(std::path::Path::new("/bin/same")),
10549                image.clone(),
10550            );
10551            assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10552            assert_eq!(verdict.image, image, "{reason:?}");
10553        }
10554    }
10555
10556    #[tokio::test]
10557    async fn malformed_control_bodies_return_invalid_control_body() {
10558        let handler = ControlHandler::default();
10559        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
10560
10561        for (corr, body) in [
10562            (56, br#"{"route_channel":1}"#.as_slice()),
10563            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
10564            (
10565                58,
10566                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
10567            ),
10568        ] {
10569            let request = Frame::build(
10570                FrameType::Request,
10571                control_flags(),
10572                0,
10573                0,
10574                corr,
10575                body.to_vec(),
10576            )
10577            .unwrap();
10578            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10579
10580            assert_eq!(response.len(), 1);
10581            assert_eq!(response[0].header.ty, FrameType::Error);
10582            assert_eq!(response[0].header.corr, corr);
10583            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
10584        }
10585    }
10586
10587    #[tokio::test]
10588    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
10589        let registry = Arc::new(Registry::default());
10590        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
10591        let router = Router::with_control_handler(Arc::clone(&control));
10592        let connection = router.begin_connection();
10593        let (ctx, mut rx) = route_ctx(connection.id());
10594
10595        router
10596            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
10597            .await
10598            .unwrap();
10599        let response = rx.recv().await.unwrap();
10600        let ack = parse_ack(&response);
10601        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
10602        let channel = 1;
10603
10604        let goodbye =
10605            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
10606        router.route_for_connection(&ctx, goodbye).await.unwrap();
10607        assert!(rx.try_recv().is_err());
10608        assert!(registry.get_module("aft").unwrap().is_none());
10609
10610        router
10611            .route_for_connection(&ctx, channel_request(channel, 13))
10612            .await
10613            .unwrap();
10614        let error_frame = rx.recv().await.unwrap();
10615        assert_eq!(error_frame.header.ty, FrameType::Error);
10616        assert_eq!(error_frame.header.channel, channel);
10617    }
10618
10619    #[tokio::test]
10620    async fn dropping_router_connection_releases_registration() {
10621        let registry = Arc::new(Registry::default());
10622        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
10623        let router = Router::with_control_handler(control);
10624        let connection = router.begin_connection();
10625        let (ctx, mut rx) = route_ctx(connection.id());
10626
10627        router
10628            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
10629            .await
10630            .unwrap();
10631        let response = rx.recv().await.unwrap();
10632        let ack = parse_ack(&response);
10633        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
10634        assert!(registry.get_module("aft").unwrap().is_some());
10635
10636        drop(connection);
10637
10638        assert!(registry.get_module("aft").unwrap().is_none());
10639        assert_eq!(registry.active_registration_count().unwrap(), 0);
10640    }
10641
10642    fn capability_manifest(
10643        module_id: &str,
10644        provides: &[&str],
10645        must_never_reach: &[&str],
10646    ) -> ModuleManifest {
10647        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
10648        manifest.capabilities = Some(CapabilityDeclarations {
10649            provides: provides
10650                .iter()
10651                .map(|capability| (*capability).to_string())
10652                .collect(),
10653            requires: Vec::new(),
10654            must_never_reach: must_never_reach
10655                .iter()
10656                .map(|capability| (*capability).to_string())
10657                .collect(),
10658        });
10659        manifest
10660    }
10661
10662    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
10663        Frame::build(
10664            FrameType::Hello,
10665            control_flags(),
10666            0,
10667            0,
10668            corr,
10669            serde_json::to_vec(&ModuleHelloBody {
10670                protocol_ver: manifest.protocol_ver,
10671                manifest,
10672                control_ops: None,
10673                launch_nonce: None,
10674            })
10675            .expect("capability test HELLO serializes"),
10676        )
10677        .expect("capability test HELLO frame builds")
10678    }
10679
10680    fn catalog_update_with_capabilities_frame(
10681        corr: u64,
10682        capabilities: CapabilityDeclarations,
10683    ) -> Frame {
10684        Frame::build(
10685            FrameType::Request,
10686            control_flags(),
10687            0,
10688            0,
10689            corr,
10690            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
10691                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
10692                capabilities: Some(capabilities),
10693                ready: None,
10694            })
10695            .expect("capability catalog.update serializes"),
10696        )
10697        .expect("capability catalog.update frame builds")
10698    }
10699
10700    async fn register_capability_manifest(
10701        handler: &ControlHandler,
10702        ctx: &RouteCtx,
10703        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10704        manifest: ModuleManifest,
10705        corr: u64,
10706    ) {
10707        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
10708    }
10709
10710    async fn open_route_for_capability_test(
10711        handler: &ControlHandler,
10712        target_ctx: &RouteCtx,
10713        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10714        client_connection_id: u64,
10715        corr: u64,
10716        target_module_id: &str,
10717        consumer_identity: Option<ConsumerIdentity>,
10718    ) -> (
10719        mpsc::Receiver<crate::router::OutboundFrame>,
10720        ModuleControlRequest,
10721    ) {
10722        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
10723        let route_handler = handler.clone();
10724        let target_module_id = target_module_id.to_string();
10725        let route_task = tokio::spawn(async move {
10726            route_handler
10727                .handle_control_frame(
10728                    &client_ctx,
10729                    route_open_frame_with_admission_facts(
10730                        corr,
10731                        &target_module_id,
10732                        unique_project_root("admission-facts"),
10733                        consumer_identity,
10734                        None,
10735                    ),
10736                )
10737                .await
10738                .expect("capability test route.open succeeds")
10739        });
10740        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
10741            .await
10742            .expect("capability test route.open must reach route.bind")
10743            .expect("target control receiver stays open");
10744        let bind_request: ModuleControlRequest =
10745            serde_json::from_slice(&bind.body).expect("route.bind decodes");
10746        handler
10747            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
10748            .await
10749            .expect("capability test route.bind ACK succeeds");
10750        assert!(route_task.await.expect("route.open task joins").is_empty());
10751        let opened = client_rx
10752            .recv()
10753            .await
10754            .expect("successful route.open publishes a response");
10755        assert!(matches!(
10756            serde_json::from_slice::<ClientControlResponse>(&opened.body),
10757            Ok(ClientControlResponse::RouteOpen { .. })
10758        ));
10759        (client_rx, bind_request)
10760    }
10761
10762    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
10763        assert_eq!(frame.header.ty, FrameType::Push);
10764        assert_eq!(frame.header.channel, 0);
10765        assert_eq!(
10766            serde_json::from_slice::<ClientControlPush>(&frame.body)
10767                .expect("route.closed control push decodes"),
10768            ClientControlPush::RouteClosed {
10769                module_id: target_module_id.to_string(),
10770                reason: RouteCloseReason::CapabilityDenied,
10771                drained: false,
10772                abandoned: 0,
10773                excluded_subscriptions: 0,
10774                terminal: Some(false),
10775            }
10776        );
10777    }
10778
10779    #[tokio::test]
10780    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
10781        let registry = Arc::new(Registry::default());
10782        let forwarding = Arc::new(ForwardingTable::default());
10783        let supervisor = SupervisorHandle::new();
10784        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10785        let handler =
10786            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10787                .with_supervisor(supervisor);
10788        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
10789        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
10790        register_capability_manifest(
10791            &handler,
10792            &target_ctx,
10793            &mut target_rx,
10794            capability_manifest("target", &["credentials-provider/v1"], &[]),
10795            1,
10796        )
10797        .await;
10798        register_capability_manifest(
10799            &handler,
10800            &opener_ctx,
10801            &mut opener_rx,
10802            capability_manifest("opener", &[], &["credentials-provider/v1"]),
10803            2,
10804        )
10805        .await;
10806
10807        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
10808        let replies = handler
10809            .handle_control_frame(
10810                &client_ctx,
10811                route_open_frame_with_admission_facts(
10812                    3,
10813                    "target",
10814                    unique_project_root("admission-facts"),
10815                    Some(ConsumerIdentity {
10816                        module_id: "opener".to_string(),
10817                        launch_nonce: "opener-nonce".to_string(),
10818                    }),
10819                    None,
10820                ),
10821            )
10822            .await
10823            .expect("denied route.open returns a typed frame");
10824        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
10825        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10826        assert!(
10827            target_rx.try_recv().is_err(),
10828            "forbidden route.open must not relay route.bind"
10829        );
10830    }
10831
10832    #[tokio::test]
10833    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
10834        let registry = Arc::new(Registry::default());
10835        let forwarding = Arc::new(ForwardingTable::default());
10836        let supervisor = SupervisorHandle::new();
10837        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10838        let handler =
10839            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10840                .with_supervisor(supervisor);
10841        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
10842        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
10843        register_capability_manifest(
10844            &handler,
10845            &target_ctx,
10846            &mut target_rx,
10847            capability_manifest("target", &["credentials-provider/v1"], &[]),
10848            1,
10849        )
10850        .await;
10851        register_capability_manifest(
10852            &handler,
10853            &old_opener_ctx,
10854            &mut old_opener_rx,
10855            capability_manifest("opener", &[], &[]),
10856            2,
10857        )
10858        .await;
10859        let (mut client_rx, _) = open_route_for_capability_test(
10860            &handler,
10861            &target_ctx,
10862            &mut target_rx,
10863            712,
10864            3,
10865            "target",
10866            Some(ConsumerIdentity {
10867                module_id: "opener".to_string(),
10868                launch_nonce: "opener-nonce".to_string(),
10869            }),
10870        )
10871        .await;
10872        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10873
10874        handler
10875            .cleanup_connection(old_opener_ctx.connection_id)
10876            .expect("old opener registration cleans up");
10877        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
10878        register_capability_manifest(
10879            &handler,
10880            &new_opener_ctx,
10881            &mut new_opener_rx,
10882            capability_manifest("opener", &[], &["credentials-provider/v1"]),
10883            4,
10884        )
10885        .await;
10886
10887        assert_capability_denied_push(
10888            client_rx
10889                .try_recv()
10890                .expect("HELLO deny addition must emit route.closed")
10891                .frame,
10892            "target",
10893        );
10894        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10895        assert!(matches!(
10896            target_rx.try_recv(),
10897            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
10898        ));
10899    }
10900
10901    #[tokio::test]
10902    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
10903        let registry = Arc::new(Registry::default());
10904        let forwarding = Arc::new(ForwardingTable::default());
10905        let supervisor = SupervisorHandle::new();
10906        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10907        let handler =
10908            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10909                .with_supervisor(supervisor);
10910        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
10911        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
10912        register_capability_manifest(
10913            &handler,
10914            &target_ctx,
10915            &mut target_rx,
10916            capability_manifest("target", &[], &[]),
10917            1,
10918        )
10919        .await;
10920        register_capability_manifest(
10921            &handler,
10922            &opener_ctx,
10923            &mut opener_rx,
10924            capability_manifest("opener", &[], &["credentials-provider/v1"]),
10925            2,
10926        )
10927        .await;
10928        let (mut client_rx, _) = open_route_for_capability_test(
10929            &handler,
10930            &target_ctx,
10931            &mut target_rx,
10932            722,
10933            3,
10934            "target",
10935            Some(ConsumerIdentity {
10936                module_id: "opener".to_string(),
10937                launch_nonce: "opener-nonce".to_string(),
10938            }),
10939        )
10940        .await;
10941        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10942
10943        let replies = handler
10944            .handle_control_frame(
10945                &target_ctx,
10946                catalog_update_with_capabilities_frame(
10947                    4,
10948                    CapabilityDeclarations {
10949                        provides: vec!["credentials-provider/v1".to_string()],
10950                        requires: Vec::new(),
10951                        must_never_reach: Vec::new(),
10952                    },
10953                ),
10954            )
10955            .await
10956            .expect("claim catalog.update succeeds");
10957        assert!(matches!(
10958            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
10959            Ok(ModuleControlResponseToModule::CatalogUpdate {})
10960        ));
10961        assert_capability_denied_push(
10962            client_rx
10963                .try_recv()
10964                .expect("claim addition must emit route.closed")
10965                .frame,
10966            "target",
10967        );
10968        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10969        assert!(matches!(
10970            target_rx.try_recv(),
10971            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
10972        ));
10973    }
10974
10975    #[tokio::test]
10976    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
10977        let registry = Arc::new(Registry::default());
10978        let forwarding = Arc::new(ForwardingTable::default());
10979        let supervisor = SupervisorHandle::new();
10980        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10981        let handler =
10982            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10983                .with_supervisor(supervisor);
10984        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
10985        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
10986        register_capability_manifest(
10987            &handler,
10988            &target_ctx,
10989            &mut target_rx,
10990            capability_manifest("target", &["credentials-provider/v1"], &[]),
10991            1,
10992        )
10993        .await;
10994        register_capability_manifest(
10995            &handler,
10996            &opener_ctx,
10997            &mut opener_rx,
10998            capability_manifest("opener", &[], &[]),
10999            2,
11000        )
11001        .await;
11002        let (mut client_rx, _) = open_route_for_capability_test(
11003            &handler,
11004            &target_ctx,
11005            &mut target_rx,
11006            732,
11007            3,
11008            "target",
11009            Some(ConsumerIdentity {
11010                module_id: "opener".to_string(),
11011                launch_nonce: "opener-nonce".to_string(),
11012            }),
11013        )
11014        .await;
11015        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11016
11017        handler
11018            .handle_control_frame(
11019                &target_ctx,
11020                catalog_update_with_capabilities_frame(
11021                    4,
11022                    CapabilityDeclarations {
11023                        provides: Vec::new(),
11024                        requires: Vec::new(),
11025                        must_never_reach: Vec::new(),
11026                    },
11027                ),
11028            )
11029            .await
11030            .expect("claim removal catalog.update succeeds");
11031        assert_eq!(
11032            forwarding.active_binding_count().unwrap(),
11033            1,
11034            "removing an attested target claim must leave the route census unchanged"
11035        );
11036        assert!(
11037            client_rx.try_recv().is_err(),
11038            "claim removal must not emit route.closed capability_denied"
11039        );
11040        assert!(
11041            target_rx.try_recv().is_err(),
11042            "claim removal must not send the target a route GOODBYE"
11043        );
11044    }
11045
11046    /// A direct client may open a route to a denied capability provider; this
11047    /// policy applies only to attested supervised module origins, not to direct clients.
11048    #[tokio::test]
11049    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
11050        let registry = Arc::new(Registry::default());
11051        let forwarding = Arc::new(ForwardingTable::default());
11052        let supervisor = SupervisorHandle::new();
11053        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11054        let handler =
11055            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11056                .with_supervisor(supervisor);
11057        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
11058        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
11059        register_capability_manifest(
11060            &handler,
11061            &target_ctx,
11062            &mut target_rx,
11063            capability_manifest("target", &["credentials-provider/v1"], &[]),
11064            1,
11065        )
11066        .await;
11067        register_capability_manifest(
11068            &handler,
11069            &opener_ctx,
11070            &mut opener_rx,
11071            capability_manifest("opener", &[], &["credentials-provider/v1"]),
11072            2,
11073        )
11074        .await;
11075
11076        let (_client_rx, bind) = open_route_for_capability_test(
11077            &handler,
11078            &target_ctx,
11079            &mut target_rx,
11080            742,
11081            3,
11082            "target",
11083            None,
11084        )
11085        .await;
11086        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11087            panic!("direct scope-honesty route must bind");
11088        };
11089        assert_eq!(principal, Some(Principal::Direct));
11090        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11091    }
11092
11093    /// A module that denies a capability receives no self-route exemption when it
11094    /// also attestedly provides that capability.
11095    #[tokio::test]
11096    async fn must_never_reach_self_route_is_capability_forbidden() {
11097        let registry = Arc::new(Registry::default());
11098        let forwarding = Arc::new(ForwardingTable::default());
11099        let supervisor = SupervisorHandle::new();
11100        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
11101        let handler =
11102            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11103                .with_supervisor(supervisor);
11104        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
11105        register_capability_manifest(
11106            &handler,
11107            &self_ctx,
11108            &mut self_rx,
11109            capability_manifest(
11110                "self-provider",
11111                &["credentials-provider/v1"],
11112                &["credentials-provider/v1"],
11113            ),
11114            1,
11115        )
11116        .await;
11117
11118        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
11119        let replies = handler
11120            .handle_control_frame(
11121                &client_ctx,
11122                route_open_frame_with_admission_facts(
11123                    2,
11124                    "self-provider",
11125                    unique_project_root("admission-facts"),
11126                    Some(ConsumerIdentity {
11127                        module_id: "self-provider".to_string(),
11128                        launch_nonce: "self-nonce".to_string(),
11129                    }),
11130                    None,
11131                ),
11132            )
11133            .await
11134            .expect("self-route refusal returns a typed frame");
11135        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
11136        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11137        assert!(
11138            self_rx.try_recv().is_err(),
11139            "self denial must not relay route.bind"
11140        );
11141    }
11142
11143    #[test]
11144    fn unsupported_channel_zero_frame_returns_error() {
11145        let handler = ControlHandler::default();
11146        let request = Frame::build(
11147            FrameType::Request,
11148            control_flags(),
11149            0,
11150            0,
11151            21,
11152            b"opaque".to_vec(),
11153        )
11154        .unwrap();
11155
11156        let response = handler
11157            .handle_control(ConnectionId::new(1), request)
11158            .unwrap();
11159
11160        assert_eq!(response[0].header.ty, FrameType::Error);
11161        assert_eq!(
11162            parse_error(&response[0])["code"],
11163            "unsupported_control_frame"
11164        );
11165    }
11166
11167    /// Blue/green swap at the control-plane boundary. The supervisor that opens
11168    /// a swap is not wired yet, so the candidate is registered here directly
11169    /// into the registry and forwarding candidate slots, the way the swap's
11170    /// HELLO admission will.
11171    mod swap {
11172        use super::*;
11173
11174        const INCUMBENT: ConnectionId = ConnectionId::new(30);
11175        const CANDIDATE: ConnectionId = ConnectionId::new(40);
11176
11177        struct Swap {
11178            registry: Arc<Registry>,
11179            forwarding: Arc<ForwardingTable>,
11180            handler: ControlHandler,
11181            incumbent_ctx: RouteCtx,
11182            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
11183            candidate_ctx: RouteCtx,
11184            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
11185        }
11186
11187        async fn swap_with_incumbent() -> Swap {
11188            let registry = Arc::new(Registry::default());
11189            let forwarding = Arc::new(ForwardingTable::default());
11190            let handler =
11191                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11192            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
11193            hello_via_sink(
11194                &handler,
11195                &incumbent_ctx,
11196                &mut incumbent_rx,
11197                hello_frame("aft", PROTOCOL_VERSION, 7),
11198            )
11199            .await;
11200            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
11201            Swap {
11202                registry,
11203                forwarding,
11204                handler,
11205                incumbent_ctx,
11206                incumbent_rx,
11207                candidate_ctx,
11208                candidate_rx,
11209            }
11210        }
11211
11212        fn register_candidate(swap: &Swap, ready: Option<bool>) {
11213            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
11214            candidate_manifest.ready = ready;
11215            let registration = swap
11216                .registry
11217                .register_candidate_with_control_ops(
11218                    candidate_manifest,
11219                    PROTOCOL_VERSION,
11220                    CANDIDATE,
11221                    module_baseline_control_ops(),
11222                )
11223                .unwrap();
11224            swap.forwarding
11225                .register_candidate_module_connection(
11226                    CANDIDATE,
11227                    "aft".to_string(),
11228                    PROTOCOL_VERSION,
11229                    manifest_concurrency(&registration.manifest),
11230                    swap.candidate_ctx.egress.clone(),
11231                )
11232                .unwrap();
11233        }
11234
11235        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
11236            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
11237            swap.registry.promote_candidate("aft").unwrap().unwrap();
11238            cutover.incumbent.unwrap()
11239        }
11240
11241        fn keyed_total(counters: &Value, key: &str) -> u64 {
11242            counters[key]
11243                .as_object()
11244                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
11245                .unwrap_or(0)
11246        }
11247
11248        /// An ack from the incumbent for a bind it was sent before cutover,
11249        /// arriving before the incumbent is drained. The incumbent is the live
11250        /// connection carrying every other client's routes, so the ack must
11251        /// not end it: the waiting client is told to retry, the reservation is
11252        /// given back, and the incumbent is told to drop just that binding.
11253        #[tokio::test]
11254        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
11255            let mut swap = swap_with_incumbent().await;
11256            let handler = swap.handler.clone();
11257
11258            // A co-tenant route, bound on the incumbent before the swap.
11259            let cotenant = ConnectionId::new(31);
11260            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
11261            let (cotenant_task, cotenant_bind) = relay_route_open(
11262                &handler,
11263                cotenant,
11264                &cotenant_ctx.egress,
11265                &mut swap.incumbent_rx,
11266                100,
11267                "aft",
11268                "swap-cotenant",
11269            )
11270            .await;
11271            handler
11272                .handle_control_frame(
11273                    &swap.incumbent_ctx,
11274                    route_bind_ack(cotenant_bind.header.corr),
11275                )
11276                .await
11277                .unwrap();
11278            assert!(cotenant_task.await.unwrap().is_empty());
11279            let (cotenant_channel, cotenant_epoch) =
11280                published_route(&cotenant_rx.recv().await.unwrap());
11281
11282            // A second route.open, relayed to the incumbent and not yet acked.
11283            let caller = ConnectionId::new(32);
11284            let (caller_ctx, mut caller_rx) = route_ctx(caller);
11285            let (caller_task, caller_bind) = relay_route_open(
11286                &handler,
11287                caller,
11288                &caller_ctx.egress,
11289                &mut swap.incumbent_rx,
11290                101,
11291                "aft",
11292                "swap-caller",
11293            )
11294            .await;
11295            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
11296
11297            register_candidate(&swap, None);
11298            cutover(&swap);
11299
11300            // The incumbent acks after promotion and before any drain.
11301            let ack = handler
11302                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
11303                .await;
11304            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
11305            if module_loop_error.is_some() {
11306                // What the connection loop does with an untranslated router
11307                // error: end the connection, releasing every route on it.
11308                handler.cleanup_connection(INCUMBENT).unwrap();
11309            }
11310
11311            // 1. The incumbent's other routes survive.
11312            assert!(
11313                cotenant_rx.try_recv().is_err(),
11314                "the co-tenant route on the incumbent was torn down by one late ack: \
11315                 {module_loop_error:?}"
11316            );
11317            assert!(matches!(
11318                swap.forwarding
11319                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
11320                    .unwrap(),
11321                DataRoute::Client(DataRouteState::Bound(_))
11322            ));
11323            assert_eq!(module_loop_error, None);
11324            assert!(swap
11325                .registry
11326                .get_module_by_connection(INCUMBENT)
11327                .unwrap()
11328                .is_some());
11329
11330            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
11331            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
11332                .await
11333                .expect("the incumbent is told to drop the abandoned binding")
11334                .unwrap()
11335                .frame;
11336            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11337            assert_eq!(goodbye.header.channel, abandoned_channel);
11338            assert_eq!(goodbye.header.epoch, abandoned_epoch);
11339            assert!(swap.incumbent_rx.try_recv().is_err());
11340
11341            // 3. The waiting client gets a retryable refusal and no route.
11342            let response = caller_task.await.unwrap();
11343            assert_eq!(response.len(), 1);
11344            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
11345            assert!(caller_rx.try_recv().is_err());
11346
11347            // 4. The reservation pair is given back, and the pending bind
11348            //    settled exactly once: one accepted open (the co-tenant) and one
11349            //    refused open (the caller), nothing counted twice.
11350            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
11351            let counters = handler.counters().snapshot();
11352            assert_eq!(
11353                keyed_total(&counters, "route_open_accepted_by_principal"),
11354                1
11355            );
11356            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
11357            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
11358        }
11359
11360        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
11361        /// id would resolve to the promoted candidate and every new route.open
11362        /// would be refused as reloading, leaving neither process routable.
11363        #[tokio::test]
11364        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
11365            let mut swap = swap_with_incumbent().await;
11366            register_candidate(&swap, None);
11367            let incumbent = cutover(&swap);
11368            swap.forwarding
11369                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
11370                .unwrap()
11371                .expect("the incumbent is still registered");
11372
11373            let client = ConnectionId::new(33);
11374            let (client_ctx, mut client_rx) = route_ctx(client);
11375            let route_handler = swap.handler.clone();
11376            let open_ctx = RouteCtx {
11377                connection_id: client,
11378                egress: client_ctx.egress.clone(),
11379            };
11380            let mut route_task = tokio::spawn(async move {
11381                route_handler
11382                    .handle_control_frame(
11383                        &open_ctx,
11384                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
11385                    )
11386                    .await
11387                    .unwrap()
11388            });
11389            let bind = tokio::select! {
11390                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
11391                response = &mut route_task => {
11392                    let response = response.unwrap();
11393                    panic!(
11394                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
11395                        parse_error(&response[0])["code"]
11396                    );
11397                }
11398            };
11399            swap.handler
11400                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
11401                .await
11402                .unwrap();
11403            assert!(route_task.await.unwrap().is_empty());
11404            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
11405            match swap
11406                .forwarding
11407                .lookup_data_route(client, channel, epoch)
11408                .unwrap()
11409            {
11410                DataRoute::Client(DataRouteState::Bound(route)) => {
11411                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
11412                }
11413                other => panic!("expected a bound route on the candidate, got {other:?}"),
11414            }
11415            assert!(swap.incumbent_rx.try_recv().is_err());
11416        }
11417
11418        /// A candidate declares itself ready with `catalog.update` on its own
11419        /// connection. If the connection-keyed registry lookups searched only the
11420        /// active slot, this would answer `not_registered` and the candidate
11421        /// would never become ready.
11422        #[tokio::test]
11423        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
11424            let swap = swap_with_incumbent().await;
11425            register_candidate(&swap, Some(false));
11426            let update = Frame::build(
11427                FrameType::Request,
11428                control_flags(),
11429                0,
11430                0,
11431                55,
11432                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
11433                    provides: manifest("aft", PROTOCOL_VERSION).provides,
11434                    capabilities: None,
11435                    ready: Some(true),
11436                })
11437                .unwrap(),
11438            )
11439            .unwrap();
11440
11441            let replies = swap
11442                .handler
11443                .handle_control_frame(&swap.candidate_ctx, update)
11444                .await
11445                .unwrap();
11446
11447            assert_eq!(replies.len(), 1);
11448            assert_eq!(
11449                replies[0].header.ty,
11450                FrameType::Response,
11451                "candidate catalog.update was refused: {:?}",
11452                serde_json::from_slice::<Value>(&replies[0].body).ok()
11453            );
11454            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
11455            assert_eq!(
11456                swap.registry
11457                    .get_module("aft")
11458                    .unwrap()
11459                    .unwrap()
11460                    .connection_id,
11461                INCUMBENT
11462            );
11463        }
11464    }
11465
11466    /// The HELLO gate while the supervisor has a swap open: only the nonce it
11467    /// minted for the candidate admits a second process, into the candidate
11468    /// slot, and that check runs ahead of the reserved-module gate.
11469    mod swap_admission {
11470        use super::*;
11471
11472        const INCUMBENT_NONCE: &str = "incumbent-nonce";
11473        const CANDIDATE_NONCE: &str = "candidate-nonce";
11474
11475        fn handler_with_incumbent(
11476            module_id: &str,
11477            reserved: bool,
11478        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
11479            let registry = Arc::new(Registry::default());
11480            let supervisor = SupervisorHandle::new();
11481            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
11482            if reserved {
11483                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
11484            }
11485            let handler =
11486                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
11487            let incumbent = handler
11488                .handle_control(
11489                    ConnectionId::new(1),
11490                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
11491                )
11492                .unwrap();
11493            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
11494            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
11495            (registry, supervisor, handler)
11496        }
11497
11498        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
11499        /// admits every nonce, so while a swap is open the swap gate is the only
11500        /// thing between a key-holder and the candidate slot. A nonce the
11501        /// supervisor did not mint, or none at all, is refused, and neither the
11502        /// incumbent's registration nor the candidate slot moves.
11503        #[test]
11504        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
11505            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
11506
11507            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
11508                let replies = handler
11509                    .handle_control(
11510                        ConnectionId::new(connection),
11511                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
11512                    )
11513                    .unwrap();
11514                assert_eq!(replies[0].header.ty, FrameType::Error);
11515                assert_eq!(
11516                    parse_error(&replies[0])["code"],
11517                    "swap_token_invalid",
11518                    "nonce {nonce:?}"
11519                );
11520            }
11521            assert!(registry.get_candidate("aft").unwrap().is_none());
11522            assert_eq!(
11523                registry.get_module("aft").unwrap().unwrap().connection_id,
11524                ConnectionId::new(1)
11525            );
11526
11527            // Control: the minted token is admitted, into the candidate slot,
11528            // and only once.
11529            let admitted = handler
11530                .handle_control(
11531                    ConnectionId::new(4),
11532                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
11533                )
11534                .unwrap();
11535            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
11536            assert_eq!(
11537                registry
11538                    .get_candidate("aft")
11539                    .unwrap()
11540                    .unwrap()
11541                    .connection_id,
11542                ConnectionId::new(4)
11543            );
11544            assert_eq!(
11545                registry.get_module("aft").unwrap().unwrap().connection_id,
11546                ConnectionId::new(1),
11547                "the candidate must not take the active slot"
11548            );
11549            let replayed = handler
11550                .handle_control(
11551                    ConnectionId::new(5),
11552                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
11553                )
11554                .unwrap();
11555            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
11556
11557            // The case only this gate covers: the incumbent has died mid-swap,
11558            // so its duplicate refusal is gone too, and without the gate a
11559            // key-holder would take the id's ACTIVE slot.
11560            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
11561            let squatter = handler
11562                .handle_control(
11563                    ConnectionId::new(6),
11564                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
11565                )
11566                .unwrap();
11567            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
11568            assert!(
11569                registry.get_module("aft").unwrap().is_none(),
11570                "a squatter took the active slot of an id being swapped"
11571            );
11572        }
11573
11574        /// Design mutation arm (iii). A reserved module's candidate presents a
11575        /// nonce the reserved gate has never seen (that gate holds the
11576        /// incumbent's), so the swap gate must run first or the candidate is
11577        /// refused `reserved_module` and a reserved module can never be swapped.
11578        #[test]
11579        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
11580            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
11581
11582            let replies = handler
11583                .handle_control(
11584                    ConnectionId::new(2),
11585                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
11586                )
11587                .unwrap();
11588
11589            assert_eq!(
11590                replies[0].header.ty,
11591                FrameType::HelloAck,
11592                "reserved candidate refused: {:?}",
11593                serde_json::from_slice::<Value>(&replies[0].body).ok()
11594            );
11595            assert_eq!(
11596                registry
11597                    .get_candidate("vault")
11598                    .unwrap()
11599                    .unwrap()
11600                    .connection_id,
11601                ConnectionId::new(2)
11602            );
11603        }
11604
11605        /// With no swap open the gate is inert: the incumbent's reserved gate
11606        /// and duplicate refusal behave exactly as before.
11607        #[test]
11608        fn without_an_open_swap_the_ordinary_gates_decide() {
11609            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
11610            supervisor.close_swap("vault");
11611
11612            let candidate = handler
11613                .handle_control(
11614                    ConnectionId::new(2),
11615                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
11616                )
11617                .unwrap();
11618            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
11619            let duplicate = handler
11620                .handle_control(
11621                    ConnectionId::new(3),
11622                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
11623                )
11624                .unwrap();
11625            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11626            assert!(registry.get_candidate("vault").unwrap().is_none());
11627        }
11628    }
11629
11630    /// `scope.sync` and `scope.describe` through the real control handler: who
11631    /// may sync is decided by the registration and launch nonce of the module
11632    /// connection, never by the request body.
11633    mod scopes {
11634        use subc_protocol::scope::{
11635            ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
11636            ScopeStatus,
11637        };
11638
11639        use super::*;
11640
11641        const OWNER: &str = "prefrontal-core";
11642
11643        fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
11644            ScopeRecord {
11645                scope_ref: scope_ref.to_string(),
11646                scope_epoch,
11647                kind: ScopeKind::Head,
11648                parent: None,
11649                child_owners: Vec::new(),
11650                carriers: Vec::new(),
11651                attributes: Default::default(),
11652            }
11653        }
11654
11655        async fn call(
11656            handler: &ControlHandler,
11657            ctx: &RouteCtx,
11658            request: &ModuleControlRequestFromModule,
11659        ) -> Frame {
11660            let body = serde_json::to_vec(request).unwrap();
11661            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
11662            let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
11663            assert_eq!(replies.len(), 1, "{replies:?}");
11664            replies.pop().unwrap()
11665        }
11666
11667        async fn sync(
11668            handler: &ControlHandler,
11669            ctx: &RouteCtx,
11670            generation: u64,
11671            scopes: Vec<ScopeRecord>,
11672        ) -> Result<ModuleControlResponseToModule, String> {
11673            let reply = call(
11674                handler,
11675                ctx,
11676                &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
11677            )
11678            .await;
11679            match reply.header.ty {
11680                FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
11681                _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
11682            }
11683        }
11684
11685        async fn describe(
11686            handler: &ControlHandler,
11687            ctx: &RouteCtx,
11688            owner: &str,
11689            scope_ref: &str,
11690        ) -> ModuleControlResponseToModule {
11691            let reply = call(
11692                handler,
11693                ctx,
11694                &ModuleControlRequestFromModule::ScopeDescribe {
11695                    owner: Principal::Reserved {
11696                        module_id: owner.to_string(),
11697                    },
11698                    scope_ref: scope_ref.to_string(),
11699                },
11700            )
11701            .await;
11702            assert_eq!(
11703                reply.header.ty,
11704                FrameType::Response,
11705                "{:?}",
11706                parse_error(&reply)
11707            );
11708            serde_json::from_slice(&reply.body).unwrap()
11709        }
11710
11711        /// Register `module_id` on `connection` with `nonce`, returning its ctx.
11712        async fn module(
11713            handler: &ControlHandler,
11714            connection: u64,
11715            module_id: &str,
11716            nonce: Option<&str>,
11717        ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
11718            let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
11719            hello_via_sink(
11720                handler,
11721                &ctx,
11722                &mut rx,
11723                hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
11724            )
11725            .await;
11726            (ctx, rx)
11727        }
11728
11729        /// `direct` and every other client connection has no registration, so
11730        /// it can neither sync nor own a scope.
11731        #[tokio::test]
11732        async fn a_client_connection_cannot_sync_or_describe() {
11733            let handler = ControlHandler::new(Arc::new(Registry::default()));
11734            let (ctx, _rx) = route_ctx(ConnectionId::new(9));
11735            for request in [
11736                ModuleControlRequestFromModule::ScopeSync {
11737                    generation: 1,
11738                    scopes: vec![head("s", 1)],
11739                },
11740                ModuleControlRequestFromModule::ScopeDescribe {
11741                    owner: Principal::Direct,
11742                    scope_ref: "s".to_string(),
11743                },
11744            ] {
11745                let reply = call(&handler, &ctx, &request).await;
11746                assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
11747            }
11748            assert!(
11749                !handler
11750                    .scopes
11751                    .read()
11752                    .unwrap()
11753                    .describe(
11754                        &Principal::Reserved {
11755                            module_id: OWNER.to_string()
11756                        },
11757                        "s"
11758                    )
11759                    .owner_synced
11760            );
11761        }
11762
11763        /// A module the supervisor did not spawn registers without a launch
11764        /// nonce, so it is never an owner's current launch.
11765        #[tokio::test]
11766        async fn a_module_without_a_supervised_launch_cannot_sync() {
11767            let handler = ControlHandler::new(Arc::new(Registry::default()));
11768            let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
11769            assert_eq!(
11770                sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
11771                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
11772            );
11773        }
11774
11775        #[tokio::test]
11776        async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
11777            let supervisor = SupervisorHandle::new();
11778            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
11779            let handler = ControlHandler::new(Arc::new(Registry::default()))
11780                .with_supervisor(supervisor.clone());
11781            let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
11782            sync(&handler, &incumbent, 1, vec![head("s", 1)])
11783                .await
11784                .expect("the current launch syncs");
11785
11786            // A swap candidate registers with the swap token and is refused
11787            // while the incumbent keeps syncing.
11788            supervisor.open_swap(OWNER, "n2".to_string());
11789            let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
11790            assert_eq!(
11791                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
11792                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
11793            );
11794            sync(&handler, &incumbent, 2, vec![head("s", 1)])
11795                .await
11796                .expect("the serving owner syncs during the swap");
11797
11798            // The swap fails and is rolled back. The candidate never held sync
11799            // authority, and still cannot sync.
11800            supervisor.close_swap(OWNER);
11801            assert_eq!(
11802                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
11803                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
11804            );
11805            sync(&handler, &incumbent, 3, vec![head("s", 1)])
11806                .await
11807                .expect("the serving owner syncs after the rollback");
11808            handler.cleanup_connection(candidate.connection_id).unwrap();
11809
11810            // A swap that cuts over. Promotion records the candidate's nonce as
11811            // the module's spawn nonce, which is what `set_spawn_nonce` does
11812            // here; the promoted connection then takes authority at any
11813            // generation and the superseded incumbent is refused.
11814            supervisor.open_swap(OWNER, "n3".to_string());
11815            let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
11816            supervisor.set_spawn_nonce(OWNER, "n3".to_string());
11817            let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
11818                .await
11819                .expect("the promoted launch takes authority");
11820            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
11821                panic!("unexpected reply {reply:?}");
11822            };
11823            assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
11824            assert_eq!(
11825                sync(&handler, &incumbent, 4, Vec::new()).await,
11826                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
11827            );
11828        }
11829
11830        /// Authority dies with its connection: the cleanup path releases it,
11831        /// so the owner's next connection takes it at any generation.
11832        #[tokio::test]
11833        async fn closing_the_authority_connection_frees_sync_authority() {
11834            let supervisor = SupervisorHandle::new();
11835            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
11836            let handler = ControlHandler::new(Arc::new(Registry::default()))
11837                .with_supervisor(supervisor.clone());
11838            let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
11839            sync(&handler, &first, 10, vec![head("s", 1)])
11840                .await
11841                .unwrap();
11842            handler.cleanup_connection(first.connection_id).unwrap();
11843
11844            let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
11845            sync(&handler, &second, 1, vec![head("s", 1)])
11846                .await
11847                .expect("the next connection takes the released authority");
11848        }
11849
11850        #[tokio::test]
11851        async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
11852            let registry = Arc::new(Registry::default());
11853            let supervisor_handle = SupervisorHandle::new();
11854            let supervisor = Supervisor::new(Arc::clone(&registry), RestartPolicy::default())
11855                .with_handle(supervisor_handle.clone())
11856                .with_daemon_incarnation("incarnation-7".to_string());
11857            // Configured with enabled: false, so the supervisor lists the
11858            // module without spawning a process for it.
11859            supervisor
11860                .supervise_configured(
11861                    ModuleSpec {
11862                        launch_nonce_env: true,
11863                        module_id: OWNER.to_string(),
11864                        program: PathBuf::from("/nonexistent/prefrontal-core"),
11865                        args: Vec::new(),
11866                        env: Vec::new(),
11867                        reserved: false,
11868                        reserved_prefixes: Vec::new(),
11869                        protocol: ModuleProtocol::Subc,
11870                        overlap: Default::default(),
11871                    },
11872                    false,
11873                )
11874                .unwrap();
11875            supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
11876            let handler =
11877                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
11878            let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
11879
11880            // Configured but not yet synced: a reader waits for the owner.
11881            let ModuleControlResponseToModule::ScopeDescribe {
11882                status,
11883                daemon_incarnation,
11884                owner_synced,
11885                owner_configured,
11886                scope,
11887                ..
11888            } = describe(&handler, &reader, OWNER, "s").await
11889            else {
11890                panic!("not a describe reply");
11891            };
11892            assert_eq!(status, ScopeStatus::NotLive);
11893            assert_eq!(daemon_incarnation, "incarnation-7");
11894            assert!(!owner_synced);
11895            assert!(owner_configured);
11896            assert!(scope.is_none());
11897
11898            // Not a supervised module: the owner will never sync, and a reader
11899            // refuses rather than waits.
11900            let ModuleControlResponseToModule::ScopeDescribe {
11901                status,
11902                owner_configured,
11903                ..
11904            } = describe(&handler, &reader, "ghost", "s").await
11905            else {
11906                panic!("not a describe reply");
11907            };
11908            assert_eq!(status, ScopeStatus::NotLive);
11909            assert!(!owner_configured);
11910
11911            // Live, with the stamp fields and the computed owner_authorized.
11912            let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
11913            sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
11914            let ModuleControlResponseToModule::ScopeDescribe {
11915                status,
11916                scope_epoch,
11917                owner_synced,
11918                scope,
11919                ..
11920            } = describe(&handler, &reader, OWNER, "s").await
11921            else {
11922                panic!("not a describe reply");
11923            };
11924            assert_eq!(status, ScopeStatus::Live);
11925            assert_eq!(scope_epoch, Some(4));
11926            assert!(owner_synced);
11927            let stamp = scope.expect("a live scope carries its stamp");
11928            assert!(
11929                stamp.owner_authorized,
11930                "prefrontal-core is the default authority"
11931            );
11932            assert_eq!(stamp.kind, ScopeKind::Head);
11933        }
11934
11935        #[tokio::test]
11936        async fn scope_authority_owners_decides_owner_authorized() {
11937            let supervisor = SupervisorHandle::new();
11938            supervisor.set_spawn_nonce("broca", "b1".to_string());
11939            let handler = ControlHandler::new(Arc::new(Registry::default()))
11940                .with_supervisor(supervisor)
11941                .with_scope_authority_owners(vec!["broca".to_string()]);
11942            let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
11943            let mut gated = head("s", 1);
11944            gated.attributes.agent_id = Some("agent".to_string());
11945            sync(&handler, &broca, 1, vec![gated]).await.unwrap();
11946            let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
11947                describe(&handler, &broca, "broca", "s").await
11948            else {
11949                panic!("not a describe reply");
11950            };
11951            assert!(scope.unwrap().owner_authorized);
11952        }
11953
11954        /// With route admission, the stamp, the commit re-check and drains in
11955        /// place, the feature is advertised: the module ops in HELLO_ACK, and
11956        /// `scopes/v1` in HELLO_ACK and `server.describe`.
11957        #[tokio::test]
11958        async fn scope_ops_and_the_scopes_capability_are_advertised() {
11959            let handler = ControlHandler::new(Arc::new(Registry::default()));
11960            let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
11961            let ack = hello_via_sink(
11962                &handler,
11963                &ctx,
11964                &mut rx,
11965                hello_frame("m", PROTOCOL_VERSION, 1),
11966            )
11967            .await;
11968            let ack = parse_ack(&ack);
11969            for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
11970                assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
11971            }
11972            assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
11973
11974            let (client, _client_rx) = route_ctx(ConnectionId::new(2));
11975            let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11976            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11977            let reply = handler
11978                .handle_control_frame(&client, frame)
11979                .await
11980                .unwrap()
11981                .pop()
11982                .unwrap();
11983            let ClientControlResponse::ServerDescribe { capabilities, .. } =
11984                serde_json::from_slice(&reply.body).unwrap()
11985            else {
11986                panic!("not a server.describe reply");
11987            };
11988            assert!(
11989                capabilities.iter().any(|c| c == CAP_SCOPES_V1),
11990                "{capabilities:?}"
11991            );
11992        }
11993
11994        // ---- route admission, stamps, commit re-check and drains ----------
11995
11996        const PLEXUS: &str = "plexus";
11997        const OTHER: &str = "other";
11998        const AFT: &str = "aft";
11999        const BROCA: &str = "broca";
12000        const MAGIC: &str = "magic-context";
12001
12002        fn nonce(module_id: &str) -> String {
12003            format!("nonce-{module_id}")
12004        }
12005
12006        fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
12007            let (tx, rx) = mpsc::channel(64);
12008            (
12009                RouteCtx {
12010                    connection_id: ConnectionId::new(connection),
12011                    egress: FrameSink::new(tx),
12012                },
12013                rx,
12014            )
12015        }
12016
12017        /// A daemon with a configured owner (prefrontal-core) registered on its
12018        /// own module connection, two routable targets (plexus, other), and
12019        /// launch nonces minted for the modules that open routes as carriers.
12020        struct Rig {
12021            handler: ControlHandler,
12022            forwarding: Arc<ForwardingTable>,
12023            owner: RouteCtx,
12024            _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12025            modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
12026            generation: u64,
12027            next_connection: u64,
12028            _supervisor: Supervisor,
12029        }
12030
12031        async fn rig() -> Rig {
12032            let registry = Arc::new(Registry::default());
12033            let forwarding = Arc::new(ForwardingTable::default());
12034            let supervisor_handle = SupervisorHandle::new();
12035            let supervisor = Supervisor::new(Arc::clone(&registry), RestartPolicy::default())
12036                .with_handle(supervisor_handle.clone());
12037            supervisor
12038                .supervise_configured(
12039                    ModuleSpec {
12040                        launch_nonce_env: true,
12041                        module_id: OWNER.to_string(),
12042                        program: PathBuf::from("/nonexistent/prefrontal-core"),
12043                        args: Vec::new(),
12044                        env: Vec::new(),
12045                        reserved: false,
12046                        reserved_prefixes: Vec::new(),
12047                        protocol: ModuleProtocol::Subc,
12048                        overlap: Default::default(),
12049                    },
12050                    false,
12051                )
12052                .unwrap();
12053            for module_id in [OWNER, AFT, BROCA, MAGIC] {
12054                supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
12055            }
12056            let handler =
12057                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12058                    .with_supervisor(supervisor_handle);
12059            let (owner, mut owner_rx) = wide_ctx(1);
12060            hello_via_sink(
12061                &handler,
12062                &owner,
12063                &mut owner_rx,
12064                hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
12065            )
12066            .await;
12067            let mut modules = BTreeMap::new();
12068            for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
12069                let (ctx, mut rx) = wide_ctx(connection);
12070                hello_via_sink(
12071                    &handler,
12072                    &ctx,
12073                    &mut rx,
12074                    hello_frame(module_id, PROTOCOL_VERSION, connection),
12075                )
12076                .await;
12077                modules.insert(module_id.to_string(), (ctx, rx));
12078            }
12079            Rig {
12080                handler,
12081                forwarding,
12082                owner,
12083                _owner_rx: owner_rx,
12084                modules,
12085                generation: 0,
12086                next_connection: 100,
12087                _supervisor: supervisor,
12088            }
12089        }
12090
12091        fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
12092            ScopeCarrier {
12093                principal: Principal::Reserved {
12094                    module_id: module_id.to_string(),
12095                },
12096                targets: targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()),
12097            }
12098        }
12099
12100        /// The scope most tests open under: aft carries to any module, broca
12101        /// only to plexus and other, and the owner delegates as agent-1.
12102        fn session(scope_epoch: u64) -> ScopeRecord {
12103            let mut record = head("s", scope_epoch);
12104            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
12105            record.attributes.agent_id = Some("agent-1".to_string());
12106            record.attributes.delegates = true;
12107            record
12108        }
12109
12110        impl Rig {
12111            async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
12112                self.generation += 1;
12113                sync(&self.handler, &self.owner, self.generation, scopes)
12114                    .await
12115                    .expect("the owner's sync is accepted");
12116            }
12117
12118            fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
12119                ScopeSelector {
12120                    owner: Principal::Reserved {
12121                        module_id: OWNER.to_string(),
12122                    },
12123                    scope_ref: scope_ref.to_string(),
12124                    scope_epoch,
12125                }
12126            }
12127
12128            fn open_frame(
12129                &mut self,
12130                opener: Option<&str>,
12131                target: &str,
12132                scope: Option<ScopeSelector>,
12133            ) -> (
12134                RouteCtx,
12135                mpsc::Receiver<crate::router::OutboundFrame>,
12136                Frame,
12137            ) {
12138                self.next_connection += 1;
12139                let (ctx, rx) = wide_ctx(self.next_connection);
12140                let root = unique_project_root("scoped-open");
12141                let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
12142                    target: RouteTarget::ToolProvider {
12143                        module_id: target.to_string(),
12144                    },
12145                    identity: BindIdentity::new(
12146                        root.path().to_path_buf(),
12147                        "unit".to_string(),
12148                        "session".to_string(),
12149                    ),
12150                    consumer_identity: opener.map(|module_id| ConsumerIdentity {
12151                        module_id: module_id.to_string(),
12152                        launch_nonce: nonce(module_id),
12153                    }),
12154                    consumer_capabilities: None,
12155                    role_versions: None,
12156                    admission_facts: None,
12157                    scope,
12158                })
12159                .unwrap();
12160                let frame = Frame::build(
12161                    FrameType::Request,
12162                    control_flags(),
12163                    0,
12164                    0,
12165                    self.next_connection,
12166                    body,
12167                )
12168                .unwrap();
12169                (ctx, rx, frame)
12170            }
12171
12172            /// Open and expect a refusal before anything is relayed.
12173            async fn refused(
12174                &mut self,
12175                opener: Option<&str>,
12176                target: &str,
12177                scope: Option<ScopeSelector>,
12178            ) -> String {
12179                let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
12180                let replies = self
12181                    .handler
12182                    .handle_control_frame(&ctx, frame)
12183                    .await
12184                    .unwrap();
12185                assert_eq!(replies.len(), 1, "{replies:?}");
12186                assert_eq!(replies[0].header.ty, FrameType::Error);
12187                let (_, module_rx) = self.modules.get_mut(target).unwrap();
12188                assert!(
12189                    module_rx.try_recv().is_err(),
12190                    "a refused open relays nothing"
12191                );
12192                parse_error(&replies[0])["code"]
12193                    .as_str()
12194                    .unwrap()
12195                    .to_string()
12196            }
12197
12198            /// Start an open and return its task and the bind the target got.
12199            async fn relayed(
12200                &mut self,
12201                opener: Option<&str>,
12202                target: &str,
12203                scope: Option<ScopeSelector>,
12204            ) -> Relayed {
12205                let (ctx, rx, frame) = self.open_frame(opener, target, scope);
12206                let handler = self.handler.clone();
12207                let task_ctx = ctx.clone();
12208                let task = tokio::spawn(async move {
12209                    handler
12210                        .handle_control_frame(&task_ctx, frame)
12211                        .await
12212                        .unwrap()
12213                });
12214                let (_, module_rx) = self.modules.get_mut(target).unwrap();
12215                let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
12216                    .await
12217                    .expect("the target receives the relayed route.bind")
12218                    .unwrap()
12219                    .frame;
12220                Relayed {
12221                    target: target.to_string(),
12222                    client: ctx,
12223                    client_rx: rx,
12224                    task,
12225                    bind,
12226                }
12227            }
12228
12229            async fn ack(&self, relayed: &Relayed) {
12230                let (module, _) = &self.modules[&relayed.target];
12231                self.handler
12232                    .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
12233                    .await
12234                    .unwrap();
12235            }
12236
12237            /// Open, ack and return the bound route.
12238            async fn bound(
12239                &mut self,
12240                opener: Option<&str>,
12241                target: &str,
12242                scope: Option<ScopeSelector>,
12243            ) -> Bound {
12244                let relayed = self.relayed(opener, target, scope).await;
12245                self.ack(&relayed).await;
12246                let Relayed {
12247                    target,
12248                    client,
12249                    mut client_rx,
12250                    task,
12251                    bind,
12252                } = relayed;
12253                assert!(
12254                    task.await.unwrap().is_empty(),
12255                    "the open is answered by commit"
12256                );
12257                let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
12258                Bound {
12259                    target,
12260                    client,
12261                    client_rx,
12262                    channel,
12263                    epoch,
12264                    bind,
12265                }
12266            }
12267
12268            fn live(&self, route: &Bound) -> bool {
12269                matches!(
12270                    self.forwarding
12271                        .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
12272                        .unwrap(),
12273                    DataRoute::Client(DataRouteState::Bound(_))
12274                )
12275            }
12276        }
12277
12278        struct Relayed {
12279            target: String,
12280            client: RouteCtx,
12281            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12282            task: tokio::task::JoinHandle<Vec<Frame>>,
12283            bind: Frame,
12284        }
12285
12286        struct Bound {
12287            target: String,
12288            client: RouteCtx,
12289            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12290            channel: u16,
12291            epoch: u32,
12292            bind: Frame,
12293        }
12294
12295        impl Bound {
12296            /// The reason of the `route.closed` this client was sent, after
12297            /// checking it also got a GOODBYE on exactly this route.
12298            fn closed_reason(&mut self) -> RouteCloseReason {
12299                let mut reason = None;
12300                let mut goodbye = false;
12301                while let Ok(outbound) = self.client_rx.try_recv() {
12302                    let frame = outbound.frame;
12303                    match frame.header.ty {
12304                        FrameType::Goodbye => {
12305                            assert_eq!(
12306                                (frame.header.channel, frame.header.epoch),
12307                                (self.channel, self.epoch)
12308                            );
12309                            goodbye = true;
12310                        }
12311                        FrameType::Push => {
12312                            let ClientControlPush::RouteClosed {
12313                                reason: r,
12314                                module_id,
12315                                ..
12316                            } = serde_json::from_slice(&frame.body).unwrap()
12317                            else {
12318                                panic!("unexpected push");
12319                            };
12320                            assert_eq!(module_id, self.target);
12321                            reason = Some(r);
12322                        }
12323                        other => panic!("unexpected frame {other:?}"),
12324                    }
12325                }
12326                assert!(goodbye, "the client is sent a GOODBYE for the closed route");
12327                reason.expect("the client is told why the route closed")
12328            }
12329
12330            fn untouched(&mut self) -> bool {
12331                self.client_rx.try_recv().is_err()
12332            }
12333
12334            fn stamp(&self) -> Option<ScopeStamp> {
12335                match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
12336                    ModuleControlRequest::RouteBind { scope, .. } => scope,
12337                    other => panic!("expected a route.bind, got {other:?}"),
12338                }
12339            }
12340        }
12341
12342        #[tokio::test]
12343        async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
12344        ) {
12345            let mut rig = rig().await;
12346            let mut record = session(1);
12347            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
12348            record.child_owners = vec![Principal::Reserved {
12349                module_id: MAGIC.to_string(),
12350            }];
12351            rig.sync(vec![record]).await;
12352            let scope = || Some(rig_selector("s", Some(1)));
12353
12354            // Admitted: the owner, a bare carrier to any module, a targeted
12355            // carrier to its listed module.
12356            rig.bound(Some(OWNER), PLEXUS, scope()).await;
12357            rig.bound(Some(AFT), OTHER, scope()).await;
12358            rig.bound(Some(BROCA), PLEXUS, scope()).await;
12359
12360            // Refused scope_not_carrier: a targeted carrier to an unlisted
12361            // module, a module that is not listed at all (a child owner is not
12362            // a carrier), and a direct key-holder.
12363            for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
12364                assert_eq!(
12365                    rig.refused(opener, target, scope()).await,
12366                    error_codes::SCOPE_NOT_CARRIER,
12367                    "{opener:?} -> {target}"
12368                );
12369            }
12370        }
12371
12372        fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
12373            ScopeSelector {
12374                owner: Principal::Reserved {
12375                    module_id: OWNER.to_string(),
12376                },
12377                scope_ref: scope_ref.to_string(),
12378                scope_epoch,
12379            }
12380        }
12381
12382        #[tokio::test]
12383        async fn an_open_without_an_epoch_is_refused_the_owners_included() {
12384            let mut rig = rig().await;
12385            rig.sync(vec![session(1)]).await;
12386            for opener in [OWNER, AFT] {
12387                assert_eq!(
12388                    rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
12389                        .await,
12390                    error_codes::SCOPE_EPOCH_REQUIRED,
12391                    "{opener}"
12392                );
12393            }
12394        }
12395
12396        #[tokio::test]
12397        async fn admission_separates_not_synced_not_live_and_ended() {
12398            let mut rig = rig().await;
12399            // Before the configured owner's first sync: retryable.
12400            let code = rig
12401                .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12402                .await;
12403            assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
12404            assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
12405
12406            // An owner that is not configured will never sync: terminal.
12407            let ghost = ScopeSelector {
12408                owner: Principal::Reserved {
12409                    module_id: "ghost".to_string(),
12410                },
12411                scope_ref: "s".to_string(),
12412                scope_epoch: Some(1),
12413            };
12414            assert_eq!(
12415                rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
12416                error_codes::SCOPE_NOT_LIVE
12417            );
12418
12419            rig.sync(vec![session(2)]).await;
12420            assert_eq!(
12421                rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
12422                    .await,
12423                error_codes::SCOPE_NOT_LIVE
12424            );
12425            for epoch in [1, 3] {
12426                assert_eq!(
12427                    rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
12428                        .await,
12429                    error_codes::SCOPE_ENDED,
12430                    "epoch {epoch}"
12431                );
12432            }
12433            // Control: the live epoch is admitted.
12434            rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
12435                .await;
12436        }
12437
12438        #[tokio::test]
12439        async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
12440            let mut rig = rig().await;
12441            rig.sync(vec![session(1)]).await;
12442            let route = rig
12443                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12444                .await;
12445            let stamp = route.stamp().expect("a scoped bind carries the stamp");
12446            assert_eq!(stamp.scope_ref, "s");
12447            assert_eq!(stamp.scope_epoch, 1);
12448            assert_eq!(stamp.kind, ScopeKind::Head);
12449            assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
12450            assert!(stamp.attributes.delegates);
12451            assert!(stamp.owner_authorized);
12452            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
12453            assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
12454
12455            // broca owns a scope of its own on its own module connection; it is
12456            // not in scope_authority_owners, so its stamp is not authorized.
12457            let (broca, mut broca_rx) = wide_ctx(50);
12458            hello_via_sink(
12459                &rig.handler,
12460                &broca,
12461                &mut broca_rx,
12462                hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
12463            )
12464            .await;
12465            sync(&rig.handler, &broca, 1, vec![head("b", 1)])
12466                .await
12467                .unwrap();
12468            let own = ScopeSelector {
12469                owner: Principal::Reserved {
12470                    module_id: BROCA.to_string(),
12471                },
12472                scope_ref: "b".to_string(),
12473                scope_epoch: Some(1),
12474            };
12475            let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
12476            assert!(!route.stamp().unwrap().owner_authorized);
12477        }
12478
12479        /// The owner's sync lands between admission and the module's ack. The
12480        /// open is refused by name, the module's other routes stay up, and the
12481        /// reserved pair is released. Changed content is retryable; an ended
12482        /// scope is not.
12483        #[tokio::test]
12484        async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
12485            let mut rig = rig().await;
12486            rig.sync(vec![session(1)]).await;
12487            let mut cotenant = rig
12488                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
12489                .await;
12490
12491            let mut changed = session(1);
12492            changed.child_owners.push(Principal::Reserved {
12493                module_id: MAGIC.to_string(),
12494            });
12495            let mut ended = None;
12496            for (code, next) in [
12497                (error_codes::SCOPE_CHANGED, vec![changed]),
12498                (error_codes::SCOPE_ENDED, Vec::new()),
12499            ] {
12500                let relayed = rig
12501                    .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12502                    .await;
12503                let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
12504                ended = Some(next.is_empty());
12505                rig.sync(next).await;
12506                rig.ack(&relayed).await;
12507                let replies = relayed.task.await.unwrap();
12508                assert_eq!(replies.len(), 1, "{replies:?}");
12509                assert_eq!(parse_error(&replies[0])["code"], code);
12510                assert_eq!(
12511                    subc_protocol::error_codes::is_retryable_route_open(code),
12512                    code == error_codes::SCOPE_CHANGED
12513                );
12514                assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
12515                // The module is told to drop just the binding it created.
12516                let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
12517                // Collected, because ending the scope also closes the co-tenant
12518                // route, whose GOODBYE comes first.
12519                let mut goodbyes = Vec::new();
12520                while let Ok(outbound) = plexus_rx.try_recv() {
12521                    assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
12522                    goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
12523                }
12524                assert!(
12525                    goodbyes.contains(&(bind_channel, bind_epoch)),
12526                    "{goodbyes:?}"
12527                );
12528                assert!(rig
12529                    .handler
12530                    .registry
12531                    .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
12532                    .unwrap()
12533                    .is_some());
12534            }
12535            assert_eq!(ended, Some(true));
12536            // The co-tenant stayed up through the change, and closed only when
12537            // the scope ended, by the drain rule rather than by the commit.
12538            assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
12539        }
12540
12541        /// Each row of the drain table on one set of routes: the owner's, a
12542        /// bare carrier's, and a targeted carrier's to each of its targets.
12543        #[tokio::test]
12544        async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
12545            struct Case {
12546                name: &'static str,
12547                change: fn(&mut ScopeRecord),
12548                /// Closed routes by index: owner->plexus, aft->plexus,
12549                /// broca->plexus, broca->other.
12550                closed: [Option<RouteCloseReason>; 4],
12551            }
12552            use RouteCloseReason::*;
12553            let cases = [
12554                Case {
12555                    name: "a carrier entry removed",
12556                    change: |r| {
12557                        r.carriers.retain(|c| {
12558                            c.principal
12559                                != Principal::Reserved {
12560                                    module_id: AFT.to_string(),
12561                                }
12562                        })
12563                    },
12564                    closed: [None, Some(ScopeCarrierRemoved), None, None],
12565                },
12566                Case {
12567                    name: "a target removed from a carrier",
12568                    change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
12569                    closed: [None, None, None, Some(ScopeCarrierRemoved)],
12570                },
12571                Case {
12572                    name: "a bare carrier narrowed to targets",
12573                    change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
12574                    closed: [None, Some(ScopeCarrierRemoved), None, None],
12575                },
12576                Case {
12577                    name: "delegates turned off",
12578                    change: |r| r.attributes.delegates = false,
12579                    closed: [Some(ScopeDelegationChanged); 4],
12580                },
12581                Case {
12582                    name: "agent_id changed",
12583                    change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
12584                    closed: [Some(ScopeDelegationChanged); 4],
12585                },
12586                Case {
12587                    name: "a carrier added, child owners changed, the record re-sent",
12588                    change: |r| {
12589                        r.carriers.push(carrier(MAGIC, None));
12590                        r.child_owners.push(Principal::Reserved {
12591                            module_id: MAGIC.to_string(),
12592                        });
12593                    },
12594                    closed: [None; 4],
12595                },
12596                Case {
12597                    name: "a target added",
12598                    change: |r| {
12599                        r.carriers[1]
12600                            .targets
12601                            .as_mut()
12602                            .unwrap()
12603                            .push("third".to_string())
12604                    },
12605                    closed: [None; 4],
12606                },
12607                Case {
12608                    name: "delegates turned on",
12609                    change: |r| r.attributes.delegates = true,
12610                    closed: [None; 4],
12611                },
12612            ];
12613            for case in cases {
12614                let mut rig = rig().await;
12615                rig.sync(vec![session(1)]).await;
12616                let scope = || Some(rig_selector("s", Some(1)));
12617                let mut routes = [
12618                    rig.bound(Some(OWNER), PLEXUS, scope()).await,
12619                    rig.bound(Some(AFT), PLEXUS, scope()).await,
12620                    rig.bound(Some(BROCA), PLEXUS, scope()).await,
12621                    rig.bound(Some(BROCA), OTHER, scope()).await,
12622                ];
12623                let mut record = session(1);
12624                (case.change)(&mut record);
12625                rig.sync(vec![record]).await;
12626                for (index, expected) in case.closed.iter().enumerate() {
12627                    let route = &mut routes[index];
12628                    match expected {
12629                        Some(reason) => {
12630                            assert!(!rig.live(route), "{}: route {index} still live", case.name);
12631                            assert_eq!(
12632                                route.closed_reason(),
12633                                *reason,
12634                                "{}: route {index}",
12635                                case.name
12636                            );
12637                        }
12638                        None => {
12639                            assert!(rig.live(route), "{}: route {index} closed", case.name);
12640                            assert!(
12641                                route.untouched(),
12642                                "{}: route {index} was told something",
12643                                case.name
12644                            );
12645                        }
12646                    }
12647                }
12648            }
12649        }
12650
12651        #[tokio::test]
12652        async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
12653            // Removed, and replaced by a higher epoch.
12654            for next in [Vec::new(), vec![session(2)]] {
12655                let mut rig = rig().await;
12656                rig.sync(vec![session(1)]).await;
12657                let mut route = rig
12658                    .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12659                    .await;
12660                rig.sync(next).await;
12661                assert!(!rig.live(&route));
12662                assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
12663            }
12664
12665            // A child whose parent ends: its routes close as parent-ended, the
12666            // child stays live, and routes under the parent close as ended.
12667            let mut rig = rig().await;
12668            let mut child = session(1);
12669            child.scope_ref = "child".to_string();
12670            child.kind = ScopeKind::Worker;
12671            child.parent = Some(ScopeParent {
12672                owner: Principal::Reserved {
12673                    module_id: OWNER.to_string(),
12674                },
12675                scope_ref: "s".to_string(),
12676                scope_epoch: 1,
12677            });
12678            rig.sync(vec![session(1), child.clone()]).await;
12679            let mut child_route = rig
12680                .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
12681                .await;
12682            assert_eq!(
12683                child_route.stamp().unwrap().parent_state,
12684                Some(ParentState::Linked)
12685            );
12686            rig.sync(vec![child]).await;
12687            assert!(!rig.live(&child_route));
12688            assert_eq!(
12689                child_route.closed_reason(),
12690                RouteCloseReason::ScopeParentEnded
12691            );
12692        }
12693
12694        #[tokio::test]
12695        async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
12696        ) {
12697            let mut rig = rig().await;
12698            rig.sync(vec![session(1)]).await;
12699            let mut route = rig
12700                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12701                .await;
12702            let before = rig.forwarding.published_scope_tag(OWNER, "s");
12703            rig.sync(vec![session(1)]).await;
12704            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
12705            assert!(rig.live(&route) && route.untouched());
12706
12707            // A call in flight on the route when another carrier is added. A
12708            // forwarded REQUEST holds one credit on the route's flow until the
12709            // module answers; the router takes it exactly like this.
12710            let DataRoute::Client(DataRouteState::Bound(binding)) = rig
12711                .forwarding
12712                .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
12713                .unwrap()
12714            else {
12715                panic!("the route is bound");
12716            };
12717            binding.flow.acquire_tagged(9, false).await.unwrap();
12718            let mut widened = session(1);
12719            widened.carriers.push(carrier(MAGIC, None));
12720            rig.sync(vec![widened]).await;
12721            assert!(rig.live(&route) && route.untouched());
12722            let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
12723            assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
12724            // The call's credit is still held on an open flow, so its answer
12725            // will be delivered: closing the route would have closed the flow.
12726            assert_eq!(binding.flow.in_flight(), 1);
12727            binding
12728                .flow
12729                .acquire_tagged(10, false)
12730                .await
12731                .expect("the flow is still open");
12732        }
12733
12734        /// A swap's superseded endpoint keeps its routes until drained; ending
12735        /// the scope closes them there too.
12736        #[tokio::test]
12737        async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
12738            let mut rig = rig().await;
12739            rig.sync(vec![session(1)]).await;
12740            let mut on_incumbent = rig
12741                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12742                .await;
12743
12744            // Swap plexus: register a candidate and cut over, leaving the
12745            // incumbent superseded with the route still on it.
12746            let (candidate, _candidate_rx) = wide_ctx(9);
12747            let registration = rig
12748                .handler
12749                .registry
12750                .register_candidate_with_control_ops(
12751                    manifest(PLEXUS, PROTOCOL_VERSION),
12752                    PROTOCOL_VERSION,
12753                    candidate.connection_id,
12754                    module_baseline_control_ops(),
12755                )
12756                .unwrap();
12757            rig.forwarding
12758                .register_candidate_module_connection(
12759                    candidate.connection_id,
12760                    PLEXUS.to_string(),
12761                    PROTOCOL_VERSION,
12762                    manifest_concurrency(&registration.manifest),
12763                    candidate.egress.clone(),
12764                )
12765                .unwrap();
12766            rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
12767            rig.handler
12768                .registry
12769                .promote_candidate(PLEXUS)
12770                .unwrap()
12771                .unwrap();
12772            assert!(rig.live(&on_incumbent), "cutover alone does not drain");
12773
12774            rig.sync(Vec::new()).await;
12775            assert!(!rig.live(&on_incumbent));
12776            assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
12777            let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
12778            let goodbye = incumbent_rx
12779                .try_recv()
12780                .expect("the superseded endpoint is told")
12781                .frame;
12782            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12783        }
12784    }
12785}
12786
12787#[cfg(test)]
12788mod concurrency_default_exposure_tests {
12789    use super::*;
12790
12791    fn hello_body(role_json: &str) -> Vec<u8> {
12792        format!(
12793            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
12794        )
12795        .into_bytes()
12796    }
12797
12798    fn manifest_from(body: &[u8]) -> ModuleManifest {
12799        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
12800        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
12801            .expect("manifest parses")
12802    }
12803
12804    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
12805
12806    #[test]
12807    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
12808        let body = hello_body(&format!(
12809            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
12810        ));
12811        let manifest = manifest_from(&body);
12812        // Precondition: serde really resolved it to the default, so the typed
12813        // manifest alone cannot answer the question this probe exists for.
12814        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
12815        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
12816    }
12817
12818    #[test]
12819    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
12820        let body = hello_body(&format!(
12821            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
12822        ));
12823        let manifest = manifest_from(&body);
12824        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
12825        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
12826    }
12827
12828    #[test]
12829    fn non_management_roles_are_never_reported() {
12830        let body = hello_body(
12831            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
12832        );
12833        let manifest = manifest_from(&body);
12834        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
12835    }
12836}