Skip to main content

subc_daemon/
control.rs

1use std::{
2    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3    fmt,
4    path::{Path, PathBuf},
5    sync::{Arc, Mutex, RwLock},
6    time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11    ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12    ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13    ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14    ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15    StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16    SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17    SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20    error_codes,
21    manifest::{
22        validate_hello_capability_grammar, validate_hello_self_signal_declarations,
23        CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
24        ProviderRole,
25    },
26    scope::{
27        ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
28        CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_DESCRIBE_OP, SCOPE_SYNC_OP,
29    },
30    session::{
31        validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
32        ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
33        MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE, ROLE_VERSIONS_FIELD,
34    },
35    BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
36    Priority, RouteTarget, PROTOCOL_VERSION,
37};
38use tokio::time::{timeout_at, Instant};
39use tracing::{debug, info, warn};
40
41use crate::{
42    capability_requirements::{
43        log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
44        CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
45        RuntimeModule,
46    },
47    daemon_config::RestartRequiredSection,
48    forwarding::{
49        CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
50        ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
51        PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
52    },
53    observability::{
54        ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
55    },
56    provenance::{
57        process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
58    },
59    registry::{ChannelState, ConnectionId, Registry, RegistryError},
60    router::{RouteCtx, RouterError},
61    scopes::{BoundScope, HelloLaunchNonces, ScopeTable},
62    server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
63    stderr_tail::{CaptureState, TailEntry},
64    supervise::{
65        validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
66        SupervisorHandle, SwapHelloAdmission,
67    },
68    ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
69};
70
71/// Lowest envelope version this subc build will negotiate.
72///
73/// Module HELLO negotiation is exact: peers must use the daemon's locked
74/// protocol version. Older and newer peers receive `version_unsupported` and
75/// are not registered.
76pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
77
78const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
79const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
80const CAP_PING_PONG: &str = "ping_pong_v1";
81const CAP_SESSION_ATTACH: &str = "session_attach_v1";
82const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
83
84const SUBC_CONTROL_OPS: &[&str] = &[
85    ops::SERVER_DESCRIBE,
86    ops::CATALOG_LIST,
87    ops::ROUTE_OPEN,
88    ops::ROUTE_POLL,
89    ops::ROUTE_CLOSING,
90    ops::ROUTE_CLOSED,
91    ops::SUPERVISOR_LIST,
92    ops::SUPERVISOR_RESTART,
93    ops::SUPERVISOR_SWAP,
94    ops::SUPERVISOR_RELOAD,
95    ops::SUPERVISOR_RESCAN,
96    ops::SUPERVISOR_RELEASE_RESERVED,
97    ops::SUPERVISOR_SET_ENABLED,
98    ops::SUPERVISOR_HEALTH_PROBE,
99    ops::SUPERVISOR_HEALTH,
100    ops::SUPERVISOR_STDERR_TAIL,
101    ops::SUPERVISOR_TERMINALS,
102    ops::SUPERVISOR_ROUTES,
103    ops::SUPERVISOR_PROVENANCE,
104    ops::SUPERVISOR_SPAWN_SNAPSHOT,
105    ops::SUPERVISOR_SPAWN_SUBSCRIBE,
106];
107
108const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
109    MODULE_TO_SUBC_OP_CATALOG_UPDATE,
110    "supervisor.live_roots",
111    SCOPE_SYNC_OP,
112    SCOPE_DESCRIBE_OP,
113];
114
115/// Module-originated ops the daemon answers but does not advertise in
116/// `HELLO_ACK`. Empty today; an op is served from here while the feature it
117/// belongs to is incomplete, so no module is told it works before it does.
118const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
119
120const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
121
122/// How long subc waits for a module to ack a relayed route.bind before returning
123/// `module_timeout`. The ack waits on the module's own configure, which for AFT
124/// includes a synchronous bounded project walk (up to ~20k files) plus gitignore
125/// and DB-open work — on a cold page cache or a large repo that legitimately
126/// exceeds a couple of seconds. The default is generous because rejecting a VALID
127/// bind is far worse than waiting on a slow one; a consumer that wants a tighter
128/// bound retries the bind itself (the sanctioned warm-bind-retry pattern).
129pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
130
131/// How many CONSECUTIVE full-budget relay timeouts against one target module
132/// open that module's bind-relay breaker.
133///
134/// Three, so that the breaker is NOT REACHABLE INSIDE ONE CLIENT CALL. Both
135/// SDKs default to a 30s request deadline and the relay budget defaults to 12s,
136/// so three consecutive full-budget timeouts take ~36s to observe: every client
137/// whose open contributed to opening the breaker had already given up on its
138/// own. That is what makes opening the breaker unable to turn a call that would
139/// have succeeded into a refusal — it can only make an already-failing module
140/// fail faster.
141///
142/// Two would be reachable inside one default deadline. One would convict a
143/// module on a single cold-cache bind, which is exactly the valid-but-slow case
144/// `DEFAULT_ROUTE_BIND_RELAY_TIMEOUT`'s own doc comment exists to protect.
145pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
146
147/// How long a module's bind-relay breaker stays open before exactly one
148/// `route.open` is let through as a probe.
149///
150/// Bounded BELOW by the relay budget: a cooldown at or under the 12s budget
151/// re-pays a full-budget stall almost continuously, and the breaker stops being
152/// a saving worth its own state. Bounded ABOVE by the SDKs' 30s default request
153/// deadline: a client that starts retrying after the module recovers has to get
154/// a probe opportunity inside its own deadline, or the breaker converts a
155/// recovered module into a failed call — the failure it exists to prevent,
156/// pointed the other way.
157///
158/// 20s sits between those with room on both sides, and it caps what a wedged
159/// module can cost at one full-budget wait per 20s ACROSS THE WHOLE DAEMON
160/// rather than one per `route.open` per connection. The stall that motivated
161/// this, with its measurements, is written up in
162/// `docs/designs/route-open-head-of-line.md`: 268 opens against one module each
163/// waited the whole budget out.
164pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
165
166const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
167const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
168
169fn reload_verdict(
170    configured: &Path,
171    spawned_from: Option<&Path>,
172    image: subc_control::RunningImageAgreement,
173) -> PendingReloadVerdict {
174    let path = match spawned_from {
175        Some(spawned_from) if configured == spawned_from => ReloadPathAgreement::Match,
176        Some(spawned_from) => ReloadPathAgreement::Mismatch {
177            configured: configured.to_path_buf(),
178            spawned_from: spawned_from.to_path_buf(),
179        },
180        None => ReloadPathAgreement::Unavailable {
181            reason: if matches!(
182                image,
183                subc_control::RunningImageAgreement::Unavailable {
184                    reason: subc_control::RunningImageUnavailableReason::NotRunning
185                }
186            ) {
187                ReloadPathUnavailableReason::NotRunning
188            } else {
189                ReloadPathUnavailableReason::SpawnedPathUnavailable
190            },
191        },
192    };
193    PendingReloadVerdict { path, image }
194}
195
196#[derive(Clone)]
197struct DaemonProvenanceFacts {
198    build: DaemonBuildProvenance,
199    pid: Option<u32>,
200    started_at_ms: Option<u64>,
201    start_clock: Option<crate::clock::StartClock>,
202    executable_path: Option<PathBuf>,
203    executable_identity: Option<SpawnedFileIdentity>,
204    process_start_time: Option<u64>,
205    probe: ExecutableIdentityProbe,
206}
207
208impl Default for DaemonProvenanceFacts {
209    fn default() -> Self {
210        Self {
211            build: DaemonBuildProvenance {
212                build_git_sha: None,
213                build_lock_digest: None,
214            },
215            pid: None,
216            started_at_ms: None,
217            start_clock: None,
218            executable_path: None,
219            executable_identity: None,
220            process_start_time: None,
221            probe: ExecutableIdentityProbe::default(),
222        }
223    }
224}
225
226#[derive(Debug, Clone)]
227struct SupervisorRescanContext {
228    supervisor: Supervisor,
229    config_path: PathBuf,
230    configured_port: Option<u16>,
231    storage_config: Option<crate::daemon_config::StorageConfig>,
232    admission_facts_carrier_module_id: Option<String>,
233    admission_facts_targets: Option<Vec<String>>,
234    scope_authority_owners: Vec<String>,
235}
236
237/// Refusal labels passed to `observe_route_open_refusal` that mean the target
238/// module is not serving right now, and so open or extend an outage in the
239/// route outage tracker. Every one of them is only reachable after the target
240/// was found in the registry, which is what keeps an arbitrary client-chosen
241/// id from ever creating tracker state.
242///
243/// Deliberately absent: `not_registered` and `removed` (the id may be
244/// anything a client sent, and a removed module is gone on purpose),
245/// `protocol_none` (such a module never serves routes, so nothing is out),
246/// `role_not_provided`, `op_not_allowed`, `bad_consumer_identity`, the
247/// capability and admission-facts refusals (they refuse the caller, not a
248/// module outage), and `relay_reservation_failed` (its code ranges over
249/// capacity limits as well as a vanished connection). Capacity, breaker,
250/// relay-timeout and module-rejection refusals do not pass through that
251/// function at all; the breaker logs its own transitions.
252///
253/// The two not-serving refusals that bypass that function record themselves
254/// at their own sites: `supervised_not_registered` and `declared_not_ready`.
255/// `required_capability_unprovided` is not tracked: the module itself is up,
256/// and the outage belongs to the missing provider.
257const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
258    "reloading",
259    "supervisor_not_live",
260    "registration_not_active",
261    "no_forwarding_connection",
262    "relay_send_failed",
263];
264
265/// Real channel-0 control handler for subc itself.
266#[derive(Clone)]
267pub struct ControlHandler {
268    registry: Arc<Registry>,
269    forwarding: Arc<ForwardingTable>,
270    process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
271    supervisor: SupervisorHandle,
272    subc_capabilities: Arc<[String]>,
273    /// Daemon-wide route.bind relay budget. Used as the fallback when the
274    /// target module has no per-module override in
275    /// `route_bind_relay_timeouts`.
276    route_bind_relay_timeout: Duration,
277    /// Per-module route.bind relay budget overrides, keyed by module id. When
278    /// `handle_route_open` resolves the deadline for a target module, a
279    /// per-module entry wins over the daemon-wide value above.
280    route_bind_relay_timeouts: BTreeMap<String, Duration>,
281    /// Per-target-module bind-relay breaker state. Shared with the forwarding
282    /// table, which is where a new module connection resets it.
283    route_bind_breakers: RouteBindBreakers,
284    /// Live relay admissions keyed by target module. Shared through the
285    /// forwarding table so cloned or separately built handlers enforce one cap.
286    route_bind_concurrency: RouteBindConcurrency,
287    /// Start and end of each module's not-serving period as seen by
288    /// `route.open`, so an outage gets one line at each edge instead of only
289    /// the per-refusal INFO lines. Taken from the forwarding table, so every
290    /// handler built over one table shares it.
291    route_outages: Arc<crate::route_outage::RouteOutageTracker>,
292    /// Consecutive relay timeouts that open a module's breaker.
293    route_bind_breaker_threshold: u32,
294    /// How long a breaker stays open before one probe is admitted.
295    route_bind_breaker_cooldown: Duration,
296    health_probe_timeout: Duration,
297    /// Central storage policy. When set, each registering module receives its
298    /// resolved storage descriptor in HELLO_ACK; `None` leaves the field absent.
299    storage_config: Option<crate::daemon_config::StorageConfig>,
300    /// The machine id established at boot, served on every HELLO_ACK and on
301    /// `server.describe`. Fixed for the daemon's lifetime: `ck machine adopt`
302    /// changes the file, never this value. `None` serves no id.
303    machine_id: Option<crate::machine_id::MachineId>,
304    admission_facts_carrier_module_id: Option<String>,
305    admission_facts_targets: Option<Vec<String>>,
306    /// Scope records with their sync authorities and tombstones; see
307    /// `crate::scopes`. Shared by clones of this handler, so every connection
308    /// reads and writes one table.
309    scopes: Arc<RwLock<ScopeTable>>,
310    /// The configured `scope_authority_owners`, kept so a rescan can report a
311    /// changed value as needing a daemon restart; rescan never applies it.
312    scope_authority_owners: Vec<String>,
313    /// The launch nonce each module connection presented at HELLO, which is how
314    /// a `scope.sync` is matched to the owner's current launch.
315    hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
316    rescan: Option<SupervisorRescanContext>,
317    connected_clients: ConnectedClients,
318    counters: DaemonCounters,
319    capability_evaluator: Arc<CapabilityRequirementEvaluator>,
320    daemon_provenance: DaemonProvenanceFacts,
321    #[cfg(test)]
322    control_dispatch_delay: Option<Duration>,
323    #[cfg(test)]
324    provenance_probe_override: Option<subc_control::RunningImageAgreement>,
325}
326
327impl fmt::Debug for ControlHandler {
328    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
329        f.debug_struct("ControlHandler")
330            .field("registry", &self.registry)
331            .field("forwarding", &self.forwarding)
332            .field("process_liveness", &self.process_liveness.is_some())
333            .field("supervisor", &self.supervisor)
334            .field("subc_capabilities", &self.subc_capabilities)
335            .finish()
336    }
337}
338
339struct RouteOpenRequest {
340    target: RouteTarget,
341    identity: BindIdentity,
342    consumer_identity: Option<ConsumerIdentity>,
343    consumer_capabilities: Option<Vec<String>>,
344    role_versions: Option<BTreeMap<String, String>>,
345    admission_facts: Option<serde_json::Value>,
346    scope: Option<ScopeSelector>,
347}
348
349struct RouteBindReservationGuard {
350    forwarding: Arc<ForwardingTable>,
351    endpoint: ModuleEndpointId,
352    relay_corr: u64,
353    armed: bool,
354}
355
356struct ModuleControlRpcGuard {
357    forwarding: Arc<ForwardingTable>,
358    endpoint: ModuleEndpointId,
359    corr: u64,
360    armed: bool,
361}
362
363impl ModuleControlRpcGuard {
364    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
365        Self {
366            forwarding,
367            endpoint,
368            corr,
369            armed: true,
370        }
371    }
372
373    fn disarm(&mut self) {
374        self.armed = false;
375    }
376}
377
378impl Drop for ModuleControlRpcGuard {
379    fn drop(&mut self) {
380        if self.armed {
381            let _ = self
382                .forwarding
383                .cancel_module_control_rpc(self.endpoint, self.corr);
384        }
385    }
386}
387
388impl RouteBindReservationGuard {
389    fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
390        Self {
391            forwarding,
392            endpoint,
393            relay_corr,
394            armed: true,
395        }
396    }
397
398    fn release_and_disarm(&mut self) {
399        if !self.armed {
400            return;
401        }
402        if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
403            self.endpoint,
404            self.relay_corr,
405            RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
406        ) {
407            send_goodbye_target_best_effort(
408                &self.forwarding.counters(),
409                &target,
410                "canceled route.bind",
411            );
412        }
413        self.armed = false;
414    }
415
416    fn disarm(&mut self) {
417        self.armed = false;
418    }
419}
420
421impl Drop for RouteBindReservationGuard {
422    fn drop(&mut self) {
423        self.release_and_disarm();
424    }
425}
426
427/// Per-target-module circuit breaker around the `route.bind` relay.
428///
429/// The connection reader is serial per connection, so a module whose `on_bind`
430/// sits on the ack blocks every LATER frame on the connections that call it,
431/// including calls to unrelated modules. This does not make any module's bind
432/// fast; it stops the daemon paying the full budget again and again for a
433/// condition it has already observed.
434///
435/// State is keyed by TARGET MODULE and shared by every connection: a wedged
436/// module wedges everyone, so what one connection learned should protect the
437/// rest.
438///
439/// THE MAP IS EMPTY WHILE THE FLEET IS HEALTHY. An entry appears only when a
440/// relay to that module has actually timed out, and is removed again when a
441/// relay is accepted or the module reconnects, so it cannot grow with traffic
442/// or with modules that behave.
443///
444/// # Why a `std` mutex here is not the head-of-line defect again
445///
446/// Acquisition never awaits. The critical section is a hash lookup plus a few
447/// integer updates, with no I/O and no `.await` inside it, so a reader task
448/// cannot be descheduled behind it the way it can behind
449/// `tokio::sync::Mutex::lock().await` or a semaphore permit. It is the same
450/// primitive, held for the same kind of work, as the refusal counter this very
451/// path already increments.
452///
453/// It is also NOT on the data-plane splice path: only `route.open` and module
454/// registration touch it, so bound-route frames gain no state check and no
455/// contention.
456#[derive(Debug, Clone, Default)]
457pub(crate) struct RouteBindBreakers {
458    modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
459}
460
461#[derive(Debug, Clone, Default)]
462pub(crate) struct RouteBindConcurrency {
463    modules: Arc<Mutex<HashMap<String, usize>>>,
464}
465
466struct RouteBindConcurrencyGuard {
467    concurrency: RouteBindConcurrency,
468    module_id: String,
469}
470
471impl RouteBindConcurrency {
472    /// Admit without waiting. Waiting here would move the bind stall from the
473    /// module reply to a semaphore and restore reader head-of-line blocking.
474    fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
475        let mut modules = self
476            .modules
477            .lock()
478            .expect("route.bind concurrency mutex poisoned");
479        let in_flight = modules.entry(module_id.to_string()).or_default();
480        if *in_flight >= limit {
481            return Err(*in_flight);
482        }
483        *in_flight += 1;
484        Ok(RouteBindConcurrencyGuard {
485            concurrency: self.clone(),
486            module_id: module_id.to_string(),
487        })
488    }
489}
490
491impl Drop for RouteBindConcurrencyGuard {
492    fn drop(&mut self) {
493        let mut modules = self
494            .concurrency
495            .modules
496            .lock()
497            .expect("route.bind concurrency mutex poisoned");
498        let remove = {
499            let in_flight = modules
500                .get_mut(&self.module_id)
501                .expect("admitted route.bind has a concurrency entry");
502            *in_flight -= 1;
503            *in_flight == 0
504        };
505        if remove {
506            modules.remove(&self.module_id);
507        }
508    }
509}
510
511#[derive(Debug, Default)]
512struct ModuleBreakerState {
513    /// Relay timeouts observed with no accepted relay in between.
514    consecutive_timeouts: u32,
515    /// `Some` while the breaker is open: the instant the cooldown expires and
516    /// the next arrival may probe. `None` means closed.
517    cooldown_until: Option<Instant>,
518    /// A half-open probe has been admitted and has not settled yet. This is
519    /// what makes the probe EXACTLY ONE: the flag is set under the same lock
520    /// that read the cooldown, so concurrent opens arriving at the moment the
521    /// cooldown expires cannot all decide that they are the probe.
522    probe_in_flight: bool,
523}
524
525/// What the breaker decided for one `route.open`, before any relay work.
526enum RouteBindAdmission<'a> {
527    Admitted {
528        guard: RouteBindBreakerGuard<'a>,
529        /// This open is the single half-open probe, so the transition is worth
530        /// one log line.
531        probe: bool,
532    },
533    Refused {
534        consecutive_timeouts: u32,
535        /// What is left of the cooldown. Zero when the refusal is because the
536        /// one probe is already in flight rather than because the cooldown has
537        /// not elapsed.
538        retry_in: Duration,
539        probe_in_flight: bool,
540    },
541}
542
543/// An outstanding admission, which must be told how its relay settled.
544///
545/// `Drop` settles it as inconclusive, so an early return between admission and
546/// the relay -- or the whole handler being cancelled when the client
547/// disconnects -- releases a half-open probe slot instead of leaving the
548/// breaker wedged half-open with no further probes.
549struct RouteBindBreakerGuard<'a> {
550    breakers: RouteBindBreakers,
551    module_id: &'a str,
552    settled: bool,
553}
554
555impl RouteBindBreakerGuard<'_> {
556    /// The module answered within the budget and took the bind. THE ONLY
557    /// OUTCOME THAT CLEARS THE COUNT. Returns true when this closed an open
558    /// breaker, which is a transition worth logging.
559    fn record_accepted(&mut self) -> bool {
560        self.settled = true;
561        self.breakers.record_accepted(self.module_id)
562    }
563
564    /// The relay burned the whole budget with no answer. THE ONLY ARM THAT
565    /// COUNTS TOWARD OPENING.
566    fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
567        self.settled = true;
568        self.breakers
569            .record_timeout(self.module_id, threshold, cooldown)
570    }
571
572    /// Everything else: the module REJECTED the bind, its connection went away
573    /// mid-relay, or the waiter was cancelled.
574    ///
575    /// None of these is evidence that a module is slow, and each already has
576    /// its own refusal with its own code. A module that rejects a bind in
577    /// microseconds is healthy and must never be convicted for it; a module
578    /// that died has said nothing about the module that replaces it. So these
579    /// neither increment nor reset the count -- they only release a probe slot.
580    fn record_inconclusive(&mut self) {
581        self.settled = true;
582        self.breakers.record_inconclusive(self.module_id);
583    }
584}
585
586impl Drop for RouteBindBreakerGuard<'_> {
587    fn drop(&mut self) {
588        if !self.settled {
589            self.breakers.record_inconclusive(self.module_id);
590        }
591    }
592}
593
594/// The breaker moved to open, reported so the caller can log it outside the
595/// lock. Opening is rare and load-bearing; the refusals that follow are
596/// frequent and are counted rather than logged.
597struct BreakerOpened {
598    consecutive_timeouts: u32,
599    /// True when a failed probe re-opened an already-open breaker, which reads
600    /// very differently in a log from a first opening.
601    reopened_after_probe: bool,
602}
603
604impl RouteBindBreakers {
605    fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
606        self.modules
607            .lock()
608            .expect("route.bind breaker mutex poisoned")
609    }
610
611    /// Decide whether this `route.open` may attempt its relay. Takes the map
612    /// lock and nothing else, and never awaits.
613    fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
614        let admitted = |probe| RouteBindAdmission::Admitted {
615            guard: RouteBindBreakerGuard {
616                breakers: self.clone(),
617                module_id,
618                settled: false,
619            },
620            probe,
621        };
622
623        let mut modules = self.lock();
624        let Some(state) = modules.get_mut(module_id) else {
625            return admitted(false);
626        };
627        let Some(cooldown_until) = state.cooldown_until else {
628            return admitted(false);
629        };
630        if state.probe_in_flight {
631            return RouteBindAdmission::Refused {
632                consecutive_timeouts: state.consecutive_timeouts,
633                retry_in: Duration::ZERO,
634                probe_in_flight: true,
635            };
636        }
637        let now = Instant::now();
638        if now < cooldown_until {
639            return RouteBindAdmission::Refused {
640                consecutive_timeouts: state.consecutive_timeouts,
641                retry_in: cooldown_until - now,
642                probe_in_flight: false,
643            };
644        }
645        state.probe_in_flight = true;
646        admitted(true)
647    }
648
649    fn record_accepted(&self, module_id: &str) -> bool {
650        self.lock()
651            .remove(module_id)
652            .is_some_and(|state| state.cooldown_until.is_some())
653    }
654
655    fn record_timeout(
656        &self,
657        module_id: &str,
658        threshold: u32,
659        cooldown: Duration,
660    ) -> Option<BreakerOpened> {
661        let mut modules = self.lock();
662        let state = modules.entry(module_id.to_string()).or_default();
663        let was_open = state.cooldown_until.is_some();
664        let was_probe = state.probe_in_flight;
665        state.probe_in_flight = false;
666        state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
667        if state.consecutive_timeouts < threshold {
668            return None;
669        }
670        state.cooldown_until = Some(Instant::now() + cooldown);
671        Some(BreakerOpened {
672            consecutive_timeouts: state.consecutive_timeouts,
673            reopened_after_probe: was_open && was_probe,
674        })
675    }
676
677    fn record_inconclusive(&self, module_id: &str) {
678        if let Some(state) = self.lock().get_mut(module_id) {
679            state.probe_in_flight = false;
680        }
681    }
682
683    /// Discard what was learned about a module, because the process it was
684    /// learned about is gone. Returns the discarded count when it was non-zero.
685    ///
686    /// A BREAKER IS A CACHED VERDICT ABOUT A PROCESS, NOT ABOUT A NAME. A
687    /// `module_id` is a configuration identity that outlives any particular
688    /// child; what the breaker observed was the process behind the module
689    /// connection of the moment. When a new connection registers under that id
690    /// the verdict's subject no longer exists, so the verdict is stale by
691    /// construction rather than merely likely to be wrong. Keeping it would
692    /// apply a dead process's record to a live one, which is the same defect
693    /// class this breaker exists to stop the daemon committing.
694    ///
695    /// A half-open probe in flight is discarded with the rest: it was a
696    /// question about the old process.
697    pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
698        self.lock()
699            .remove(module_id)
700            .map(|state| state.consecutive_timeouts)
701            .filter(|discarded| *discarded > 0)
702    }
703
704    /// Open breakers, for the `server.describe` counters object. `None` when
705    /// none is open, so the key stays absent rather than present-and-empty.
706    ///
707    /// This is the operator's answer to "is this module refusing instantly or
708    /// is it fine?", which look identical from a client that retries and then
709    /// succeeds.
710    fn open_snapshot(&self) -> Option<serde_json::Value> {
711        let now = Instant::now();
712        let modules = self.lock();
713        let open = modules
714            .iter()
715            .filter_map(|(module_id, state)| {
716                let cooldown_until = state.cooldown_until?;
717                Some((
718                    module_id.clone(),
719                    serde_json::json!({
720                        "consecutive_timeouts": state.consecutive_timeouts,
721                        "cooldown_remaining_ms":
722                            cooldown_until.saturating_duration_since(now).as_millis() as u64,
723                        "probe_in_flight": state.probe_in_flight,
724                    }),
725                ))
726            })
727            .collect::<serde_json::Map<String, serde_json::Value>>();
728        (!open.is_empty()).then_some(serde_json::Value::Object(open))
729    }
730}
731
732impl ControlHandler {
733    pub fn new(registry: Arc<Registry>) -> Self {
734        Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
735    }
736
737    pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
738        let counters = forwarding.counters();
739        // Taken from the forwarding table rather than created here, so that the
740        // breaker a `route.open` consults is the same one a module's
741        // registration resets, however many handlers are built over one table.
742        let route_bind_breakers = forwarding.route_bind_breakers();
743        let route_bind_concurrency = forwarding.route_bind_concurrency();
744        let route_outages = forwarding.route_outages();
745        Self {
746            registry,
747            forwarding,
748            process_liveness: None,
749            supervisor: SupervisorHandle::new(),
750            subc_capabilities: Arc::from([
751                CAP_MANIFEST_REGISTRATION.to_string(),
752                CAP_CHANNEL_LIFECYCLE.to_string(),
753                CAP_PING_PONG.to_string(),
754                CAP_SESSION_ATTACH.to_string(),
755                CAP_ADMISSION_FACTS_RELAY.to_string(),
756                CAP_SCOPES_V1.to_string(),
757                CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
758            ]),
759            route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
760            route_bind_relay_timeouts: BTreeMap::new(),
761            route_bind_breakers,
762            route_bind_concurrency,
763            route_outages,
764            route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
765            route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
766            health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
767            storage_config: None,
768            machine_id: None,
769            admission_facts_carrier_module_id: None,
770            admission_facts_targets: None,
771            scopes: Arc::new(RwLock::new(ScopeTable::new(
772                crate::daemon_config::default_scope_authority_owners(),
773            ))),
774            scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
775            hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
776            rescan: None,
777            connected_clients: ConnectedClients::new(),
778            counters,
779            capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
780            daemon_provenance: DaemonProvenanceFacts::default(),
781            #[cfg(test)]
782            control_dispatch_delay: None,
783            #[cfg(test)]
784            provenance_probe_override: None,
785        }
786    }
787
788    /// Set the central storage policy: registering modules then receive their
789    /// resolved storage descriptor in HELLO_ACK.
790    pub fn with_storage_config(
791        mut self,
792        storage_config: Option<crate::daemon_config::StorageConfig>,
793    ) -> Self {
794        self.storage_config = storage_config;
795        self
796    }
797
798    /// Set the machine id served to every registering module (HELLO_ACK) and on
799    /// `server.describe`.
800    pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
801        self.machine_id = machine_id;
802        self
803    }
804
805    /// Configure the exact reserved module and target ids permitted to relay
806    /// opaque admission facts. Config-file loading validates this authority;
807    /// this builder keeps the same policy available to embedded test daemons.
808    pub fn with_admission_facts_config(
809        mut self,
810        carrier_module_id: Option<String>,
811        targets: Option<Vec<String>>,
812    ) -> Self {
813        self.admission_facts_carrier_module_id = carrier_module_id;
814        self.admission_facts_targets = targets;
815        self
816    }
817
818    /// Set the module ids whose scopes may carry `agent_id` and `delegates`.
819    /// Replaces the scope table with an empty one under the new list, so call it
820    /// while building the handler, before any module can sync.
821    pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
822        self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
823        self.scope_authority_owners = owners;
824        self
825    }
826
827    /// Override the route.bind relay timeout. Used by tests that assert the
828    /// timeout path so they don't block on the production-safe default.
829    pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
830        self.route_bind_relay_timeout = timeout;
831        self
832    }
833
834    /// Install per-module route.bind relay budget overrides. A module id
835    /// listed here wins over the daemon-wide default set via
836    /// `with_route_bind_relay_timeout`. Values are pre-resolved at parse time
837    /// from `subc.jsonc` (per-module > daemon-wide > absent), so callers pass
838    /// the same `Duration` the bind path will use.
839    pub fn with_route_bind_relay_timeouts(
840        mut self,
841        timeouts: impl IntoIterator<Item = (String, Duration)>,
842    ) -> Self {
843        self.route_bind_relay_timeouts = timeouts.into_iter().collect();
844        self
845    }
846
847    /// Resolve the route.bind relay budget for a specific target module id.
848    /// Per-module overrides win; the daemon-wide value (set via
849    /// `with_route_bind_relay_timeout` or the built-in default) is the
850    /// fallback. Exposed so config-aware callers (bootstrap, tests) can audit
851    /// the same resolution `handle_route_open` will use.
852    pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
853        self.route_bind_relay_timeouts
854            .get(module_id)
855            .copied()
856            .unwrap_or(self.route_bind_relay_timeout)
857    }
858
859    /// Override the per-module bind-relay breaker policy.
860    ///
861    /// Used by tests, which cannot spend three production budgets opening a
862    /// breaker or twenty seconds waiting for its cooldown. The production
863    /// values are `DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD` and
864    /// `DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN`, whose doc comments carry the
865    /// reasoning for the numbers.
866    pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
867        self.route_bind_breaker_threshold = threshold.max(1);
868        self.route_bind_breaker_cooldown = cooldown;
869        self
870    }
871
872    #[cfg(test)]
873    pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
874        self.health_probe_timeout = timeout;
875        self
876    }
877
878    #[cfg(test)]
879    pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
880        self.control_dispatch_delay = Some(delay);
881        self
882    }
883
884    pub fn with_process_liveness(
885        mut self,
886        process_liveness: Arc<dyn ModuleProcessLiveness>,
887    ) -> Self {
888        self.process_liveness = Some(process_liveness);
889        self
890    }
891
892    pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
893        self.supervisor = supervisor;
894        self
895    }
896
897    pub fn with_daemon_provenance(
898        mut self,
899        pid: u32,
900        started_at_ms: u64,
901        executable_path: Option<PathBuf>,
902        build_git_sha: Option<String>,
903        build_lock_digest: Option<String>,
904    ) -> Self {
905        let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
906        let process_start_time = process_start_time(pid);
907        self.daemon_provenance = DaemonProvenanceFacts {
908            build: DaemonBuildProvenance {
909                build_git_sha,
910                build_lock_digest,
911            },
912            pid: Some(pid),
913            started_at_ms: Some(started_at_ms),
914            start_clock: None,
915            executable_path,
916            executable_identity,
917            process_start_time,
918            probe: ExecutableIdentityProbe::default(),
919        };
920        self
921    }
922
923    pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
924        self.daemon_provenance.start_clock = Some(clock);
925        self
926    }
927
928    #[cfg(test)]
929    fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
930        self.provenance_probe_override = Some(result);
931        self
932    }
933
934    /// Install the configured module set and its reserved capability bindings.
935    /// Bindings are configuration-scoped and may point at a provider that has not
936    /// been installed yet, so this does not require the bound module to exist.
937    pub fn with_capability_config(
938        self,
939        modules: impl IntoIterator<Item = (String, bool)>,
940        reserved_capabilities: BTreeMap<String, String>,
941    ) -> Self {
942        self.capability_evaluator
943            .configure(modules, reserved_capabilities);
944        self
945    }
946
947    pub fn with_supervisor_rescan(
948        mut self,
949        supervisor: Supervisor,
950        config_path: impl Into<PathBuf>,
951        configured_port: Option<u16>,
952    ) -> Self {
953        self.rescan = Some(SupervisorRescanContext {
954            supervisor,
955            config_path: config_path.into(),
956            configured_port,
957            storage_config: self.storage_config.clone(),
958            admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
959            admission_facts_targets: self.admission_facts_targets.clone(),
960            scope_authority_owners: self.scope_authority_owners.clone(),
961        });
962        self
963    }
964
965    pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
966        self.connected_clients = connected_clients;
967        self
968    }
969
970    pub fn forwarding(&self) -> Arc<ForwardingTable> {
971        Arc::clone(&self.forwarding)
972    }
973
974    pub(crate) fn counters(&self) -> DaemonCounters {
975        self.counters.clone()
976    }
977
978    /// Wake at each candidate's own deadline so a stalled fresh exec emits its
979    /// requirement event without depending on an operator polling a status command.
980    pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
981        tokio::spawn(async move {
982            loop {
983                self.capability_evaluator
984                    .wait_for_change_or_deadline()
985                    .await;
986                self.refresh_capability_requirements();
987            }
988        });
989    }
990
991    fn runtime_capability_snapshot(
992        &self,
993    ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
994        let runtime = self
995            .supervisor
996            .list()
997            .into_iter()
998            .map(|module| {
999                let status = module.status().map_err(|err| {
1000                    RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1001                })?;
1002                Ok(RuntimeModule {
1003                    module_id: status.module_id,
1004                    state: status.state,
1005                    enabled: status.enabled,
1006                })
1007            })
1008            .collect::<Result<Vec<_>, RouterError>>()?;
1009        let (_, registrations) = self.registry.list_modules().map_err(|err| {
1010            RouterError::backend(
1011                0,
1012                0,
1013                format!("failed to list capability registrations: {err}"),
1014            )
1015        })?;
1016        let registrations = registrations
1017            .into_iter()
1018            .map(|registration| RegisteredModule {
1019                module_id: registration.manifest.module_id,
1020                module_version: registration.manifest.module_version,
1021                capabilities: registration.manifest.capabilities,
1022            })
1023            .collect();
1024        Ok((runtime, registrations))
1025    }
1026
1027    /// The capability side effects of a module becoming the active registration
1028    /// for its id: cache its manifest (warning if its claims drifted), run the
1029    /// deny census when its declarations call for one, and recompute the
1030    /// requirement statuses. An ordinary HELLO does this as it registers; a swap
1031    /// candidate's does not, and the supervisor does it at promotion instead,
1032    /// through [`crate::supervise::SwapPromotionObserver`].
1033    fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1034        let cached_registration = RegisteredModule {
1035            module_id: registration.manifest.module_id.clone(),
1036            module_version: registration.manifest.module_version.clone(),
1037            capabilities: registration.manifest.capabilities.clone(),
1038        };
1039        if self.capability_evaluator.record_hello(&cached_registration) {
1040            warn!(
1041                module_id = %cached_registration.module_id,
1042                "capability claims drifted from the cached manifest"
1043            );
1044        }
1045        if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1046            self.enforce_capability_denies();
1047        }
1048        self.refresh_capability_requirements();
1049    }
1050
1051    /// Point the shared supervisor handle at this handler for swap promotions.
1052    /// Called wherever a handler is put behind the `Arc` the router serves, so
1053    /// it can be held weakly.
1054    pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1055        let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1056            Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1057        self.supervisor.set_swap_promotion_observer(observer);
1058    }
1059
1060    pub fn refresh_capability_requirements(&self) {
1061        match self.runtime_capability_snapshot() {
1062            Ok((runtime, registrations)) => {
1063                log_requirement_events(
1064                    self.capability_evaluator
1065                        .evaluate_now(&runtime, &registrations),
1066                );
1067            }
1068            Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1069        }
1070    }
1071
1072    /// Reconcile only live, attested route bindings after a capability deny edge
1073    /// or target claim was added. This is deliberately a control-plane census:
1074    /// the opaque forwarding hot path must not grow a per-frame capability check.
1075    fn enforce_capability_denies(&self) {
1076        let (_, registrations) = match self.registry.list_modules() {
1077            Ok(snapshot) => snapshot,
1078            Err(err) => {
1079                warn!(error = %err, "failed to read registrations for capability deny census");
1080                return;
1081            }
1082        };
1083        let manifests = registrations
1084            .into_iter()
1085            .map(|registration| {
1086                (
1087                    registration.manifest.module_id.clone(),
1088                    registration.manifest,
1089                )
1090            })
1091            .collect::<BTreeMap<_, _>>();
1092        let census = match self.forwarding.route_census(None) {
1093            Ok(census) => census,
1094            Err(err) => {
1095                warn!(error = %err, "failed to read route census for capability deny enforcement");
1096                return;
1097            }
1098        };
1099
1100        for (target_module_id, routes) in census {
1101            let Some(target_manifest) = manifests.get(&target_module_id) else {
1102                continue;
1103            };
1104            let mut closed_routes = Vec::new();
1105            let mut module_goodbyes = Vec::new();
1106            for route in routes {
1107                let Principal::Reserved {
1108                    module_id: opening_module_id,
1109                } = &route.principal
1110                else {
1111                    continue;
1112                };
1113                let Some(opening_manifest) = manifests.get(opening_module_id) else {
1114                    continue;
1115                };
1116                let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1117                    continue;
1118                };
1119
1120                match self.forwarding.release_client_route(
1121                    route.goodbye_target.connection_id,
1122                    route.goodbye_target.channel,
1123                    route.goodbye_target.epoch,
1124                ) {
1125                    Ok(RouteRelease::Removed(module_goodbye)) => {
1126                        warn!(
1127                            opening_module_id,
1128                            target_module_id,
1129                            capability,
1130                            "force-closing route because an attested capability deny edge now matches"
1131                        );
1132                        closed_routes.push(route);
1133                        module_goodbyes.push(module_goodbye);
1134                    }
1135                    Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1136                    Err(err) => warn!(
1137                        opening_module_id,
1138                        target_module_id,
1139                        capability,
1140                        error = %err,
1141                        "failed to force-close capability-denied route"
1142                    ),
1143                }
1144            }
1145
1146            if closed_routes.is_empty() {
1147                continue;
1148            }
1149            send_route_control_pushes(
1150                &self.forwarding,
1151                closed_routes,
1152                ClientControlPush::RouteClosed {
1153                    module_id: target_module_id,
1154                    channels: Vec::new(),
1155                    reason: RouteCloseReason::CapabilityDenied,
1156                    drained: false,
1157                    abandoned: 0,
1158                    excluded_subscriptions: 0,
1159                    terminal: Some(false),
1160                },
1161            );
1162            self.emit_route_goodbyes(module_goodbyes);
1163        }
1164    }
1165
1166    /// Why a registered module is not accepting new route binds, or `None` when
1167    /// it is. This is the module's effective readiness: its declared readiness
1168    /// first, then every `need: required` capability it declares evaluating to
1169    /// `provided`. `route.open` and `catalog.list` both read it here so the
1170    /// catalog never reports a module routable that `route.open` would refuse.
1171    fn not_ready_reason(
1172        &self,
1173        registration: &crate::registry::ModuleRegistration,
1174    ) -> Option<NotReadyReason> {
1175        if !registration.ready {
1176            return Some(NotReadyReason {
1177                reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1178                capability: None,
1179            });
1180        }
1181        self.first_unprovided_required_capability(registration)
1182            .map(|capability| NotReadyReason {
1183                reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1184                capability: Some(capability),
1185            })
1186    }
1187
1188    /// The lexicographically first capability this registration declares
1189    /// `need: required` whose evaluator verdict is not `provided`.
1190    ///
1191    /// The verdicts are the capability evaluator's own; nothing here decides
1192    /// what "provided" means. The evaluator counts a capability provided as
1193    /// soon as a module claiming it has REGISTERED, not once that module is
1194    /// ready. That distinction is what keeps two modules that require each
1195    /// other's capabilities from deadlocking: if "provided" meant "the claimant
1196    /// is ready", each would wait for the other to become ready first and
1197    /// neither ever would. Do not tighten it to readiness.
1198    ///
1199    /// A required capability with no verdict at all means this registration's
1200    /// HELLO or catalog.update landed after the last recompute; recompute once
1201    /// rather than let a missing verdict read as either answer. If it is still
1202    /// missing (the recompute itself failed) the capability counts as
1203    /// unprovided: the refusal is retryable, and routing a module whose
1204    /// required provider is unknown is the outcome this check exists to stop.
1205    fn first_unprovided_required_capability(
1206        &self,
1207        registration: &crate::registry::ModuleRegistration,
1208    ) -> Option<String> {
1209        let required = registration
1210            .manifest
1211            .capabilities
1212            .iter()
1213            .flat_map(|declarations| declarations.requires.iter())
1214            .filter(|requirement| requirement.need == CapabilityNeed::Required)
1215            .map(|requirement| requirement.capability.as_str())
1216            .collect::<BTreeSet<_>>();
1217        if required.is_empty() {
1218            return None;
1219        }
1220        let module_id = registration.manifest.module_id.as_str();
1221        let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1222        if required
1223            .iter()
1224            .any(|capability| verdict(capability).is_none())
1225        {
1226            self.refresh_capability_requirements();
1227        }
1228        required
1229            .into_iter()
1230            .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1231            .map(str::to_string)
1232    }
1233
1234    fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1235        self.capability_evaluator
1236            .statuses()
1237            .into_iter()
1238            .map(capability_requirement_status)
1239            .collect()
1240    }
1241
1242    /// Remove a connection's registry entries WITHOUT signalling the supervisor's
1243    /// registration-release watch. The signal is what the supervisor waits on
1244    /// before spawning a replacement, so it must only fire once forwarding
1245    /// teardown is also done (see [`Self::cleanup_connection`] /
1246    /// [`Self::handle_goodbye`]). Used directly only where there is no forwarding
1247    /// state to tear down (a HELLO that failed before module registration).
1248    fn deregister_connection(
1249        &self,
1250        connection_id: ConnectionId,
1251    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1252        self.registry.deregister_connection(connection_id)
1253    }
1254
1255    pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1256        if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1257            return None;
1258        }
1259        let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1260            parse_client_control_request(&frame.body)
1261        else {
1262            return None;
1263        };
1264        Some(target_module_id(&target).to_string())
1265    }
1266
1267    pub(crate) fn route_open_capacity_refusal(
1268        &self,
1269        ctx: &RouteCtx,
1270        frame: &Frame,
1271        target_module_id: &str,
1272        in_flight: usize,
1273        limit: usize,
1274    ) -> Result<Frame, RouterError> {
1275        self.route_open_admission_refusal_frame(
1276            ctx,
1277            frame,
1278            target_module_id,
1279            "open_admission_full",
1280            (in_flight, limit),
1281            format!(
1282                "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1283            ),
1284        )
1285    }
1286
1287    fn route_open_target_capacity_refusal(
1288        &self,
1289        ctx: &RouteCtx,
1290        frame: &Frame,
1291        target_module_id: &str,
1292        in_flight: usize,
1293    ) -> Result<Frame, RouterError> {
1294        self.route_open_admission_refusal_frame(
1295            ctx,
1296            frame,
1297            target_module_id,
1298            "target_binds_full",
1299            (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1300            format!(
1301                "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1302            ),
1303        )
1304    }
1305
1306    /// Admission pressure clears as existing binds settle, so its refusal must
1307    /// remain in the deployed SDKs' closed retryable set: `unknown_module`,
1308    /// `module_reloading`, `module_warming`, `target_unavailable`, or
1309    /// `module_timeout`. `target_unavailable` is honest for an attempt that
1310    /// cannot currently reach its target; `module_timeout` would falsely claim
1311    /// that a wait expired. A new, cleaner code would be terminal to deployed
1312    /// clients, so it requires a client-tolerance rollout before daemon emission.
1313    fn route_open_admission_refusal_frame(
1314        &self,
1315        ctx: &RouteCtx,
1316        frame: &Frame,
1317        target_module_id: &str,
1318        reason: &'static str,
1319        (in_flight, limit): (usize, usize),
1320        message: impl Into<String>,
1321    ) -> Result<Frame, RouterError> {
1322        let code = error_codes::TARGET_UNAVAILABLE;
1323        self.counters.increment_route_open_refused(code);
1324        info!(
1325            target: "control",
1326            code,
1327            reason,
1328            module_id = ?target_module_id,
1329            connection_id = ctx.connection_id.get(),
1330            in_flight,
1331            limit,
1332            "route.open refused"
1333        );
1334        control_error_frame(frame, code, message.into())
1335    }
1336
1337    /// Test-only compatibility entry point for unit control handling that does not have a socket sink.
1338    ///
1339    /// The real server path uses [`Self::handle_control_frame`] so module HELLO registration can
1340    /// record the module connection's [`crate::FrameSink`] and session attach can await the module
1341    /// relay response. This seam stays cfg(test) so production has only one channel-0 path.
1342    #[cfg(test)]
1343    pub fn handle_control(
1344        &self,
1345        connection_id: ConnectionId,
1346        frame: Frame,
1347    ) -> Result<Vec<Frame>, RouterError> {
1348        match frame.header.ty {
1349            FrameType::Ping => Ok(vec![pong(&frame)?]),
1350            FrameType::Hello => self.handle_hello(connection_id, None, frame),
1351            FrameType::Goodbye => self.handle_goodbye(connection_id),
1352            ty => Ok(vec![control_error_frame(
1353                &frame,
1354                "unsupported_control_frame",
1355                format!("unsupported channel-0 frame {ty:?}"),
1356            )?]),
1357        }
1358    }
1359
1360    pub async fn handle_control_frame(
1361        &self,
1362        ctx: &RouteCtx,
1363        frame: Frame,
1364    ) -> Result<Vec<Frame>, RouterError> {
1365        self.handle_control_frame_timed(ctx, frame, None).await
1366    }
1367
1368    pub(crate) async fn handle_control_frame_timed(
1369        &self,
1370        ctx: &RouteCtx,
1371        frame: Frame,
1372        dispatch_started_at: Option<StdInstant>,
1373    ) -> Result<Vec<Frame>, RouterError> {
1374        match frame.header.ty {
1375            FrameType::Ping => Ok(vec![pong(&frame)?]),
1376            FrameType::Hello => {
1377                self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1378            }
1379            FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1380            FrameType::Cancel => {
1381                if self
1382                    .supervisor
1383                    .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1384                {
1385                    Ok(Vec::new())
1386                } else {
1387                    Ok(vec![control_error_frame(
1388                        &frame,
1389                        "unknown_subscription",
1390                        "no supervisor spawn subscription has this correlation id",
1391                    )?])
1392                }
1393            }
1394            FrameType::Request => {
1395                if self
1396                    .forwarding
1397                    .module_endpoint_for_connection(ctx.connection_id)
1398                    .map_err(RouterError::Forwarding)?
1399                    .is_some()
1400                {
1401                    if !is_known_module_request_op(&frame.body) {
1402                        return Ok(vec![control_error_frame(
1403                            &frame,
1404                            "unsupported_control_frame",
1405                            "module-originated channel-0 REQUEST is not supported",
1406                        )?]);
1407                    }
1408                    let request = match parse_module_control_request_from_module(&frame.body) {
1409                        Ok(request) => request,
1410                        Err((err, ControlRequestBodyError::UnknownOp)) => {
1411                            return Ok(vec![control_error_frame(
1412                                &frame,
1413                                "unsupported_control_frame",
1414                                format!("unsupported module-originated channel-0 REQUEST: {err}"),
1415                            )?])
1416                        }
1417                        Err((err, ControlRequestBodyError::InvalidBody)) => {
1418                            return Ok(vec![control_error_frame(
1419                                &frame,
1420                                "invalid_control_body",
1421                                format!("malformed module control body: {err}"),
1422                            )?])
1423                        }
1424                    };
1425                    let op = module_control_request_op(&request);
1426                    let corr = frame.header.corr;
1427                    log_control_dispatch_arrival(op, ctx.connection_id, corr);
1428                    let result =
1429                        self.handle_module_control_request(ctx.connection_id, frame, request);
1430                    log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1431                    return result;
1432                }
1433
1434                if is_known_module_request_op(&frame.body) {
1435                    return Ok(vec![control_error_frame(
1436                        &frame,
1437                        "not_registered",
1438                        "catalog.update requires an active module registration owned by this connection",
1439                    )?]);
1440                }
1441
1442                let request = match parse_client_control_request(&frame.body) {
1443                    Ok(request) => request,
1444                    Err((err, ControlRequestBodyError::UnknownOp)) => {
1445                        return Ok(vec![control_error_frame(
1446                            &frame,
1447                            "unknown_control_op",
1448                            format!("unknown client control op: {err}"),
1449                        )?])
1450                    }
1451                    Err((err, ControlRequestBodyError::InvalidBody)) => {
1452                        return Ok(vec![control_error_frame(
1453                            &frame,
1454                            "invalid_control_body",
1455                            format!("malformed client control body: {err}"),
1456                        )?])
1457                    }
1458                };
1459                let op = client_control_request_op(&request);
1460                let corr = frame.header.corr;
1461                log_control_dispatch_arrival(op, ctx.connection_id, corr);
1462                #[cfg(test)]
1463                if let Some(delay) = self.control_dispatch_delay {
1464                    tokio::time::sleep(delay).await;
1465                }
1466                let result = self
1467                    .handle_client_control_request(ctx, frame, request)
1468                    .await;
1469                log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1470                result
1471            }
1472            FrameType::Push => {
1473                let Some(endpoint) = self
1474                    .forwarding
1475                    .module_endpoint_for_connection(ctx.connection_id)
1476                    .map_err(RouterError::Forwarding)?
1477                else {
1478                    return Ok(vec![control_error_frame(
1479                        &frame,
1480                        "unsupported_control_frame",
1481                        "client-originated channel-0 PUSH is not supported",
1482                    )?]);
1483                };
1484                self.handle_status_update(endpoint, frame)
1485            }
1486            FrameType::Response | FrameType::Error
1487                if self
1488                    .forwarding
1489                    .module_endpoint_for_connection(ctx.connection_id)
1490                    .map_err(RouterError::Forwarding)?
1491                    .is_some() =>
1492            {
1493                self.handle_module_relay_response(ctx.connection_id, frame)
1494            }
1495            ty => Ok(vec![control_error_frame(
1496                &frame,
1497                "unsupported_control_frame",
1498                format!("unsupported channel-0 frame {ty:?}"),
1499            )?]),
1500        }
1501    }
1502
1503    pub fn cleanup_connection(
1504        &self,
1505        connection_id: ConnectionId,
1506    ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1507        let crash_closed = self
1508            .registry
1509            .get_module_by_connection(connection_id)?
1510            .and_then(|registration| {
1511                self.forwarding
1512                    .module_endpoint_for_connection(connection_id)
1513                    .ok()
1514                    .flatten()
1515                    .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1516                    .map(|routes| (registration.manifest.module_id, routes))
1517            });
1518        let crash_closed = crash_closed.map(|(module_id, routes)| {
1519            let terminal = match self.supervisor.get(&module_id) {
1520                None => false,
1521                Some(module) => match module.will_recover_after_connection_loss() {
1522                    Ok(will_recover) => !will_recover,
1523                    Err(err) => {
1524                        warn!(
1525                            %module_id,
1526                            error = %err,
1527                            "failed to read crash recovery verdict; reporting non-terminal conservatively"
1528                        );
1529                        false
1530                    }
1531                },
1532            };
1533            // The forwarding table gates all providers at the start of daemon
1534            // shutdown, before their connections are closed. An ordinary
1535            // module disconnect still reports crash if that gate is not set.
1536            let reason = match self.forwarding.is_daemon_draining() {
1537                Ok(true) => RouteCloseReason::Restart,
1538                Ok(false) => RouteCloseReason::Crash,
1539                Err(err) => {
1540                    warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1541                    RouteCloseReason::Crash
1542                }
1543            };
1544            (module_id, routes, reason, terminal)
1545        });
1546        let registrations = self.deregister_connection(connection_id);
1547        let cleanup = self.forwarding.cleanup_connection_counted(connection_id);
1548        // The route.closed push waits for forwarding teardown because only
1549        // teardown knows how many pending route.bind relays it aborted. It still
1550        // goes out before the GOODBYEs for the released routes, and its targets
1551        // were captured above, before teardown removed those routes.
1552        if let Some((module_id, routes, reason, terminal)) = crash_closed {
1553            let abandoned = cleanup
1554                .as_ref()
1555                .map_or(0, |cleanup| cleanup.abandoned_relays);
1556            send_route_control_pushes(
1557                &self.forwarding,
1558                routes,
1559                ClientControlPush::RouteClosed {
1560                    module_id,
1561                    channels: Vec::new(),
1562                    reason,
1563                    drained: false,
1564                    abandoned,
1565                    excluded_subscriptions: 0,
1566                    terminal: Some(terminal),
1567                },
1568            );
1569        }
1570        if let Ok(cleanup) = cleanup {
1571            self.emit_route_goodbyes(cleanup.released);
1572        }
1573        // Signal the registration-release watch only now that BOTH registry and
1574        // forwarding teardown are done, so a supervisor waiting to spawn a
1575        // replacement never observes release while old routes still exist.
1576        if matches!(&registrations, Ok(r) if !r.is_empty()) {
1577            crate::supervise::notify_registration_release();
1578            self.capability_evaluator.wake_deadline_loop();
1579            self.refresh_capability_requirements();
1580        }
1581        self.supervisor.remove_spawn_subscribers(connection_id);
1582        // Sync authority dies with its connection, so the owner's next
1583        // connection can take it; the owner's scopes stay as they are.
1584        self.hello_launch_nonces
1585            .lock()
1586            .unwrap_or_else(|poisoned| poisoned.into_inner())
1587            .forget(connection_id);
1588        self.scopes
1589            .write()
1590            .unwrap_or_else(|poisoned| poisoned.into_inner())
1591            .release_connection(connection_id);
1592        registrations
1593    }
1594
1595    pub(crate) fn handle_route_goodbye(
1596        &self,
1597        connection_id: ConnectionId,
1598        route_channel: u16,
1599        route_epoch: u32,
1600    ) -> Result<bool, RouterError> {
1601        debug!(
1602            connection_id = connection_id.get(),
1603            route_channel, route_epoch, "handling route GOODBYE"
1604        );
1605        let RouteRelease::Removed(released_route) = self
1606            .forwarding
1607            .release_client_route(connection_id, route_channel, route_epoch)
1608            .map_err(RouterError::Forwarding)?
1609        else {
1610            return Ok(false);
1611        };
1612        self.emit_route_goodbyes(vec![released_route]);
1613        Ok(true)
1614    }
1615
1616    fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1617        for released in released_routes {
1618            let frame = match Frame::build_with_version(
1619                released.negotiated_ver,
1620                FrameType::Goodbye,
1621                control_flags(),
1622                released.channel,
1623                released.epoch,
1624                0,
1625                Vec::new(),
1626            ) {
1627                Ok(frame) => frame,
1628                Err(err) => {
1629                    warn!(
1630                        route_channel = released.channel,
1631                        error = %err,
1632                        "failed to build route GOODBYE frame"
1633                    );
1634                    continue;
1635                }
1636            };
1637            if !released.close_on_delivery_failure() {
1638                crate::forwarding::send_module_route_goodbye(
1639                    &self.counters,
1640                    &released.sink,
1641                    frame,
1642                    released.module_id.as_deref(),
1643                    "client route released",
1644                );
1645                continue;
1646            }
1647            if let Err(err) = released.sink.try_send(frame) {
1648                warn!(
1649                    target_connection_id = released.connection_id.get(),
1650                    route_channel = released.channel,
1651                    error = %err,
1652                    "route GOODBYE was not delivered to client; closing target connection"
1653                );
1654                if self
1655                    .forwarding
1656                    .escalate_client_delivery_failure(
1657                        released.connection_id,
1658                        released.channel,
1659                        released.epoch,
1660                        CloseReason::new(
1661                            "route_goodbye_delivery_failed",
1662                            format!(
1663                                "failed to enqueue route GOODBYE for channel {}: {err}",
1664                                released.channel
1665                            ),
1666                        ),
1667                        crate::forwarding::UndeliveredFrame {
1668                            module_id: released.module_id.as_deref(),
1669                            sink: &released.sink,
1670                        },
1671                    )
1672                    .unwrap_or(false)
1673                {
1674                    self.counters.increment_goodbye_relay_client_failed();
1675                }
1676            }
1677        }
1678    }
1679
1680    /// Best-effort GOODBYE to a module for a route channel subc reserved but then
1681    /// abandoned (route.bind relay timed out, its waiter was cancelled, or subc's
1682    /// own commit failed after the module had already accepted). Without this, a
1683    /// module that accepts late keeps a binding subc has torn down, so a later
1684    /// frame on that module channel could misdeliver if the channel is reused.
1685    ///
1686    /// Never closes the shared module connection on failure: a dropped notification
1687    /// only wastes a bounded amount of warm module-side state, which the module's
1688    /// own idle reaper reclaims. Only call this once the route.bind relay was
1689    /// actually enqueued to the module — if the relay send itself failed, the
1690    /// module never created a binding and there is nothing to tear down.
1691    fn send_abandoned_route_bind_goodbye(
1692        &self,
1693        module_sink: &crate::FrameSink,
1694        negotiated_ver: u8,
1695        module_channel: u16,
1696        module_epoch: u32,
1697    ) {
1698        let frame = match Frame::build_with_version(
1699            negotiated_ver,
1700            FrameType::Goodbye,
1701            control_flags(),
1702            module_channel,
1703            module_epoch,
1704            0,
1705            Vec::new(),
1706        ) {
1707            Ok(frame) => frame,
1708            Err(err) => {
1709                warn!(
1710                    route_channel = module_channel,
1711                    error = %err,
1712                    "failed to build GOODBYE for abandoned route.bind"
1713                );
1714                return;
1715            }
1716        };
1717        crate::forwarding::send_module_route_goodbye(
1718            &self.counters,
1719            module_sink,
1720            frame,
1721            None,
1722            "abandoned route.bind",
1723        );
1724    }
1725
1726    fn handle_hello(
1727        &self,
1728        connection_id: ConnectionId,
1729        sink: Option<crate::FrameSink>,
1730        frame: Frame,
1731    ) -> Result<Vec<Frame>, RouterError> {
1732        debug!(
1733            connection_id = connection_id.get(),
1734            corr = frame.header.corr,
1735            "handling HELLO"
1736        );
1737        let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1738            Ok(value) => value,
1739            Err(err) => {
1740                return Ok(vec![control_error_frame(
1741                    &frame,
1742                    "invalid_hello",
1743                    format!("malformed HELLO body: {err}"),
1744                )?])
1745            }
1746        };
1747        if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1748            return Ok(vec![control_error_frame(
1749                &frame,
1750                "invalid_capability_grammar",
1751                err.to_string(),
1752            )?]);
1753        }
1754        if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
1755            return Ok(vec![control_error_frame(
1756                &frame,
1757                "invalid_manifest",
1758                err.to_string(),
1759            )?]);
1760        }
1761        if let Some(provenance) = hello_value
1762            .get("manifest")
1763            .and_then(|manifest| manifest.get("provenance"))
1764        {
1765            if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
1766                return Ok(vec![control_error_frame(
1767                    &frame,
1768                    "invalid_manifest",
1769                    format!("malformed manifest provenance: {err}"),
1770                )?]);
1771            }
1772        }
1773        let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
1774            Ok(hello) => hello,
1775            Err(err) => {
1776                return Ok(vec![control_error_frame(
1777                    &frame,
1778                    "invalid_hello",
1779                    format!("malformed HELLO body: {err}"),
1780                )?])
1781            }
1782        };
1783
1784        if hello.protocol_ver != hello.manifest.protocol_ver {
1785            return Ok(vec![control_error_frame(
1786                &frame,
1787                "invalid_manifest",
1788                format!(
1789                    "HELLO protocol_ver {} does not match manifest protocol_ver {}",
1790                    hello.protocol_ver, hello.manifest.protocol_ver
1791                ),
1792            )?]);
1793        }
1794
1795        if hello.manifest.module_id.trim().is_empty() {
1796            return Ok(vec![control_error_frame(
1797                &frame,
1798                "invalid_manifest",
1799                "manifest module_id must not be empty",
1800            )?]);
1801        }
1802
1803        let negotiated_ver = match negotiate_version(hello.protocol_ver) {
1804            Ok(negotiated_ver) => negotiated_ver,
1805            Err(message) => {
1806                return Ok(vec![control_error_frame(
1807                    &frame,
1808                    "version_unsupported",
1809                    message,
1810                )?])
1811            }
1812        };
1813
1814        // Swap gate, ahead of the reserved gate on purpose. While a blue/green
1815        // swap is open for this id, the only HELLO admitted as a second process
1816        // is the one carrying the candidate's launch nonce (the swap token), and
1817        // it registers into the candidate slot rather than being refused as a
1818        // duplicate. Run after the reserved gate, a reserved module's candidate
1819        // would be refused `reserved_module` for presenting a nonce that gate
1820        // does not know. See `SupervisorHandle::swap_hello_admission`.
1821        let swap_admission = self
1822            .supervisor
1823            .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
1824        if swap_admission == SwapHelloAdmission::Refused {
1825            warn!(
1826                module_id = %hello.manifest.module_id,
1827                connection_id = connection_id.get(),
1828                "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
1829            );
1830            return Ok(vec![control_error_frame(
1831                &frame,
1832                "swap_token_invalid",
1833                format!(
1834                    "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
1835                    hello.manifest.module_id
1836                ),
1837            )?]);
1838        }
1839        let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
1840
1841        // Reserved-module identity gate: a module_id configured `reserved` may be
1842        // registered ONLY by the process subc spawned for it, proven by echoing the
1843        // one-time launch nonce subc injected. A non-reserved id has no recorded
1844        // nonce and always passes. This blocks a key-holder from impersonating a
1845        // security-boundary module (e.g. the credential vault) while the real one is
1846        // down/restarting and its registration slot is momentarily free. A swap
1847        // candidate has already proven the same thing with its own nonce above.
1848        if let Some(rejection) = (!swap_candidate)
1849            .then(|| {
1850                self.supervisor.reserved_hello_rejection(
1851                    &hello.manifest.module_id,
1852                    hello.launch_nonce.as_deref(),
1853                )
1854            })
1855            .flatten()
1856        {
1857            let message = match rejection {
1858                ReservedHelloRejection::Exact { module_id } => format!(
1859                    "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
1860                ),
1861                ReservedHelloRejection::Prefix {
1862                    prefix,
1863                    owner_module_id,
1864                } => format!(
1865                    "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
1866                    hello.manifest.module_id
1867                ),
1868            };
1869            return Ok(vec![control_error_frame(
1870                &frame,
1871                "reserved_module",
1872                message,
1873            )?]);
1874        }
1875
1876        let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
1877            &hello.manifest.module_id,
1878            hello.manifest.capabilities.as_ref(),
1879        );
1880        if let Some(refusal) = reserved_capability_refusals.first() {
1881            let capability = refusal.capability.clone();
1882            let bound_module = refusal.claimants[0].clone();
1883            log_duplicate_claim_events(reserved_capability_refusals);
1884            return Ok(vec![control_error_frame(
1885                &frame,
1886                "reserved_capability",
1887                format!(
1888                    "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
1889                    capability, bound_module, hello.manifest.module_id
1890                ),
1891            )?]);
1892        }
1893
1894        // A connection that already opened client routes must not also register as
1895        // a module: cleanup would then release only one side and leak the other.
1896        if self
1897            .forwarding
1898            .connection_has_client_routes(connection_id)
1899            .map_err(RouterError::Forwarding)?
1900        {
1901            return Ok(vec![control_error_frame(
1902                &frame,
1903                "invalid_hello",
1904                "connection has open client routes and cannot also register as a module",
1905            )?]);
1906        }
1907
1908        // Kept for scope sync authority, which goes only to the connection that
1909        // presented the module's current launch nonce. Recorded before the
1910        // registration is attempted: a connection whose registration then fails
1911        // has no registration, so it cannot sync anyway, and cleanup forgets it.
1912        self.hello_launch_nonces
1913            .lock()
1914            .unwrap_or_else(|poisoned| poisoned.into_inner())
1915            .record(connection_id, hello.launch_nonce.as_deref());
1916        let control_ops = effective_module_control_ops(hello.control_ops);
1917        // Built before anything is registered so an encoding failure leaves no
1918        // registry or forwarding state behind.
1919        let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
1920        if swap_candidate {
1921            return self.register_swap_candidate(
1922                connection_id,
1923                sink,
1924                &frame,
1925                hello.manifest,
1926                negotiated_ver,
1927                control_ops,
1928                hello_ack,
1929            );
1930        }
1931        let registration = match self.registry.register_with_control_ops(
1932            hello.manifest,
1933            negotiated_ver,
1934            connection_id,
1935            control_ops,
1936        ) {
1937            Ok(registration) => registration,
1938            Err(RegistryError::DuplicateModuleId { module_id }) => {
1939                return Ok(vec![control_error_frame(
1940                    &frame,
1941                    "duplicate_module_id",
1942                    format!(
1943                        "module_id '{module_id}' is already registered; duplicate HELLO rejected"
1944                    ),
1945                )?])
1946            }
1947            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
1948                return Ok(vec![control_error_frame(
1949                    &frame,
1950                    "invalid_module_id",
1951                    err.to_string(),
1952                )?])
1953            }
1954            Err(err) => {
1955                return Ok(vec![control_error_frame(
1956                    &frame,
1957                    "registry_error",
1958                    err.to_string(),
1959                )?])
1960            }
1961        };
1962
1963        let reply = if let Some(sink) = sink {
1964            // The forwarding table's module store is also the daemon-to-module
1965            // control-RPC lane, so every HELLO gets a live endpoint even when the
1966            // manifest has no routable provider role. Non-routable modules still
1967            // cannot receive route.bind in production: `handle_route_open` checks
1968            // the registry manifest with `target_has_required_role` before the
1969            // only production call to `begin_route_bind_relay_for` below that
1970            // route.open path. The remaining direct relay callers are unit tests
1971            // and benchmark harnesses that construct forwarding state explicitly.
1972            //
1973            // The HELLO_ACK is queued by the forwarding table itself, before the
1974            // endpoint becomes visible, and is NOT returned as a reply. A module
1975            // reads HELLO_ACK first and exits on anything else; a reply is only
1976            // written after this handler returns, by which time a route.open on
1977            // another connection could already have queued a route.bind request
1978            // for this module ahead of it.
1979            let concurrency = manifest_concurrency(&registration.manifest);
1980            if let Err(err) = self.forwarding.register_module_connection_acked(
1981                connection_id,
1982                registration.manifest.module_id.clone(),
1983                negotiated_ver,
1984                concurrency,
1985                sink,
1986                hello_ack,
1987            ) {
1988                // Forwarding registration failed, so there is no forwarding
1989                // state to tear down. Remove the registry entry and signal the
1990                // release watch directly.
1991                if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
1992                    crate::supervise::notify_registration_release();
1993                }
1994                return Ok(vec![control_error_frame(
1995                    &frame,
1996                    forwarding_error_code(&err),
1997                    err.to_string(),
1998                )?]);
1999            }
2000            Vec::new()
2001        } else {
2002            // No sink means no forwarding endpoint, so nothing can be routed
2003            // ahead of the ack; it goes out as the reply.
2004            vec![hello_ack]
2005        };
2006
2007        // Exposure over assumption: Concurrency's serde default is pinned to the
2008        // pre-field behavior (ModuleManaged), so a management surface that is
2009        // genuinely Serial and just never declared it inherits concurrent
2010        // delivery silently. Logging which registrations RESOLVED BY DEFAULT
2011        // turns "no module has been bitten yet" into the checkable claim "no
2012        // module is exposed" -- one read of the boot log instead of a fleet
2013        // audit. Detected from the raw HELLO bytes because the serde default
2014        // deliberately erases the absent/declared distinction from the type.
2015        if manifest_concurrency_was_defaulted(&frame.body, &registration.manifest) {
2016            info!(
2017                module_id = %registration.manifest.module_id,
2018                "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2019            );
2020        }
2021
2022        self.apply_registration_capabilities(&registration);
2023
2024        info!(
2025            module_id = %registration.manifest.module_id,
2026            module_version = %registration.manifest.module_version,
2027            negotiated_ver,
2028            routable_provider = manifest_provides_routable_role(&registration.manifest),
2029            connection_id = connection_id.get(),
2030            "module registered"
2031        );
2032
2033        Ok(reply)
2034    }
2035
2036    /// Register a HELLO the swap gate admitted into the candidate slot of the
2037    /// registry and of forwarding, where it is reachable over its own
2038    /// connection (its `catalog.update` finds it) but by no by-id lookup, so
2039    /// nothing routes to it until the supervisor cuts over.
2040    ///
2041    /// Registry first, then forwarding, the same order as an ordinary HELLO;
2042    /// a forwarding failure removes the registry entry again. The capability
2043    /// census is not run: it describes routable modules, and this one is not
2044    /// routable until promotion.
2045    #[allow(clippy::too_many_arguments)]
2046    fn register_swap_candidate(
2047        &self,
2048        connection_id: ConnectionId,
2049        sink: Option<crate::FrameSink>,
2050        frame: &Frame,
2051        manifest: ModuleManifest,
2052        negotiated_ver: u8,
2053        control_ops: Vec<String>,
2054        hello_ack: Frame,
2055    ) -> Result<Vec<Frame>, RouterError> {
2056        let module_id = manifest.module_id.clone();
2057        let registration = match self.registry.register_candidate_with_control_ops(
2058            manifest,
2059            negotiated_ver,
2060            connection_id,
2061            control_ops,
2062        ) {
2063            Ok(registration) => registration,
2064            Err(RegistryError::DuplicateModuleId { module_id }) => {
2065                return Ok(vec![control_error_frame(
2066                    frame,
2067                    "duplicate_module_id",
2068                    format!(
2069                        "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2070                    ),
2071                )?])
2072            }
2073            Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2074                return Ok(vec![control_error_frame(
2075                    frame,
2076                    "invalid_module_id",
2077                    err.to_string(),
2078                )?])
2079            }
2080            Err(err) => {
2081                return Ok(vec![control_error_frame(
2082                    frame,
2083                    "registry_error",
2084                    err.to_string(),
2085                )?])
2086            }
2087        };
2088        let reply = if let Some(sink) = sink {
2089            // Same ordering as an ordinary HELLO: the forwarding table queues
2090            // the HELLO_ACK before the candidate endpoint is inserted, because
2091            // a module exits if its first frame after HELLO is anything else.
2092            let concurrency = manifest_concurrency(&registration.manifest);
2093            if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2094                connection_id,
2095                module_id.clone(),
2096                negotiated_ver,
2097                concurrency,
2098                sink,
2099                hello_ack,
2100            ) {
2101                if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
2102                    crate::supervise::notify_registration_release();
2103                }
2104                return Ok(vec![control_error_frame(
2105                    frame,
2106                    forwarding_error_code(&err),
2107                    err.to_string(),
2108                )?]);
2109            }
2110            Vec::new()
2111        } else {
2112            vec![hello_ack]
2113        };
2114        self.supervisor.mark_swap_candidate_admitted(&module_id);
2115        info!(
2116            module_id = %module_id,
2117            module_version = %registration.manifest.module_version,
2118            negotiated_ver,
2119            ready = registration.ready,
2120            connection_id = connection_id.get(),
2121            "swap candidate registered; not routable until cutover"
2122        );
2123        Ok(reply)
2124    }
2125
2126    fn build_hello_ack(
2127        &self,
2128        frame: &Frame,
2129        negotiated_ver: u8,
2130        module_id: &str,
2131    ) -> Result<Frame, RouterError> {
2132        let ack = ModuleHelloAckBody {
2133            negotiated_ver,
2134            subc_ops: module_subc_ops(),
2135            subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2136            storage: self
2137                .storage_config
2138                .as_ref()
2139                .map(|cfg| cfg.descriptor_for(module_id)),
2140            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2141        };
2142        let body = serde_json::to_vec(&ack).map_err(|err| {
2143            RouterError::backend(
2144                0,
2145                frame.header.corr,
2146                format!("failed to encode HELLO_ACK: {err}"),
2147            )
2148        })?;
2149
2150        Frame::build_with_version(
2151            negotiated_ver,
2152            FrameType::HelloAck,
2153            control_flags(),
2154            0,
2155            0,
2156            frame.header.corr,
2157            body,
2158        )
2159        .map_err(RouterError::FrameBuild)
2160    }
2161
2162    async fn handle_client_control_request(
2163        &self,
2164        ctx: &RouteCtx,
2165        frame: Frame,
2166        request: ClientControlRequest,
2167    ) -> Result<Vec<Frame>, RouterError> {
2168        match request {
2169            ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2170            ClientControlRequest::CatalogList { module_id } => {
2171                self.handle_catalog_list(frame, module_id)
2172            }
2173            ClientControlRequest::RouteOpen {
2174                target,
2175                identity,
2176                consumer_identity,
2177                consumer_capabilities,
2178                role_versions,
2179                admission_facts,
2180                scope,
2181            } => {
2182                self.handle_route_open(
2183                    ctx,
2184                    frame,
2185                    RouteOpenRequest {
2186                        target,
2187                        identity,
2188                        consumer_identity,
2189                        consumer_capabilities,
2190                        role_versions,
2191                        admission_facts,
2192                        scope,
2193                    },
2194                )
2195                .await
2196            }
2197            ClientControlRequest::RoutePoll {
2198                route_channel,
2199                route_epoch,
2200                kind,
2201            } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2202            ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2203            ClientControlRequest::SupervisorSpawnSnapshot {} => {
2204                self.handle_supervisor_spawn_snapshot(frame)
2205            }
2206            ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2207                self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2208            }
2209            ClientControlRequest::SupervisorRestart {
2210                module_id,
2211                drain_timeout_ms,
2212            } => {
2213                self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2214                    .await
2215            }
2216            ClientControlRequest::SupervisorSwap {
2217                module_id,
2218                ready_timeout_ms,
2219            } => {
2220                self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2221                    .await
2222            }
2223            ClientControlRequest::SupervisorReload { module_id } => {
2224                self.handle_supervisor_reload(frame, module_id).await
2225            }
2226            ClientControlRequest::SupervisorRescan { preview } => {
2227                self.handle_supervisor_rescan(frame, preview).await
2228            }
2229            ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2230                self.handle_supervisor_release_reserved(frame, module_id)
2231                    .await
2232            }
2233            ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2234                self.handle_supervisor_set_enabled(frame, module_id, enabled)
2235                    .await
2236            }
2237            ClientControlRequest::SupervisorHealthProbe { module_id } => {
2238                self.handle_supervisor_health_probe(frame, module_id).await
2239            }
2240            ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2241            ClientControlRequest::SupervisorRoutes { module_id } => {
2242                self.handle_supervisor_routes(frame, module_id)
2243            }
2244            ClientControlRequest::SupervisorProvenance { module_id } => {
2245                self.handle_supervisor_provenance(frame, module_id).await
2246            }
2247            ClientControlRequest::SupervisorStderrTail {
2248                module_id,
2249                max_lines,
2250                max_bytes,
2251            } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2252            ClientControlRequest::SupervisorTerminals { module_id } => {
2253                self.handle_supervisor_terminals(frame, module_id).await
2254            }
2255        }
2256    }
2257
2258    fn handle_module_control_request(
2259        &self,
2260        connection_id: ConnectionId,
2261        frame: Frame,
2262        request: ModuleControlRequestFromModule,
2263    ) -> Result<Vec<Frame>, RouterError> {
2264        match request {
2265            ModuleControlRequestFromModule::CatalogUpdate {
2266                provides,
2267                capabilities,
2268                ready,
2269            } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2270            ModuleControlRequestFromModule::LiveRoots {} => {
2271                let registered = self
2272                    .registry
2273                    .get_module_by_connection(connection_id)
2274                    .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2275                let Some(registration) = registered else {
2276                    return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2277                };
2278                let response = self
2279                    .forwarding
2280                    .live_roots(&registration.manifest.module_id)
2281                    .map_err(RouterError::Forwarding)?;
2282                Ok(vec![control_response_body_frame(
2283                    &frame,
2284                    &response,
2285                    "ModuleControlResponseToModule::LiveRoots",
2286                )?])
2287            }
2288            ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2289                self.handle_scope_sync(connection_id, frame, generation, scopes)
2290            }
2291            ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2292                self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2293            }
2294        }
2295    }
2296
2297    /// `scope.sync`: the owner is the module registered on this connection.
2298    /// A connection with no registration (every client connection, `direct`
2299    /// included) is refused `not_registered` before the table is consulted.
2300    fn handle_scope_sync(
2301        &self,
2302        connection_id: ConnectionId,
2303        frame: Frame,
2304        generation: u64,
2305        scopes: Vec<ScopeRecord>,
2306    ) -> Result<Vec<Frame>, RouterError> {
2307        let Some(registration) = self
2308            .registry
2309            .get_module_by_connection(connection_id)
2310            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2311        else {
2312            return Ok(vec![control_error_frame(
2313                &frame,
2314                "not_registered",
2315                "scope.sync requires an active module registration owned by this connection",
2316            )?]);
2317        };
2318        let owner = registration.manifest.module_id;
2319        let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
2320        let is_current_launch = |connection: ConnectionId| {
2321            self.hello_launch_nonces
2322                .lock()
2323                .unwrap_or_else(|poisoned| poisoned.into_inner())
2324                .presented(connection, current_nonce.as_deref())
2325        };
2326        // Lock order is the scope table, then the forwarding table: the new
2327        // tags are published, and the routes the change closes are selected,
2328        // while the scope table is still write-locked, so no admission can read
2329        // a record whose tag is not yet published.
2330        let mut table = self
2331            .scopes
2332            .write()
2333            .unwrap_or_else(|poisoned| poisoned.into_inner());
2334        let outcome = table.sync(&owner, connection_id, is_current_launch, generation, scopes);
2335        let drained = match &outcome {
2336            Ok(applied) => self
2337                .forwarding
2338                .publish_scope_changes(&applied.tag_changes)
2339                .map_err(RouterError::Forwarding)?,
2340            Err(_) => Vec::new(),
2341        };
2342        drop(table);
2343        match outcome {
2344            Ok(applied) => {
2345                let counts = ScopeOutcomeCounts::of(&applied.results);
2346                info!(
2347                    owner = %owner,
2348                    generation,
2349                    records = applied.results.len(),
2350                    created = counts.created,
2351                    replaced = counts.replaced,
2352                    updated = counts.updated,
2353                    unchanged = counts.unchanged,
2354                    refused = counts.refused,
2355                    ended = applied.ended.len(),
2356                    tag_changes = applied.tag_changes.len(),
2357                    routes_closed = drained.len(),
2358                    "scope sync accepted"
2359                );
2360                // An accepted sync can still refuse individual records, and the
2361                // owner is the only party that sees the reply. Name them here so
2362                // an operator can tell a refused session from a missing one
2363                // without the owner's logs. Capped so a sync that refuses
2364                // thousands cannot flood the log; the count above is complete.
2365                for refused in applied
2366                    .results
2367                    .iter()
2368                    .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
2369                    .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
2370                {
2371                    warn!(
2372                        owner = %owner,
2373                        generation,
2374                        scope_ref = %refused.scope_ref,
2375                        scope_epoch = refused.scope_epoch,
2376                        code = refused.code.as_deref().unwrap_or(""),
2377                        "scope record refused"
2378                    );
2379                }
2380                self.close_scope_drained_routes(drained);
2381                let response = ModuleControlResponseToModule::ScopeSync {
2382                    generation,
2383                    results: applied.results,
2384                    ended: applied.ended,
2385                };
2386                Ok(vec![control_response_body_frame(
2387                    &frame,
2388                    &response,
2389                    "ModuleControlResponseToModule::ScopeSync",
2390                )?])
2391            }
2392            Err(refusal) => {
2393                info!(
2394                    owner = %owner,
2395                    generation,
2396                    code = refusal.code,
2397                    "scope sync refused"
2398                );
2399                Ok(vec![control_error_frame(
2400                    &frame,
2401                    refusal.code,
2402                    refusal.message,
2403                )?])
2404            }
2405        }
2406    }
2407
2408    /// Tell both ends of each route a scope change closed. The module gets a
2409    /// channel-scoped GOODBYE and so does the client: the GOODBYE is what ends
2410    /// the client's route handle. The client also gets `route.closed` with the
2411    /// scope reason, one push per module and reason, so it can tell a revoked
2412    /// route from an ordinary close and not reopen it.
2413    fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
2414        if drained.is_empty() {
2415            return;
2416        }
2417        let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
2418            BTreeMap::new();
2419        let mut goodbyes = Vec::with_capacity(drained.len() * 2);
2420        for route in drained {
2421            warn!(
2422                module_id = %route.module_id,
2423                reason = ?route.reason,
2424                client_connection_id = route.client.connection_id.get(),
2425                route_channel = route.client.channel,
2426                "closing route because its scope changed"
2427            );
2428            pushes
2429                .entry((route.module_id.clone(), format!("{:?}", route.reason)))
2430                .or_insert_with(|| (route.reason, Vec::new()))
2431                .1
2432                .push(EndpointRoute {
2433                    goodbye_target: route.client.clone(),
2434                    principal: Principal::Unverified,
2435                    bound_at: Instant::now(),
2436                    draining: false,
2437                    drain_reason: None,
2438                });
2439            goodbyes.push(route.module);
2440            goodbyes.push(route.client);
2441        }
2442        for ((module_id, _), (reason, routes)) in pushes {
2443            send_route_control_pushes(
2444                &self.forwarding,
2445                routes,
2446                ClientControlPush::RouteClosed {
2447                    module_id,
2448                    channels: Vec::new(),
2449                    reason,
2450                    drained: false,
2451                    abandoned: 0,
2452                    excluded_subscriptions: 0,
2453                    terminal: Some(false),
2454                },
2455            );
2456        }
2457        self.emit_route_goodbyes(goodbyes);
2458    }
2459
2460    /// `scope.describe`: any registered module may read any scope, because a
2461    /// provider must read the scope a route it serves is stamped with.
2462    fn handle_scope_describe(
2463        &self,
2464        connection_id: ConnectionId,
2465        frame: Frame,
2466        owner: Principal,
2467        scope_ref: String,
2468    ) -> Result<Vec<Frame>, RouterError> {
2469        let registered = self
2470            .registry
2471            .get_module_by_connection(connection_id)
2472            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2473        if registered.is_none() {
2474            return Ok(vec![control_error_frame(
2475                &frame,
2476                "not_registered",
2477                "scope.describe requires an active module registration owned by this connection",
2478            )?]);
2479        }
2480        let description = self
2481            .scopes
2482            .read()
2483            .unwrap_or_else(|poisoned| poisoned.into_inner())
2484            .describe(&owner, &scope_ref);
2485        let owner_configured = match &owner {
2486            Principal::Reserved { module_id } => self.supervisor.get(module_id).is_some(),
2487            _ => false,
2488        };
2489        let response = ModuleControlResponseToModule::ScopeDescribe {
2490            status: description.status,
2491            scope_epoch: description.scope_epoch,
2492            daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
2493            owner_synced: description.owner_synced,
2494            owner_configured,
2495            scope: description.stamp,
2496        };
2497        Ok(vec![control_response_body_frame(
2498            &frame,
2499            &response,
2500            "ModuleControlResponseToModule::ScopeDescribe",
2501        )?])
2502    }
2503
2504    fn handle_catalog_update(
2505        &self,
2506        connection_id: ConnectionId,
2507        frame: Frame,
2508        provides: Vec<ProviderRole>,
2509        capabilities: Option<CapabilityDeclarations>,
2510        ready: Option<bool>,
2511    ) -> Result<Vec<Frame>, RouterError> {
2512        self.refresh_capability_requirements();
2513        let Some(registration) = self
2514            .registry
2515            .get_module_by_connection(connection_id)
2516            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2517        else {
2518            return Ok(vec![control_error_frame(
2519                &frame,
2520                "not_registered",
2521                "catalog.update requires an active module registration owned by this connection",
2522            )?]);
2523        };
2524
2525        if let Some(message) =
2526            catalog_update_frozen_field_message(&registration.manifest, &provides)
2527        {
2528            return Ok(vec![control_error_frame(
2529                &frame,
2530                "catalog_update_frozen_field",
2531                message,
2532            )?]);
2533        }
2534
2535        let mut candidate = registration.manifest.clone();
2536        candidate.provides = provides.clone();
2537        candidate.capabilities = capabilities
2538            .clone()
2539            .or_else(|| registration.manifest.capabilities.clone());
2540        if let Err(err) = candidate.validate_capability_grammar() {
2541            return Ok(vec![control_error_frame(
2542                &frame,
2543                "invalid_capability_grammar",
2544                err.to_string(),
2545            )?]);
2546        }
2547
2548        let updated = self
2549            .registry
2550            .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
2551            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2552        if updated.is_none() {
2553            return Ok(vec![control_error_frame(
2554                &frame,
2555                "not_registered",
2556                "catalog.update requires an active module registration owned by this connection",
2557            )?]);
2558        }
2559        if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
2560            log_duplicate_claim_events(
2561                self.capability_evaluator
2562                    .duplicate_claims(DuplicateClaimSource::CatalogUpdate, &registrations),
2563            );
2564        }
2565        if capability_census_trigger(
2566            registration.manifest.capabilities.as_ref(),
2567            updated
2568                .as_ref()
2569                .and_then(|entry| entry.manifest.capabilities.as_ref()),
2570        ) {
2571            self.enforce_capability_denies();
2572        }
2573        self.refresh_capability_requirements();
2574
2575        let response = ModuleControlResponseToModule::CatalogUpdate {};
2576        control_response_body_frame(
2577            &frame,
2578            &response,
2579            "ModuleControlResponseToModule::CatalogUpdate",
2580        )
2581        .map(|frame| vec![frame])
2582    }
2583
2584    fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2585        self.refresh_capability_requirements();
2586        // A bare connection count is ambiguous between many clients holding a
2587        // route each and one client accumulating hundreds, so publish the
2588        // concentration alongside it. Route state is best-effort here: a
2589        // diagnostic endpoint must still answer if the forwarding lock is
2590        // contended.
2591        let mut counters = self.counters.snapshot();
2592        if let (Ok((connections_with_routes, max)), Some(obj)) = (
2593            self.forwarding.client_route_concentration(),
2594            counters.as_object_mut(),
2595        ) {
2596            obj.insert(
2597                "client_connections_with_routes".into(),
2598                connections_with_routes.into(),
2599            );
2600            obj.insert("max_routes_on_one_connection".into(), max.into());
2601        }
2602        // A module that is being fast-refused and a module that is fine look
2603        // identical from a client that retries and succeeds, so name the open
2604        // breakers here. This rides the existing free-form counters object
2605        // rather than a new wire field, so no sibling that deserializes
2606        // `ServerDescribe` has to be rebuilt to keep reading it.
2607        if let (Some(open_breakers), Some(obj)) = (
2608            self.route_bind_breakers.open_snapshot(),
2609            counters.as_object_mut(),
2610        ) {
2611            obj.insert("route_bind_breakers_open".into(), open_breakers);
2612        }
2613        let response = ClientControlResponse::ServerDescribe {
2614            protocol_ver: PROTOCOL_VERSION,
2615            subc_ops: subc_ops(),
2616            capabilities: self.subc_capabilities.as_ref().to_vec(),
2617            connected_clients: self.connected_clients.count(),
2618            counters: Some(counters),
2619            build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
2620            build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
2621            capability_requirements: self.capability_requirement_statuses(),
2622            machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2623        };
2624        Ok(vec![control_response_body_frame(
2625            &frame,
2626            &response,
2627            "ClientControlResponse::ServerDescribe",
2628        )?])
2629    }
2630
2631    fn handle_catalog_list(
2632        &self,
2633        frame: Frame,
2634        module_id: Option<String>,
2635    ) -> Result<Vec<Frame>, RouterError> {
2636        let (generation, modules) = self.registry.list_modules().map_err(|err| {
2637            RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
2638        })?;
2639        let entries = modules
2640            .into_iter()
2641            .filter(|registration| {
2642                module_id
2643                    .as_deref()
2644                    .map(|wanted| registration.manifest.module_id == wanted)
2645                    .unwrap_or(true)
2646            })
2647            .map(|registration| {
2648                let not_ready = self.not_ready_reason(&registration);
2649                let roles = registration.manifest.provides;
2650                CatalogEntry {
2651                    module_id: registration.manifest.module_id,
2652                    ready: not_ready.is_none(),
2653                    not_ready,
2654                    module_version: Some(registration.manifest.module_version),
2655                    roles,
2656                    control_ops: registration.control_ops,
2657                    capabilities: registration.manifest.capabilities,
2658                    self_signals: registration.manifest.self_signals,
2659                }
2660            })
2661            .collect();
2662        let response = ClientControlResponse::CatalogList {
2663            generation,
2664            modules: entries,
2665            subc_ops: subc_ops(),
2666        };
2667        Ok(vec![control_response_body_frame(
2668            &frame,
2669            &response,
2670            "ClientControlResponse::CatalogList",
2671        )?])
2672    }
2673
2674    fn route_open_principal(
2675        &self,
2676        frame: &Frame,
2677        consumer_identity: Option<ConsumerIdentity>,
2678    ) -> Result<Result<Principal, Frame>, RouterError> {
2679        let Some(consumer_identity) = consumer_identity else {
2680            return Ok(Ok(Principal::Direct));
2681        };
2682
2683        if self.supervisor.spawned_consumer_authorized(
2684            &consumer_identity.module_id,
2685            &consumer_identity.launch_nonce,
2686        ) {
2687            return Ok(Ok(Principal::Reserved {
2688                module_id: consumer_identity.module_id,
2689            }));
2690        }
2691
2692        Ok(Err(control_error_frame(
2693            frame,
2694            "bad_consumer_identity",
2695            format!(
2696                "consumer_identity for module_id '{}' did not match a supervised launch nonce",
2697                consumer_identity.module_id
2698            ),
2699        )?))
2700    }
2701
2702    /// Ordinary `route.open` refusals go through here; admission and breaker
2703    /// refusals log separately with their capacity or breaker state. The daemon can
2704    /// attest which code it sent: without the event, a client's "the daemon
2705    /// refused me" and the daemon's own view could only be reconciled by
2706    /// argument. Malformed input (`invalid_project_root`) does not come here;
2707    /// rejecting a request that was never a valid open is not a refusal of one.
2708    fn route_open_refusal_frame(
2709        &self,
2710        ctx: &RouteCtx,
2711        frame: &Frame,
2712        module_id: &str,
2713        reason: &'static str,
2714        code: &'static str,
2715        message: impl Into<String>,
2716    ) -> Result<Frame, RouterError> {
2717        self.observe_route_open_refusal(ctx, module_id, reason, code);
2718        control_error_frame(frame, code, message.into())
2719    }
2720
2721    /// Refuse a `route.open` because the target module's bind-relay breaker is
2722    /// open, without attempting the relay.
2723    ///
2724    /// The wire code is `module_timeout`, which is the truth (the module has
2725    /// not been answering binds) and which both SDKs already classify as
2726    /// retryable with capped backoff. Reusing it is what keeps this change out
2727    /// of both SDKs; the daemon-side distinction lives in the counter key
2728    /// instead.
2729    ///
2730    /// DELIBERATELY NOT LOGGED PER OCCURRENCE, unlike every other refusal.
2731    /// While a breaker is open this fires on every open to that module, and the
2732    /// stall written up in `docs/designs/route-open-head-of-line.md` already
2733    /// produced 261 lines about a single module inside 3000 lines of daemon
2734    /// log. The rare transitions are logged at warn/info instead and the volume
2735    /// is carried by the counter, so the evidence survives without the flood.
2736    /// The debug line keeps a per-refusal record reachable for whoever turns
2737    /// the level up.
2738    fn route_open_breaker_refusal_frame(
2739        &self,
2740        ctx: &RouteCtx,
2741        frame: &Frame,
2742        module_id: &str,
2743        consecutive_timeouts: u32,
2744        retry_in: Duration,
2745        probe_in_flight: bool,
2746    ) -> Result<Frame, RouterError> {
2747        self.counters
2748            .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
2749        debug!(
2750            target: "control",
2751            code = "module_timeout",
2752            module_id = ?module_id,
2753            connection_id = ctx.connection_id.get(),
2754            consecutive_timeouts,
2755            retry_in_ms = retry_in.as_millis() as u64,
2756            probe_in_flight,
2757            "route.open refused by open bind-relay breaker"
2758        );
2759        let detail = if probe_in_flight {
2760            "one probe bind is already in flight; retry once it settles".to_string()
2761        } else {
2762            format!("not relaying for another {retry_in:?}")
2763        };
2764        control_error_frame(
2765            frame,
2766            "module_timeout",
2767            format!(
2768                "module_id '{module_id}' failed {consecutive_timeouts} consecutive route.bind \
2769                 relays; {detail}"
2770            ),
2771        )
2772    }
2773
2774    /// `code` is daemon vocabulary and prints plainly; `module_id` is the
2775    /// requester's bytes (an unknown target is whatever the client sent) and
2776    /// is Debug-formatted so control characters land in the log escaped
2777    /// rather than as terminal sequences for whoever tails it.
2778    ///
2779    /// `reason` names the check that refused, because one wire code has
2780    /// several senders: after a module registers, `target_unavailable` can
2781    /// come from a missing role, an inactive registration, a supervisor that
2782    /// has not marked the process live, a missing forwarding connection, or a
2783    /// failed relay, and a log that records only the code cannot say which of
2784    /// them fired. It is a static, daemon-chosen label per branch, so it is
2785    /// safe to print plainly and stays a closed set.
2786    fn observe_route_open_refusal(
2787        &self,
2788        ctx: &RouteCtx,
2789        module_id: &str,
2790        reason: &'static str,
2791        code: &'static str,
2792    ) {
2793        self.counters.increment_route_open_refused(code);
2794        info!(
2795            target: "control",
2796            code,
2797            reason,
2798            module_id = ?module_id,
2799            connection_id = ctx.connection_id.get(),
2800            "route.open refused"
2801        );
2802        if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
2803            self.route_outages.record_not_serving(module_id, reason);
2804        }
2805    }
2806
2807    /// Record an ACCEPTED route.open.
2808    ///
2809    /// Refusals have been logged and counted since the attestation work; accepts
2810    /// were invisible, so the daemon knew every principal it stamped and wrote
2811    /// none of them down. The party that attests the identity was the only party
2812    /// not recording it, which left a credential vault unable to name the sender
2813    /// of a call that reached it (claustrum #43) and left the launch-nonce
2814    /// concurrency question unanswerable from the outside.
2815    ///
2816    /// FIELD NAMES MATCH `route.open refused` DELIBERATELY, so one grep over
2817    /// `code`/`module_id`/`connection_id` returns both directions of the same
2818    /// decision rather than two shapes a reader has to join by hand.
2819    ///
2820    /// `module_id` IS RENDERED BARE HERE AND DEBUG-ESCAPED ON THE REFUSAL PATH,
2821    /// and the difference carries information rather than being an
2822    /// inconsistency. This line is only reachable after a successful bind to a
2823    /// REGISTERED module, so the value has already passed HELLO validation
2824    /// including the path-hazard refusal and cannot contain control bytes. A
2825    /// refused id may be arbitrary attacker-chosen bytes and must stay escaped.
2826    /// So A QUOTED `module_id` IN THE LOG MEANS THE VALUE WAS NEVER VALIDATED.
2827    ///
2828    /// Bare is also what every other daemon line already emits (`module
2829    /// registered`, `configured module supervised`). Shipping `?module_id` here
2830    /// made this instrument the only one in the file whose ids did not answer
2831    /// `grep module_id=broca` -- 3 hits against 342 for the escaped form, in a
2832    /// line whose whole purpose is being grepped beside its sibling.
2833    ///
2834    /// THIS RENDERING IS UNFENCED AND THE REASON IS WORTH KNOWING: the in-crate
2835    /// `EventCapture` test layer implements only `record_debug`, so `Visit`
2836    /// forwards every field type through it and a bare `&str` and a `?`-escaped
2837    /// one are recorded identically. A test written against that harness passes
2838    /// either way -- I wrote one, measured it, and deleted it rather than ship a
2839    /// green assertion that cannot fail. The same limit applies to the escaping
2840    /// assertion in `route_open_supervised_absence_emits_refusal_fields_and_counts_code`:
2841    /// it reads as a guard on the Debug escaping and cannot detect its removal.
2842    /// Fencing either needs the real formatter, not the capture layer.
2843    ///
2844    /// `peer_addr` is NOT here and cannot be: `SO_PEERCRED`/`LOCAL_PEERPID` are
2845    /// unix-socket options and subc is loopback TCP, so there is no peer identity
2846    /// to record. The ephemeral port would decay within minutes and answer only a
2847    /// live question. The identity question is instead answered by counting
2848    /// distinct live connections presenting one module's `consumer_identity` --
2849    /// "is anyone else holding this secret" rather than "is this the right
2850    /// process".
2851    fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
2852        self.route_outages.record_accepted(module_id);
2853        self.counters.increment_route_open_accepted(principal);
2854        info!(
2855            target: "control",
2856            principal,
2857            module_id,
2858            connection_id = ctx.connection_id.get(),
2859            "route.open accepted"
2860        );
2861    }
2862
2863    fn supervised_absent_route_open_refusal_frame(
2864        &self,
2865        ctx: &RouteCtx,
2866        frame: &Frame,
2867        module_id: &str,
2868        code: &'static str,
2869        status: &crate::supervise::ModuleStatus,
2870    ) -> Result<Frame, RouterError> {
2871        self.counters.increment_route_open_refused(code);
2872        info!(
2873            target: "control",
2874            code,
2875            reason = "supervised_not_registered",
2876            module_id = ?module_id,
2877            connection_id = ctx.connection_id.get(),
2878            state = %status.state,
2879            enabled = status.enabled,
2880            live = status.live,
2881            "route.open refused"
2882        );
2883        // A supervised module whose process has not registered is not
2884        // serving, whatever the reason; the supervisor knows this id, so it is
2885        // safe to track.
2886        self.route_outages
2887            .record_not_serving(module_id, "supervised_not_registered");
2888        control_error_frame(
2889            frame,
2890            code,
2891            format!(
2892                "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
2893                status.state, status.enabled, status.live
2894            ),
2895        )
2896    }
2897
2898    async fn handle_route_open(
2899        &self,
2900        ctx: &RouteCtx,
2901        frame: Frame,
2902        request: RouteOpenRequest,
2903    ) -> Result<Vec<Frame>, RouterError> {
2904        let RouteOpenRequest {
2905            target,
2906            mut identity,
2907            consumer_identity,
2908            consumer_capabilities,
2909            role_versions,
2910            admission_facts,
2911            scope,
2912        } = request;
2913        let target_module_id = target_module_id(&target).to_string();
2914        debug!(
2915            connection_id = ctx.connection_id.get(),
2916            corr = frame.header.corr,
2917            module_id = %target_module_id,
2918            "handling route.open"
2919        );
2920
2921        // A malformed declaration is refused first, before anything about the
2922        // target is looked up: the same body would be refused against any
2923        // module, so the caller learns nothing by retrying or waiting. An empty
2924        // map declares nothing and travels as no field at all, so a provider
2925        // only ever sees a missing field or a non-empty one.
2926        let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
2927        if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
2928            self.observe_route_open_refusal(
2929                ctx,
2930                &target_module_id,
2931                "invalid_role_versions",
2932                error_codes::INVALID_REQUEST,
2933            );
2934            return Ok(vec![control_error_body_frame(
2935                &frame,
2936                ErrorBody {
2937                    code: error_codes::INVALID_REQUEST.to_string(),
2938                    message: error.to_string(),
2939                    detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
2940                },
2941            )?]);
2942        }
2943
2944        // WHY THESE REPLIES DISCRIMINATE FREELY, since the usual rule is the
2945        // opposite. Below, a caller learns whether a module is unregistered,
2946        // supervised-but-down (with state/enabled/live), or registered without the
2947        // requested role. Elsewhere that is an enumeration leak: a probe learning
2948        // the shape of a fleet it cannot otherwise see.
2949        //
2950        // It is not one here, and the reason is the ACCESS MODEL rather than
2951        // anything about these errors. Reaching route.open requires the
2952        // pre-envelope HMAC handshake, whose key lives in a 0600 user-owned
2953        // connection file, so any caller who completes it already runs as this
2954        // user -- and can read subc.jsonc for the module list and `ck module
2955        // status` for live state. The reply discloses nothing the caller cannot
2956        // read more easily from disk, while the precision is load-bearing:
2957        // `unknown_module` is retryable and a missing role is not.
2958        //
2959        // IF THE HANDSHAKE EVER ADMITS A PRINCIPAL THAT IS NOT THIS USER -- a
2960        // remote transport, a sandboxed caller, a shared-host mode -- THAT
2961        // PREMISE DIES AND THESE THREE REPLIES MUST COLLAPSE INTO ONE.
2962        let Some(registration) = self
2963            .registry
2964            .get_module(&target_module_id)
2965            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2966        else {
2967            if let Some((status, warming)) =
2968                self.supervisor_status(&target_module_id, frame.header.corr)?
2969            {
2970                // BEFORE the two availability codes below, because for a module
2971                // that speaks no subc wire both of them are false comfort: they
2972                // say "not right now" and are retried, and this module will
2973                // never register no matter how long the caller waits. The
2974                // absence here is the declaration being honoured, not a module
2975                // that is late.
2976                if status.protocol == ModuleProtocol::None {
2977                    return Ok(vec![self.route_open_refusal_frame(
2978                        ctx,
2979                        &frame,
2980                        &target_module_id,
2981                        "protocol_none",
2982                        error_codes::MODULE_NO_PROTOCOL,
2983                        format!(
2984                            "module_id '{target_module_id}' is declared protocol: none; \
2985                             it speaks no subc wire and serves no routes"
2986                        ),
2987                    )?]);
2988                }
2989                let code = if warming {
2990                    "module_warming"
2991                } else {
2992                    "target_unavailable"
2993                };
2994                return Ok(vec![self.supervised_absent_route_open_refusal_frame(
2995                    ctx,
2996                    &frame,
2997                    &target_module_id,
2998                    code,
2999                    &status,
3000                )?]);
3001            }
3002            if let Some(removed_ago_ms) =
3003                self.supervisor.removal_tombstone_age_ms(&target_module_id)
3004            {
3005                return Ok(vec![self.route_open_refusal_frame(
3006                    ctx,
3007                    &frame,
3008                    &target_module_id,
3009                    "removed",
3010                    error_codes::MODULE_REMOVED,
3011                    format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
3012                )?]);
3013            }
3014            return Ok(vec![self.route_open_refusal_frame(
3015                ctx,
3016                &frame,
3017                &target_module_id,
3018                "not_registered",
3019                error_codes::UNKNOWN_MODULE,
3020                format!("module_id '{target_module_id}' is not registered"),
3021            )?]);
3022        };
3023
3024        // Best-effort only: registry readiness and forwarding reservation use
3025        // different locks, so a module can flip readiness between this read and
3026        // the relay. Modules must still tolerate an `on_bind` while not ready.
3027        if !registration.ready {
3028            self.counters
3029                .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
3030            info!(
3031                target: "control",
3032                code = error_codes::MODULE_WARMING,
3033                module_id = ?target_module_id,
3034                connection_id = ctx.connection_id.get(),
3035                reason = "declared_not_ready",
3036                "route.open refused"
3037            );
3038            // The module is registered but says it cannot take work, which is
3039            // an outage from the caller's side even though its process is up.
3040            self.route_outages
3041                .record_not_serving(&target_module_id, "declared_not_ready");
3042            return Ok(vec![control_error_body_frame(
3043                &frame,
3044                ErrorBody {
3045                    code: error_codes::MODULE_WARMING.to_string(),
3046                    message: format!(
3047                        "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
3048                    ),
3049                    detail: Some(serde_json::json!({
3050                        "reason": "declared_not_ready"
3051                    })),
3052                },
3053            )?]);
3054        }
3055
3056        // Effective readiness, second half: a module that declares a capability
3057        // `need: required` is not routable while that capability has no
3058        // registered provider. It is enforced HERE, as a retryable routing
3059        // refusal, and deliberately not as spawn ordering or a boot block. The
3060        // module is still started and registered and can make its own calls;
3061        // spawn ordering is a promise that cannot be kept once a provider
3062        // crashes at runtime, and refusing to boot would stop the whole
3063        // machine, including the tools needed to fix its configuration.
3064        //
3065        // "Provided" is the evaluator's verdict, which counts a provider as
3066        // soon as it has REGISTERED, not once it is ready. Two modules that
3067        // require each other's capabilities are therefore both routable once
3068        // both register; counting readiness instead would deadlock them.
3069        //
3070        // Only new opens are refused. Routes already bound when a provider
3071        // goes away stay bound: nothing here tears them down, and the module
3072        // answers them as it can. Like the readiness read above this is
3073        // best-effort against a provider registering or leaving concurrently.
3074        if let Some(capability) = self.first_unprovided_required_capability(&registration) {
3075            self.counters
3076                .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
3077            info!(
3078                target: "control",
3079                code = error_codes::MODULE_WARMING,
3080                module_id = ?target_module_id,
3081                connection_id = ctx.connection_id.get(),
3082                reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3083                capability = %capability,
3084                "route.open refused"
3085            );
3086            return Ok(vec![control_error_body_frame(
3087                &frame,
3088                ErrorBody {
3089                    code: error_codes::MODULE_WARMING.to_string(),
3090                    message: format!(
3091                        "module_id '{target_module_id}' requires capability '{capability}', \
3092                         which no registered module provides; retry"
3093                    ),
3094                    detail: Some(serde_json::json!({
3095                        "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3096                        "capability": capability,
3097                    })),
3098                },
3099            )?]);
3100        }
3101
3102        if !target_has_required_role(&target, &registration.manifest.provides) {
3103            return Ok(vec![self.route_open_refusal_frame(
3104                ctx,
3105                &frame,
3106                &target_module_id,
3107                "role_not_provided",
3108                "target_unavailable",
3109                format!("module_id '{target_module_id}' does not provide the requested target"),
3110            )?]);
3111        }
3112
3113        if registration.state != ChannelState::Active {
3114            return Ok(vec![self.route_open_refusal_frame(
3115                ctx,
3116                &frame,
3117                &target_module_id,
3118                "registration_not_active",
3119                "target_unavailable",
3120                format!("module_id '{target_module_id}' is not active"),
3121            )?]);
3122        }
3123
3124        if self
3125            .forwarding
3126            .module_is_draining(&target_module_id)
3127            .map_err(RouterError::Forwarding)?
3128        {
3129            return Ok(vec![self.route_open_refusal_frame(
3130                ctx,
3131                &frame,
3132                &target_module_id,
3133                "reloading",
3134                "module_reloading",
3135                format!("module_id '{target_module_id}' is reloading"),
3136            )?]);
3137        }
3138
3139        if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
3140            process_liveness.process_live(&target_module_id) == Some(false)
3141        }) {
3142            // A module the supervisor is restarting or reloading can still hold
3143            // a registration: the old process before its connection closes, or
3144            // a new one that registered while the supervisor was draining. The
3145            // forwarding table does not see that as draining, but the consumer
3146            // should still be told to retry soon, exactly as for the drain
3147            // above, rather than that the target is unavailable.
3148            if process_liveness.process_replacing(&target_module_id) {
3149                return Ok(vec![self.route_open_refusal_frame(
3150                    ctx,
3151                    &frame,
3152                    &target_module_id,
3153                    "reloading",
3154                    "module_reloading",
3155                    format!("module_id '{target_module_id}' is reloading"),
3156                )?]);
3157            }
3158            return Ok(vec![self.route_open_refusal_frame(
3159                ctx,
3160                &frame,
3161                &target_module_id,
3162                "supervisor_not_live",
3163                "target_unavailable",
3164                format!("module_id '{target_module_id}' is not live"),
3165            )?]);
3166        }
3167
3168        if !self
3169            .forwarding
3170            .has_live_module_connection(&target_module_id)
3171            .map_err(RouterError::Forwarding)?
3172        {
3173            return Ok(vec![self.route_open_refusal_frame(
3174                ctx,
3175                &frame,
3176                &target_module_id,
3177                "no_forwarding_connection",
3178                "target_unavailable",
3179                format!("module_id '{target_module_id}' has no live forwarding connection"),
3180            )?]);
3181        }
3182
3183        if let Some(error) =
3184            self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
3185        {
3186            self.observe_route_open_refusal(
3187                ctx,
3188                &target_module_id,
3189                "op_not_allowed",
3190                "op_not_allowed",
3191            );
3192            return Ok(vec![error]);
3193        }
3194
3195        let principal = match self.route_open_principal(&frame, consumer_identity)? {
3196            Ok(principal) => principal,
3197            Err(error) => {
3198                self.observe_route_open_refusal(
3199                    ctx,
3200                    &target_module_id,
3201                    "bad_consumer_identity",
3202                    "bad_consumer_identity",
3203                );
3204                return Ok(vec![error]);
3205            }
3206        };
3207
3208        // This is attested, control-plane policy for supervised module origins.
3209        // Keep it before route reservation and out of the opaque forwarding hot
3210        // path: data frames must never acquire a per-frame capability check.
3211        if let Principal::Reserved {
3212            module_id: opening_module_id,
3213        } = &principal
3214        {
3215            if let Some(opening_registration) = self
3216                .registry
3217                .get_module(opening_module_id)
3218                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3219            {
3220                if let Some(capability) =
3221                    denied_capability(&opening_registration.manifest, &registration.manifest)
3222                {
3223                    warn!(
3224                        opening_module_id,
3225                        target_module_id,
3226                        capability,
3227                        "refusing route.open because an attested capability deny edge matches"
3228                    );
3229                    return Ok(vec![self.route_open_refusal_frame(
3230                        ctx,
3231                        &frame,
3232                        &target_module_id,
3233                        "capability_deny_edge",
3234                        "capability_forbidden",
3235                        format!(
3236                            "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
3237                        ),
3238                    )?]);
3239                }
3240            }
3241        }
3242
3243        if admission_facts.is_some() {
3244            let carrier_matches = matches!(
3245                &principal,
3246                Principal::Reserved { module_id }
3247                    if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
3248            );
3249            if !carrier_matches {
3250                return Ok(vec![self.route_open_refusal_frame(
3251                    ctx,
3252                    &frame,
3253                    &target_module_id,
3254                    "admission_facts_carrier_not_permitted",
3255                    "admission_facts_not_permitted",
3256                    "admission facts may only be carried by the configured reserved module",
3257                )?]);
3258            }
3259
3260            let target_allowed = self
3261                .admission_facts_targets
3262                .as_ref()
3263                .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
3264            if !target_allowed {
3265                return Ok(vec![self.route_open_refusal_frame(
3266                    ctx,
3267                    &frame,
3268                    &target_module_id,
3269                    "admission_facts_target_not_listed",
3270                    "admission_facts_target_not_allowed",
3271                    format!(
3272                        "admission facts are not permitted for target module_id '{target_module_id}'"
3273                    ),
3274                )?]);
3275            }
3276
3277            // Keep the value opaque to subc. The downstream admission validator owns
3278            // schema and semantic checks; this daemon only enforces carrier authority
3279            // and the configured destination allowlist.
3280        }
3281
3282        // Scope admission, on the attested principal above and never on the
3283        // request body. The tag read here travels with the pending bind and is
3284        // compared with the published one at commit, so a sync between here
3285        // and the module's ack refuses the open instead of binding a stamp
3286        // that is no longer true.
3287        let (bound_scope, scope_stamp) = match scope {
3288            None => (None, None),
3289            Some(selector) => {
3290                let owner_configured = match &selector.owner {
3291                    Principal::Reserved { module_id } => self.supervisor.get(module_id).is_some(),
3292                    _ => false,
3293                };
3294                let admitted = self
3295                    .scopes
3296                    .read()
3297                    .unwrap_or_else(|poisoned| poisoned.into_inner())
3298                    .admit(&principal, &target_module_id, &selector, owner_configured);
3299                match admitted {
3300                    Ok(admission) => (
3301                        Some(BoundScope {
3302                            owner: admission.owner,
3303                            scope_ref: admission.stamp.scope_ref.clone(),
3304                            tag: admission.tag,
3305                        }),
3306                        Some(admission.stamp),
3307                    ),
3308                    Err(refusal) => {
3309                        return Ok(vec![self.route_open_refusal_frame(
3310                            ctx,
3311                            &frame,
3312                            &target_module_id,
3313                            refusal.code,
3314                            refusal.code,
3315                            refusal.message,
3316                        )?]);
3317                    }
3318                }
3319            }
3320        };
3321
3322        // Bind admits a root that no longer exists on disk, because refusing here
3323        // closes the only exit from a paused run: cancel needs a bound route, and a
3324        // renamed or reclaimed directory makes that route unopenable forever. The
3325        // run itself is intact and still addressable by its recorded identity.
3326        //
3327        // This does NOT relax the rule the strict constructor protects. That rule is
3328        // that no root is ever aliased into NEW durable state -- a missing component
3329        // can reappear as a symlink elsewhere, which would move the identity and
3330        // split a session's history across two of them. The engine now refuses the
3331        // two operations that create such state (send and import) at admission,
3332        // which is a narrower way to hold the same invariant: reads and terminations
3333        // are admitted, writes are not. That refusal had to ship before this line
3334        // changed, or there is an interval where a send commits under a provisional
3335        // identity -- the exact failure the original policy existed to prevent.
3336        //
3337        // Resolution follows realpath rather than lexical cleanup: the longest
3338        // existing ancestor is canonicalized and the missing tail re-appended, so a
3339        // live root is unchanged and a vanished leaf keeps the identity it was
3340        // admitted under. Lexical cleanup would mint a DIFFERENT identity for the
3341        // same caller the moment the directory vanished, which strands the run more
3342        // quietly than refusing it.
3343        let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
3344            Ok(project_root) => project_root,
3345            Err(err) => {
3346                return Ok(vec![control_error_frame(
3347                    &frame,
3348                    "invalid_project_root",
3349                    err.to_string(),
3350                )?])
3351            }
3352        };
3353        identity.project_root = project_root.as_path().to_path_buf();
3354
3355        // Last gate before any relay work, and deliberately after the cheap
3356        // registry and availability checks above: those name a more precise
3357        // condition (unknown, removed, reloading) and a caller is better served
3358        // by the precise code than by this one.
3359        //
3360        // Everything below this point costs an egress permit, a reserved handle
3361        // pair and, if the module does not answer, the whole relay budget. The
3362        // reader no longer waits for that budget, so cap each target explicitly;
3363        // serial dispatch used to provide the accidental cap of one relay per
3364        // connection. Admission is a mutex-protected count and never waits.
3365        let _concurrency_guard = match self
3366            .route_bind_concurrency
3367            .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
3368        {
3369            Ok(guard) => guard,
3370            Err(in_flight) => {
3371                return Ok(vec![self.route_open_target_capacity_refusal(
3372                    ctx,
3373                    &frame,
3374                    &target_module_id,
3375                    in_flight,
3376                )?]);
3377            }
3378        };
3379
3380        // A module that has already burned the whole budget `threshold` times
3381        // in a row does not get to charge it again until a probe says it recovered.
3382        let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
3383            RouteBindAdmission::Admitted { guard, probe } => {
3384                if probe {
3385                    info!(
3386                        module_id = %target_module_id,
3387                        connection_id = ctx.connection_id.get(),
3388                        "route.bind breaker half-open: admitting one probe"
3389                    );
3390                }
3391                guard
3392            }
3393            RouteBindAdmission::Refused {
3394                consecutive_timeouts,
3395                retry_in,
3396                probe_in_flight,
3397            } => {
3398                return Ok(vec![self.route_open_breaker_refusal_frame(
3399                    ctx,
3400                    &frame,
3401                    &target_module_id,
3402                    consecutive_timeouts,
3403                    retry_in,
3404                    probe_in_flight,
3405                )?]);
3406            }
3407        };
3408
3409        // Resolve the per-module budget here so the wait matches the operator's
3410        // intent for this specific target. A per-module override in
3411        // `subc.jsonc` (or `with_route_bind_relay_timeouts` for embedded
3412        // daemons) wins over the daemon-wide default.
3413        let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
3414        let relay_deadline = Instant::now() + route_bind_relay_timeout;
3415        let pending = match self
3416            .forwarding
3417            .begin_route_bind_relay_for(
3418                ctx.connection_id,
3419                ctx.egress.clone(),
3420                response_version(&frame),
3421                frame.header.corr,
3422                &target_module_id,
3423                principal.clone(),
3424                bound_scope,
3425                Some(project_root),
3426                relay_deadline,
3427            )
3428            .await
3429        {
3430            Ok(pending) => pending,
3431            Err(err) => {
3432                return Ok(vec![self.route_open_refusal_frame(
3433                    ctx,
3434                    &frame,
3435                    &target_module_id,
3436                    "relay_reservation_failed",
3437                    forwarding_error_code(&err),
3438                    err.to_string(),
3439                )?])
3440            }
3441        };
3442        let crate::forwarding::PendingRouteBindRelay {
3443            endpoint,
3444            module_sink,
3445            negotiated_ver,
3446            client_channel,
3447            client_epoch,
3448            module_channel,
3449            module_epoch,
3450            corr: relay_corr,
3451            receiver,
3452        } = pending;
3453        let mut reservation =
3454            RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
3455
3456        debug!(
3457            connection_id = ctx.connection_id.get(),
3458            client_channel,
3459            client_epoch,
3460            module_channel,
3461            module_epoch,
3462            "reserved route handle pair"
3463        );
3464        // Rendered BEFORE the move into the relay, because the accept arm below
3465        // is where it is logged and the principal is gone by then.
3466        let principal_label = match &principal {
3467            Principal::Reserved { module_id } => format!("reserved:{module_id}"),
3468            Principal::Direct => "direct".to_string(),
3469            other => format!("{other:?}"),
3470        };
3471        let relay = ModuleControlRequest::RouteBind {
3472            route_channel: module_channel,
3473            epoch: module_epoch,
3474            target,
3475            identity,
3476            principal: Some(principal),
3477            consumer_capabilities,
3478            role_versions,
3479            admission_facts,
3480            scope: scope_stamp,
3481        };
3482        let relay_body = serde_json::to_vec(&relay).map_err(|err| {
3483            RouterError::backend(
3484                0,
3485                frame.header.corr,
3486                format!("failed to encode route.bind request: {err}"),
3487            )
3488        })?;
3489        let relay_frame = Frame::build_with_version(
3490            negotiated_ver,
3491            FrameType::Request,
3492            control_flags(),
3493            0,
3494            0,
3495            relay_corr,
3496            relay_body,
3497        )
3498        .map_err(RouterError::FrameBuild)?;
3499
3500        if let Err(err) = module_sink.send(relay_frame).await {
3501            reservation.release_and_disarm();
3502            return Ok(vec![self.route_open_refusal_frame(
3503                ctx,
3504                &frame,
3505                &target_module_id,
3506                "relay_send_failed",
3507                "target_unavailable",
3508                err.to_string(),
3509            )?]);
3510        }
3511
3512        if !self
3513            .forwarding
3514            .mark_route_bind_relay_enqueued(endpoint, relay_corr)
3515            .map_err(RouterError::Forwarding)?
3516        {
3517            self.send_abandoned_route_bind_goodbye(
3518                &module_sink,
3519                negotiated_ver,
3520                module_channel,
3521                module_epoch,
3522            );
3523        }
3524
3525        match timeout_at(relay_deadline, receiver).await {
3526            Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
3527                reservation.disarm();
3528                if breaker.record_accepted() {
3529                    info!(
3530                        module_id = %target_module_id,
3531                        "route.bind breaker closed: the probe was accepted"
3532                    );
3533                }
3534                self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
3535                Ok(Vec::new())
3536            }
3537            Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
3538                reservation.release_and_disarm();
3539                // A module that says no in microseconds is healthy. Rejection
3540                // is a different condition with its own refusal and must not
3541                // move the breaker.
3542                breaker.record_inconclusive();
3543                // The daemon's own commit re-check refused the bind because the
3544                // scope ended or changed after admission. The module accepted;
3545                // counting it as a module rejection would blame the module.
3546                let scope_code = match body.code.as_str() {
3547                    error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
3548                    error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
3549                    _ => None,
3550                };
3551                if let Some(code) = scope_code {
3552                    self.observe_route_open_refusal(
3553                        ctx,
3554                        &target_module_id,
3555                        "scope_changed_before_commit",
3556                        code,
3557                    );
3558                    return Ok(vec![control_error_body_frame(&frame, body)?]);
3559                }
3560                self.counters
3561                    .increment_route_open_refused("module_rejected");
3562                info!(
3563                    target: "control",
3564                    code = "module_rejected",
3565                    module_code = ?body.code,
3566                    module_id = ?target_module_id,
3567                    connection_id = ctx.connection_id.get(),
3568                    "route.open refused"
3569                );
3570                Ok(vec![control_error_body_frame(&frame, body)?])
3571            }
3572            Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
3573                reservation.release_and_disarm();
3574                breaker.record_inconclusive();
3575                // Fires when the module's connection closes while a relayed
3576                // bind is pending -- typically a caller racing a module restart
3577                // whose bind was relayed BEFORE the drain mark went up. Logged
3578                // because the caller sees only its own error and the fleet has
3579                // already spent one diagnosis round unable to tell this arm
3580                // from a relay timeout without daemon-side evidence.
3581                tracing::warn!(
3582                    module_id = %target_module_id,
3583                    "route.bind relay abandoned: {message}"
3584                );
3585                Ok(vec![self.route_open_refusal_frame(
3586                    ctx,
3587                    &frame,
3588                    &target_module_id,
3589                    "relay_abandoned",
3590                    "target_unavailable",
3591                    message,
3592                )?])
3593            }
3594            Ok(Err(_)) => {
3595                reservation.release_and_disarm();
3596                breaker.record_inconclusive();
3597                Ok(vec![self.route_open_refusal_frame(
3598                    ctx,
3599                    &frame,
3600                    &target_module_id,
3601                    "relay_waiter_canceled",
3602                    "target_unavailable",
3603                    "route.bind relay waiter was canceled before the module responded",
3604                )?])
3605            }
3606            Err(_) => {
3607                reservation.release_and_disarm();
3608                // THE ONLY ARM THAT MOVES THE BREAKER. Budget exhausted with no
3609                // answer at all is the one condition a fast refusal can
3610                // usefully stand in for; every other arm already answered.
3611                if let Some(opened) = breaker.record_timeout(
3612                    self.route_bind_breaker_threshold,
3613                    self.route_bind_breaker_cooldown,
3614                ) {
3615                    warn!(
3616                        module_id = %target_module_id,
3617                        consecutive_timeouts = opened.consecutive_timeouts,
3618                        cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
3619                        reopened_after_probe = opened.reopened_after_probe,
3620                        "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
3621                    );
3622                }
3623                // The generous budget just burned to no answer: the module is
3624                // registered and its connection is up, but its bind handler sat
3625                // on the ack for the full budget (warm-on-bind, cold configure,
3626                // or a wedged handler). Every earlier unavailability shape
3627                // fast-refuses BEFORE the relay, so this arm firing means the
3628                // slowness is module-side -- log it so the per-module timeline
3629                // is reconstructable without client audit rows.
3630                tracing::warn!(
3631                    module_id = %target_module_id,
3632                    timeout_ms = route_bind_relay_timeout.as_millis() as u64,
3633                    "route.bind relay timed out: module did not ack within budget"
3634                );
3635                Ok(vec![self.route_open_refusal_frame(
3636                    ctx,
3637                    &frame,
3638                    &target_module_id,
3639                    "relay_timed_out",
3640                    "module_timeout",
3641                    format!(
3642                        "module_id '{target_module_id}' did not answer route.bind within {:?}",
3643                        route_bind_relay_timeout
3644                    ),
3645                )?])
3646            }
3647        }
3648    }
3649
3650    fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3651        let response = ClientControlResponse::SupervisorSpawnSnapshot {
3652            snapshot: self.supervisor.spawn_snapshot(),
3653        };
3654        Ok(vec![control_response_body_frame(
3655            &frame,
3656            &response,
3657            "ClientControlResponse::SupervisorSpawnSnapshot",
3658        )?])
3659    }
3660
3661    fn handle_supervisor_spawn_subscribe(
3662        &self,
3663        ctx: &RouteCtx,
3664        frame: Frame,
3665        since: Option<SpawnCursor>,
3666    ) -> Result<Vec<Frame>, RouterError> {
3667        match self.supervisor.subscribe_spawns(
3668            ctx.connection_id,
3669            frame.header.corr,
3670            response_version(&frame),
3671            since,
3672            ctx.egress.clone(),
3673        ) {
3674            Ok(()) => Ok(Vec::new()),
3675            Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
3676                Ok(vec![control_error_body_frame(
3677                    &frame,
3678                    ErrorBody {
3679                        code: "spawn_cursor_incarnation_mismatch".to_string(),
3680                        message: "spawn cursor belongs to a different daemon incarnation"
3681                            .to_string(),
3682                        detail: Some(serde_json::json!({
3683                            "current_daemon_incarnation": current
3684                        })),
3685                    },
3686                )?])
3687            }
3688            Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
3689                &frame,
3690                ErrorBody {
3691                    code: "spawn_cursor_too_old".to_string(),
3692                    message: "spawn cursor predates the retained event ring".to_string(),
3693                    detail: Some(serde_json::json!({
3694                        "oldest_retained_cursor": oldest
3695                    })),
3696                },
3697            )?]),
3698            Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
3699                0,
3700                frame.header.corr,
3701                format!("failed to open supervisor spawn subscription: {error}"),
3702            )),
3703        }
3704    }
3705
3706    async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3707        let generation = self
3708            .registry
3709            .generation()
3710            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3711        let mut modules = Vec::new();
3712        for module in self.supervisor.list() {
3713            let status = module.status_for_control("list").map_err(|err| {
3714                RouterError::backend(
3715                    0,
3716                    frame.header.corr,
3717                    format!("failed to read supervisor status: {err}"),
3718                )
3719            })?;
3720            let (configured, _) = module.configuration().map_err(|err| {
3721                RouterError::backend(
3722                    0,
3723                    frame.header.corr,
3724                    format!("failed to read module configuration: {err}"),
3725                )
3726            })?;
3727            // Status and configuration snapshots release their locks before the image probe awaits.
3728            let image = module.running_image_agreement().await;
3729            // Read per request so the figure is current when the operator asks;
3730            // the daemon samples nothing in between.
3731            let resources = Some(module.child_resource_usage());
3732            let pending_reload = Some(reload_verdict(
3733                &configured.program,
3734                status.spawned_from.as_deref(),
3735                image,
3736            ));
3737            modules.push(SupervisorEntry {
3738                // Keep the retired policy field on the wire for one release so
3739                // existing status consumers still receive the platform policy.
3740                launch_nonce_env: Some(!cfg!(unix)),
3741                module_id: status.module_id,
3742                state: status.state.to_string(),
3743                enabled: status.enabled,
3744                live: status.live,
3745                protocol: status.protocol,
3746                health: status.health.status,
3747                pending_reload,
3748                last_probe_ms: status.health.last_probe_ms,
3749                last_exit_code: status.last_exit.as_ref().and_then(|e| e.code),
3750                last_exit_signal: status.last_exit.as_ref().and_then(|e| e.signal),
3751                last_exit_ms: status.last_exit.as_ref().map(|e| e.at_ms),
3752                last_exit_kind: status.last_exit.as_ref().map(|e| e.kind.into()),
3753                restart_count: Some(status.restart_count),
3754                max_restarts: Some(status.max_restarts),
3755                lifetime_restarts: Some(status.lifetime_restarts),
3756                spawn_generation: Some(status.spawn_generation),
3757                restart_window_secs: Some(status.restart_window.as_secs()),
3758                drain_timeout_ms: Some(status.drain_timeout.as_millis() as u64),
3759                restart_backoff_ms: Some(status.restart_backoff.as_millis() as u64),
3760                restart_max_backoff_ms: Some(status.restart_max_backoff.as_millis() as u64),
3761                resources,
3762            });
3763        }
3764        let response = ClientControlResponse::SupervisorList {
3765            generation,
3766            modules,
3767        };
3768        Ok(vec![control_response_body_frame(
3769            &frame,
3770            &response,
3771            "ClientControlResponse::SupervisorList",
3772        )?])
3773    }
3774
3775    fn handle_supervisor_stderr_tail(
3776        &self,
3777        frame: Frame,
3778        module_id: String,
3779        max_lines: Option<u32>,
3780        max_bytes: Option<u32>,
3781    ) -> Result<Vec<Frame>, RouterError> {
3782        let Some(module) = self.supervisor.get(&module_id) else {
3783            return Ok(vec![control_error_frame(
3784                &frame,
3785                "unknown_module",
3786                format!("module_id '{module_id}' is not supervised"),
3787            )?]);
3788        };
3789
3790        let snapshot = module.stderr_tail(
3791            max_lines.map(|value| value as usize),
3792            max_bytes.map(|value| value as usize),
3793        );
3794
3795        let response = ClientControlResponse::SupervisorStderrTail {
3796            module_id,
3797            tail: StderrTail {
3798                capture: match snapshot.capture {
3799                    CaptureState::Captured => StderrCaptureState::Captured,
3800                    CaptureState::Incomplete { reason } => {
3801                        StderrCaptureState::Incomplete { reason }
3802                    }
3803                    CaptureState::NotCaptured { reason } => {
3804                        StderrCaptureState::NotCaptured { reason }
3805                    }
3806                },
3807                entries: snapshot
3808                    .entries
3809                    .into_iter()
3810                    .map(|entry| match entry {
3811                        TailEntry::Line {
3812                            text,
3813                            truncated,
3814                            at_ms,
3815                        } => StderrTailEntry::Line {
3816                            text,
3817                            truncated,
3818                            at_ms,
3819                        },
3820                        TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
3821                    })
3822                    .collect(),
3823                dropped_lines: snapshot.dropped_lines,
3824            },
3825        };
3826        Ok(vec![control_response_body_frame(
3827            &frame,
3828            &response,
3829            "ClientControlResponse::SupervisorStderrTail",
3830        )?])
3831    }
3832
3833    async fn handle_supervisor_terminals(
3834        &self,
3835        frame: Frame,
3836        module_id: String,
3837    ) -> Result<Vec<Frame>, RouterError> {
3838        let Some(module) = self.supervisor.get(&module_id) else {
3839            return Ok(vec![control_error_frame(
3840                &frame,
3841                "unknown_module",
3842                format!("module_id '{module_id}' is not supervised"),
3843            )?]);
3844        };
3845
3846        // The journal read runs on a blocking thread: it can be megabytes of
3847        // file I/O and must not occupy a runtime worker.
3848        let terminals = module
3849            .read_durable_terminal_history()
3850            .await
3851            .map_err(|error| {
3852                RouterError::backend(
3853                    0,
3854                    frame.header.corr,
3855                    format!("failed to read terminal history: {error}"),
3856                )
3857            })?;
3858        let response = ClientControlResponse::SupervisorTerminals {
3859            module_id,
3860            terminals,
3861        };
3862        Ok(vec![control_response_body_frame(
3863            &frame,
3864            &response,
3865            "ClientControlResponse::SupervisorTerminals",
3866        )?])
3867    }
3868
3869    fn handle_supervisor_routes(
3870        &self,
3871        frame: Frame,
3872        module_id: Option<String>,
3873    ) -> Result<Vec<Frame>, RouterError> {
3874        let modules = self
3875            .forwarding
3876            .route_census(module_id.as_deref())
3877            .map_err(RouterError::Forwarding)?
3878            .into_iter()
3879            .map(|(module_id, routes)| SupervisorRouteModule {
3880                module_id,
3881                routes: routes
3882                    .into_iter()
3883                    .map(|route| SupervisorRoute {
3884                        consumer: match route.principal {
3885                            Principal::Reserved { module_id } => {
3886                                SupervisorRouteConsumer::Reserved { module_id }
3887                            }
3888                            Principal::Direct | Principal::Unverified => {
3889                                SupervisorRouteConsumer::Direct {
3890                                    connection_id: route.goodbye_target.connection_id.get(),
3891                                }
3892                            }
3893                        },
3894                        age_ms: Instant::now()
3895                            .saturating_duration_since(route.bound_at)
3896                            .as_millis()
3897                            .try_into()
3898                            .unwrap_or(u64::MAX),
3899                        draining: route.draining,
3900                        drain_reason: route.drain_reason,
3901                    })
3902                    .collect(),
3903            })
3904            .collect();
3905        let response = ClientControlResponse::SupervisorRoutes { modules };
3906        Ok(vec![control_response_body_frame(
3907            &frame,
3908            &response,
3909            "ClientControlResponse::SupervisorRoutes",
3910        )?])
3911    }
3912
3913    async fn handle_supervisor_provenance(
3914        &self,
3915        frame: Frame,
3916        module_id: Option<String>,
3917    ) -> Result<Vec<Frame>, RouterError> {
3918        let mut selected = if let Some(module_id) = module_id {
3919            let Some(module) = self.supervisor.get(&module_id) else {
3920                return Ok(vec![control_error_frame(
3921                    &frame,
3922                    "unknown_module",
3923                    format!("module_id '{module_id}' is not supervised"),
3924                )?]);
3925            };
3926            vec![module]
3927        } else {
3928            self.supervisor.list()
3929        };
3930
3931        let mut modules = Vec::with_capacity(selected.len());
3932        for module in selected.drain(..) {
3933            let status = module.status().map_err(|err| {
3934                RouterError::backend(
3935                    0,
3936                    frame.header.corr,
3937                    format!("failed to read supervisor status: {err}"),
3938                )
3939            })?;
3940            let module_declared = self
3941                .registry
3942                .get_module(&status.module_id)
3943                .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3944                .and_then(|registration| registration.manifest.provenance)
3945                .map(|build| ModuleDeclaredProvenance::Reported { build })
3946                .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
3947            #[cfg(test)]
3948            let running_image = match &self.provenance_probe_override {
3949                Some(result) => result.clone(),
3950                None => module.running_image_agreement().await,
3951            };
3952            #[cfg(not(test))]
3953            let running_image = module.running_image_agreement().await;
3954            modules.push(SupervisorModuleProvenance {
3955                module_id: status.module_id,
3956                module_declared,
3957                daemon_observed: SupervisorObservedProcess {
3958                    pid: status.pid,
3959                    spawned_at_ms: status.spawned_at_ms,
3960                    spawned_from: status.spawned_from,
3961                    running_image,
3962                },
3963            });
3964        }
3965        let daemon = SupervisorDaemonProvenance {
3966            daemon_build: self.daemon_provenance.build.clone(),
3967            daemon_observed: DaemonObservedProcess {
3968                pid: self.daemon_provenance.pid,
3969                started_at_ms: self
3970                    .daemon_provenance
3971                    .start_clock
3972                    .map(|clock| clock.started_at_ms())
3973                    .or(self.daemon_provenance.started_at_ms),
3974                running_image: self
3975                    .daemon_provenance
3976                    .probe
3977                    .observe(
3978                        self.daemon_provenance.pid,
3979                        self.daemon_provenance.executable_path.as_deref(),
3980                        self.daemon_provenance.executable_identity,
3981                        self.daemon_provenance.process_start_time,
3982                    )
3983                    .await,
3984            },
3985        };
3986        let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
3987        Ok(vec![control_response_body_frame(
3988            &frame,
3989            &response,
3990            "ClientControlResponse::SupervisorProvenance",
3991        )?])
3992    }
3993
3994    fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3995        self.refresh_capability_requirements();
3996        let generation = self
3997            .registry
3998            .generation()
3999            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4000        let modules = self
4001            .supervisor
4002            .list()
4003            .into_iter()
4004            .map(|module| {
4005                let status = module.status_for_control("health").map_err(|err| {
4006                    RouterError::backend(
4007                        0,
4008                        frame.header.corr,
4009                        format!("failed to read supervisor health: {err}"),
4010                    )
4011                })?;
4012                let module_id = status.module_id;
4013                let capability_detail = self
4014                    .capability_evaluator
4015                    .required_problem_detail(&module_id);
4016                Ok(SupervisorHealthEntry {
4017                    module_id,
4018                    status: status.health.status,
4019                    detail: append_capability_problem_detail(
4020                        status.health.detail,
4021                        capability_detail,
4022                    ),
4023                    metrics: status.health.metrics,
4024                    consecutive_failures: status.health.consecutive_failures,
4025                    late_answer_count: status.health.late_answer_count,
4026                    last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
4027                    last_action: status.health.last_action,
4028                    last_action_ms: status.health.last_action_ms,
4029                    last_probe_ms: status.health.last_probe_ms,
4030                })
4031            })
4032            .collect::<Result<Vec<_>, RouterError>>()?;
4033        let response = ClientControlResponse::SupervisorHealth {
4034            generation,
4035            modules,
4036        };
4037        Ok(vec![control_response_body_frame(
4038            &frame,
4039            &response,
4040            "ClientControlResponse::SupervisorHealth",
4041        )?])
4042    }
4043
4044    async fn handle_supervisor_restart(
4045        &self,
4046        frame: Frame,
4047        module_id: String,
4048        drain_timeout_ms: Option<u64>,
4049    ) -> Result<Vec<Frame>, RouterError> {
4050        let operation_lock = self.supervisor.operation_lock();
4051        let _operation_guard = operation_lock.lock().await;
4052        let Some(module) = self.supervisor.get(&module_id) else {
4053            return Ok(vec![control_error_frame(
4054                &frame,
4055                "unknown_module",
4056                format!("module_id '{module_id}' is not supervised"),
4057            )?]);
4058        };
4059
4060        self.route_outages.mark_operator_action(&module_id);
4061        if let Err(err) = module.restart(drain_timeout_ms).await {
4062            self.route_outages
4063                .operator_action_ended_unrefused(&module_id);
4064            let (code, message) = match err {
4065                crate::supervise::SuperviseError::Disabled { .. } => {
4066                    ("module_disabled", err.to_string())
4067                }
4068                crate::supervise::SuperviseError::SwapInProgress { .. } => {
4069                    ("swap_in_progress", err.to_string())
4070                }
4071                _ => (
4072                    "target_unavailable",
4073                    format!("failed to restart module_id '{module_id}': {err}"),
4074                ),
4075            };
4076            return Ok(vec![control_error_frame(&frame, code, message)?]);
4077        }
4078
4079        let response = ClientControlResponse::SupervisorAck {
4080            module_id,
4081            applied: true,
4082        };
4083        Ok(vec![control_response_body_frame(
4084            &frame,
4085            &response,
4086            "ClientControlResponse::SupervisorAck",
4087        )?])
4088    }
4089
4090    /// `supervisor.swap`. Answered when the swap has cut over or failed, not
4091    /// when the old process has finished draining: a caller whose own lane
4092    /// rides the old process must get its reply before that drain waits on it.
4093    async fn handle_supervisor_swap(
4094        &self,
4095        frame: Frame,
4096        module_id: String,
4097        ready_timeout_ms: Option<u64>,
4098    ) -> Result<Vec<Frame>, RouterError> {
4099        // The daemon-wide operation lock is held only to resolve the handle,
4100        // not across the swap. The swap can take its whole readiness budget,
4101        // and `supervisor.set_enabled` (ck module stop) takes the same lock:
4102        // holding it here would park an operator's stop behind the swap it is
4103        // meant to abort. A rescan or stop that reaches the module during the
4104        // swap is served by the swap itself (see `supervise_swap`).
4105        let module = {
4106            let operation_lock = self.supervisor.operation_lock();
4107            let _operation_guard = operation_lock.lock().await;
4108            self.supervisor.get(&module_id)
4109        };
4110        let Some(module) = module else {
4111            return Ok(vec![control_error_frame(
4112                &frame,
4113                "unknown_module",
4114                format!("module_id '{module_id}' is not supervised"),
4115            )?]);
4116        };
4117
4118        self.route_outages.mark_operator_action(&module_id);
4119        if let Err(err) = module
4120            .swap(ready_timeout_ms.map(Duration::from_millis))
4121            .await
4122        {
4123            self.route_outages
4124                .operator_action_ended_unrefused(&module_id);
4125            use crate::supervise::SuperviseError;
4126            let message = err.to_string();
4127            let error = match err {
4128                SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
4129                SuperviseError::SwapRefused { reason, .. } => ErrorBody {
4130                    code: "swap_refused".to_string(),
4131                    message,
4132                    detail: Some(serde_json::json!({ "reason": reason.as_str() })),
4133                },
4134                SuperviseError::SwapFailed {
4135                    arm,
4136                    candidate_exit,
4137                    ..
4138                } => ErrorBody {
4139                    code: "swap_failed".to_string(),
4140                    message,
4141                    detail: Some(serde_json::json!({
4142                        "arm": arm.as_str(),
4143                        "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
4144                        "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
4145                    })),
4146                },
4147                _ => ErrorBody::new(
4148                    "target_unavailable",
4149                    format!("failed to swap module_id '{module_id}': {message}"),
4150                ),
4151            };
4152            return Ok(vec![control_error_body_frame(&frame, error)?]);
4153        }
4154        // A completed swap kept the incumbent serving until cutover, so it
4155        // usually opened no outage; a mark left behind would make the next,
4156        // unrelated outage read as requested.
4157        self.route_outages
4158            .operator_action_ended_unrefused(&module_id);
4159
4160        let response = ClientControlResponse::SupervisorAck {
4161            module_id,
4162            applied: true,
4163        };
4164        Ok(vec![control_response_body_frame(
4165            &frame,
4166            &response,
4167            "ClientControlResponse::SupervisorAck",
4168        )?])
4169    }
4170
4171    async fn handle_supervisor_reload(
4172        &self,
4173        frame: Frame,
4174        module_id: String,
4175    ) -> Result<Vec<Frame>, RouterError> {
4176        let operation_lock = self.supervisor.operation_lock();
4177        let _operation_guard = operation_lock.lock().await;
4178        let Some(module) = self.supervisor.get(&module_id) else {
4179            return Ok(vec![control_error_frame(
4180                &frame,
4181                "unknown_module",
4182                format!("module_id '{module_id}' is not supervised"),
4183            )?]);
4184        };
4185
4186        self.route_outages.mark_operator_action(&module_id);
4187        if let Err(err) = module.reload().await {
4188            self.route_outages
4189                .operator_action_ended_unrefused(&module_id);
4190            let (code, message) = match err {
4191                crate::supervise::SuperviseError::Disabled { .. } => {
4192                    ("module_disabled", err.to_string())
4193                }
4194                crate::supervise::SuperviseError::SwapInProgress { .. } => {
4195                    ("swap_in_progress", err.to_string())
4196                }
4197                _ => (
4198                    "reload_failed",
4199                    format!("failed to reload module_id '{module_id}': {err}"),
4200                ),
4201            };
4202            return Ok(vec![control_error_frame(&frame, code, message)?]);
4203        }
4204
4205        let response = ClientControlResponse::SupervisorAck {
4206            module_id,
4207            applied: true,
4208        };
4209        Ok(vec![control_response_body_frame(
4210            &frame,
4211            &response,
4212            "ClientControlResponse::SupervisorAck",
4213        )?])
4214    }
4215
4216    async fn handle_supervisor_rescan(
4217        &self,
4218        frame: Frame,
4219        preview: bool,
4220    ) -> Result<Vec<Frame>, RouterError> {
4221        let Some(context) = self.rescan.clone() else {
4222            return Ok(vec![control_error_frame(
4223                &frame,
4224                "rescan_unavailable",
4225                "the daemon was not started with a reloadable config path".to_string(),
4226            )?]);
4227        };
4228
4229        let operation_lock = self.supervisor.operation_lock();
4230        let _operation_guard = operation_lock.lock().await;
4231        let loaded = match crate::daemon_config::load(&context.config_path) {
4232            Ok(config) => config,
4233            Err(err) => {
4234                return Ok(vec![control_error_frame(
4235                    &frame,
4236                    "invalid_daemon_config",
4237                    format!("supervisor rescan rejected daemon config: {err}"),
4238                )?])
4239            }
4240        };
4241        // `load` reports a missing file as Ok(None), which is correct at boot
4242        // (no config, nothing to supervise) and catastrophic here: rescan treats
4243        // "not in the config" as "remove it", so an absent file would read as an
4244        // empty module list and retire the entire running fleet. An editor
4245        // writing via write-new-then-rename, or a half-finished edit, is enough
4246        // to open that window. Refuse instead: a config that cannot be read
4247        // carries no instruction to remove anything.
4248        let Some(config) = loaded else {
4249            return Ok(vec![control_error_frame(
4250                &frame,
4251                "invalid_daemon_config",
4252                format!(
4253                    "daemon config not found at {}; refusing to rescan (an absent config would \
4254                     retire every supervised module)",
4255                    context.config_path.display()
4256                ),
4257            )?]);
4258        };
4259        let (
4260            configured_port,
4261            storage_config,
4262            admission_facts_carrier_module_id,
4263            admission_facts_targets,
4264            scope_authority_owners,
4265            modules,
4266            reserved_capabilities,
4267        ) = (
4268            config.port,
4269            config.storage,
4270            config.admission_facts_carrier_module_id,
4271            config.admission_facts_targets,
4272            config.scope_authority_owners,
4273            config.modules,
4274            config.reserved_capabilities,
4275        );
4276
4277        // Collect the sections rescan cannot apply, so the REPLY carries them.
4278        //
4279        // The warning below has always been correct and has always gone only to
4280        // the journal -- addressed to whoever reads logs, while the person who
4281        // just edited the config is looking at the CLI. Naming each section
4282        // individually rather than setting a flag: "something outside modules
4283        // changed" sends the operator back to diffing their own file, which is
4284        // the work this is meant to save.
4285        let mut restart_required = Vec::new();
4286        for section in RestartRequiredSection::ALL {
4287            let changed = match section {
4288                RestartRequiredSection::Port => configured_port != context.configured_port,
4289                RestartRequiredSection::Storage => storage_config != context.storage_config,
4290                RestartRequiredSection::AdmissionFactsCarrierModuleId => {
4291                    admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
4292                }
4293                RestartRequiredSection::AdmissionFactsTargets => {
4294                    admission_facts_targets != context.admission_facts_targets
4295                }
4296                RestartRequiredSection::ScopeAuthorityOwners => {
4297                    scope_authority_owners != context.scope_authority_owners
4298                }
4299            };
4300            if changed {
4301                restart_required.push(section.label().to_string());
4302            }
4303        }
4304        if !restart_required.is_empty() {
4305            warn!(
4306                config_path = %context.config_path.display(),
4307                sections = %restart_required.join(", "),
4308                "daemon config changed outside the modules section; restart the daemon to apply those changes"
4309            );
4310        }
4311
4312        for configured in &modules {
4313            if let Err(err) = validate_spec(&configured.module_spec()) {
4314                return Ok(vec![control_error_frame(
4315                    &frame,
4316                    "invalid_daemon_config",
4317                    format!("supervisor rescan rejected daemon config: {err}"),
4318                )?]);
4319            }
4320        }
4321
4322        let configured_capabilities = modules
4323            .iter()
4324            .map(|module| (module.module_id.clone(), module.enabled))
4325            .collect::<Vec<_>>();
4326        let preview_capability_warnings = if preview {
4327            let (_, registrations) = self.runtime_capability_snapshot()?;
4328            let current_modules = self
4329                .supervisor
4330                .list()
4331                .into_iter()
4332                .map(|module| module.module_id().to_string())
4333                .collect::<BTreeSet<_>>();
4334            let resulting_modules = configured_capabilities.clone();
4335            let removed = current_modules
4336                .into_iter()
4337                .filter(|module_id| {
4338                    !resulting_modules
4339                        .iter()
4340                        .any(|(configured_id, _)| configured_id == module_id)
4341                })
4342                .collect::<Vec<_>>();
4343            self.capability_evaluator.preview_removal_warnings(
4344                resulting_modules,
4345                &removed,
4346                &registrations,
4347            )
4348        } else {
4349            Vec::new()
4350        };
4351        let result = match self
4352            .reconcile_supervised_modules(&context.supervisor, modules, preview)
4353            .await
4354        {
4355            Ok(result) => result,
4356            Err(message) => {
4357                return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
4358            }
4359        };
4360        if !preview {
4361            self.capability_evaluator
4362                .configure(configured_capabilities, reserved_capabilities);
4363            self.capability_evaluator.wake_deadline_loop();
4364            self.refresh_capability_requirements();
4365        }
4366        let mut result = result;
4367        result.restart_required = restart_required;
4368        result.capability_warnings = preview_capability_warnings;
4369        let response = ClientControlResponse::SupervisorRescan { result };
4370        Ok(vec![control_response_body_frame(
4371            &frame,
4372            &response,
4373            "ClientControlResponse::SupervisorRescan",
4374        )?])
4375    }
4376
4377    async fn handle_supervisor_release_reserved(
4378        &self,
4379        frame: Frame,
4380        module_id: String,
4381    ) -> Result<Vec<Frame>, RouterError> {
4382        let Some(context) = self.rescan.clone() else {
4383            return Ok(vec![control_error_frame(
4384                &frame,
4385                "release_unavailable",
4386                "reserved-id release requires a daemon started with a reloadable config path",
4387            )?]);
4388        };
4389        let operation_lock = self.supervisor.operation_lock();
4390        let _operation_guard = operation_lock.lock().await;
4391        let loaded = match crate::daemon_config::load(&context.config_path) {
4392            Ok(Some(config)) => config,
4393            Ok(None) => {
4394                return Ok(vec![control_error_frame(
4395                    &frame,
4396                    "invalid_daemon_config",
4397                    format!(
4398                        "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
4399                        context.config_path.display()
4400                    ),
4401                )?])
4402            }
4403            Err(err) => {
4404                return Ok(vec![control_error_frame(
4405                    &frame,
4406                    "invalid_daemon_config",
4407                    format!("unable to verify reserved-id release against daemon config: {err}"),
4408                )?])
4409            }
4410        };
4411        if loaded
4412            .modules
4413            .iter()
4414            .any(|configured| configured.module_id == module_id)
4415        {
4416            return Ok(vec![control_error_frame(
4417                &frame,
4418                "reserved_module_configured",
4419                format!(
4420                    "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
4421                ),
4422            )?]);
4423        }
4424        if !self.supervisor.release_retained_reserved_gate(&module_id) {
4425            return Ok(vec![control_error_frame(
4426                &frame,
4427                "reserved_gate_not_retained",
4428                format!(
4429                    "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
4430                ),
4431            )?]);
4432        }
4433
4434        let response = ClientControlResponse::SupervisorAck {
4435            module_id,
4436            applied: true,
4437        };
4438        Ok(vec![control_response_body_frame(
4439            &frame,
4440            &response,
4441            "ClientControlResponse::SupervisorAck",
4442        )?])
4443    }
4444
4445    /// Reconcile the running module set against the configured one.
4446    ///
4447    /// With `preview` set, the diff is computed and returned WITHOUT applying any
4448    /// of it: nothing is retired, reconfigured, enabled or spawned. The preview
4449    /// deliberately shares this function with the executing path rather than
4450    /// computing the same diff somewhere else -- two implementations of one
4451    /// decision agree until they do not, and the whole value of a preview is that
4452    /// it describes the operation that will actually run.
4453    async fn reconcile_supervised_modules(
4454        &self,
4455        supervisor: &Supervisor,
4456        configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
4457        preview: bool,
4458    ) -> Result<SupervisorRescanResult, String> {
4459        let mut current = BTreeMap::new();
4460        for module in self.supervisor.list() {
4461            let (spec, health) = module.configuration().map_err(|err| {
4462                format!(
4463                    "failed to read configuration for module_id '{}': {err}",
4464                    module.module_id()
4465                )
4466            })?;
4467            let enabled = module
4468                .status()
4469                .map_err(|err| {
4470                    format!(
4471                        "failed to read status for module_id '{}': {err}",
4472                        module.module_id()
4473                    )
4474                })?
4475                .enabled;
4476            current.insert(
4477                module.module_id().to_string(),
4478                (module, spec, health, enabled),
4479            );
4480        }
4481        let configured = configured_modules
4482            .into_iter()
4483            .map(|module| (module.module_id.clone(), module))
4484            .collect::<BTreeMap<_, _>>();
4485
4486        let added = configured
4487            .keys()
4488            .filter(|module_id| !current.contains_key(*module_id))
4489            .cloned()
4490            .collect::<Vec<_>>();
4491        let removed = current
4492            .keys()
4493            .filter(|module_id| !configured.contains_key(*module_id))
4494            .cloned()
4495            .collect::<Vec<_>>();
4496        let mut changed_pending_reload = Vec::new();
4497        let mut configuration_changes = BTreeSet::new();
4498        let mut enabled_changes = BTreeSet::new();
4499        let mut unchanged = 0_u32;
4500
4501        for (module_id, configured_module) in &configured {
4502            let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
4503            else {
4504                continue;
4505            };
4506            let configuration_changed = *current_spec != configured_module.module_spec()
4507                || *current_health != configured_module.health;
4508            let enabled_changed = *current_enabled != configured_module.enabled;
4509            if configuration_changed {
4510                configuration_changes.insert(module_id.clone());
4511                changed_pending_reload.push(module_id.clone());
4512            }
4513            if enabled_changed {
4514                enabled_changes.insert(module_id.clone());
4515            }
4516            if !configuration_changed && !enabled_changed {
4517                unchanged = unchanged.saturating_add(1);
4518            }
4519        }
4520
4521        // Everything above this point is pure computation over two snapshots.
4522        // Everything below MUTATES. The preview returns here so the boundary is a
4523        // single early return rather than a condition repeated at each mutation
4524        // site, where one missed guard would apply part of a change the caller was
4525        // told would not happen.
4526        if preview {
4527            return Ok(SupervisorRescanResult {
4528                added,
4529                removed,
4530                changed_pending_reload,
4531                enabled_changes: enabled_changes.iter().cloned().collect(),
4532                unchanged,
4533                preview: true,
4534                // Filled by the caller on both paths, so the preview reports
4535                // restart-required sections identically to an executed rescan --
4536                // the preview is where an operator is most likely to be looking.
4537                restart_required: Vec::new(),
4538                capability_warnings: Vec::new(),
4539            });
4540        }
4541
4542        for module_id in &removed {
4543            let module = &current
4544                .get(module_id)
4545                .expect("removed module came from current supervisor state")
4546                .0;
4547            module.retire().await.map_err(|err| {
4548                format!("failed to retire module_id '{module_id}' during rescan: {err}")
4549            })?;
4550            // TOMBSTONE BEFORE RETIRE, and the order is the whole fix.
4551            //
4552            // `handle_route_open` resolves an absent module in three steps:
4553            // registry, then supervisor status, then tombstone. Retiring first
4554            // opens a window where ALL THREE ARE ABSENT -- the registry entry
4555            // went with the teardown above, the supervisor entry went with
4556            // `retire`, and the tombstone does not exist yet -- so a route.open
4557            // landing in it gets `unknown_module` (RETRYABLE, "never heard of
4558            // it") for a module that was deliberately removed and whose caller
4559            // should get `module_removed` (TERMINAL, carrying a removal age).
4560            //
4561            // Writing the tombstone first closes it: during the window the
4562            // supervisor entry still answers, so the caller gets
4563            // `target_unavailable` -- retryable, and TRUE, because the module
4564            // is mid-teardown. After both statements it is `module_removed`.
4565            // No instant remains where a removed module reads as one that
4566            // never existed.
4567            //
4568            // NOT DETERMINISTICALLY TESTABLE FROM HERE, said plainly because
4569            // the absence of a test beside a fix invites deletion: these are
4570            // two sync statements with no await between them, so reaching the
4571            // window needs a second worker thread to land exactly between them
4572            // and there is no hook to force it. MEASURED: the 25 daemon_config
4573            // tests pass identically with the old order and the new one, so
4574            // the existing suite cannot see this and a green run is not
4575            // evidence either way. What the suite does hold is the
4576            // post-condition -- a removed module answers `module_removed` --
4577            // which this preserves.
4578            //
4579            // Found by an Athena panel reading the shipped tree against a
4580            // design note (2026-09-19), as the one concrete instance of that
4581            // note's class that survived contact with source. Direction is
4582            // benign: retryable where terminal was intended, never the reverse.
4583            self.supervisor.record_rescan_removal(module_id);
4584            self.supervisor.retire(module_id);
4585            self.route_outages.forget(module_id);
4586        }
4587
4588        for module_id in configured.keys() {
4589            let Some((module, _, _, _)) = current.get(module_id) else {
4590                continue;
4591            };
4592            let configured_module = configured
4593                .get(module_id)
4594                .expect("configured module id came from configured map");
4595            if configuration_changes.contains(module_id) {
4596                module
4597                    .update_configuration(
4598                        configured_module.module_spec(),
4599                        configured_module.health,
4600                        configured_module.drain_timeout_ms,
4601                    )
4602                    .await
4603                    .map_err(|err| {
4604                        format!(
4605                            "failed to update module_id '{module_id}' configuration during rescan: {err}"
4606                        )
4607                    })?;
4608            }
4609            if enabled_changes.contains(module_id) {
4610                // A rescan that starts or stops a module applies an operator's
4611                // edit to the config, so the resulting outage was asked for.
4612                self.route_outages.mark_operator_action(module_id);
4613                module
4614                    .set_enabled(configured_module.enabled)
4615                    .await
4616                    .map_err(|err| {
4617                        self.route_outages.operator_action_ended_unrefused(module_id);
4618                        format!(
4619                            "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
4620                            configured_module.enabled
4621                        )
4622                    })?;
4623            }
4624        }
4625
4626        for module_id in &added {
4627            let configured_module = configured
4628                .get(module_id)
4629                .expect("added module id came from configured map");
4630            supervisor
4631                .supervise_configured_with_health(
4632                    configured_module.module_spec(),
4633                    configured_module.enabled,
4634                    configured_module.health,
4635                    configured_module.drain_timeout_ms,
4636                    configured_module.restart,
4637                )
4638                .map_err(|err| {
4639                    format!("failed to add module_id '{module_id}' during rescan: {err}")
4640                })?;
4641        }
4642
4643        Ok(SupervisorRescanResult {
4644            added,
4645            removed,
4646            changed_pending_reload,
4647            enabled_changes: enabled_changes.iter().cloned().collect(),
4648            unchanged,
4649            preview: false,
4650            // Filled by the caller, which is the only layer that can see the
4651            // previous config to diff against.
4652            restart_required: Vec::new(),
4653            capability_warnings: Vec::new(),
4654        })
4655    }
4656
4657    async fn handle_supervisor_set_enabled(
4658        &self,
4659        frame: Frame,
4660        module_id: String,
4661        enabled: bool,
4662    ) -> Result<Vec<Frame>, RouterError> {
4663        let operation_lock = self.supervisor.operation_lock();
4664        let _operation_guard = operation_lock.lock().await;
4665        let Some(module) = self.supervisor.get(&module_id) else {
4666            return Ok(vec![control_error_frame(
4667                &frame,
4668                "unknown_module",
4669                format!("module_id '{module_id}' is not supervised"),
4670            )?]);
4671        };
4672
4673        // Enabling counts as well as disabling: a module an operator starts
4674        // is refused until it registers, and that wait was asked for.
4675        self.route_outages.mark_operator_action(&module_id);
4676        let applied = match module.set_enabled(enabled).await {
4677            Ok(applied) => applied,
4678            Err(err) => {
4679                self.route_outages
4680                    .operator_action_ended_unrefused(&module_id);
4681                return Ok(vec![control_error_frame(
4682                    &frame,
4683                    "target_unavailable",
4684                    format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
4685                )?]);
4686            }
4687        };
4688        if !applied {
4689            // Already in the requested state: nothing was made unavailable,
4690            // so the mark must not outlive this request.
4691            self.route_outages
4692                .operator_action_ended_unrefused(&module_id);
4693        }
4694
4695        self.capability_evaluator.wake_deadline_loop();
4696        self.refresh_capability_requirements();
4697        let response = ClientControlResponse::SupervisorAck { module_id, applied };
4698        Ok(vec![control_response_body_frame(
4699            &frame,
4700            &response,
4701            "ClientControlResponse::SupervisorAck",
4702        )?])
4703    }
4704
4705    async fn handle_supervisor_health_probe(
4706        &self,
4707        frame: Frame,
4708        module_id: String,
4709    ) -> Result<Vec<Frame>, RouterError> {
4710        self.refresh_capability_requirements();
4711        let Some(registration) = self
4712            .registry
4713            .get_module(&module_id)
4714            .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4715        else {
4716            return Ok(vec![control_error_frame(
4717                &frame,
4718                "unknown_module",
4719                format!("module_id '{module_id}' is not registered"),
4720            )?]);
4721        };
4722
4723        // This guard's ACCEPT direction is fenced, but only INCIDENTALLY: no test is
4724        // named for it. Making `module_registration_grants_op` return false
4725        // unconditionally reddens five tests, and every one is named for something
4726        // else -- capability relay, probe/bind demultiplexing, supervision-only
4727        // probing. They exercise a successful advertisement check on the way to their
4728        // own subject.
4729        //
4730        // Real protection, fragile in a specific way: narrowing any of those tests to
4731        // focus on its stated subject would silently remove coverage nobody knows
4732        // they are carrying. Recorded here rather than as a sixth test, because the
4733        // useful fact is WHICH tests hold the guard up -- a new test would add
4734        // coverage without telling the next person what the existing ones quietly do.
4735        if !module_registration_grants_op(&registration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
4736        {
4737            return Ok(vec![control_error_frame(
4738                &frame,
4739                "health_not_advertised",
4740                format!("module_id '{module_id}' did not advertise health.check"),
4741            )?]);
4742        }
4743
4744        let deadline = Instant::now() + self.health_probe_timeout;
4745        let pending = match self.forwarding.begin_module_control_rpc_for(
4746            &module_id,
4747            MODULE_CONTROL_OP_HEALTH_CHECK,
4748            deadline,
4749        ) {
4750            Ok(pending) => pending,
4751            Err(err) => {
4752                return Ok(vec![control_error_frame(
4753                    &frame,
4754                    forwarding_error_code(&err),
4755                    err.to_string(),
4756                )?])
4757            }
4758        };
4759
4760        let PendingModuleControlRpc {
4761            endpoint,
4762            module_sink,
4763            negotiated_ver,
4764            corr: probe_corr,
4765            receiver,
4766        } = pending;
4767        let mut guard =
4768            ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
4769        let probe_body =
4770            serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
4771                RouterError::backend(
4772                    0,
4773                    frame.header.corr,
4774                    format!("failed to encode health.check request: {err}"),
4775                )
4776            })?;
4777        let probe_frame = Frame::build_with_version(
4778            negotiated_ver,
4779            FrameType::Request,
4780            control_flags(),
4781            0,
4782            0,
4783            probe_corr,
4784            probe_body,
4785        )
4786        .map_err(RouterError::FrameBuild)?;
4787
4788        if let Err(err) = module_sink.send(probe_frame).await {
4789            return Ok(vec![control_error_frame(
4790                &frame,
4791                "target_unavailable",
4792                err.to_string(),
4793            )?]);
4794        }
4795
4796        match timeout_at(deadline, receiver).await {
4797            Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
4798                guard.disarm();
4799                let Some(report) = response.health_report() else {
4800                    return Ok(vec![control_error_frame(
4801                        &frame,
4802                        "invalid_control_body",
4803                        "health.check RPC returned a non-health response",
4804                    )?]);
4805                };
4806                // Metrics go out whole here. The supervisor's cached snapshot
4807                // caps this blob (see truncate_health_metrics), and this path
4808                // exists precisely to answer without that cap -- so applying it
4809                // here would leave no way to see what the cached view drops.
4810                let HealthReport {
4811                    status,
4812                    detail,
4813                    metrics,
4814                } = report;
4815                let capability_detail = self
4816                    .capability_evaluator
4817                    .required_problem_detail(&module_id);
4818                let response = ClientControlResponse::SupervisorHealthProbe {
4819                    module_id,
4820                    status,
4821                    detail: append_capability_problem_detail(detail, capability_detail),
4822                    metrics,
4823                };
4824                Ok(vec![control_response_body_frame(
4825                    &frame,
4826                    &response,
4827                    "ClientControlResponse::SupervisorHealthProbe",
4828                )?])
4829            }
4830            Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
4831                guard.disarm();
4832                Ok(vec![control_error_body_frame(&frame, body)?])
4833            }
4834            Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
4835                guard.disarm();
4836                Ok(vec![control_error_frame(
4837                    &frame,
4838                    "target_unavailable",
4839                    message,
4840                )?])
4841            }
4842            Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
4843                guard.disarm();
4844                Ok(vec![control_error_frame(
4845                    &frame,
4846                    "invalid_control_body",
4847                    message,
4848                )?])
4849            }
4850            Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
4851                guard.disarm();
4852                Ok(vec![control_error_frame(
4853                    &frame,
4854                    "invalid_control_body",
4855                    format!("expected module-control op '{expected}', got '{actual}'"),
4856                )?])
4857            }
4858            Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
4859                guard.disarm();
4860                Ok(vec![control_error_frame(
4861                    &frame,
4862                    "module_timeout",
4863                    format!(
4864                        "module_id '{module_id}' answered health.check after {:?}",
4865                        self.health_probe_timeout
4866                    ),
4867                )?])
4868            }
4869            Ok(Err(_)) => Ok(vec![control_error_frame(
4870                &frame,
4871                "target_unavailable",
4872                "health.check waiter was canceled before the module responded",
4873            )?]),
4874            Err(_) => Ok(vec![control_error_frame(
4875                &frame,
4876                "module_timeout",
4877                format!(
4878                    "module_id '{module_id}' did not answer health.check within {:?}",
4879                    self.health_probe_timeout
4880                ),
4881            )?]),
4882        }
4883    }
4884
4885    fn supervisor_status(
4886        &self,
4887        module_id: &str,
4888        corr: u64,
4889    ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
4890        self.supervisor
4891            .get(module_id)
4892            .map(|module| {
4893                let warming = module.is_warming_for_control("status").map_err(|err| {
4894                    RouterError::backend(
4895                        0,
4896                        corr,
4897                        format!(
4898                            "failed to read supervisor warming state for module_id '{module_id}': {err}"
4899                        ),
4900                    )
4901                })?;
4902                module.status_for_control("status").map_err(|err| {
4903                    RouterError::backend(
4904                        0,
4905                        corr,
4906                        format!(
4907                            "failed to read supervisor status for module_id '{module_id}': {err}"
4908                        ),
4909                    )
4910                }).map(|status| (status, warming))
4911            })
4912            .transpose()
4913    }
4914
4915    fn guard_module_control_op(
4916        &self,
4917        frame: &Frame,
4918        module_id: &str,
4919        op: &str,
4920    ) -> Result<Option<Frame>, RouterError> {
4921        if self.module_grants_op(module_id, op, frame.header.corr)? {
4922            return Ok(None);
4923        }
4924
4925        Ok(Some(control_error_frame(
4926            frame,
4927            "op_not_allowed",
4928            format!("module_id '{module_id}' did not grant control op '{op}'"),
4929        )?))
4930    }
4931
4932    fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
4933        let Some(registration) = self
4934            .registry
4935            .get_module(module_id)
4936            .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
4937        else {
4938            return Ok(false);
4939        };
4940        Ok(module_registration_grants_op(&registration.control_ops, op))
4941    }
4942
4943    fn handle_status_update(
4944        &self,
4945        endpoint: ModuleEndpointId,
4946        frame: Frame,
4947    ) -> Result<Vec<Frame>, RouterError> {
4948        let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
4949            Ok(update) => update,
4950            Err(err) => {
4951                // Forward-compat: a newer module may push a channel-0 op this subc
4952                // version doesn't know. The control contract says unknown push ops
4953                // are IGNORED, never answered with an error. Only a malformed body
4954                // for an op we DO know is a real error worth surfacing.
4955                if is_known_module_push_op(&frame.body) {
4956                    return Ok(vec![control_error_frame(
4957                        &frame,
4958                        "invalid_control_body",
4959                        format!("malformed module control push body: {err}"),
4960                    )?]);
4961                }
4962                return Ok(Vec::new());
4963            }
4964        };
4965
4966        match update {
4967            ModuleControlPush::RouteStatus {
4968                route_channel,
4969                route_epoch,
4970                status,
4971            } => {
4972                self.forwarding
4973                    .cache_status(endpoint, route_channel, route_epoch, status)
4974                    .map_err(RouterError::Forwarding)?;
4975            }
4976        }
4977        Ok(Vec::new())
4978    }
4979
4980    fn handle_route_poll(
4981        &self,
4982        ctx: &RouteCtx,
4983        frame: Frame,
4984        route_channel: u16,
4985        route_epoch: u32,
4986        kind: PollKind,
4987    ) -> Result<Vec<Frame>, RouterError> {
4988        let snapshot = self
4989            .forwarding
4990            .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
4991            .map_err(RouterError::Forwarding)?;
4992        let response = match (kind, snapshot) {
4993            (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
4994                ClientControlResponse::RoutePoll {
4995                    route_channel,
4996                    route_epoch,
4997                    status,
4998                    live: None,
4999                }
5000            }
5001            (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5002                route_channel,
5003                route_epoch,
5004                status: None,
5005                live: None,
5006            },
5007            (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
5008                // ABSENCE HERE MEANS "NOT SUPERVISED", NOT "UNKNOWN", and that
5009                // is what makes reporting `true` correct rather than a
5010                // confident guess. `process_live` returns None only when the
5011                // module id has no supervisor snapshot at all -- an
5012                // externally-started module the daemon did not spawn -- and
5013                // for those the supervisor has no opinion to offer, ever. It
5014                // is never None for a supervised module in an unknown state:
5015                // a supervised module always has a snapshot, and the answer
5016                // comes from `state == Running && process_alive`.
5017                //
5018                // The route is Bound, so the module completed a HELLO on a
5019                // live connection; "the process this route points at is
5020                // running" is therefore attested by the binding rather than
5021                // assumed. Reporting `false` for an unsupervised module would
5022                // be the actual lie -- it would tell a client its healthy
5023                // route is dead because the daemon does not manage the
5024                // process.
5025                //
5026                // IF `process_live` EVER GAINS A THIRD CASE -- a supervised
5027                // module whose liveness is genuinely unknown, e.g. a snapshot
5028                // that has not been populated yet -- THIS DEFAULT BECOMES
5029                // WRONG and must split: unsupervised stays true, unknown
5030                // becomes null so the client can tell the two apart. The
5031                // response field is already `Option<bool>`, so the wire can
5032                // carry that distinction today.
5033                let live = self
5034                    .process_liveness
5035                    .as_ref()
5036                    .and_then(|source| source.process_live(&module_id))
5037                    .unwrap_or(true);
5038                ClientControlResponse::RoutePoll {
5039                    route_channel,
5040                    route_epoch,
5041                    status: None,
5042                    live: Some(live),
5043                }
5044            }
5045            (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5046                route_channel,
5047                route_epoch,
5048                status: None,
5049                live: Some(false),
5050            },
5051        };
5052
5053        Ok(vec![control_response_body_frame(
5054            &frame,
5055            &response,
5056            "ClientControlResponse::RoutePoll",
5057        )?])
5058    }
5059
5060    pub(crate) fn observe_module_control_completion(
5061        &self,
5062        completion: ModuleControlRpcCompletion,
5063    ) -> bool {
5064        match completion {
5065            ModuleControlRpcCompletion::Unknown => false,
5066            ModuleControlRpcCompletion::Settled => true,
5067            ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
5068                let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
5069                info!(
5070                    module_id = %module_id,
5071                    latency_ms,
5072                    "late health.check answer proves the module is alive"
5073                );
5074                match self
5075                    .supervisor
5076                    .record_late_health_answer(&module_id, latency_ms)
5077                {
5078                    Ok(true) => {}
5079                    Ok(false) => debug!(
5080                        module_id = %module_id,
5081                        latency_ms,
5082                        "late health.check answer has no active supervisor snapshot"
5083                    ),
5084                    Err(err) => warn!(
5085                        module_id = %module_id,
5086                        latency_ms,
5087                        error = %err,
5088                        "failed to record late health.check answer"
5089                    ),
5090                }
5091                true
5092            }
5093        }
5094    }
5095
5096    /// Decide whether a failure while settling a relayed `route.bind` belongs to
5097    /// the module connection whose frame is being handled, or to the client that
5098    /// relay was opened for.
5099    ///
5100    /// This runs on the MODULE connection's frame handler, where returning `Err`
5101    /// ends that connection -- and a module connection carries every client's
5102    /// routes to that module, so ending it costs the whole fleet its tools.
5103    /// `ConnectionClosing` carries the id of the connection that is closing, and
5104    /// when that id is a CLIENT's, the condition is entirely about that one
5105    /// client's route.open. A client-scoped condition has no authority over a
5106    /// shared module connection, so it is logged and the single relay is dropped:
5107    /// the client is going away, and `complete_pending_relay` already removed the
5108    /// relay before failing, so there is nothing left to settle. Anything that
5109    /// relay still reserved is released by that client's own connection teardown,
5110    /// which is already under way -- that is what "closing" means.
5111    ///
5112    /// Every other failure is a statement about THIS connection and stays fatal:
5113    /// a poisoned forwarding lock, a stale module endpoint, and the module's own
5114    /// id in `ConnectionClosing` all mean this connection cannot keep serving
5115    /// frames correctly.
5116    fn refuse_to_end_module_connection_for_a_client(
5117        &self,
5118        module_connection_id: ConnectionId,
5119        corr: u64,
5120        err: ForwardingError,
5121    ) -> Result<(), RouterError> {
5122        if let ForwardingError::ConnectionClosing { connection_id } = err {
5123            if connection_id != module_connection_id {
5124                warn!(
5125                    module_connection_id = module_connection_id.get(),
5126                    client_connection_id = connection_id.get(),
5127                    corr,
5128                    "dropping a route.bind response for a closing client; the module connection keeps serving"
5129                );
5130                return Ok(());
5131            }
5132        }
5133        Err(RouterError::Forwarding(err))
5134    }
5135
5136    fn handle_module_relay_response(
5137        &self,
5138        connection_id: ConnectionId,
5139        frame: Frame,
5140    ) -> Result<Vec<Frame>, RouterError> {
5141        let mut secondary_error = None;
5142        let outcome = match frame.header.ty {
5143            FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
5144                Ok(probe) if probe.op == "route.bind" => {
5145                    match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
5146                        Ok(ModuleControlResponse::RouteBindAck {}) => {
5147                            RouteBindRelayOutcome::Accepted
5148                        }
5149                        Ok(other) => {
5150                            let message =
5151                                format!("route.bind response carried unexpected body: {other:?}");
5152                            secondary_error = Some(control_error_frame(
5153                                &frame,
5154                                "invalid_control_body",
5155                                message.clone(),
5156                            )?);
5157                            RouteBindRelayOutcome::ModuleGone(message)
5158                        }
5159                        Err(err) => {
5160                            let message = format!("malformed route.bind response body: {err}");
5161                            secondary_error = Some(control_error_frame(
5162                                &frame,
5163                                "invalid_control_body",
5164                                message.clone(),
5165                            )?);
5166                            RouteBindRelayOutcome::ModuleGone(message)
5167                        }
5168                    }
5169                }
5170                Ok(probe) => {
5171                    let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
5172                    {
5173                        Ok(response) => ModuleControlRpcOutcome::Response(response),
5174                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5175                            "malformed {} response body: {err}",
5176                            probe.op
5177                        )),
5178                    };
5179                    let completion = self
5180                        .forwarding
5181                        .complete_module_control_rpc(
5182                            connection_id,
5183                            frame.header.corr,
5184                            Some(&probe.op),
5185                            outcome,
5186                        )
5187                        .map_err(RouterError::Forwarding)?;
5188                    if !self.observe_module_control_completion(completion) {
5189                        debug!(
5190                            connection_id = connection_id.get(),
5191                            corr = frame.header.corr,
5192                            op = %probe.op,
5193                            "dropping late or unknown module-control RPC response"
5194                        );
5195                    }
5196                    return Ok(Vec::new());
5197                }
5198                Err(err) => {
5199                    if let Some(expected_op) = self
5200                        .forwarding
5201                        .pending_module_control_op(connection_id, frame.header.corr)
5202                        .map_err(RouterError::Forwarding)?
5203                    {
5204                        let completion = self
5205                            .forwarding
5206                            .complete_module_control_rpc(
5207                                connection_id,
5208                                frame.header.corr,
5209                                None,
5210                                ModuleControlRpcOutcome::MalformedResponse(format!(
5211                                    "malformed {expected_op} response body: {err}"
5212                                )),
5213                            )
5214                            .map_err(RouterError::Forwarding)?;
5215                        if !self.observe_module_control_completion(completion) {
5216                            debug!(
5217                                connection_id = connection_id.get(),
5218                                corr = frame.header.corr,
5219                                "dropping late malformed module-control RPC response"
5220                            );
5221                        }
5222                        return Ok(Vec::new());
5223                    }
5224                    let message = format!("malformed route.bind response body: {err}");
5225                    secondary_error = Some(control_error_frame(
5226                        &frame,
5227                        "invalid_control_body",
5228                        message.clone(),
5229                    )?);
5230                    RouteBindRelayOutcome::ModuleGone(message)
5231                }
5232            },
5233            FrameType::Error => {
5234                if self
5235                    .forwarding
5236                    .pending_module_control_op(connection_id, frame.header.corr)
5237                    .map_err(RouterError::Forwarding)?
5238                    .is_some()
5239                {
5240                    let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
5241                        Ok(body) => ModuleControlRpcOutcome::Rejected(body),
5242                        Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5243                            "malformed module-control ERROR body: {err}"
5244                        )),
5245                    };
5246                    let completion = self
5247                        .forwarding
5248                        .complete_module_control_rpc(
5249                            connection_id,
5250                            frame.header.corr,
5251                            None,
5252                            outcome,
5253                        )
5254                        .map_err(RouterError::Forwarding)?;
5255                    if !self.observe_module_control_completion(completion) {
5256                        debug!(
5257                            connection_id = connection_id.get(),
5258                            corr = frame.header.corr,
5259                            "dropping late or unknown module-control RPC error"
5260                        );
5261                    }
5262                    return Ok(Vec::new());
5263                }
5264                match serde_json::from_slice::<ErrorBody>(&frame.body) {
5265                    Ok(body) => RouteBindRelayOutcome::Rejected(body),
5266                    Err(err) => {
5267                        let message = format!("malformed route.bind ERROR body: {err}");
5268                        secondary_error = Some(control_error_frame(
5269                            &frame,
5270                            "invalid_control_body",
5271                            message.clone(),
5272                        )?);
5273                        RouteBindRelayOutcome::ModuleGone(message)
5274                    }
5275                }
5276            }
5277            ty => {
5278                return Ok(vec![control_error_frame(
5279                    &frame,
5280                    "unsupported_control_frame",
5281                    format!("unsupported module channel-0 frame {ty:?}"),
5282                )?])
5283            }
5284        };
5285
5286        let settled =
5287            self.forwarding
5288                .complete_pending_relay(connection_id, frame.header.corr, outcome);
5289        let completion = match settled {
5290            Ok(completion) => completion,
5291            Err(err) => {
5292                self.refuse_to_end_module_connection_for_a_client(
5293                    connection_id,
5294                    frame.header.corr,
5295                    err,
5296                )?;
5297                return Ok(secondary_error.into_iter().collect());
5298            }
5299        };
5300        if let Some(target) = completion.abandoned.as_ref() {
5301            send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
5302        }
5303        if !completion.settled {
5304            debug!(
5305                connection_id = connection_id.get(),
5306                corr = frame.header.corr,
5307                frame_type = ?frame.header.ty,
5308                "dropping late or unknown route.bind relay response"
5309            );
5310        }
5311        Ok(secondary_error.into_iter().collect())
5312    }
5313
5314    fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
5315        debug!(connection_id = connection_id.get(), "handling GOODBYE");
5316        let registrations = self
5317            .deregister_connection(connection_id)
5318            .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
5319        let released_routes = self
5320            .forwarding
5321            .cleanup_connection(connection_id)
5322            .map_err(RouterError::Forwarding)?;
5323        self.emit_route_goodbyes(released_routes);
5324        // Notify only after forwarding teardown completes (see cleanup_connection).
5325        if !registrations.is_empty() {
5326            crate::supervise::notify_registration_release();
5327        }
5328        Ok(Vec::new())
5329    }
5330}
5331
5332impl Default for ControlHandler {
5333    fn default() -> Self {
5334        Self::new(Arc::new(Registry::default()))
5335    }
5336}
5337
5338impl crate::supervise::SwapPromotionObserver for ControlHandler {
5339    fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
5340        self.apply_registration_capabilities(registration);
5341    }
5342}
5343
5344fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
5345    CapabilityRequirementStatus {
5346        consumer: status.consumer,
5347        capability: status.capability,
5348        need: match status.need {
5349            subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
5350            subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
5351        },
5352        verdict: status.verdict.as_str().to_string(),
5353        episode_seq: status.episode_seq,
5354        config_satisfiable: status.config_satisfiable,
5355        runtime_available: status.runtime_available,
5356        detail: status.detail,
5357    }
5358}
5359
5360fn append_capability_problem_detail(
5361    detail: Option<String>,
5362    capability_detail: Option<String>,
5363) -> Option<String> {
5364    match (detail, capability_detail) {
5365        (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
5366        (Some(detail), None) => Some(detail),
5367        (None, Some(capability_detail)) => Some(capability_detail),
5368        (None, None) => None,
5369    }
5370}
5371
5372fn subc_ops() -> Vec<String> {
5373    SUBC_CONTROL_OPS
5374        .iter()
5375        .map(|op| (*op).to_string())
5376        .collect()
5377}
5378
5379fn module_subc_ops() -> Vec<String> {
5380    SUBC_CONTROL_OPS
5381        .iter()
5382        .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
5383        .map(|op| (*op).to_string())
5384        .collect()
5385}
5386
5387#[cfg(test)]
5388fn module_baseline_control_ops() -> Vec<String> {
5389    MODULE_BASELINE_CONTROL_OPS
5390        .iter()
5391        .map(|op| (*op).to_string())
5392        .collect()
5393}
5394
5395fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
5396    let mut seen = HashSet::new();
5397    let mut effective = Vec::new();
5398    for op in MODULE_BASELINE_CONTROL_OPS {
5399        if seen.insert((*op).to_string()) {
5400            effective.push((*op).to_string());
5401        }
5402    }
5403    for op in declared.unwrap_or_default() {
5404        if seen.insert(op.clone()) {
5405            effective.push(op);
5406        }
5407    }
5408    effective
5409}
5410
5411fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
5412    MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
5413}
5414
5415fn target_module_id(target: &RouteTarget) -> &str {
5416    match target {
5417        RouteTarget::ToolProvider { module_id }
5418        | RouteTarget::ManagementSurface { module_id }
5419        | RouteTarget::InternalService { module_id, .. } => module_id,
5420    }
5421}
5422
5423fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
5424    roles.iter().any(|role| match (target, role) {
5425        (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
5426        (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
5427        (
5428            RouteTarget::InternalService { service_id, .. },
5429            ProviderRole::InternalService {
5430                service_id: provided,
5431                ..
5432            },
5433        ) => service_id == provided,
5434        _ => false,
5435    })
5436}
5437
5438fn is_routable_role(role: &ProviderRole) -> bool {
5439    matches!(
5440        role,
5441        ProviderRole::ToolProvider { .. }
5442            | ProviderRole::ManagementSurface { .. }
5443            | ProviderRole::InternalService { .. }
5444    )
5445}
5446
5447#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5448enum ControlRequestBodyError {
5449    UnknownOp,
5450    InvalidBody,
5451}
5452
5453#[derive(Debug, Deserialize)]
5454struct ControlOpProbe {
5455    op: String,
5456}
5457
5458/// Channel-0 push ops this subc version understands. A push whose `op` is not in
5459/// this set is treated as a forward-compat unknown and ignored rather than errored.
5460const MODULE_PUSH_OPS: &[&str] = &["route.status"];
5461
5462fn is_known_module_push_op(body: &[u8]) -> bool {
5463    serde_json::from_slice::<ControlOpProbe>(body)
5464        .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
5465        .unwrap_or(false)
5466}
5467
5468fn is_known_module_request_op(body: &[u8]) -> bool {
5469    serde_json::from_slice::<ControlOpProbe>(body)
5470        .map(|probe| is_module_to_subc_op(&probe.op))
5471        .unwrap_or(false)
5472}
5473
5474fn is_module_to_subc_op(op: &str) -> bool {
5475    MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
5476}
5477
5478fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
5479    debug!(
5480        op = %op,
5481        connection_id = connection_id.get(),
5482        corr,
5483        "control dispatch"
5484    );
5485}
5486
5487fn log_slow_control_dispatch(
5488    dispatch_started_at: Option<StdInstant>,
5489    op: &'static str,
5490    connection_id: ConnectionId,
5491    corr: u64,
5492) {
5493    let Some(dispatch_started_at) = dispatch_started_at else {
5494        return;
5495    };
5496    let elapsed = dispatch_started_at.elapsed();
5497    if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
5498        warn!(
5499            op = %op,
5500            connection_id = connection_id.get(),
5501            corr,
5502            elapsed_ms = elapsed.as_millis() as u64,
5503            "slow control dispatch"
5504        );
5505    }
5506}
5507
5508fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
5509    match request {
5510        ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
5511        ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
5512        ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
5513        ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
5514        ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
5515        ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
5516        ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
5517        ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
5518        ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
5519        ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
5520        ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
5521        ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
5522        ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
5523        ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
5524        ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
5525        ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
5526        ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
5527        ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
5528        ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
5529    }
5530}
5531
5532fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
5533    match request {
5534        ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
5535        ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
5536        ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
5537        ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
5538    }
5539}
5540
5541fn parse_client_control_request(
5542    body: &[u8],
5543) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
5544    serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
5545        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5546            Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5547                ControlRequestBodyError::InvalidBody
5548            }
5549            Ok(_) => ControlRequestBodyError::UnknownOp,
5550            Err(_) => ControlRequestBodyError::InvalidBody,
5551        };
5552        (err, classification)
5553    })
5554}
5555
5556fn parse_module_control_request_from_module(
5557    body: &[u8],
5558) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
5559    serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
5560        let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5561            Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
5562            Ok(_) => ControlRequestBodyError::UnknownOp,
5563            Err(_) => ControlRequestBodyError::InvalidBody,
5564        };
5565        (err, classification)
5566    })
5567}
5568
5569#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
5570enum ProviderRoleKind {
5571    ToolProvider,
5572    PipelineStage,
5573    ManagementSurface,
5574    InternalService,
5575}
5576
5577fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
5578    match role {
5579        ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
5580        ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
5581        ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
5582        ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
5583    }
5584}
5585
5586fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
5587    roles.iter().map(provider_role_kind).collect()
5588}
5589
5590/// Most refused scope records named individually in the log per sync; the
5591/// `refused` count on the accepted line is always complete.
5592const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
5593
5594/// Per-outcome counts of one accepted `scope.sync`, for its log line.
5595#[derive(Debug, Default, PartialEq, Eq)]
5596struct ScopeOutcomeCounts {
5597    created: usize,
5598    replaced: usize,
5599    updated: usize,
5600    unchanged: usize,
5601    refused: usize,
5602}
5603
5604impl ScopeOutcomeCounts {
5605    fn of(results: &[ScopeRecordResult]) -> Self {
5606        let mut counts = Self::default();
5607        for result in results {
5608            let slot = match result.outcome {
5609                ScopeRecordOutcome::Created => &mut counts.created,
5610                ScopeRecordOutcome::Replaced => &mut counts.replaced,
5611                ScopeRecordOutcome::Updated => &mut counts.updated,
5612                ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
5613                ScopeRecordOutcome::Refused => &mut counts.refused,
5614            };
5615            *slot += 1;
5616        }
5617        counts
5618    }
5619}
5620
5621#[cfg(test)]
5622mod scope_outcome_count_tests {
5623    use super::*;
5624
5625    fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
5626        ScopeRecordResult {
5627            scope_ref: "r".to_string(),
5628            scope_epoch: 1,
5629            outcome,
5630            code: None,
5631            message: None,
5632            version: None,
5633            parent_state: None,
5634        }
5635    }
5636
5637    /// Each outcome lands in its own count, so a refused record can never be
5638    /// hidden inside the total the log already printed.
5639    #[test]
5640    fn every_outcome_is_counted_in_its_own_field() {
5641        let results = [
5642            result(ScopeRecordOutcome::Created),
5643            result(ScopeRecordOutcome::Created),
5644            result(ScopeRecordOutcome::Replaced),
5645            result(ScopeRecordOutcome::Updated),
5646            result(ScopeRecordOutcome::Unchanged),
5647            result(ScopeRecordOutcome::Refused),
5648            result(ScopeRecordOutcome::Refused),
5649            result(ScopeRecordOutcome::Refused),
5650        ];
5651        assert_eq!(
5652            ScopeOutcomeCounts::of(&results),
5653            ScopeOutcomeCounts {
5654                created: 2,
5655                replaced: 1,
5656                updated: 1,
5657                unchanged: 1,
5658                refused: 3,
5659            }
5660        );
5661    }
5662}
5663
5664/// Return whether a catalog change can create a newly violating live route.
5665/// Removing an attested claim is intentionally excluded: it makes fewer routes
5666/// forbidden and therefore must leave the existing route census untouched.
5667fn capability_census_trigger(
5668    old: Option<&CapabilityDeclarations>,
5669    new: Option<&CapabilityDeclarations>,
5670) -> bool {
5671    let old_provides = old
5672        .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
5673        .unwrap_or_default();
5674    let old_denies = old
5675        .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
5676        .unwrap_or_default();
5677    let new = new.cloned().unwrap_or(CapabilityDeclarations {
5678        provides: Vec::new(),
5679        requires: Vec::new(),
5680        must_never_reach: Vec::new(),
5681    });
5682
5683    new.provides
5684        .iter()
5685        .any(|capability| !old_provides.contains(capability))
5686        || new
5687            .must_never_reach
5688            .iter()
5689            .any(|capability| !old_denies.contains(capability))
5690}
5691
5692/// Find the first capability an attested opener denies that an attested target
5693/// claims. Both manifests are live registry records, never cached or client data.
5694fn denied_capability<'a>(
5695    opening_manifest: &'a ModuleManifest,
5696    target_manifest: &ModuleManifest,
5697) -> Option<&'a str> {
5698    let opening_capabilities = opening_manifest.capabilities.as_ref()?;
5699    let target_capabilities = target_manifest.capabilities.as_ref()?;
5700    opening_capabilities
5701        .must_never_reach
5702        .iter()
5703        .find(|denied| {
5704            target_capabilities
5705                .provides
5706                .iter()
5707                .any(|provided| provided == *denied)
5708        })
5709        .map(String::as_str)
5710}
5711
5712fn catalog_update_frozen_field_message(
5713    registered: &ModuleManifest,
5714    provides: &[ProviderRole],
5715) -> Option<String> {
5716    let old_has_provides = !registered.provides.is_empty();
5717    let new_has_provides = !provides.is_empty();
5718    if old_has_provides != new_has_provides {
5719        return Some(format!(
5720            "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
5721            registered.module_id
5722        ));
5723    }
5724
5725    if provider_role_kind_set(&registered.provides) != provider_role_kind_set(provides) {
5726        return Some(format!(
5727            "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
5728            registered.module_id
5729        ));
5730    }
5731
5732    let registered_concurrency = manifest_concurrency(registered);
5733    let mut candidate = registered.clone();
5734    candidate.provides = provides.to_vec();
5735    let candidate_concurrency = manifest_concurrency(&candidate);
5736    if candidate_concurrency != registered_concurrency {
5737        return Some(format!(
5738            "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
5739            registered.module_id, registered_concurrency, candidate_concurrency
5740        ));
5741    }
5742
5743    // control_ops live beside the manifest in the HELLO body, not inside
5744    // ModuleManifest, so a provides-only catalog.update cannot change them.
5745    None
5746}
5747
5748fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
5749    manifest.provides.iter().any(is_routable_role)
5750}
5751
5752/// Returns the routable-provider concurrency subc should enforce for this manifest.
5753///
5754/// ToolProvider and ManagementSurface store their delivery concurrency directly.
5755/// InternalService has no role-specific concurrency field, so it retains the
5756/// existing ModuleManaged default for backward compatibility.
5757fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
5758    manifest
5759        .provides
5760        .iter()
5761        .find_map(|provider| match provider {
5762            ProviderRole::ToolProvider { concurrency, .. }
5763            | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
5764            ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
5765        })
5766        .unwrap_or(Concurrency::ModuleManaged)
5767}
5768
5769/// True when the manifest carries a ManagementSurface role whose concurrency
5770/// was RESOLVED BY SERDE DEFAULT rather than declared. Reads the raw HELLO
5771/// bytes because the typed manifest deliberately erases that distinction: the
5772/// default exists for wire compatibility, and this probe exists so the default
5773/// stays observable. Any parse irregularity returns false -- the caller only
5774/// logs, and a malformed body already failed registration upstream.
5775fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
5776    let has_management_surface = manifest
5777        .provides
5778        .iter()
5779        .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
5780    if !has_management_surface {
5781        return false;
5782    }
5783    let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
5784        return false;
5785    };
5786    let Some(provides) = raw
5787        .get("manifest")
5788        .and_then(|manifest| manifest.get("provides"))
5789        .and_then(serde_json::Value::as_array)
5790    else {
5791        return false;
5792    };
5793    // ProviderRole is internally tagged (`tag = "role"`), so the wire shape is
5794    // flat: {"role": "management_surface", ..., "concurrency": ...} -- verified
5795    // against the management_surface_manifest_without_concurrency golden, not
5796    // recalled (the externally-tagged guess was this function's first bug).
5797    provides.iter().any(|role| {
5798        role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
5799            && role.get("concurrency").is_none()
5800    })
5801}
5802
5803fn negotiate_version(peer_version: u8) -> Result<u8, String> {
5804    if peer_version != PROTOCOL_VERSION {
5805        return Err(format!(
5806            "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
5807        ));
5808    }
5809    Ok(PROTOCOL_VERSION)
5810}
5811
5812fn pong(frame: &Frame) -> Result<Frame, RouterError> {
5813    Frame::build_with_version(
5814        response_version(frame),
5815        FrameType::Pong,
5816        frame.header.flags,
5817        0,
5818        0,
5819        frame.header.corr,
5820        Vec::new(),
5821    )
5822    .map_err(RouterError::FrameBuild)
5823}
5824
5825fn control_error_frame(
5826    frame: &Frame,
5827    code: &'static str,
5828    message: impl Into<String>,
5829) -> Result<Frame, RouterError> {
5830    control_error_body_frame(
5831        frame,
5832        ErrorBody {
5833            code: code.to_string(),
5834            message: message.into(),
5835            detail: None,
5836        },
5837    )
5838}
5839
5840fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
5841    let body = serde_json::to_vec(&error).map_err(|err| {
5842        RouterError::backend(
5843            0,
5844            frame.header.corr,
5845            format!("failed to encode control ERROR: {err}"),
5846        )
5847    })?;
5848
5849    Frame::build_with_version(
5850        response_version(frame),
5851        FrameType::Error,
5852        control_flags(),
5853        0,
5854        0,
5855        frame.header.corr,
5856        body,
5857    )
5858    .map_err(RouterError::FrameBuild)
5859}
5860
5861fn control_response_body_frame<T: Serialize>(
5862    frame: &Frame,
5863    reply: &T,
5864    label: &'static str,
5865) -> Result<Frame, RouterError> {
5866    let body = serde_json::to_vec(reply).map_err(|err| {
5867        RouterError::backend(
5868            0,
5869            frame.header.corr,
5870            format!("failed to encode {label}: {err}"),
5871        )
5872    })?;
5873
5874    Frame::build_with_version(
5875        response_version(frame),
5876        FrameType::Response,
5877        control_flags(),
5878        0,
5879        0,
5880        frame.header.corr,
5881        body,
5882    )
5883    .map_err(RouterError::FrameBuild)
5884}
5885
5886/// Map a forwarding failure to the wire code a client sees.
5887///
5888/// The code is not a label: clients BRANCH on it. Both SDKs decide "retry in
5889/// place" with `subc_protocol::error_codes::is_retryable_route_open`, so a code
5890/// chosen here decides whether a caller retries or gives up.
5891///
5892/// That makes attribution the load-bearing property, not merely having a code. A
5893/// permanent fault published as a retryable one produces a fleet-wide retry storm
5894/// against something that can never recover; a transient fault published as
5895/// permanent gives up on work that would have succeeded. Both look correct in a
5896/// log, which is why `retryability_of_forwarding_codes_matches_the_failure` pins
5897/// the mapping per variant rather than merely asserting that some code exists.
5898///
5899/// That fence partitions by RETRYABILITY, which is coarser than identity: swapping
5900/// two codes on the same side of the boundary passes it. Measured rather than
5901/// assumed — `NoModuleConnection` re-pointed at `module_reloading` is caught only
5902/// by `supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up`,
5903/// a test named for something else that happens to assert the string.
5904///
5905/// That accidental coverage is deliberately left alone rather than promoted to a
5906/// named test, because it guards a property this function does not promise.
5907/// Checked at source: every consumer branches on the RETRYABLE SET and none on a
5908/// specific code within a class, so identity is free to change and only the
5909/// partition is a contract. Splitting it out would assert a guarantee nothing
5910/// depends on — and a suite that promises more than the code does is the harder
5911/// thing to correct later, because the next reader cannot tell which assertions
5912/// are load-bearing.
5913///
5914/// Pin identity here the moment a consumer branches on a specific code.
5915fn forwarding_error_code(err: &ForwardingError) -> &'static str {
5916    match err {
5917        ForwardingError::NoModuleConnection => "target_unavailable",
5918        ForwardingError::ModuleReloading { .. } => "module_reloading",
5919        ForwardingError::ClientRouteChannelExhausted { .. }
5920        | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
5921        ForwardingError::StaleModuleEndpoint
5922        | ForwardingError::UnknownReservation { .. }
5923        | ForwardingError::ConnectionClosing { .. }
5924        | ForwardingError::ClientEgressClosed { .. }
5925        | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
5926        // Only a swap candidate's registration can produce this, and it means
5927        // exactly what a second active HELLO for a live id means.
5928        ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
5929        ForwardingError::RelayCorrelationExhausted
5930        | ForwardingError::RouteOpenBuild(_)
5931        | ForwardingError::Poisoned => "forwarding_error",
5932    }
5933}
5934
5935fn response_version(frame: &Frame) -> u8 {
5936    if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
5937        frame.header.ver
5938    } else {
5939        PROTOCOL_VERSION
5940    }
5941}
5942
5943fn control_flags() -> Flags {
5944    Flags::new(false, Priority::Passive, false)
5945}
5946
5947/// GOODBYE for a route.bind the daemon gave up on after reserving the module's
5948/// channel. The target is the module (a client never saw the route), so this
5949/// takes the module path: delivered late rather than dropped when the module's
5950/// queue is momentarily full, and never closing its connection.
5951fn send_goodbye_target_best_effort(
5952    counters: &DaemonCounters,
5953    target: &GoodbyeTarget,
5954    context: &'static str,
5955) {
5956    let Ok(frame) = Frame::build_with_version(
5957        target.negotiated_ver,
5958        FrameType::Goodbye,
5959        control_flags(),
5960        target.channel,
5961        target.epoch,
5962        0,
5963        Vec::new(),
5964    ) else {
5965        return;
5966    };
5967    crate::forwarding::send_module_route_goodbye(
5968        counters,
5969        &target.sink,
5970        frame,
5971        target.module_id.as_deref(),
5972        context,
5973    );
5974}
5975
5976pub(crate) fn send_route_control_pushes(
5977    forwarding: &ForwardingTable,
5978    routes: Vec<EndpointRoute>,
5979    push: ClientControlPush,
5980) {
5981    let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
5982    for route in routes {
5983        let target = route.goodbye_target;
5984        if let Some((existing, channels)) = targets
5985            .iter_mut()
5986            .find(|(existing, _)| existing.connection_id == target.connection_id)
5987        {
5988            debug_assert_eq!(
5989                existing.negotiated_ver, target.negotiated_ver,
5990                "one connection cannot negotiate multiple frame versions"
5991            );
5992            if !channels.contains(&target.channel) {
5993                channels.push(target.channel);
5994            }
5995            continue;
5996        }
5997        let channel = target.channel;
5998        targets.push((target, vec![channel]));
5999    }
6000    for (target, mut channels) in targets {
6001        channels.sort_unstable();
6002        let mut push = push.clone();
6003        match &mut push {
6004            ClientControlPush::RouteClosing {
6005                channels: covered, ..
6006            }
6007            | ClientControlPush::RouteClosed {
6008                channels: covered, ..
6009            } => *covered = channels,
6010        }
6011        let body = match serde_json::to_vec(&push) {
6012            Ok(body) => body,
6013            Err(err) => {
6014                warn!(error = %err, "failed to serialize route lifecycle control PUSH");
6015                continue;
6016            }
6017        };
6018        let frame = match Frame::build_with_version(
6019            target.negotiated_ver,
6020            FrameType::Push,
6021            control_flags(),
6022            0,
6023            0,
6024            0,
6025            body.clone(),
6026        ) {
6027            Ok(frame) => frame,
6028            Err(err) => {
6029                warn!(
6030                    route_channel = target.channel,
6031                    error = %err,
6032                    "failed to build route lifecycle control PUSH frame"
6033                );
6034                continue;
6035            }
6036        };
6037        if let Err(err) = target.sink.try_send(frame) {
6038            if target.close_on_delivery_failure() {
6039                warn!(
6040                    target_connection_id = target.connection_id.get(),
6041                    route_channel = target.channel,
6042                    error = %err,
6043                    "route lifecycle control PUSH was not delivered to client; closing target connection"
6044                );
6045                let _ = forwarding.escalate_client_delivery_failure(
6046                    target.connection_id,
6047                    target.channel,
6048                    target.epoch,
6049                    CloseReason::new(
6050                        "route_lifecycle_push_delivery_failed",
6051                        format!(
6052                            "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
6053                            target.channel
6054                        ),
6055                    ),
6056                    crate::forwarding::UndeliveredFrame {
6057                        module_id: target.module_id.as_deref(),
6058                        sink: &target.sink,
6059                    },
6060                );
6061            }
6062        }
6063    }
6064}
6065
6066#[cfg(test)]
6067mod tests {
6068    use std::{
6069        collections::BTreeMap,
6070        fmt,
6071        path::PathBuf,
6072        sync::{Arc, Mutex},
6073        time::Duration,
6074    };
6075    use subc_test_support::TestTempDir;
6076
6077    use serde_json::{json, Value};
6078    use subc_protocol::{
6079        manifest::{
6080            Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
6081            ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
6082        },
6083        session::HealthStatus,
6084        FrameType,
6085    };
6086
6087    use super::*;
6088    use crate::{
6089        forwarding::{DataRoute, DataRouteState},
6090        registry::ChannelState,
6091        router::FrameSink,
6092        stderr_tail::DEFAULT_MAX_LINE_BYTES,
6093        supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
6094        RouteCtx, Router,
6095    };
6096    use tokio::{
6097        sync::mpsc,
6098        time::{sleep, Instant},
6099    };
6100    use tracing::{
6101        field::{Field, Visit},
6102        Event, Subscriber,
6103    };
6104    use tracing_subscriber::{layer::Context, prelude::*, Layer};
6105
6106    /// Locates the `fake-aft-stub` binary from a `src/lib.rs` unit test.
6107    ///
6108    /// `CARGO_BIN_EXE_*` (compile-time `env!` and runtime `std::env::var` alike)
6109    /// is only populated for `tests/*.rs` integration test binaries -- this file
6110    /// compiles as part of the library target, which gets neither. This test's
6111    /// own executable path is `<target-dir>/<profile>/deps/subc_core-<hash>`,
6112    /// and the sibling binary lives two directories up at
6113    /// `<target-dir>/<profile>/fake-aft-stub`.
6114    ///
6115    /// THE BINARY IS NOT ALWAYS THERE, and the existence check below is why.
6116    /// `cargo test -p subc-core` builds every target including `[[bin]]`, so the
6117    /// stub is on disk; `cargo test -p subc-core --lib` builds ONLY the library
6118    /// test and leaves the stub unbuilt. A bare spawn then fails with a raw
6119    /// `NotFound`, which reads as a broken test rather than an unbuilt
6120    /// dependency -- so state the cause and the remedy instead. Deliberately a
6121    /// panic and not a silent skip: a test that quietly passes when it could not
6122    /// run is worse than one that fails, because it reports health it never
6123    /// verified.
6124    fn fake_aft_stub_path() -> PathBuf {
6125        let mut path = std::env::current_exe().expect("current_exe available in tests");
6126        path.pop(); // .../deps/
6127        path.pop(); // .../<profile>/
6128        path.push(if cfg!(windows) {
6129            "fake-aft-stub.exe"
6130        } else {
6131            "fake-aft-stub"
6132        });
6133        assert!(
6134            path.exists(),
6135            "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
6136             [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
6137            path.display()
6138        );
6139        path
6140    }
6141
6142    /// Whether clients retry `code` in place: the predicate itself, never a copy
6143    /// of its set. A copied list breaks silently when a code is added to or
6144    /// removed from the real one, and a stale copy here would let exactly the
6145    /// failure this test exists to catch pass.
6146    fn client_retries(code: &str) -> bool {
6147        subc_protocol::error_codes::is_retryable_route_open(code)
6148    }
6149
6150    /// A code is not a label — clients branch on it, so publishing the wrong KIND
6151    /// of failure is worse than publishing none. A permanent fault dressed as
6152    /// retryable makes every client in the fleet retry forever against something
6153    /// that cannot recover; a transient fault dressed as permanent abandons work
6154    /// that would have succeeded.
6155    ///
6156    /// Asserting "a code exists" cannot catch either, because the string is free
6157    /// to say anything. This enumerates every variant and pins which side of the
6158    /// retry boundary it lands on, so a new variant must be classified here
6159    /// deliberately rather than inheriting whichever arm it was appended to.
6160    #[test]
6161    fn retryability_of_forwarding_codes_matches_the_failure() {
6162        // Transient by nature: the target is booting, reloading, or its endpoint
6163        // was swapped mid-flight. Retrying is how these resolve.
6164        let transient = [
6165            ForwardingError::NoModuleConnection,
6166            ForwardingError::ModuleReloading {
6167                module_id: "m".into(),
6168            },
6169            ForwardingError::StaleModuleEndpoint,
6170            ForwardingError::UnknownReservation {
6171                client_channel: 1,
6172                module_channel: 1,
6173            },
6174            ForwardingError::ConnectionClosing {
6175                connection_id: ConnectionId::new(1),
6176            },
6177            ForwardingError::ClientEgressClosed {
6178                connection_id: ConnectionId::new(1),
6179            },
6180            ForwardingError::ModuleEgressUnavailable {
6181                connection_id: ConnectionId::new(1),
6182            },
6183        ];
6184        for err in transient {
6185            let code = forwarding_error_code(&err);
6186            assert!(
6187                client_retries(code),
6188                "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
6189            );
6190        }
6191
6192        // Not fixed by retrying. Channel and correlation exhaustion need the
6193        // caller to close routes, and a poisoned lock is a daemon that cannot
6194        // recover at all — the worst thing to advertise as retryable, since every
6195        // client would storm a daemon that will never answer.
6196        let permanent = [
6197            ForwardingError::ClientRouteChannelExhausted {
6198                connection_id: ConnectionId::new(1),
6199            },
6200            ForwardingError::ModuleRouteChannelExhausted {
6201                endpoint: ModuleEndpointId {
6202                    connection_id: ConnectionId::new(1),
6203                    generation: 1,
6204                },
6205            },
6206            ForwardingError::RelayCorrelationExhausted,
6207            ForwardingError::RouteOpenBuild("x".into()),
6208            ForwardingError::Poisoned,
6209        ];
6210        for err in permanent {
6211            let code = forwarding_error_code(&err);
6212            assert!(
6213                !client_retries(code),
6214                "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
6215            );
6216        }
6217    }
6218
6219    /// The principal is the daemon's answer to "who is calling", and modules
6220    /// branch on it: aft gates bash on it, cerebellum gates browser control,
6221    /// plexus gates connector invocation. So a stamp is an authorization input in
6222    /// another process, not a label — and both possible answers SUCCEED, which is
6223    /// what makes a wrong one quiet. An unattested caller stamped `Reserved` hands
6224    /// first-party capability to something that never proved it; a supervised one
6225    /// stamped `Direct` silently strips a module of capability it is entitled to.
6226    ///
6227    /// Neither shows up in a test that only checks the bind succeeded. Before this
6228    /// test the only coverage was accidental —
6229    /// `route_open_round_trip_via_tagged_shape_forwards_through_stub` asserts the
6230    /// stamped principal on its way past, so narrowing that wire-shape test to its
6231    /// stated subject would have deleted the last assertion on this value. It
6232    /// still asserts the stamp, which is now redundancy rather than the only
6233    /// guard: both fail under the same mutation, and this one names the reason.
6234    /// SCOPE: this handler's supervisor has spawned nothing, so
6235    /// `spawned_consumer_authorized` can only ever return false and the GRANT arm
6236    /// is unreachable here. Both assertions below are refusals, and a mutant that
6237    /// refuses everything would satisfy them.
6238    ///
6239    /// The grant side is covered where a real nonce exists: `tests/forwarding.rs`
6240    /// spawns a supervised consumer, reads its live nonce, and asserts the module
6241    /// observed `principal.kind == "reserved"` carrying that module_id — verified
6242    /// at source rather than assumed, since a citation is a claim about another
6243    /// file and ages like one. Recorded because a harness that structurally
6244    /// cannot reach an arm reports "none" for that arm identically to one that
6245    /// covers it and found nothing.
6246    #[tokio::test]
6247    async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
6248        let handler = ControlHandler::default();
6249        let frame =
6250            Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
6251
6252        // Absent consumer_identity is the ordinary case: a human at a terminal, or
6253        // any process holding the connection file. Nothing was proved, so nothing
6254        // may be granted beyond the unattested floor.
6255        let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
6256        assert_eq!(
6257            stamped,
6258            Principal::Direct,
6259            "a caller that proved nothing must not be stamped as a supervised module"
6260        );
6261
6262        // A claimed module_id with a nonce no supervised child was given is a
6263        // forgery attempt, not a weaker caller: it must be REFUSED rather than
6264        // quietly demoted to Direct, or an impersonation attempt looks identical
6265        // to an ordinary unattested connection.
6266        let forged = handler
6267            .route_open_principal(
6268                &frame,
6269                Some(ConsumerIdentity {
6270                    module_id: "aft".to_string(),
6271                    launch_nonce: "not-a-real-nonce".to_string(),
6272                }),
6273            )
6274            .unwrap();
6275        let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
6276        assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
6277    }
6278
6279    /// The test above hands `route_open_principal` an identity it built itself,
6280    /// which proves the stamping rule and nothing about where the identity comes
6281    /// from. The real producer is a wire body, and the two are joined by a serde
6282    /// field name that nothing else asserts.
6283    ///
6284    /// That join fails quietly in one specific way: an unrecognised key is simply
6285    /// absent after parsing, so a renamed or misspelled `consumer_identity`
6286    /// yields `None` and every supervised module silently drops to `Direct`.
6287    /// Capability-wise that is the safe direction, but it surfaces far from its
6288    /// cause — as a module mysteriously refused bash — and it would pass every
6289    /// test that builds its own input.
6290    ///
6291    /// Deliberately NOT closed with `deny_unknown_fields`: refusing unknown keys
6292    /// would break every client the moment the daemon gains a field, trading a
6293    /// quiet demotion for a hard refusal on additive change. Asserting the join
6294    /// instead means a rename breaks a test here rather than the fleet.
6295    #[test]
6296    fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
6297        let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
6298        let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
6299        let ClientControlRequest::RouteOpen {
6300            consumer_identity, ..
6301        } = parsed
6302        else {
6303            panic!("route.open body must parse as RouteOpen");
6304        };
6305        assert_eq!(
6306            consumer_identity,
6307            Some(ConsumerIdentity {
6308                module_id: "aft".to_string(),
6309                launch_nonce: "n".to_string(),
6310            }),
6311            "the wire field name must reach the value route_open_principal reads"
6312        );
6313    }
6314
6315    fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
6316        ModuleManifest::builder(module_id, "0.1.0")
6317            .protocol_ver(protocol_ver)
6318            .provides(vec![ProviderRole::ToolProvider {
6319                tools: vec![Tool {
6320                    name: "read".to_string(),
6321                    description: None,
6322                    execution_mode: ExecutionMode::Pure,
6323                    schema: json!({"type": "object"}),
6324                }],
6325                identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
6326                concurrency: Concurrency::ModuleManaged,
6327                emits_push: true,
6328                sub_supervises: true,
6329            }])
6330            .build()
6331    }
6332
6333    fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
6334        hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
6335    }
6336
6337    fn hello_frame_with_control_ops(
6338        module_id: &str,
6339        protocol_ver: u8,
6340        corr: u64,
6341        control_ops: Option<Vec<String>>,
6342    ) -> Frame {
6343        hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
6344    }
6345
6346    fn hello_frame_with_nonce(
6347        module_id: &str,
6348        protocol_ver: u8,
6349        corr: u64,
6350        launch_nonce: Option<&str>,
6351    ) -> Frame {
6352        hello_frame_full(
6353            module_id,
6354            protocol_ver,
6355            corr,
6356            None,
6357            launch_nonce.map(ToOwned::to_owned),
6358        )
6359    }
6360
6361    fn hello_frame_full(
6362        module_id: &str,
6363        protocol_ver: u8,
6364        corr: u64,
6365        control_ops: Option<Vec<String>>,
6366        launch_nonce: Option<String>,
6367    ) -> Frame {
6368        let body = serde_json::to_vec(&ModuleHelloBody {
6369            manifest: manifest(module_id, protocol_ver),
6370            protocol_ver,
6371            control_ops,
6372            launch_nonce,
6373        })
6374        .unwrap();
6375        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6376    }
6377
6378    fn non_routable_hello_frame_with_control_ops(
6379        module_id: &str,
6380        corr: u64,
6381        control_ops: Option<Vec<String>>,
6382    ) -> Frame {
6383        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
6384        manifest.provides.clear();
6385        let body = serde_json::to_vec(&ModuleHelloBody {
6386            manifest,
6387            protocol_ver: PROTOCOL_VERSION,
6388            control_ops,
6389            launch_nonce: None,
6390        })
6391        .unwrap();
6392        Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6393    }
6394
6395    fn capability_grammar_hello_frame(
6396        capabilities: Value,
6397        runtime_computed: Option<Value>,
6398        corr: u64,
6399    ) -> Frame {
6400        let mut body = serde_json::to_value(ModuleHelloBody {
6401            manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
6402            protocol_ver: PROTOCOL_VERSION,
6403            control_ops: None,
6404            launch_nonce: None,
6405        })
6406        .expect("HELLO body serializes");
6407        body["manifest"]["capabilities"] = capabilities;
6408        if let Some(runtime_computed) = runtime_computed {
6409            body["runtime_computed"] = runtime_computed;
6410        }
6411        Frame::build(
6412            FrameType::Hello,
6413            control_flags(),
6414            0,
6415            0,
6416            corr,
6417            serde_json::to_vec(&body).expect("HELLO body reserializes"),
6418        )
6419        .expect("HELLO frame builds")
6420    }
6421
6422    fn channel_request(channel: u16, corr: u64) -> Frame {
6423        Frame::build(
6424            FrameType::Request,
6425            Flags::new(true, Priority::Interactive, false),
6426            channel,
6427            0,
6428            corr,
6429            b"opaque".to_vec(),
6430        )
6431        .unwrap()
6432    }
6433
6434    fn route_ctx(
6435        connection_id: ConnectionId,
6436    ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
6437        let (tx, rx) = mpsc::channel(8);
6438        (
6439            RouteCtx {
6440                connection_id,
6441                egress: FrameSink::new(tx),
6442            },
6443            rx,
6444        )
6445    }
6446
6447    fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
6448        serde_json::from_slice(&frame.body).unwrap()
6449    }
6450
6451    /// Register a module over a connection that has a sink and return the
6452    /// HELLO_ACK the module reads. A successful HELLO queues its ack on the
6453    /// module's own sink rather than returning it as a reply, so the ack is
6454    /// taken off `rx` here and whatever the test reads next is what followed it.
6455    async fn hello_via_sink(
6456        handler: &ControlHandler,
6457        ctx: &RouteCtx,
6458        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
6459        hello: Frame,
6460    ) -> Frame {
6461        let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
6462        assert!(
6463            replies.is_empty(),
6464            "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
6465        );
6466        let ack = rx
6467            .try_recv()
6468            .expect("HELLO_ACK is queued on the module sink")
6469            .frame;
6470        assert_eq!(ack.header.ty, FrameType::HelloAck);
6471        ack
6472    }
6473
6474    fn parse_error(frame: &Frame) -> Value {
6475        serde_json::from_slice(&frame.body).unwrap()
6476    }
6477
6478    fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
6479        serde_json::from_slice(&frame.body).unwrap()
6480    }
6481
6482    fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
6483        let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
6484            route_channel,
6485            route_epoch: 0,
6486            kind,
6487        })
6488        .unwrap();
6489        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6490    }
6491
6492    fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
6493        let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
6494            module_id: module_id.to_string(),
6495        })
6496        .unwrap();
6497        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6498    }
6499
6500    fn route_open_frame(corr: u64, module_id: &str, project_root: TestTempDir) -> Frame {
6501        route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
6502    }
6503
6504    fn route_open_frame_with_consumer_capabilities(
6505        corr: u64,
6506        module_id: &str,
6507        project_root: TestTempDir,
6508        consumer_capabilities: Option<Vec<String>>,
6509    ) -> Frame {
6510        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6511            target: RouteTarget::ToolProvider {
6512                module_id: module_id.to_string(),
6513            },
6514            identity: BindIdentity::new(
6515                project_root.path().to_path_buf(),
6516                "unit".to_string(),
6517                "session".to_string(),
6518            ),
6519            consumer_identity: None,
6520            consumer_capabilities,
6521            role_versions: None,
6522            admission_facts: None,
6523            scope: None,
6524        })
6525        .unwrap();
6526        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6527    }
6528
6529    fn route_open_frame_with_role_versions(
6530        corr: u64,
6531        module_id: &str,
6532        project_root: TestTempDir,
6533        role_versions: Option<BTreeMap<String, String>>,
6534    ) -> Frame {
6535        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6536            target: RouteTarget::ToolProvider {
6537                module_id: module_id.to_string(),
6538            },
6539            identity: BindIdentity::new(
6540                project_root.path().to_path_buf(),
6541                "unit".to_string(),
6542                format!("session-{corr}"),
6543            ),
6544            consumer_identity: None,
6545            consumer_capabilities: None,
6546            role_versions,
6547            admission_facts: None,
6548            scope: None,
6549        })
6550        .unwrap();
6551        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6552    }
6553
6554    fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
6555        entries
6556            .iter()
6557            .map(|(role, version)| (role.to_string(), version.to_string()))
6558            .collect()
6559    }
6560
6561    fn route_open_frame_with_admission_facts(
6562        corr: u64,
6563        module_id: &str,
6564        project_root: TestTempDir,
6565        consumer_identity: Option<subc_control::ConsumerIdentity>,
6566        facts: Option<Value>,
6567    ) -> Frame {
6568        let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6569            target: RouteTarget::ToolProvider {
6570                module_id: module_id.to_string(),
6571            },
6572            identity: BindIdentity::new(
6573                project_root.path().to_path_buf(),
6574                "unit".to_string(),
6575                format!("session-{corr}"),
6576            ),
6577            consumer_identity,
6578            consumer_capabilities: None,
6579            role_versions: None,
6580            admission_facts: facts,
6581            scope: None,
6582        })
6583        .unwrap();
6584        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6585    }
6586
6587    #[derive(Clone, Default)]
6588    struct EventCapture {
6589        events: Arc<Mutex<Vec<CapturedEvent>>>,
6590    }
6591
6592    #[derive(Clone, Debug)]
6593    struct CapturedEvent {
6594        target: String,
6595        level: tracing::Level,
6596        fields: BTreeMap<String, String>,
6597    }
6598
6599    impl EventCapture {
6600        fn events(&self) -> Vec<CapturedEvent> {
6601            self.events.lock().unwrap().clone()
6602        }
6603    }
6604
6605    impl<S> Layer<S> for EventCapture
6606    where
6607        S: Subscriber,
6608    {
6609        fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
6610            let mut visitor = EventFieldVisitor::default();
6611            event.record(&mut visitor);
6612            self.events.lock().unwrap().push(CapturedEvent {
6613                target: event.metadata().target().to_string(),
6614                level: *event.metadata().level(),
6615                fields: visitor.fields,
6616            });
6617        }
6618    }
6619
6620    #[derive(Default)]
6621    struct EventFieldVisitor {
6622        fields: BTreeMap<String, String>,
6623    }
6624
6625    impl Visit for EventFieldVisitor {
6626        fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
6627            self.fields
6628                .insert(field.name().to_string(), format!("{value:?}"));
6629        }
6630    }
6631
6632    fn health_response(corr: u64, status: HealthStatus) -> Frame {
6633        let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
6634            status,
6635            detail: Some("warming".to_string()),
6636            metrics: Some(json!({"queue_depth": 3})),
6637        })
6638        .unwrap();
6639        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6640    }
6641
6642    fn route_bind_ack(corr: u64) -> Frame {
6643        let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
6644        Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6645    }
6646
6647    fn unique_project_root(label: &str) -> TestTempDir {
6648        TestTempDir::new(label)
6649    }
6650
6651    fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
6652        match parse_route_poll(frame) {
6653            ClientControlResponse::RoutePoll {
6654                status: None,
6655                live: Some(live),
6656                ..
6657            } => assert_eq!(live, expected_live),
6658            other => panic!("unexpected route.poll response: {other:?}"),
6659        }
6660    }
6661
6662    fn bind_liveness_route(
6663        registry: &Registry,
6664        forwarding: &ForwardingTable,
6665        module_id: &str,
6666    ) -> (RouteCtx, u16, u32) {
6667        let module_connection = ConnectionId::new(101);
6668        let client_connection = ConnectionId::new(202);
6669        let registration = registry
6670            .register_with_control_ops(
6671                manifest(module_id, PROTOCOL_VERSION),
6672                PROTOCOL_VERSION,
6673                module_connection,
6674                module_baseline_control_ops(),
6675            )
6676            .unwrap();
6677        let (module_tx, _module_rx) = mpsc::channel(8);
6678        let endpoint = forwarding
6679            .register_module_connection(
6680                module_connection,
6681                module_id.to_string(),
6682                PROTOCOL_VERSION,
6683                manifest_concurrency(&registration.manifest),
6684                FrameSink::new(module_tx),
6685            )
6686            .unwrap();
6687        let (client_ctx, _client_rx) = route_ctx(client_connection);
6688        let pending = forwarding
6689            .begin_route_bind_relay_for_test(
6690                client_connection,
6691                client_ctx.egress.clone(),
6692                1,
6693                module_id,
6694            )
6695            .unwrap();
6696        assert_eq!(pending.endpoint, endpoint);
6697        let route_channel = pending.client_channel;
6698        let route_epoch = pending.client_epoch;
6699        forwarding
6700            .complete_pending_relay(
6701                module_connection,
6702                pending.corr,
6703                RouteBindRelayOutcome::Accepted,
6704            )
6705            .unwrap();
6706        (client_ctx, route_channel, route_epoch)
6707    }
6708
6709    struct FakeProcessLiveness {
6710        live: Option<bool>,
6711    }
6712
6713    impl ModuleProcessLiveness for FakeProcessLiveness {
6714        fn process_live(&self, _module_id: &str) -> Option<bool> {
6715            self.live
6716        }
6717    }
6718
6719    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6720    async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
6721    {
6722        let registry = Arc::new(Registry::default());
6723        let supervisor_handle = SupervisorHandle::new();
6724        let supervisor = Supervisor::new(
6725            Arc::clone(&registry),
6726            RestartPolicy::new(1, Duration::from_millis(10)),
6727        )
6728        .with_handle(supervisor_handle.clone());
6729        let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
6730        let module = supervisor
6731            .spawn(ModuleSpec {
6732                module_id: "stderr-tail-wire".to_string(),
6733                program: fake_aft_stub_path(),
6734                args: Vec::new(),
6735                env: vec![
6736                    ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
6737                    ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
6738                ],
6739                reserved: false,
6740                reserved_prefixes: Vec::new(),
6741                protocol: ModuleProtocol::Subc,
6742                overlap: Default::default(),
6743            })
6744            .unwrap();
6745
6746        let deadline = Instant::now() + Duration::from_secs(5);
6747        loop {
6748            let tail = module.stderr_tail(None, None);
6749            if tail
6750                .entries
6751                .iter()
6752                .any(|entry| matches!(entry, TailEntry::ProcessStart))
6753                && tail.entries.iter().any(|entry| {
6754                    matches!(
6755                        entry,
6756                        TailEntry::Line {
6757                            truncated: true,
6758                            ..
6759                        }
6760                    )
6761                })
6762            {
6763                break;
6764            }
6765            assert!(
6766                Instant::now() < deadline,
6767                "module did not produce a truncated line and restart boundary: {tail:?}"
6768            );
6769            sleep(Duration::from_millis(10)).await;
6770        }
6771
6772        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
6773        let request = ClientControlRequest::SupervisorStderrTail {
6774            module_id: "stderr-tail-wire".to_string(),
6775            max_lines: None,
6776            max_bytes: None,
6777        };
6778        let frame = Frame::build(
6779            FrameType::Request,
6780            control_flags(),
6781            0,
6782            0,
6783            1,
6784            serde_json::to_vec(&request).unwrap(),
6785        )
6786        .unwrap();
6787        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6788        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6789        let ClientControlResponse::SupervisorStderrTail { tail, .. } =
6790            serde_json::from_slice(&responses[0].body).unwrap()
6791        else {
6792            panic!("expected supervisor.stderr_tail response");
6793        };
6794
6795        assert!(
6796            tail.entries
6797                .iter()
6798                .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
6799            "the control response lost the restart boundary"
6800        );
6801        let Some(StderrTailEntry::Line {
6802            text,
6803            truncated,
6804            at_ms,
6805        }) = tail.entries.iter().find(|entry| {
6806            matches!(
6807                entry,
6808                StderrTailEntry::Line {
6809                    truncated: true,
6810                    ..
6811                }
6812            )
6813        })
6814        else {
6815            panic!("the control response lost the truncated line");
6816        };
6817        assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
6818        assert!(*truncated);
6819        assert!(
6820            at_ms.is_some(),
6821            "the control response lost the line's capture time"
6822        );
6823    }
6824
6825    /// `supervisor.terminals` reads journal files. On a single-worker runtime a
6826    /// read done on the worker thread would stall every other task until it
6827    /// finished; the read must run off the worker so this test's own task keeps
6828    /// running while the read is paused.
6829    #[tokio::test(flavor = "current_thread")]
6830    async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
6831        let dir = TestTempDir::new("terminals-off-worker");
6832        let journal_path = dir.join("terminals.jsonl");
6833        let registry = Arc::new(Registry::default());
6834        let supervisor_handle = SupervisorHandle::new();
6835        let supervisor =
6836            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
6837                .with_handle(supervisor_handle.clone())
6838                .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
6839        let module = supervisor
6840            .spawn(ModuleSpec {
6841                module_id: "terminal-off-worker".to_string(),
6842                program: fake_aft_stub_path(),
6843                args: Vec::new(),
6844                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6845                reserved: false,
6846                reserved_prefixes: Vec::new(),
6847                protocol: ModuleProtocol::Subc,
6848                overlap: Default::default(),
6849            })
6850            .unwrap();
6851        let deadline = Instant::now() + Duration::from_secs(5);
6852        while module.terminal_history().entries.len() != 2 {
6853            assert!(Instant::now() < deadline, "module did not record two exits");
6854            sleep(Duration::from_millis(10)).await;
6855        }
6856
6857        let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
6858        let handler =
6859            Arc::new(ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle));
6860        let frame = Frame::build(
6861            FrameType::Request,
6862            control_flags(),
6863            0,
6864            0,
6865            1,
6866            serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
6867                module_id: "terminal-off-worker".to_string(),
6868            })
6869            .unwrap(),
6870        )
6871        .unwrap();
6872        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6873        let spawned_at = std::time::Instant::now();
6874        let read = tokio::spawn({
6875            let handler = Arc::clone(&handler);
6876            async move { handler.handle_control_frame(&ctx, frame).await }
6877        });
6878        // Waiting for the pause from a blocking thread keeps this task pending,
6879        // so the runtime's single worker is free to run the read task.
6880        tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
6881            .await
6882            .unwrap()
6883            .expect("the history read reached its pause");
6884        let elapsed = spawned_at.elapsed();
6885        assert!(
6886            elapsed < Duration::from_secs(2) && !read.is_finished(),
6887            "this task could not run while the history read was paused \
6888             (resumed after {elapsed:?}, read finished: {})",
6889            read.is_finished()
6890        );
6891
6892        drop(release);
6893        let responses = read.await.unwrap().unwrap();
6894        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6895        let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
6896            panic!("expected supervisor.terminals response");
6897        };
6898        assert_eq!(terminals.entries.len(), 2);
6899        assert_eq!(terminals.journal_skipped_lines, 0);
6900        assert_eq!(terminals.journal_read_errors, 0);
6901    }
6902
6903    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6904    async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
6905        let registry = Arc::new(Registry::default());
6906        let supervisor_handle = SupervisorHandle::new();
6907        let supervisor =
6908            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(1, Duration::ZERO))
6909                .with_handle(supervisor_handle.clone());
6910        let module = supervisor
6911            .spawn(ModuleSpec {
6912                module_id: "terminal-golden".to_string(),
6913                program: fake_aft_stub_path(),
6914                args: Vec::new(),
6915                env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6916                reserved: false,
6917                reserved_prefixes: Vec::new(),
6918                protocol: ModuleProtocol::Subc,
6919                overlap: Default::default(),
6920            })
6921            .unwrap();
6922
6923        let deadline = Instant::now() + Duration::from_secs(5);
6924        while module.terminal_history().entries.len() != 2 {
6925            assert!(
6926                Instant::now() < deadline,
6927                "module did not retain two terminal exits: {:?}",
6928                module.terminal_history()
6929            );
6930            sleep(Duration::from_millis(10)).await;
6931        }
6932
6933        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
6934        let request = ClientControlRequest::SupervisorTerminals {
6935            module_id: "terminal-golden".to_string(),
6936        };
6937        let frame = Frame::build(
6938            FrameType::Request,
6939            control_flags(),
6940            0,
6941            0,
6942            1,
6943            serde_json::to_vec(&request).unwrap(),
6944        )
6945        .unwrap();
6946        let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6947        let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6948        let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6949        let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
6950            panic!("expected supervisor.terminals response");
6951        };
6952        assert_eq!(terminals.entries.len(), 2);
6953        assert_eq!(terminals.dropped, 0);
6954
6955        let mut rendered = serde_json::to_value(response).unwrap();
6956        // Wall-clock fields are the observation contract, but not stable fixture
6957        // bytes; normalize only them after the real handler has shaped the response.
6958        rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
6959        for (index, entry) in rendered["entries"]
6960            .as_array_mut()
6961            .expect("terminal response entries array")
6962            .iter_mut()
6963            .enumerate()
6964        {
6965            entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
6966        }
6967
6968        let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
6969            .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
6970        let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
6971        if std::env::var_os("UPDATE_GOLDEN").is_some() {
6972            std::fs::write(&golden_path, &serialized).unwrap();
6973        }
6974        let expected: Value =
6975            serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
6976        assert_eq!(rendered, expected);
6977    }
6978
6979    #[test]
6980    fn hello_registers_manifest_and_returns_ack() {
6981        let registry = Arc::new(Registry::default());
6982        let handler = ControlHandler::new(Arc::clone(&registry));
6983        let conn = ConnectionId::new(1);
6984
6985        let responses = handler
6986            .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
6987            .unwrap();
6988
6989        assert_eq!(responses.len(), 1);
6990        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6991        assert_eq!(responses[0].header.channel, 0);
6992        assert_eq!(responses[0].header.corr, 7);
6993        let ack = parse_ack(&responses[0]);
6994        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
6995        assert!(ack
6996            .subc_capabilities
6997            .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
6998        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
6999        assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
7000        assert!(ack
7001            .subc_ops
7002            .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
7003        assert!(ack
7004            .subc_ops
7005            .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
7006
7007        let registration = registry.get_module("aft").unwrap().unwrap();
7008        assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
7009        assert_eq!(registration.state, ChannelState::Active);
7010        assert_eq!(registration.connection_id, conn);
7011        assert_eq!(registration.control_ops, module_baseline_control_ops());
7012    }
7013
7014    #[test]
7015    fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
7016        let invalid_identifiers = [
7017            ("case_change", "credentials-Provider/v1"),
7018            ("leading_zero", "credentials-provider/v01"),
7019            ("trailing_hyphen", "credentials-provider-/v1"),
7020            ("consecutive_hyphens", "credentials--provider/v1"),
7021            ("uppercase", "Credentials-provider/v1"),
7022            ("missing_v", "credentials-provider/1"),
7023            ("whitespace", "credentials provider/v1"),
7024            ("zero_version", "credentials-provider/v0"),
7025            ("out_of_range_version", "credentials-provider/v4294967296"),
7026            (
7027                "overlength_name",
7028                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
7029            ),
7030        ];
7031        let mut cases = invalid_identifiers
7032            .into_iter()
7033            .map(|(name, identifier)| {
7034                (
7035                    format!("identifier_{name}"),
7036                    "capabilities.provides[0]".to_string(),
7037                    identifier.to_string(),
7038                    json!({ "provides": [identifier] }),
7039                    None,
7040                )
7041            })
7042            .collect::<Vec<_>>();
7043        cases.extend([
7044            (
7045                "unknown_need".to_string(),
7046                "capabilities.requires[0].need".to_string(),
7047                "deferred".to_string(),
7048                json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
7049                None,
7050            ),
7051            (
7052                "duplicate_provides".to_string(),
7053                "capabilities.provides[1]".to_string(),
7054                "credentials-provider/v1".to_string(),
7055                json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
7056                None,
7057            ),
7058            (
7059                "duplicate_must_never_reach".to_string(),
7060                "capabilities.must_never_reach[1]".to_string(),
7061                "credentials-provider/v1".to_string(),
7062                json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
7063                None,
7064            ),
7065            (
7066                "duplicate_requires_same_need".to_string(),
7067                "capabilities.requires[1]".to_string(),
7068                "credentials-provider/v1".to_string(),
7069                json!({ "requires": [
7070                    { "capability": "credentials-provider/v1", "need": "required" },
7071                    { "capability": "credentials-provider/v1", "need": "required" }
7072                ] }),
7073                None,
7074            ),
7075            (
7076                "duplicate_requires_conflicting_need".to_string(),
7077                "capabilities.requires[1]".to_string(),
7078                "credentials-provider/v1".to_string(),
7079                json!({ "requires": [
7080                    { "capability": "credentials-provider/v1", "need": "required" },
7081                    { "capability": "credentials-provider/v1", "need": "optional" }
7082                ] }),
7083                None,
7084            ),
7085            (
7086                "capabilities_root_pointer".to_string(),
7087                "runtime_computed[0]".to_string(),
7088                "/capabilities".to_string(),
7089                json!({}),
7090                Some(json!(["/capabilities"])),
7091            ),
7092            (
7093                "capabilities_descendant_pointer".to_string(),
7094                "runtime_computed[0]".to_string(),
7095                "/capabilities/provides".to_string(),
7096                json!({}),
7097                Some(json!(["/capabilities/provides"])),
7098            ),
7099            (
7100                "malformed_pointer_without_leading_slash".to_string(),
7101                "runtime_computed[0]".to_string(),
7102                "capabilities".to_string(),
7103                json!({}),
7104                Some(json!(["capabilities"])),
7105            ),
7106            (
7107                "malformed_pointer_escape".to_string(),
7108                "runtime_computed[0]".to_string(),
7109                "/roles/~2/tools".to_string(),
7110                json!({}),
7111                Some(json!(["/roles/~2/tools"])),
7112            ),
7113            (
7114                "unknown_capabilities_field".to_string(),
7115                "capabilities.future".to_string(),
7116                "<array>".to_string(),
7117                json!({ "future": [] }),
7118                None,
7119            ),
7120        ]);
7121
7122        for (index, (name, field, value, capabilities, runtime_computed)) in
7123            cases.into_iter().enumerate()
7124        {
7125            let registry = Arc::new(Registry::default());
7126            let handler = ControlHandler::new(Arc::clone(&registry));
7127            let response = handler
7128                .handle_control(
7129                    ConnectionId::new((index + 1) as u64),
7130                    capability_grammar_hello_frame(
7131                        capabilities,
7132                        runtime_computed,
7133                        index as u64 + 1,
7134                    ),
7135                )
7136                .expect("invalid HELLO returns a refusal");
7137
7138            assert_eq!(response.len(), 1, "{name} must emit one refusal");
7139            let error = parse_error(&response[0]);
7140            assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
7141            let message = error["message"]
7142                .as_str()
7143                .expect("error message is a string");
7144            assert!(
7145                message.contains(&field),
7146                "{name}: field missing from {message}"
7147            );
7148            assert!(
7149                message.contains(&value),
7150                "{name}: value missing from {message}"
7151            );
7152            assert_eq!(
7153                registry
7154                    .active_registration_count()
7155                    .expect("registry reads"),
7156                0,
7157                "{name}: refused HELLO must not create a catalog entry"
7158            );
7159        }
7160    }
7161
7162    #[test]
7163    fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
7164        let registry = Arc::new(Registry::default());
7165        let handler = ControlHandler::new(Arc::clone(&registry));
7166        let capabilities = json!({
7167            "provides": ["credentials-provider/v1"],
7168            "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
7169            "must_never_reach": ["federation-transport/v1"]
7170        });
7171        let response = handler
7172            .handle_control(
7173                ConnectionId::new(99),
7174                capability_grammar_hello_frame(
7175                    capabilities.clone(),
7176                    Some(json!(["/roles/0/tools"])),
7177                    99,
7178                ),
7179            )
7180            .expect("valid HELLO registers");
7181        assert_eq!(response[0].header.ty, FrameType::HelloAck);
7182
7183        let request = Frame::build(
7184            FrameType::Request,
7185            control_flags(),
7186            0,
7187            0,
7188            100,
7189            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7190                .expect("catalog request serializes"),
7191        )
7192        .expect("catalog request frame builds");
7193        let response = handler
7194            .handle_catalog_list(request, None)
7195            .expect("catalog list succeeds");
7196        let ClientControlResponse::CatalogList { modules, .. } =
7197            serde_json::from_slice(&response[0].body).expect("catalog response decodes")
7198        else {
7199            panic!("catalog request must return catalog.list");
7200        };
7201        assert_eq!(modules.len(), 1);
7202        assert_eq!(
7203            serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
7204            capabilities
7205        );
7206    }
7207
7208    #[test]
7209    fn catalog_list_mirrors_management_operation_description() {
7210        let registry = Arc::new(Registry::default());
7211        let handler = ControlHandler::new(Arc::clone(&registry));
7212        let description = "List managed records and return their identifiers and metadata.";
7213        let mut manifest = manifest("described-management", PROTOCOL_VERSION);
7214        manifest.provides = vec![ProviderRole::ManagementSurface {
7215            operations: vec![ManagementOperation {
7216                name: "records.list".to_string(),
7217                kind: ManagementOperationKind::Query,
7218                description: Some(description.to_string()),
7219            }],
7220            config_schema: json!({"type": "object"}),
7221            observability: vec![ObservabilitySurface {
7222                name: "records.stats".to_string(),
7223                kind: ObservabilityKind::Snapshot,
7224            }],
7225            identity_scope: vec![IdentityScope::Project],
7226            concurrency: Concurrency::ModuleManaged,
7227        }];
7228        registry
7229            .register_with_control_ops(
7230                manifest,
7231                PROTOCOL_VERSION,
7232                ConnectionId::new(99),
7233                Vec::new(),
7234            )
7235            .expect("described management manifest registers");
7236
7237        let request = Frame::build(
7238            FrameType::Request,
7239            control_flags(),
7240            0,
7241            0,
7242            100,
7243            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7244                .expect("catalog request serializes"),
7245        )
7246        .expect("catalog request frame builds");
7247        let response = handler
7248            .handle_catalog_list(request, None)
7249            .expect("catalog list succeeds");
7250        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7251        assert_eq!(
7252            body["modules"][0]["roles"][0]["operations"][0]["description"], description,
7253            "catalog.list must preserve the declared operation description verbatim"
7254        );
7255    }
7256
7257    #[test]
7258    fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
7259        let registry = Arc::new(Registry::default());
7260        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
7261            [("vault".to_string(), true), ("squatter".to_string(), true)],
7262            BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
7263        );
7264        let mut squatter = manifest("squatter", PROTOCOL_VERSION);
7265        squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7266            provides: vec!["credentials-provider/v1".to_string()],
7267            requires: Vec::new(),
7268            must_never_reach: Vec::new(),
7269        });
7270        let frame = Frame::build(
7271            FrameType::Hello,
7272            control_flags(),
7273            0,
7274            0,
7275            77,
7276            serde_json::to_vec(&ModuleHelloBody {
7277                manifest: squatter,
7278                protocol_ver: PROTOCOL_VERSION,
7279                control_ops: None,
7280                launch_nonce: None,
7281            })
7282            .expect("HELLO serializes"),
7283        )
7284        .expect("HELLO frame builds");
7285        let response = handler
7286            .handle_control(ConnectionId::new(77), frame)
7287            .expect("reserved claim receives a typed refusal");
7288        assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
7289        assert_eq!(
7290            registry
7291                .active_registration_count()
7292                .expect("registry reads"),
7293            0,
7294            "a reserved capability refusal must not leave a catalog entry"
7295        );
7296    }
7297
7298    #[test]
7299    fn server_describe_surfaces_required_capability_verdict_fields() {
7300        let registry = Arc::new(Registry::default());
7301        let handler = ControlHandler::new(Arc::clone(&registry)).with_capability_config(
7302            [
7303                ("consumer".to_string(), true),
7304                ("provider".to_string(), false),
7305            ],
7306            BTreeMap::new(),
7307        );
7308        let mut consumer = manifest("consumer", PROTOCOL_VERSION);
7309        consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7310            provides: Vec::new(),
7311            requires: vec![subc_protocol::manifest::CapabilityRequirement {
7312                capability: "credentials-provider/v1".to_string(),
7313                need: subc_protocol::manifest::CapabilityNeed::Required,
7314            }],
7315            must_never_reach: Vec::new(),
7316        });
7317        let hello = Frame::build(
7318            FrameType::Hello,
7319            control_flags(),
7320            0,
7321            0,
7322            78,
7323            serde_json::to_vec(&ModuleHelloBody {
7324                manifest: consumer,
7325                protocol_ver: PROTOCOL_VERSION,
7326                control_ops: None,
7327                launch_nonce: None,
7328            })
7329            .expect("HELLO serializes"),
7330        )
7331        .expect("HELLO frame builds");
7332        handler
7333            .handle_control(ConnectionId::new(78), hello)
7334            .expect("consumer registers");
7335        let describe = Frame::build(
7336            FrameType::Request,
7337            control_flags(),
7338            0,
7339            0,
7340            79,
7341            serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
7342                .expect("request serializes"),
7343        )
7344        .expect("describe frame builds");
7345        let response = handler
7346            .handle_server_describe(describe)
7347            .expect("server.describe succeeds");
7348        let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
7349        let requirement = &rendered["capability_requirements"][0];
7350        assert_eq!(requirement["consumer"], "consumer");
7351        assert_eq!(requirement["verdict"], "never_provided");
7352        assert_eq!(requirement["episode_seq"], 1);
7353        assert_eq!(requirement["config_satisfiable"], false);
7354        assert_eq!(requirement["runtime_available"], false);
7355        assert!(requirement["detail"]
7356            .as_str()
7357            .expect("detail string")
7358            .contains("credentials-provider/v1"));
7359    }
7360
7361    #[test]
7362    fn catalog_list_omits_capabilities_for_legacy_manifest() {
7363        let registry = Arc::new(Registry::default());
7364        let handler = ControlHandler::new(Arc::clone(&registry));
7365        let hello = handler
7366            .handle_control(
7367                ConnectionId::new(101),
7368                hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
7369            )
7370            .expect("legacy HELLO registers");
7371        assert_eq!(hello[0].header.ty, FrameType::HelloAck);
7372
7373        let request = Frame::build(
7374            FrameType::Request,
7375            control_flags(),
7376            0,
7377            0,
7378            102,
7379            serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7380                .expect("catalog request serializes"),
7381        )
7382        .expect("catalog request frame builds");
7383        let response = handler
7384            .handle_catalog_list(request, None)
7385            .expect("catalog list succeeds");
7386        let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7387        assert!(
7388            body["modules"][0].get("capabilities").is_none(),
7389            "legacy manifest must retain an absent capabilities field on catalog.list"
7390        );
7391    }
7392
7393    #[test]
7394    fn hello_ack_omits_storage_when_no_storage_config() {
7395        let registry = Arc::new(Registry::default());
7396        let handler = ControlHandler::new(Arc::clone(&registry));
7397        let responses = handler
7398            .handle_control(
7399                ConnectionId::new(1),
7400                hello_frame("aft", PROTOCOL_VERSION, 7),
7401            )
7402            .unwrap();
7403        let ack = parse_ack(&responses[0]);
7404        assert_eq!(ack.storage, None, "no storage config -> no descriptor");
7405        assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
7406    }
7407
7408    #[tokio::test]
7409    async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
7410        let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
7411        let registry = Arc::new(Registry::default());
7412        let handler = ControlHandler::new(Arc::clone(&registry)).with_machine_id(Some(id.clone()));
7413        let responses = handler
7414            .handle_control(
7415                ConnectionId::new(1),
7416                hello_frame("aft", PROTOCOL_VERSION, 7),
7417            )
7418            .unwrap();
7419        let ack = parse_ack(&responses[0]);
7420        assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
7421
7422        let described = handler
7423            .handle_control_frame(
7424                &route_ctx(ConnectionId::new(2)).0,
7425                Frame::build(
7426                    FrameType::Request,
7427                    control_flags(),
7428                    0,
7429                    0,
7430                    9,
7431                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
7432                )
7433                .unwrap(),
7434            )
7435            .await
7436            .unwrap();
7437        let ClientControlResponse::ServerDescribe { machine_id, .. } =
7438            serde_json::from_slice(&described[0].body).unwrap()
7439        else {
7440            panic!("server.describe answered with another shape");
7441        };
7442        assert_eq!(machine_id.as_deref(), Some(id.as_str()));
7443    }
7444
7445    #[test]
7446    fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
7447        // With a central sqlite storage policy, each registering module gets its
7448        // own resolved descriptor in HELLO_ACK, keyed by its module id.
7449        let registry = Arc::new(Registry::default());
7450        let handler = ControlHandler::new(Arc::clone(&registry)).with_storage_config(Some(
7451            crate::daemon_config::StorageConfig::Sqlite {
7452                data_home: std::path::PathBuf::from("/data"),
7453            },
7454        ));
7455
7456        let responses = handler
7457            .handle_control(
7458                ConnectionId::new(1),
7459                hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
7460            )
7461            .unwrap();
7462        let ack = parse_ack(&responses[0]);
7463        assert_eq!(
7464            ack.storage,
7465            Some(serde_json::json!({
7466                "module_id": "alfonso-routing",
7467                "storage_namespace": "default",
7468                "isolation": { "kind": "module" },
7469                "backend": {
7470                    "backend": "sqlite",
7471                    "path": "/data/cortexkit/alfonso-routing/store.db"
7472                }
7473            })),
7474            "the delivered descriptor is the module's own sqlite store path"
7475        );
7476    }
7477
7478    #[test]
7479    fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
7480        let registry = Arc::new(Registry::default());
7481        let handler = ControlHandler::new(Arc::clone(&registry));
7482        let conn = ConnectionId::new(1);
7483        let responses = handler
7484            .handle_control(
7485                conn,
7486                hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
7487            )
7488            .unwrap();
7489        assert_eq!(responses[0].header.ty, FrameType::HelloAck);
7490        let registration = registry.get_module("aft").unwrap().unwrap();
7491        assert_eq!(registration.control_ops, module_baseline_control_ops());
7492
7493        let frame =
7494            Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
7495        assert!(handler
7496            .guard_module_control_op(&frame, "aft", "route.bind")
7497            .unwrap()
7498            .is_none());
7499        let error = handler
7500            .guard_module_control_op(&frame, "aft", "test.synthetic")
7501            .unwrap()
7502            .expect("synthetic ungranted op should be rejected");
7503        assert_eq!(error.header.ty, FrameType::Error);
7504        assert_eq!(parse_error(&error)["code"], "op_not_allowed");
7505    }
7506
7507    #[test]
7508    fn hello_control_ops_some_adds_optional_grants() {
7509        let registry = Arc::new(Registry::default());
7510        let handler = ControlHandler::new(Arc::clone(&registry));
7511        handler
7512            .handle_control(
7513                ConnectionId::new(1),
7514                hello_frame_with_control_ops(
7515                    "aft",
7516                    PROTOCOL_VERSION,
7517                    7,
7518                    Some(vec![
7519                        "future.synthetic".to_string(),
7520                        "route.bind".to_string(),
7521                    ]),
7522                ),
7523            )
7524            .unwrap();
7525        let registration = registry.get_module("aft").unwrap().unwrap();
7526        assert_eq!(
7527            registration.control_ops,
7528            vec![
7529                "route.bind".to_string(),
7530                "route.status".to_string(),
7531                "future.synthetic".to_string(),
7532            ]
7533        );
7534        let frame =
7535            Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
7536        assert!(handler
7537            .guard_module_control_op(&frame, "aft", "future.synthetic")
7538            .unwrap()
7539            .is_none());
7540    }
7541
7542    #[tokio::test]
7543    async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
7544        let registry = Arc::new(Registry::default());
7545        let forwarding = Arc::new(ForwardingTable::default());
7546        let handler =
7547            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
7548        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
7549        hello_via_sink(
7550            &handler,
7551            &module_ctx,
7552            &mut module_rx,
7553            hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
7554        )
7555        .await;
7556
7557        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
7558        let responses = handler
7559            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
7560            .await
7561            .unwrap();
7562        assert_eq!(responses.len(), 1);
7563        assert_eq!(responses[0].header.ty, FrameType::Error);
7564        assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
7565        assert!(module_rx.try_recv().is_err());
7566    }
7567
7568    #[tokio::test]
7569    async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
7570        let registry = Arc::new(Registry::default());
7571        let forwarding = Arc::new(ForwardingTable::default());
7572        let handler =
7573            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
7574        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
7575        hello_via_sink(
7576            &handler,
7577            &module_ctx,
7578            &mut module_rx,
7579            hello_frame_with_control_ops(
7580                "aft",
7581                PROTOCOL_VERSION,
7582                7,
7583                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
7584            ),
7585        )
7586        .await;
7587
7588        let project_root = unique_project_root("demux");
7589        let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
7590        let route_handler = handler.clone();
7591        let route_task = tokio::spawn(async move {
7592            route_handler
7593                .handle_control_frame(
7594                    &route_client_ctx,
7595                    route_open_frame(100, "aft", project_root),
7596                )
7597                .await
7598                .unwrap()
7599        });
7600        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7601            .await
7602            .unwrap()
7603            .unwrap();
7604        assert!(matches!(
7605            serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
7606            ModuleControlRequest::RouteBind { .. }
7607        ));
7608
7609        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
7610        let health_handler = handler.clone();
7611        let health_task = tokio::spawn(async move {
7612            health_handler
7613                .handle_control_frame(
7614                    &health_client_ctx,
7615                    supervisor_health_probe_frame(101, "aft"),
7616                )
7617                .await
7618                .unwrap()
7619        });
7620        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7621            .await
7622            .unwrap()
7623            .unwrap();
7624        assert_eq!(
7625            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
7626            ModuleControlRequest::HealthCheck {}
7627        );
7628
7629        handler
7630            .handle_control_frame(
7631                &module_ctx,
7632                health_response(health_frame.header.corr, HealthStatus::Degraded),
7633            )
7634            .await
7635            .unwrap();
7636        let health_response = health_task.await.unwrap();
7637        assert_eq!(health_response.len(), 1);
7638        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
7639            ClientControlResponse::SupervisorHealthProbe {
7640                module_id,
7641                status,
7642                detail,
7643                metrics,
7644            } => {
7645                assert_eq!(module_id, "aft");
7646                assert_eq!(status, HealthStatus::Degraded);
7647                assert_eq!(detail.as_deref(), Some("warming"));
7648                assert_eq!(metrics, Some(json!({"queue_depth": 3})));
7649            }
7650            other => panic!("unexpected health response: {other:?}"),
7651        }
7652
7653        handler
7654            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
7655            .await
7656            .unwrap();
7657        let route_response = route_task.await.unwrap();
7658        assert!(route_response.is_empty());
7659        let published = route_client_rx.recv().await.unwrap();
7660        assert!(matches!(
7661            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
7662            ClientControlResponse::RouteOpen { .. }
7663        ));
7664    }
7665
7666    /// Start one `route.open` on `client_connection` and return its still-running
7667    /// handler task together with the `route.bind` the module received for it.
7668    /// The handler blocks until the module answers, so it has to run as a task
7669    /// while the test drives the module side.
7670    async fn relay_route_open(
7671        handler: &ControlHandler,
7672        client_connection: ConnectionId,
7673        client_egress: &FrameSink,
7674        module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7675        corr: u64,
7676        module_id: &str,
7677        project_root_label: &str,
7678    ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
7679        let ctx = RouteCtx {
7680            connection_id: client_connection,
7681            egress: client_egress.clone(),
7682        };
7683        let handler = handler.clone();
7684        let project_root = unique_project_root(project_root_label);
7685        let module_id = module_id.to_string();
7686        let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
7687        let task = tokio::spawn(async move {
7688            let _guard = tracing::dispatcher::set_default(&dispatch);
7689            handler
7690                .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
7691                .await
7692                .unwrap()
7693        });
7694        let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
7695            .await
7696            .expect("module receives the relayed route.bind")
7697            .expect("module egress is open");
7698        (task, bind.frame)
7699    }
7700
7701    fn route_bind_channel(frame: &Frame) -> (u16, u32) {
7702        match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
7703            ModuleControlRequest::RouteBind {
7704                route_channel,
7705                epoch,
7706                ..
7707            } => (route_channel, epoch),
7708            other => panic!("expected a route.bind request, got {other:?}"),
7709        }
7710    }
7711
7712    fn published_route(frame: &Frame) -> (u16, u32) {
7713        match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
7714            ClientControlResponse::RouteOpen {
7715                route_channel,
7716                route_epoch,
7717            } => (route_channel, route_epoch),
7718            other => panic!("expected a route.open response, got {other:?}"),
7719        }
7720    }
7721
7722    /// Reproduction of a production outage. A client had `route.open`s in
7723    /// flight to a module and was already marked closing -- its egress had refused a
7724    /// module frame, so the daemon asked its connection to end -- while its sink
7725    /// was still open. When the module acked those binds, the daemon refused to
7726    /// commit a route for a closing client, and that refusal was returned from
7727    /// the MODULE connection's frame handler, where a router error that has no
7728    /// ERROR-frame translation ends the connection. The module saw EOF, exited 0,
7729    /// the supervisor correctly did not respawn a clean exit, and every seat lost
7730    /// its tools for hours -- one client's teardown took down a connection
7731    /// carrying ~170 other routes.
7732    ///
7733    /// The window is opened here by calling the production path that opens it
7734    /// (`escalate_client_delivery_failure`) rather than by closing a socket. The
7735    /// state that matters is "in `closing_connections`, sink still open, relay
7736    /// still pending", and it lasts only from the close request until the
7737    /// connection loop reacts to it; a socket-level test can flood a client into
7738    /// that escalation but cannot pin the module's ack inside the window. Closing
7739    /// the socket instead takes the other path entirely -- connection teardown
7740    /// removes the pending relay under the same lock, so the ack finds nothing.
7741    #[tokio::test]
7742    async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
7743        let registry = Arc::new(Registry::default());
7744        let forwarding = Arc::new(ForwardingTable::default());
7745        let handler =
7746            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
7747
7748        let module_connection = ConnectionId::new(30);
7749        let (module_ctx, mut module_rx) = route_ctx(module_connection);
7750        hello_via_sink(
7751            &handler,
7752            &module_ctx,
7753            &mut module_rx,
7754            hello_frame("aft", PROTOCOL_VERSION, 7),
7755        )
7756        .await;
7757
7758        let dying_client = ConnectionId::new(31);
7759        let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
7760
7761        // A published route on the dying client. The escalation below only marks
7762        // a connection closing for a route it has already published.
7763        let (first_task, first_bind) = relay_route_open(
7764            &handler,
7765            dying_client,
7766            &dying_ctx.egress,
7767            &mut module_rx,
7768            100,
7769            "aft",
7770            "closing-first",
7771        )
7772        .await;
7773        handler
7774            .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
7775            .await
7776            .unwrap();
7777        assert!(first_task.await.unwrap().is_empty());
7778        let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
7779
7780        // A second route.open from the same client, relayed and awaiting its ack.
7781        let (second_task, second_bind) = relay_route_open(
7782            &handler,
7783            dying_client,
7784            &dying_ctx.egress,
7785            &mut module_rx,
7786            101,
7787            "aft",
7788            "closing-second",
7789        )
7790        .await;
7791        let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
7792
7793        // The window: the client is closing, its sink is still open, and its
7794        // second bind is still pending.
7795        assert!(forwarding
7796            .escalate_client_delivery_failure(
7797                dying_client,
7798                first_channel,
7799                first_epoch,
7800                CloseReason::new(
7801                    "module_to_client_delivery_failed",
7802                    "client egress refused a module frame",
7803                ),
7804                crate::forwarding::UndeliveredFrame {
7805                    module_id: None,
7806                    sink: &dying_ctx.egress,
7807                },
7808            )
7809            .unwrap());
7810        assert!(!dying_ctx.egress.is_closed());
7811
7812        // The frame that used to end the module connection.
7813        let ack = handler
7814            .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
7815            .await;
7816        let module_loop_error = ack.as_ref().err().map(ToString::to_string);
7817        if module_loop_error.is_some() {
7818            // What the server's connection loop does with a router error that has
7819            // no ERROR-frame translation: end the connection, which releases the
7820            // module's registration and every route on it.
7821            handler.cleanup_connection(module_connection).unwrap();
7822        }
7823        // Read the module's next frame before opening the co-tenant's route, so
7824        // the GOODBYE assertion below is about THIS ack and not about later
7825        // traffic. `None` means the module was told nothing.
7826        let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7827            .await
7828            .ok()
7829            .flatten();
7830
7831        // 1. The module connection is still registered.
7832        assert!(
7833            registry
7834                .get_module_by_connection(module_connection)
7835                .unwrap()
7836                .is_some(),
7837            "one client's closing connection ended the shared module connection: \
7838             {module_loop_error:?}"
7839        );
7840        // ...and still serving: another client can open and use a route on it.
7841        let cotenant = ConnectionId::new(32);
7842        let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
7843        let (cotenant_task, cotenant_bind) = relay_route_open(
7844            &handler,
7845            cotenant,
7846            &cotenant_ctx.egress,
7847            &mut module_rx,
7848            102,
7849            "aft",
7850            "closing-cotenant",
7851        )
7852        .await;
7853        handler
7854            .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
7855            .await
7856            .unwrap();
7857        assert!(cotenant_task.await.unwrap().is_empty());
7858        let (cotenant_channel, cotenant_epoch) =
7859            published_route(&cotenant_rx.recv().await.unwrap());
7860        assert!(matches!(
7861            forwarding
7862                .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
7863                .unwrap(),
7864            DataRoute::Client(DataRouteState::Bound(_))
7865        ));
7866
7867        // 2. The module was told to drop the binding it created for the route
7868        //    that will never be published.
7869        let goodbye = post_ack_module_frame
7870            .expect("module receives a GOODBYE for the abandoned route channel");
7871        assert_eq!(goodbye.header.ty, FrameType::Goodbye);
7872        assert_eq!(goodbye.header.channel, abandoned_channel);
7873        assert_eq!(goodbye.header.epoch, abandoned_epoch);
7874
7875        // 3. The dying client received nothing: no route was ever published to
7876        //    it. Its route.open is answered as unavailable, which the connection
7877        //    loop would write to a socket that is already going away.
7878        assert!(dying_rx.try_recv().is_err());
7879        let second_response = second_task.await.unwrap();
7880        assert_eq!(second_response.len(), 1);
7881        assert_eq!(
7882            parse_error(&second_response[0])["code"],
7883            "target_unavailable"
7884        );
7885    }
7886
7887    /// The fence at the module-loop boundary, stated as its own contract: which
7888    /// forwarding failures are allowed to end the module connection that is being
7889    /// served. A `ConnectionClosing` naming some client is about that client, and
7890    /// a module connection is shared; the same error naming the module's own
7891    /// connection is about this connection and must stay fatal, as must failures
7892    /// that are about the forwarding table itself.
7893    #[test]
7894    fn only_the_modules_own_closing_connection_ends_the_module_loop() {
7895        let handler = ControlHandler::default();
7896        let module_connection = ConnectionId::new(30);
7897        let client_connection = ConnectionId::new(31);
7898
7899        handler
7900            .refuse_to_end_module_connection_for_a_client(
7901                module_connection,
7902                77,
7903                ForwardingError::ConnectionClosing {
7904                    connection_id: client_connection,
7905                },
7906            )
7907            .expect("a closing client must never end the module connection");
7908
7909        assert!(matches!(
7910            handler.refuse_to_end_module_connection_for_a_client(
7911                module_connection,
7912                78,
7913                ForwardingError::ConnectionClosing {
7914                    connection_id: module_connection,
7915                },
7916            ),
7917            Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
7918                connection_id
7919            })) if connection_id == module_connection
7920        ));
7921        assert!(matches!(
7922            handler.refuse_to_end_module_connection_for_a_client(
7923                module_connection,
7924                79,
7925                ForwardingError::Poisoned,
7926            ),
7927            Err(RouterError::Forwarding(ForwardingError::Poisoned))
7928        ));
7929        assert!(matches!(
7930            handler.refuse_to_end_module_connection_for_a_client(
7931                module_connection,
7932                80,
7933                ForwardingError::StaleModuleEndpoint,
7934            ),
7935            Err(RouterError::Forwarding(
7936                ForwardingError::StaleModuleEndpoint
7937            ))
7938        ));
7939    }
7940
7941    /// The spawn-attestation guard is what stops a connected module from claiming
7942    /// another module's identity and being stamped `Reserved` for it. Every other
7943    /// test that supplies a consumer_identity supplies a CORRECT one, because a
7944    /// correct one is what the rest of the flow needs -- so the guard's rejection
7945    /// branch was never the subject of an assertion, only its acceptance branch.
7946    ///
7947    /// Deleting the guard's EFFECT (granting Reserved unconditionally) leaves the
7948    /// whole subc-core library suite green; only the forwarding integration tests
7949    /// notice, and they notice for unrelated reasons. This test exists so the
7950    /// refusal itself is asserted where the guard lives: it fails if the identity
7951    /// check stops refusing, which is the direction that matters, since a guard
7952    /// that wrongly ACCEPTS is silent while one that wrongly REJECTS is loud.
7953    #[tokio::test]
7954    async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
7955        let registry = Arc::new(Registry::default());
7956        let forwarding = Arc::new(ForwardingTable::default());
7957        let supervisor = SupervisorHandle::new();
7958        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7959        let handler =
7960            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
7961                .with_supervisor(supervisor);
7962
7963        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
7964        hello_via_sink(
7965            &handler,
7966            &target_ctx,
7967            &mut target_rx,
7968            hello_frame("target", PROTOCOL_VERSION, 1),
7969        )
7970        .await;
7971
7972        // A real supervised module id presenting the wrong nonce. This is the
7973        // impersonation case: the attacker knows a privileged module_id, which is
7974        // public, and guesses at the nonce, which is not.
7975        let wrong_nonce = handler
7976            .handle_control_frame(
7977                &route_ctx(ConnectionId::new(91)).0,
7978                route_open_frame_with_admission_facts(
7979                    20,
7980                    "target",
7981                    unique_project_root("admission-facts"),
7982                    Some(subc_control::ConsumerIdentity {
7983                        module_id: "fed".to_string(),
7984                        launch_nonce: "not-the-real-nonce".to_string(),
7985                    }),
7986                    None,
7987                ),
7988            )
7989            .await
7990            .unwrap();
7991        assert_eq!(
7992            parse_error(&wrong_nonce[0])["code"],
7993            "bad_consumer_identity",
7994            "a mismatched launch nonce must be refused, not stamped Reserved"
7995        );
7996
7997        // A module id the supervisor never spawned at all, so no nonce exists to
7998        // compare against. An implementation that treats "no record" as "nothing
7999        // to check" fails open here while passing the case above.
8000        let never_spawned = handler
8001            .handle_control_frame(
8002                &route_ctx(ConnectionId::new(92)).0,
8003                route_open_frame_with_admission_facts(
8004                    21,
8005                    "target",
8006                    unique_project_root("admission-facts"),
8007                    Some(subc_control::ConsumerIdentity {
8008                        module_id: "never-spawned".to_string(),
8009                        launch_nonce: "any-nonce".to_string(),
8010                    }),
8011                    None,
8012                ),
8013            )
8014            .await
8015            .unwrap();
8016        assert_eq!(
8017            parse_error(&never_spawned[0])["code"],
8018            "bad_consumer_identity",
8019            "an unspawned module_id must be refused rather than accepted for lack of a record"
8020        );
8021    }
8022
8023    /// The refusal test above proves the guard says NO. Nothing proved it can say
8024    /// YES, and the difference is not academic: replacing the whole authorization
8025    /// with `false` -- admitting no consumer identity at all, revoking Reserved
8026    /// standing for every supervised module in the fleet -- leaves 110 of the 111
8027    /// library tests GREEN. The one that notices does so by HANGING, because it
8028    /// waits for a bind that can no longer happen.
8029    ///
8030    /// A hang is the weakest signal a suite can produce. In CI it reads as a slow
8031    /// or flaky test, invites a RETRY rather than an investigation, and the retry
8032    /// hangs too and gets blamed on the runner. So a total revocation of the
8033    /// daemon's trust grant would have shipped behind a symptom nobody attributes
8034    /// to code.
8035    ///
8036    /// The bias is structural rather than accidental. A REFUSAL looks like a
8037    /// failure someone writes a test for; a GRANT looks like the happy path. Every
8038    /// binary-outcome guard whose STRICTNESS is the point acquires a refusal-heavy
8039    /// suite for that reason, and this one is the purest case in the daemon.
8040    ///
8041    /// This test asserts the EFFECT rather than the absence of an error: the module
8042    /// receives a RouteBind and it carries `Reserved` naming the attested module.
8043    /// A guard that admitted nobody would produce no bind at all; one that admitted
8044    /// everybody would stamp the wrong principal, which the refusal test catches.
8045    #[tokio::test]
8046    async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
8047        let registry = Arc::new(Registry::default());
8048        let forwarding = Arc::new(ForwardingTable::default());
8049        let supervisor = SupervisorHandle::new();
8050        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8051        let handler =
8052            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8053                .with_supervisor(supervisor);
8054
8055        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
8056        hello_via_sink(
8057            &handler,
8058            &target_ctx,
8059            &mut target_rx,
8060            hello_frame("target", PROTOCOL_VERSION, 1),
8061        )
8062        .await;
8063
8064        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
8065        let route_handler = handler.clone();
8066        let route_task = tokio::spawn(async move {
8067            route_handler
8068                .handle_control_frame(
8069                    &client_ctx,
8070                    route_open_frame_with_admission_facts(
8071                        30,
8072                        "target",
8073                        unique_project_root("admission-facts"),
8074                        Some(subc_control::ConsumerIdentity {
8075                            module_id: "fed".to_string(),
8076                            launch_nonce: "fed-nonce".to_string(),
8077                        }),
8078                        None,
8079                    ),
8080                )
8081                .await
8082                .unwrap()
8083        });
8084
8085        // BOUND THE WAIT. The first version of this test recv'd unbounded, and under
8086        // the very mutation it exists to catch -- a guard that admits nobody -- no
8087        // bind is ever sent, so it HUNG rather than failing. That reproduces the
8088        // exact defect being fixed: a total revocation detected only as a stalled
8089        // suite, which reads as flakiness and invites a retry. An acceptance test
8090        // that waits for an effect must bound the wait, or a red becomes a hang.
8091        let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8092            .await
8093            .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
8094            .expect("module control channel closed before route.bind");
8095        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8096        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
8097            panic!("expected route.bind")
8098        };
8099        assert_eq!(
8100            principal,
8101            Some(Principal::Reserved {
8102                module_id: "fed".to_string()
8103            }),
8104            "a correctly attested consumer must be stamped Reserved for its own id"
8105        );
8106
8107        handler
8108            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8109            .await
8110            .unwrap();
8111        assert!(route_task.await.unwrap().is_empty());
8112        assert!(
8113            matches!(
8114                serde_json::from_slice::<ClientControlResponse>(
8115                    &client_rx.recv().await.unwrap().body
8116                )
8117                .unwrap(),
8118                ClientControlResponse::RouteOpen { .. }
8119            ),
8120            "the route must actually open, not merely avoid an error"
8121        );
8122    }
8123
8124    #[tokio::test(start_paused = true)]
8125    async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
8126        let registry = Arc::new(Registry::default());
8127        let forwarding = Arc::new(ForwardingTable::default());
8128        let supervisor = SupervisorHandle::new();
8129        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8130        let handler =
8131            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8132                .with_supervisor(supervisor);
8133
8134        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
8135        hello_via_sink(
8136            &handler,
8137            &target_ctx,
8138            &mut target_rx,
8139            hello_frame("target", PROTOCOL_VERSION, 1),
8140        )
8141        .await;
8142
8143        let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
8144        let direct_handler = handler.clone();
8145        let direct_open = tokio::spawn(async move {
8146            direct_handler
8147                .handle_control_frame(
8148                    &direct_ctx,
8149                    route_open_frame(2, "target", unique_project_root("route-census-direct")),
8150                )
8151                .await
8152                .unwrap()
8153        });
8154        let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8155            .await
8156            .expect("no direct route.bind within 5s")
8157            .expect("target control channel closed before direct route.bind");
8158        handler
8159            .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
8160            .await
8161            .unwrap();
8162        assert!(direct_open.await.unwrap().is_empty());
8163        let _ = direct_rx.recv().await.unwrap();
8164
8165        let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
8166        let reserved_handler = handler.clone();
8167        let reserved_open = tokio::spawn(async move {
8168            reserved_handler
8169                .handle_control_frame(
8170                    &reserved_ctx,
8171                    route_open_frame_with_admission_facts(
8172                        3,
8173                        "target",
8174                        unique_project_root("admission-facts"),
8175                        Some(ConsumerIdentity {
8176                            module_id: "fed".to_string(),
8177                            launch_nonce: "fed-nonce".to_string(),
8178                        }),
8179                        None,
8180                    ),
8181                )
8182                .await
8183                .unwrap()
8184        });
8185        let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8186            .await
8187            .expect("no reserved route.bind within 5s")
8188            .expect("target control channel closed before reserved route.bind");
8189        handler
8190            .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
8191            .await
8192            .unwrap();
8193        assert!(reserved_open.await.unwrap().is_empty());
8194        let _ = reserved_rx.recv().await.unwrap();
8195
8196        forwarding
8197            .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
8198            .unwrap();
8199        let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
8200        let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
8201            module_id: Some("target".to_string()),
8202        })
8203        .unwrap();
8204        let census_frame =
8205            Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
8206        let response = handler
8207            .handle_control_frame(&census_ctx, census_frame)
8208            .await
8209            .unwrap()
8210            .pop()
8211            .unwrap();
8212        let actual: Value = serde_json::from_slice(&response.body).unwrap();
8213        let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
8214        assert!(matches!(
8215            decoded,
8216            ClientControlResponse::SupervisorRoutes { .. }
8217        ));
8218        let routes = actual["modules"][0]["routes"].as_array().unwrap();
8219        assert_eq!(routes.len(), 2);
8220        assert!(routes.iter().all(|route| route["draining"] == true));
8221        // The census carries WHY: the reason the drain was begun with, in the
8222        // route.closing vocabulary, on every draining route this drain marked.
8223        assert!(
8224            routes.iter().all(|route| route["drain_reason"] == "reload"),
8225            "draining routes must name the drain's reason: {routes:?}"
8226        );
8227        assert!(routes.iter().any(|route| {
8228            route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
8229        }));
8230        assert!(routes.iter().any(|route| {
8231            route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
8232        }));
8233
8234        let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8235            .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
8236        if std::env::var_os("UPDATE_GOLDEN").is_some() {
8237            std::fs::write(
8238                &golden_path,
8239                format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
8240            )
8241            .unwrap();
8242        }
8243        let expected: Value =
8244            serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
8245        assert_eq!(actual, expected);
8246    }
8247
8248    async fn query_live_roots(
8249        handler: &ControlHandler,
8250        module_ctx: &RouteCtx,
8251    ) -> ModuleControlResponseToModule {
8252        let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
8253        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
8254        let response = handler
8255            .handle_control_frame(module_ctx, frame)
8256            .await
8257            .unwrap()
8258            .pop()
8259            .unwrap();
8260        serde_json::from_slice(&response.body).unwrap()
8261    }
8262
8263    #[tokio::test(start_paused = true)]
8264    async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
8265        let registry = Arc::new(Registry::default());
8266        let forwarding = Arc::new(ForwardingTable::default());
8267        let handler = ControlHandler::with_forwarding(registry, forwarding);
8268        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
8269        hello_via_sink(
8270            &handler,
8271            &target_ctx,
8272            &mut target_rx,
8273            hello_frame("target", PROTOCOL_VERSION, 1),
8274        )
8275        .await;
8276        let root = unique_project_root("live-roots-known");
8277        let path = ProjectRootId::from_path_allowing_missing(root.path())
8278            .unwrap()
8279            .as_path()
8280            .to_path_buf();
8281        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
8282        let open_handler = handler.clone();
8283        let opened = tokio::spawn(async move {
8284            open_handler
8285                .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
8286                .await
8287                .unwrap()
8288        });
8289        let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8290            .await
8291            .unwrap()
8292            .unwrap();
8293        handler
8294            .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
8295            .await
8296            .unwrap();
8297        assert!(opened.await.unwrap().is_empty());
8298        let _ = client_rx.recv().await.unwrap();
8299
8300        let root = unique_project_root("live-roots-pending");
8301        let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
8302            .unwrap()
8303            .as_path()
8304            .to_path_buf();
8305        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
8306        let open_handler = handler.clone();
8307        let pending = tokio::spawn(async move {
8308            open_handler
8309                .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
8310                .await
8311                .unwrap()
8312        });
8313        let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8314            .await
8315            .unwrap()
8316            .unwrap();
8317        let actual = query_live_roots(&handler, &target_ctx).await;
8318        let ModuleControlResponseToModule::LiveRoots {
8319            roots,
8320            unknown_root_bindings,
8321            total_bindings,
8322        } = actual
8323        else {
8324            panic!("expected live roots")
8325        };
8326        assert_eq!(total_bindings, 2, "root-known arm must count live routes");
8327        assert_eq!(unknown_root_bindings, 0);
8328        assert_eq!(
8329            roots.len(),
8330            2,
8331            "root-known arm must retain each canonical root"
8332        );
8333        assert_eq!(
8334            total_bindings,
8335            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8336        );
8337        let counts = roots
8338            .iter()
8339            .map(|root| (root.project_root.clone(), root.bound, root.pending))
8340            .collect::<Vec<_>>();
8341        let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
8342        expected.sort_by(|a, b| a.0.cmp(&b.0));
8343        assert_eq!(
8344            counts, expected,
8345            "roots must sort by path and count pending separately"
8346        );
8347        handler
8348            .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
8349            .await
8350            .unwrap();
8351        assert!(pending.await.unwrap().is_empty());
8352    }
8353
8354    #[tokio::test(start_paused = true)]
8355    async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
8356        let forwarding = Arc::new(ForwardingTable::default());
8357        let handler =
8358            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
8359        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
8360        hello_via_sink(
8361            &handler,
8362            &target_ctx,
8363            &mut target_rx,
8364            hello_frame("target", PROTOCOL_VERSION, 1),
8365        )
8366        .await;
8367        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
8368        let pending = forwarding
8369            .begin_route_bind_relay_for_test(
8370                client_ctx.connection_id,
8371                client_ctx.egress.clone(),
8372                2,
8373                "target",
8374            )
8375            .unwrap();
8376        forwarding
8377            .complete_pending_relay(
8378                target_ctx.connection_id,
8379                pending.corr,
8380                RouteBindRelayOutcome::Accepted,
8381            )
8382            .unwrap();
8383        let actual = query_live_roots(&handler, &target_ctx).await;
8384        let ModuleControlResponseToModule::LiveRoots {
8385            roots,
8386            unknown_root_bindings,
8387            total_bindings,
8388        } = actual
8389        else {
8390            panic!("expected live roots")
8391        };
8392        assert!(roots.is_empty(), "unknown-root arm must not invent a root");
8393        assert_eq!(
8394            unknown_root_bindings, 1,
8395            "unknown-root arm must not read as no bindings"
8396        );
8397        assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
8398        assert_eq!(
8399            total_bindings,
8400            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8401        );
8402    }
8403
8404    /// A module reads HELLO_ACK as its first frame and exits on anything else,
8405    /// so the ack has to be on its outbound queue before the module is
8406    /// routable. The connection loop writes a handler's replies only after the
8407    /// handler returns; this test stops in exactly that gap, runs a real
8408    /// route.open from another connection, and only then writes whatever the
8409    /// HELLO handler returned, the way the loop would. If the ack were still a
8410    /// reply, the route.bind request would reach the module first.
8411    #[tokio::test(start_paused = true)]
8412    async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
8413        let forwarding = Arc::new(ForwardingTable::default());
8414        let handler =
8415            ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
8416        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
8417        let replies = handler
8418            .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
8419            .await
8420            .unwrap();
8421        let queued_by_hello = module_rx.len();
8422
8423        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
8424        let open_handler = handler.clone();
8425        let open = tokio::spawn(async move {
8426            open_handler
8427                .handle_control_frame(
8428                    &client_ctx,
8429                    route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
8430                )
8431                .await
8432                .unwrap()
8433        });
8434        // Let the route.open run until its route.bind is on the module's queue.
8435        let mut spins = 0;
8436        while module_rx.len() == queued_by_hello {
8437            spins += 1;
8438            assert!(spins < 10_000, "route.open never queued a route.bind");
8439            tokio::task::yield_now().await;
8440        }
8441
8442        // Now the connection loop's half: write the HELLO handler's replies.
8443        for reply in replies {
8444            module_ctx.egress.send(reply).await.unwrap();
8445        }
8446
8447        let first = module_rx.recv().await.unwrap().frame;
8448        assert_eq!(
8449            first.header.ty,
8450            FrameType::HelloAck,
8451            "the first frame a registering module reads must be its HELLO_ACK"
8452        );
8453        assert_eq!(first.header.corr, 7);
8454        let second = module_rx.recv().await.unwrap().frame;
8455        assert_eq!(second.header.ty, FrameType::Request);
8456        assert!(
8457            matches!(
8458                serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
8459                ModuleControlRequest::RouteBind { .. }
8460            ),
8461            "the route.bind follows the ack"
8462        );
8463        assert!(module_rx.try_recv().is_err(), "nothing else was queued");
8464
8465        handler
8466            .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
8467            .await
8468            .unwrap();
8469        assert!(open.await.unwrap().is_empty());
8470        let _ = client_rx.recv().await.unwrap();
8471    }
8472
8473    #[tokio::test(start_paused = true)]
8474    async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
8475        let handler = ControlHandler::with_forwarding(
8476            Arc::new(Registry::default()),
8477            Arc::new(ForwardingTable::default()),
8478        );
8479        let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
8480        let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
8481        hello_via_sink(
8482            &handler,
8483            &first_ctx,
8484            &mut first_rx,
8485            hello_frame("first", PROTOCOL_VERSION, 1),
8486        )
8487        .await;
8488        hello_via_sink(
8489            &handler,
8490            &second_ctx,
8491            &mut second_rx,
8492            hello_frame("second", PROTOCOL_VERSION, 2),
8493        )
8494        .await;
8495        let root = unique_project_root("second-only");
8496        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
8497        let cloned = handler.clone();
8498        let open = tokio::spawn(async move {
8499            cloned
8500                .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
8501                .await
8502                .unwrap()
8503        });
8504        let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
8505            .await
8506            .unwrap()
8507            .unwrap();
8508        let first = query_live_roots(&handler, &first_ctx).await;
8509        let second = query_live_roots(&handler, &second_ctx).await;
8510        assert!(
8511            matches!(
8512                first,
8513                ModuleControlResponseToModule::LiveRoots {
8514                    total_bindings: 0,
8515                    ..
8516                }
8517            ),
8518            "cross-module scope must not expose another module's roots"
8519        );
8520        assert!(
8521            matches!(
8522                second,
8523                ModuleControlResponseToModule::LiveRoots {
8524                    total_bindings: 1,
8525                    ..
8526                }
8527            ),
8528            "second module must see its pending route"
8529        );
8530        handler
8531            .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
8532            .await
8533            .unwrap();
8534        assert!(open.await.unwrap().is_empty());
8535    }
8536
8537    #[tokio::test(start_paused = true)]
8538    async fn supervisor_live_roots_no_bindings_arm_is_empty() {
8539        let handler = ControlHandler::with_forwarding(
8540            Arc::new(Registry::default()),
8541            Arc::new(ForwardingTable::default()),
8542        );
8543        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
8544        hello_via_sink(
8545            &handler,
8546            &target_ctx,
8547            &mut target_rx,
8548            hello_frame("target", PROTOCOL_VERSION, 1),
8549        )
8550        .await;
8551        let actual = query_live_roots(&handler, &target_ctx).await;
8552        let ModuleControlResponseToModule::LiveRoots {
8553            roots,
8554            unknown_root_bindings,
8555            total_bindings,
8556        } = actual
8557        else {
8558            panic!("expected live roots")
8559        };
8560        assert!(roots.is_empty());
8561        assert_eq!(unknown_root_bindings, 0);
8562        assert_eq!(total_bindings, 0);
8563        assert_eq!(
8564            total_bindings,
8565            roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8566        );
8567    }
8568
8569    /// Read the vendored fed corpus rather than hand-building a package.
8570    ///
8571    /// A hand-built object encodes what the test author believed the carrier
8572    /// emits. These vectors are what it actually emits, and one of them exists
8573    /// specifically to pin OUR side of the seam: its note reads "SUBC relay
8574    /// ignores additive unknown fields at the traversal emit terminus."
8575    fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
8576        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8577            .join("tests/fixtures/fed/admission-facts-emit.jsonl");
8578        let text = std::fs::read_to_string(&path)
8579            .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
8580        let vectors: Vec<(String, Value)> = text
8581            .lines()
8582            .filter(|line| !line.trim().is_empty())
8583            .map(|line| {
8584                let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
8585                let id = entry["corpus_id"]
8586                    .as_str()
8587                    .expect("every vector carries a corpus_id")
8588                    .to_string();
8589                (id, entry["package"].clone())
8590            })
8591            .collect();
8592        // Pin the count: a corpus that silently shrinks would take its coverage
8593        // with it, and a suite reading N-1 vectors reports the same clean pass
8594        // as one reading N.
8595        assert_eq!(
8596            vectors.len(),
8597            3,
8598            "vendored fed corpus changed size; re-sync from subc-federation"
8599        );
8600
8601        // Pin what makes the corpus DISCRIMINATING, not just present.
8602        //
8603        // The relay test below takes its expected value from the corpus, so the
8604        // corpus supplies the test's power to detect a lossy relay rather than
8605        // its correctness. A relay that dropped unrecognised fields would still
8606        // be caught -- but only by a package carrying fields it does not know.
8607        // Shrink every package to the handful of keys any implementation would
8608        // recognise and the test keeps passing over an input that can no longer
8609        // fail, which is the same clean green as a corpus that shrank away.
8610        //
8611        // So assert the precondition rather than duplicating the packages here:
8612        // at least one vector must carry a field beyond the small common set.
8613        // That is one claim to maintain instead of nine, and it fails loudly if
8614        // a re-sync ever flattens the corpus.
8615        const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
8616        let richest = vectors
8617            .iter()
8618            .filter_map(|(_, package)| package.as_object())
8619            .map(|object| {
8620                object
8621                    .keys()
8622                    .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
8623                    .count()
8624            })
8625            .max()
8626            .unwrap_or(0);
8627        assert!(
8628            richest >= 2,
8629            "vendored corpus no longer carries a package with unmodelled fields, \
8630             so the relay test can no longer distinguish a verbatim relay from a lossy one"
8631        );
8632
8633        vectors
8634    }
8635
8636    /// The relay must carry the carrier's package through BYTE-FOR-BYTE.
8637    ///
8638    /// The gate test below proves the ACCESS RULE (who may send facts, to whom).
8639    /// This proves the PAYLOAD RULE, which the gate cannot: it hand-builds a
8640    /// three-key object, so a relay that quietly dropped fields it did not
8641    /// recognise would satisfy it. These vectors carry nine keys including ones
8642    /// this crate has no type for, so a typed relay fails here and only here.
8643    #[tokio::test]
8644    async fn admission_facts_relay_carries_vendored_packages_verbatim() {
8645        for (corpus_id, package) in fed_admission_facts_vectors() {
8646            let registry = Arc::new(Registry::default());
8647            let forwarding = Arc::new(ForwardingTable::default());
8648            let supervisor = SupervisorHandle::new();
8649            supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8650            let handler =
8651                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8652                    .with_supervisor(supervisor)
8653                    .with_admission_facts_config(
8654                        Some("fed".to_string()),
8655                        Some(vec!["target".to_string()]),
8656                    );
8657
8658            let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
8659            hello_via_sink(
8660                &handler,
8661                &target_ctx,
8662                &mut target_rx,
8663                hello_frame("target", PROTOCOL_VERSION, 1),
8664            )
8665            .await;
8666
8667            let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
8668            let route_handler = handler.clone();
8669            let expected = package.clone();
8670            let route_task = tokio::spawn(async move {
8671                route_handler
8672                    .handle_control_frame(
8673                        &client_ctx,
8674                        route_open_frame_with_admission_facts(
8675                            20,
8676                            "target",
8677                            unique_project_root("admission-facts"),
8678                            Some(subc_control::ConsumerIdentity {
8679                                module_id: "fed".to_string(),
8680                                launch_nonce: "fed-nonce".to_string(),
8681                            }),
8682                            Some(package),
8683                        ),
8684                    )
8685                    .await
8686                    .unwrap()
8687            });
8688
8689            let bind_frame = target_rx.recv().await.unwrap();
8690            let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8691            let ModuleControlRequest::RouteBind {
8692                admission_facts, ..
8693            } = bind
8694            else {
8695                panic!("{corpus_id}: expected route.bind")
8696            };
8697            assert_eq!(
8698                admission_facts,
8699                Some(expected),
8700                "{corpus_id}: relay must not add, drop or reshape any field"
8701            );
8702
8703            handler
8704                .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8705                .await
8706                .unwrap();
8707            route_task.await.unwrap();
8708        }
8709    }
8710
8711    #[tokio::test]
8712    async fn admission_facts_gate_checks_carrier_target_and_precedence() {
8713        let registry = Arc::new(Registry::default());
8714        let forwarding = Arc::new(ForwardingTable::default());
8715        let supervisor = SupervisorHandle::new();
8716        supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8717        supervisor.set_spawn_nonce("other", "other-nonce".to_string());
8718        let handler =
8719            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
8720                .with_supervisor(supervisor)
8721                .with_admission_facts_config(
8722                    Some("fed".to_string()),
8723                    Some(vec!["target".to_string()]),
8724                );
8725
8726        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
8727        hello_via_sink(
8728            &handler,
8729            &target_ctx,
8730            &mut target_rx,
8731            hello_frame("target", PROTOCOL_VERSION, 1),
8732        )
8733        .await;
8734        let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
8735        hello_via_sink(
8736            &handler,
8737            &other_ctx,
8738            &mut other_rx,
8739            hello_frame("other", PROTOCOL_VERSION, 2),
8740        )
8741        .await;
8742
8743        let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
8744        let expected_facts = facts.clone();
8745        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
8746        let route_handler = handler.clone();
8747        let route_task = tokio::spawn(async move {
8748            route_handler
8749                .handle_control_frame(
8750                    &client_ctx,
8751                    route_open_frame_with_admission_facts(
8752                        10,
8753                        "target",
8754                        unique_project_root("admission-facts"),
8755                        Some(subc_control::ConsumerIdentity {
8756                            module_id: "fed".to_string(),
8757                            launch_nonce: "fed-nonce".to_string(),
8758                        }),
8759                        Some(facts.clone()),
8760                    ),
8761                )
8762                .await
8763                .unwrap()
8764        });
8765        let bind_frame = target_rx.recv().await.unwrap();
8766        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8767        let ModuleControlRequest::RouteBind {
8768            admission_facts, ..
8769        } = bind
8770        else {
8771            panic!("expected route.bind")
8772        };
8773        assert_eq!(admission_facts, Some(expected_facts));
8774        handler
8775            .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8776            .await
8777            .unwrap();
8778        assert!(route_task.await.unwrap().is_empty());
8779        assert!(matches!(
8780            serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
8781                .unwrap(),
8782            ClientControlResponse::RouteOpen { .. }
8783        ));
8784
8785        let direct = handler
8786            .handle_control_frame(
8787                &route_ctx(ConnectionId::new(73)).0,
8788                route_open_frame_with_admission_facts(
8789                    11,
8790                    "target",
8791                    unique_project_root("admission-facts"),
8792                    None,
8793                    Some(json!({"x": 1})),
8794                ),
8795            )
8796            .await
8797            .unwrap();
8798        assert_eq!(
8799            parse_error(&direct[0])["code"],
8800            "admission_facts_not_permitted"
8801        );
8802
8803        let different_reserved = handler
8804            .handle_control_frame(
8805                &route_ctx(ConnectionId::new(77)).0,
8806                route_open_frame_with_admission_facts(
8807                    15,
8808                    "target",
8809                    unique_project_root("admission-facts"),
8810                    Some(subc_control::ConsumerIdentity {
8811                        module_id: "other".to_string(),
8812                        launch_nonce: "other-nonce".to_string(),
8813                    }),
8814                    Some(json!({"x": 1})),
8815                ),
8816            )
8817            .await
8818            .unwrap();
8819        assert_eq!(
8820            parse_error(&different_reserved[0])["code"],
8821            "admission_facts_not_permitted"
8822        );
8823
8824        let other_target = handler
8825            .handle_control_frame(
8826                &route_ctx(ConnectionId::new(74)).0,
8827                route_open_frame_with_admission_facts(
8828                    12,
8829                    "other",
8830                    unique_project_root("admission-facts"),
8831                    Some(subc_control::ConsumerIdentity {
8832                        module_id: "fed".to_string(),
8833                        launch_nonce: "fed-nonce".to_string(),
8834                    }),
8835                    Some(json!({"x": 1})),
8836                ),
8837            )
8838            .await
8839            .unwrap();
8840        assert_eq!(
8841            parse_error(&other_target[0])["code"],
8842            "admission_facts_target_not_allowed"
8843        );
8844
8845        let nonexistent = handler
8846            .handle_control_frame(
8847                &route_ctx(ConnectionId::new(75)).0,
8848                route_open_frame_with_admission_facts(
8849                    13,
8850                    "missing",
8851                    unique_project_root("admission-facts"),
8852                    None,
8853                    Some(json!({"x": 1})),
8854                ),
8855            )
8856            .await
8857            .unwrap();
8858        assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
8859
8860        let described = handler
8861            .handle_control_frame(
8862                &route_ctx(ConnectionId::new(76)).0,
8863                Frame::build(
8864                    FrameType::Request,
8865                    control_flags(),
8866                    0,
8867                    0,
8868                    14,
8869                    serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8870                )
8871                .unwrap(),
8872            )
8873            .await
8874            .unwrap();
8875        let ClientControlResponse::ServerDescribe { capabilities, .. } =
8876            serde_json::from_slice(&described[0].body).unwrap()
8877        else {
8878            panic!("expected server.describe response")
8879        };
8880        assert!(capabilities
8881            .iter()
8882            .any(|cap| cap == "admission_facts_relay_v1"));
8883    }
8884
8885    #[tokio::test]
8886    async fn admission_facts_without_configured_carrier_are_rejected() {
8887        let registry = Arc::new(Registry::default());
8888        let forwarding = Arc::new(ForwardingTable::default());
8889        let handler = ControlHandler::with_forwarding(registry, forwarding);
8890        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
8891        hello_via_sink(
8892            &handler,
8893            &target_ctx,
8894            &mut target_rx,
8895            hello_frame("target", PROTOCOL_VERSION, 1),
8896        )
8897        .await;
8898
8899        let responses = handler
8900            .handle_control_frame(
8901                &route_ctx(ConnectionId::new(79)).0,
8902                route_open_frame_with_admission_facts(
8903                    16,
8904                    "target",
8905                    unique_project_root("admission-facts"),
8906                    None,
8907                    Some(json!({"x": 1})),
8908                ),
8909            )
8910            .await
8911            .unwrap();
8912        assert_eq!(
8913            parse_error(&responses[0])["code"],
8914            "admission_facts_not_permitted"
8915        );
8916    }
8917
8918    #[tokio::test]
8919    async fn route_open_relays_consumer_capabilities_verbatim() {
8920        let registry = Arc::new(Registry::default());
8921        let forwarding = Arc::new(ForwardingTable::default());
8922        let handler =
8923            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
8924        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
8925        hello_via_sink(
8926            &handler,
8927            &module_ctx,
8928            &mut module_rx,
8929            hello_frame("aft", PROTOCOL_VERSION, 7),
8930        )
8931        .await;
8932
8933        let expected = vec!["elicitation".to_string(), "roots".to_string()];
8934        let expected_for_request = expected.clone();
8935        let project_root = unique_project_root("consumer-capabilities-present");
8936        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
8937        let route_handler = handler.clone();
8938        let route_task = tokio::spawn(async move {
8939            route_handler
8940                .handle_control_frame(
8941                    &client_ctx,
8942                    route_open_frame_with_consumer_capabilities(
8943                        401,
8944                        "aft",
8945                        project_root,
8946                        Some(expected_for_request),
8947                    ),
8948                )
8949                .await
8950                .unwrap()
8951        });
8952        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8953            .await
8954            .unwrap()
8955            .unwrap();
8956        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8957        let ModuleControlRequest::RouteBind {
8958            consumer_capabilities,
8959            ..
8960        } = bind
8961        else {
8962            panic!("expected route.bind request, got {bind:?}");
8963        };
8964        assert_eq!(consumer_capabilities, Some(expected.clone()));
8965
8966        handler
8967            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8968            .await
8969            .unwrap();
8970        let route_response = route_task.await.unwrap();
8971        assert!(route_response.is_empty());
8972        let published = client_rx.recv().await.unwrap();
8973        assert!(matches!(
8974            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8975            ClientControlResponse::RouteOpen { .. }
8976        ));
8977    }
8978
8979    #[tokio::test]
8980    async fn route_open_without_consumer_capabilities_relays_none() {
8981        let registry = Arc::new(Registry::default());
8982        let forwarding = Arc::new(ForwardingTable::default());
8983        let handler =
8984            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
8985        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
8986        hello_via_sink(
8987            &handler,
8988            &module_ctx,
8989            &mut module_rx,
8990            hello_frame("aft", PROTOCOL_VERSION, 7),
8991        )
8992        .await;
8993
8994        let project_root = unique_project_root("consumer-capabilities-absent");
8995        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
8996        let route_handler = handler.clone();
8997        let route_task = tokio::spawn(async move {
8998            route_handler
8999                .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
9000                .await
9001                .unwrap()
9002        });
9003        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9004            .await
9005            .unwrap()
9006            .unwrap();
9007        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9008        let ModuleControlRequest::RouteBind {
9009            consumer_capabilities,
9010            ..
9011        } = bind
9012        else {
9013            panic!("expected route.bind request, got {bind:?}");
9014        };
9015        assert_eq!(consumer_capabilities, None);
9016
9017        handler
9018            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9019            .await
9020            .unwrap();
9021        let route_response = route_task.await.unwrap();
9022        assert!(route_response.is_empty());
9023        let published = client_rx.recv().await.unwrap();
9024        assert!(matches!(
9025            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9026            ClientControlResponse::RouteOpen { .. }
9027        ));
9028    }
9029
9030    /// Opens a route to a freshly registered `aft` with `sent` as the
9031    /// route.open's role_versions, acks the bind, and returns the role_versions
9032    /// the module's bind carried.
9033    async fn bind_role_versions_for(
9034        sent: Option<BTreeMap<String, String>>,
9035        connection: u64,
9036    ) -> Option<BTreeMap<String, String>> {
9037        let registry = Arc::new(Registry::default());
9038        let forwarding = Arc::new(ForwardingTable::default());
9039        let handler =
9040            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9041        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
9042        hello_via_sink(
9043            &handler,
9044            &module_ctx,
9045            &mut module_rx,
9046            hello_frame("aft", PROTOCOL_VERSION, 7),
9047        )
9048        .await;
9049        let project_root = unique_project_root("role-versions");
9050        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
9051        let route_handler = handler.clone();
9052        let route_task = tokio::spawn(async move {
9053            route_handler
9054                .handle_control_frame(
9055                    &client_ctx,
9056                    route_open_frame_with_role_versions(403, "aft", project_root, sent),
9057                )
9058                .await
9059                .unwrap()
9060        });
9061        let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9062            .await
9063            .expect("a well-formed route.open reaches the module as a bind")
9064            .unwrap();
9065        let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9066        let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
9067            panic!("expected route.bind request, got {bind:?}");
9068        };
9069        handler
9070            .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9071            .await
9072            .unwrap();
9073        assert!(route_task.await.unwrap().is_empty());
9074        let published = client_rx.recv().await.unwrap();
9075        assert!(matches!(
9076            serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9077            ClientControlResponse::RouteOpen { .. }
9078        ));
9079        role_versions
9080    }
9081
9082    #[tokio::test]
9083    async fn route_open_relays_role_versions_verbatim() {
9084        let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
9085        assert_eq!(
9086            bind_role_versions_for(Some(sent.clone()), 141).await,
9087            Some(sent)
9088        );
9089    }
9090
9091    /// An empty map declares nothing, so the provider sees no field rather
9092    /// than an empty object it would have to treat as a second "none".
9093    #[tokio::test]
9094    async fn route_open_with_empty_or_absent_role_versions_relays_none() {
9095        assert_eq!(bind_role_versions_for(None, 143).await, None);
9096        assert_eq!(
9097            bind_role_versions_for(Some(BTreeMap::new()), 145).await,
9098            None
9099        );
9100    }
9101
9102    #[tokio::test]
9103    async fn route_open_refuses_malformed_role_versions_before_any_bind() {
9104        let registry = Arc::new(Registry::default());
9105        let forwarding = Arc::new(ForwardingTable::default());
9106        let handler =
9107            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9108        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
9109        hello_via_sink(
9110            &handler,
9111            &module_ctx,
9112            &mut module_rx,
9113            hello_frame("aft", PROTOCOL_VERSION, 7),
9114        )
9115        .await;
9116
9117        let nine: BTreeMap<String, String> = (0..9)
9118            .map(|index| (format!("role-{index}"), "v1".to_string()))
9119            .collect();
9120        for (label, malformed) in [
9121            (
9122                "invalid role name",
9123                role_versions(&[("Tool_Provider", "v1")]),
9124            ),
9125            ("invalid version", role_versions(&[("tool-provider", "v0")])),
9126            ("nine entries", nine),
9127        ] {
9128            // A refused open answers at once; one that reached the module
9129            // would wait for its bind ack and trip this timeout.
9130            let responses = tokio::time::timeout(
9131                Duration::from_secs(1),
9132                handler.handle_control_frame(
9133                    &route_ctx(ConnectionId::new(148)).0,
9134                    route_open_frame_with_role_versions(
9135                        404,
9136                        "aft",
9137                        unique_project_root("role-versions-malformed"),
9138                        Some(malformed),
9139                    ),
9140                ),
9141            )
9142            .await
9143            .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
9144            .unwrap();
9145            assert_eq!(responses.len(), 1, "{label}");
9146            assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
9147            let error = parse_error(&responses[0]);
9148            assert_eq!(error["code"], "invalid_request", "{label}: {error}");
9149            assert_eq!(
9150                error["detail"]["field"], "role_versions",
9151                "{label}: {error}"
9152            );
9153            assert!(
9154                !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
9155                "{label}: a malformed declaration is terminal"
9156            );
9157            assert!(
9158                module_rx.try_recv().is_err(),
9159                "{label}: the module must never see a bind"
9160            );
9161        }
9162    }
9163
9164    /// `route-role-versions/v1` is in HELLO_ACK and `server.describe`, so a
9165    /// consumer can tell this daemon forwards the field from one that would
9166    /// drop it.
9167    #[tokio::test]
9168    async fn route_role_versions_capability_is_advertised() {
9169        let handler = ControlHandler::new(Arc::new(Registry::default()));
9170        let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
9171        let ack = hello_via_sink(
9172            &handler,
9173            &ctx,
9174            &mut rx,
9175            hello_frame("m", PROTOCOL_VERSION, 1),
9176        )
9177        .await;
9178        let ack = parse_ack(&ack);
9179        assert!(
9180            ack.subc_capabilities
9181                .iter()
9182                .any(|c| c == "route-role-versions/v1"),
9183            "{:?}",
9184            ack.subc_capabilities
9185        );
9186
9187        let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
9188        let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
9189        let reply = handler
9190            .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
9191            .await
9192            .unwrap()
9193            .pop()
9194            .unwrap();
9195        let ClientControlResponse::ServerDescribe { capabilities, .. } =
9196            serde_json::from_slice(&reply.body).unwrap()
9197        else {
9198            panic!("not a server.describe reply");
9199        };
9200        assert!(
9201            capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
9202            "{capabilities:?}"
9203        );
9204    }
9205
9206    #[tokio::test]
9207    async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
9208        let registry = Arc::new(Registry::default());
9209        let forwarding = Arc::new(ForwardingTable::default());
9210        let handler =
9211            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9212                .with_health_probe_timeout(Duration::from_secs(5));
9213        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
9214        hello_via_sink(
9215            &handler,
9216            &module_ctx,
9217            &mut module_rx,
9218            non_routable_hello_frame_with_control_ops(
9219                "mcp",
9220                300,
9221                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9222            ),
9223        )
9224        .await;
9225        assert!(registry
9226            .get_module("mcp")
9227            .unwrap()
9228            .unwrap()
9229            .manifest
9230            .provides
9231            .is_empty());
9232
9233        let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
9234        let route_response = handler
9235            .handle_control_frame(
9236                &route_client_ctx,
9237                route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
9238            )
9239            .await
9240            .unwrap();
9241        assert_eq!(route_response[0].header.ty, FrameType::Error);
9242        assert_eq!(
9243            parse_error(&route_response[0])["code"],
9244            "target_unavailable"
9245        );
9246        assert!(parse_error(&route_response[0])["message"]
9247            .as_str()
9248            .unwrap()
9249            .contains("does not provide the requested target"));
9250        assert!(module_rx.try_recv().is_err());
9251
9252        let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
9253        let health_handler = handler.clone();
9254        let health_task = tokio::spawn(async move {
9255            health_handler
9256                .handle_control_frame(
9257                    &health_client_ctx,
9258                    supervisor_health_probe_frame(302, "mcp"),
9259                )
9260                .await
9261                .unwrap()
9262        });
9263        let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9264            .await
9265            .unwrap()
9266            .unwrap();
9267        assert_eq!(
9268            serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
9269            ModuleControlRequest::HealthCheck {}
9270        );
9271        handler
9272            .handle_control_frame(
9273                &module_ctx,
9274                health_response(health_frame.header.corr, HealthStatus::Ok),
9275            )
9276            .await
9277            .unwrap();
9278        let health_response = health_task.await.unwrap();
9279        assert_eq!(health_response[0].header.ty, FrameType::Response);
9280        match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
9281            ClientControlResponse::SupervisorHealthProbe {
9282                module_id, status, ..
9283            } => {
9284                assert_eq!(module_id, "mcp");
9285                assert_eq!(status, HealthStatus::Ok);
9286            }
9287            other => panic!("unexpected health response: {other:?}"),
9288        }
9289
9290        // Exercise the forwarding cleanup path directly while leaving the registry
9291        // advertisement in place. If cleanup leaves a stale control sink behind,
9292        // the next probe will enqueue onto it and wait for the long probe timeout
9293        // instead of returning an immediate no-connection error.
9294        forwarding
9295            .cleanup_connection(module_ctx.connection_id)
9296            .unwrap();
9297        let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
9298        let cleanup_response = tokio::time::timeout(
9299            Duration::from_millis(200),
9300            handler.handle_control_frame(
9301                &cleanup_probe_ctx,
9302                supervisor_health_probe_frame(303, "mcp"),
9303            ),
9304        )
9305        .await
9306        .expect("probe should fail immediately when the control lane is gone")
9307        .unwrap();
9308        assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
9309        assert_eq!(
9310            parse_error(&cleanup_response[0])["code"],
9311            "target_unavailable"
9312        );
9313        assert!(parse_error(&cleanup_response[0])["message"]
9314            .as_str()
9315            .unwrap()
9316            .contains("no module connection"));
9317
9318        handler
9319            .cleanup_connection(module_ctx.connection_id)
9320            .unwrap();
9321    }
9322
9323    #[tokio::test]
9324    async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
9325        let registry = Arc::new(Registry::default());
9326        let supervisor_handle = SupervisorHandle::new();
9327        let supervisor =
9328            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
9329                .with_handle(supervisor_handle.clone())
9330                .with_connection_file_path(
9331                    std::env::temp_dir()
9332                        .join(format!("subc-route-open-warming-{}", std::process::id())),
9333                );
9334        let module = supervisor
9335            .supervise_configured(
9336                ModuleSpec {
9337                    module_id: "warming".to_string(),
9338                    program: fake_aft_stub_path(),
9339                    args: Vec::new(),
9340                    env: Vec::new(),
9341                    reserved: false,
9342                    reserved_prefixes: Vec::new(),
9343                    protocol: ModuleProtocol::Subc,
9344                    overlap: Default::default(),
9345                },
9346                true,
9347            )
9348            .unwrap();
9349        assert_eq!(module.state().unwrap(), ModuleState::Running);
9350
9351        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9352        let (ctx, _rx) = route_ctx(ConnectionId::new(39));
9353        let response = handler
9354            .handle_control_frame(
9355                &ctx,
9356                route_open_frame(304, "warming", unique_project_root("warming")),
9357            )
9358            .await
9359            .unwrap();
9360        module.stop().await.unwrap();
9361
9362        assert_eq!(response[0].header.ty, FrameType::Error);
9363        let error = parse_error(&response[0]);
9364        assert_eq!(error["code"], "module_warming");
9365        assert!(error["message"]
9366            .as_str()
9367            .unwrap()
9368            .contains("state=running, enabled=true, live=false"));
9369    }
9370
9371    #[test]
9372    fn route_open_connection_cap_logs_admission_reason_and_capacity() {
9373        let handler = ControlHandler::new(Arc::new(Registry::default()));
9374        let capture = EventCapture::default();
9375        let _subscriber =
9376            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9377        let (ctx, _rx) = route_ctx(ConnectionId::new(96));
9378        let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
9379        let pending = (0..limit).collect::<Vec<_>>();
9380        let response = handler
9381            .route_open_capacity_refusal(
9382                &ctx,
9383                &route_open_frame(396, "busy", unique_project_root("connection-cap")),
9384                "busy",
9385                pending.len(),
9386                limit,
9387            )
9388            .unwrap();
9389        assert_eq!(parse_error(&response)["code"], "target_unavailable");
9390        let event = capture
9391            .events()
9392            .into_iter()
9393            .find(|event| {
9394                event.target == "control"
9395                    && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
9396            })
9397            .expect("connection admission refusal event");
9398        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
9399        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
9400    }
9401
9402    #[test]
9403    fn route_open_target_cap_logs_admission_reason_and_capacity() {
9404        let handler = ControlHandler::new(Arc::new(Registry::default()));
9405        let capture = EventCapture::default();
9406        let _subscriber =
9407            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9408        let (ctx, _rx) = route_ctx(ConnectionId::new(97));
9409        let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
9410        let guards = (0..limit)
9411            .map(|_| {
9412                handler
9413                    .route_bind_concurrency
9414                    .try_admit("busy", limit)
9415                    .unwrap()
9416            })
9417            .collect::<Vec<_>>();
9418        let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
9419            Err(in_flight) => in_flight,
9420            Ok(_) => panic!("target cap must refuse after {limit} admissions"),
9421        };
9422        let response = handler
9423            .route_open_target_capacity_refusal(
9424                &ctx,
9425                &route_open_frame(397, "busy", unique_project_root("target-cap")),
9426                "busy",
9427                in_flight,
9428            )
9429            .unwrap();
9430        assert_eq!(parse_error(&response)["code"], "target_unavailable");
9431        let event = capture
9432            .events()
9433            .into_iter()
9434            .find(|event| {
9435                event.target == "control"
9436                    && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
9437            })
9438            .expect("target admission refusal event");
9439        assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
9440        assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
9441        drop(guards);
9442    }
9443
9444    /// One wire code has several senders, so the refusal line names the check
9445    /// that refused. This drives the shared refusal path for ordinary refusals
9446    /// with an unregistered
9447    /// target and requires the branch label on the event.
9448    #[tokio::test]
9449    async fn route_open_refusal_names_the_check_that_refused() {
9450        let handler = ControlHandler::new(Arc::new(Registry::default()));
9451        let capture = EventCapture::default();
9452        let _subscriber =
9453            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9454        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
9455        let response = handler
9456            .handle_control_frame(
9457                &ctx,
9458                route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
9459            )
9460            .await
9461            .unwrap();
9462
9463        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9464        let event = capture
9465            .events()
9466            .into_iter()
9467            .find(|event| {
9468                event.target == "control"
9469                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
9470            })
9471            .expect("route.open refusal event");
9472        assert_eq!(
9473            event.fields.get("reason"),
9474            Some(&"\"not_registered\"".to_string())
9475        );
9476    }
9477
9478    #[tokio::test]
9479    async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
9480        let registry = Arc::new(Registry::default());
9481        let supervisor_handle = SupervisorHandle::new();
9482        let supervisor =
9483            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
9484                .with_handle(supervisor_handle.clone())
9485                .with_connection_file_path(std::env::temp_dir().join(format!(
9486                    "subc-route-open-refusal-info-{}",
9487                    std::process::id()
9488                )));
9489        let module = supervisor
9490            .supervise_configured(
9491                ModuleSpec {
9492                    module_id: "warming".to_string(),
9493                    program: fake_aft_stub_path(),
9494                    args: Vec::new(),
9495                    env: Vec::new(),
9496                    reserved: false,
9497                    reserved_prefixes: Vec::new(),
9498                    protocol: ModuleProtocol::Subc,
9499                    overlap: Default::default(),
9500                },
9501                true,
9502            )
9503            .unwrap();
9504        assert_eq!(module.state().unwrap(), ModuleState::Running);
9505
9506        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9507        assert!(handler
9508            .counters()
9509            .snapshot()
9510            .get("route_open_refused_by_code")
9511            .is_none());
9512        let capture = EventCapture::default();
9513        let _subscriber =
9514            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9515        let (ctx, _rx) = route_ctx(ConnectionId::new(94));
9516        let response = handler
9517            .handle_control_frame(
9518                &ctx,
9519                route_open_frame(394, "warming", unique_project_root("refusal-info")),
9520            )
9521            .await
9522            .unwrap();
9523        module.stop().await.unwrap();
9524
9525        assert_eq!(parse_error(&response[0])["code"], "module_warming");
9526        let event = capture
9527            .events()
9528            .into_iter()
9529            .find(|event| {
9530                event.target == "control"
9531                    && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
9532            })
9533            .expect("route.open refusal event");
9534        assert_eq!(
9535            event.fields.get("module_id"),
9536            Some(&"\"warming\"".to_string())
9537        );
9538        assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
9539        assert_eq!(
9540            event.fields.get("reason"),
9541            Some(&"\"supervised_not_registered\"".to_string())
9542        );
9543        assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
9544        assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
9545        assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
9546        assert_eq!(
9547            handler.counters().snapshot()["route_open_refused_by_code"],
9548            json!({ "module_warming": 1 })
9549        );
9550    }
9551
9552    const OUTAGE_START: &str = "route.open refusing module: not serving";
9553    const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
9554
9555    fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
9556        capture
9557            .events()
9558            .into_iter()
9559            .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
9560            .collect()
9561    }
9562
9563    fn supervise_stub(
9564        registry: &Arc<Registry>,
9565        module_id: &str,
9566        enabled: bool,
9567    ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
9568        let supervisor_handle = SupervisorHandle::new();
9569        let supervisor =
9570            Supervisor::new(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
9571                .with_handle(supervisor_handle.clone())
9572                .with_connection_file_path(std::env::temp_dir().join(format!(
9573                    "subc-route-outage-{module_id}-{}",
9574                    std::process::id()
9575                )));
9576        let module = supervisor
9577            .supervise_configured(
9578                ModuleSpec {
9579                    module_id: module_id.to_string(),
9580                    program: fake_aft_stub_path(),
9581                    args: Vec::new(),
9582                    env: Vec::new(),
9583                    reserved: false,
9584                    reserved_prefixes: Vec::new(),
9585                    protocol: ModuleProtocol::Subc,
9586                    overlap: Default::default(),
9587                },
9588                enabled,
9589            )
9590            .unwrap();
9591        (supervisor_handle, module)
9592    }
9593
9594    fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
9595        let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
9596            module_id: module_id.to_string(),
9597            drain_timeout_ms: Some(50),
9598        })
9599        .unwrap();
9600        Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9601    }
9602
9603    /// Two handlers built over one forwarding table must share one outage
9604    /// tracker; separate trackers would each log their own opening line for
9605    /// the same outage.
9606    #[test]
9607    fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
9608        let registry = Arc::new(Registry::default());
9609        let forwarding = Arc::new(ForwardingTable::default());
9610        let first = ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9611        let second = ControlHandler::with_forwarding(registry, forwarding);
9612        assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
9613    }
9614
9615    /// A client can name any module id it likes. Refusing an unknown one,
9616    /// however often, must not create outage state or outage lines, or the
9617    /// tracker would be a memory sink any client could fill.
9618    #[tokio::test(flavor = "current_thread")]
9619    async fn route_open_unknown_module_refusals_add_no_outage_state() {
9620        let handler = ControlHandler::new(Arc::new(Registry::default()));
9621        let capture = EventCapture::default();
9622        let _subscriber =
9623            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9624        let (ctx, _rx) = route_ctx(ConnectionId::new(90));
9625        for corr in 0..8 {
9626            let response = handler
9627                .handle_control_frame(
9628                    &ctx,
9629                    route_open_frame(
9630                        380 + corr,
9631                        &format!("nobody-{corr}"),
9632                        unique_project_root("outage-unknown"),
9633                    ),
9634                )
9635                .await
9636                .unwrap();
9637            assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9638        }
9639
9640        assert_eq!(handler.route_outages.tracked_module_count(), 0);
9641        assert!(outage_lines(&capture, OUTAGE_START).is_empty());
9642        assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
9643    }
9644
9645    /// Drives the refusal path end to end: a supervised module that served
9646    /// before and stopped being registered with no instruction to stop is a
9647    /// WARN, and the same module refused after an operator `supervisor.restart`
9648    /// is an INFO.
9649    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9650    async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
9651        let registry = Arc::new(Registry::default());
9652        let (supervisor_handle, module) = supervise_stub(&registry, "outage-restart", true);
9653        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9654        let capture = EventCapture::default();
9655        let _subscriber =
9656            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9657        let (ctx, _rx) = route_ctx(ConnectionId::new(91));
9658        // The stub never registers, so pretend it served once: otherwise every
9659        // refusal would fall in its startup window.
9660        handler.route_outages.record_accepted("outage-restart");
9661
9662        let response = handler
9663            .handle_control_frame(
9664                &ctx,
9665                route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
9666            )
9667            .await
9668            .unwrap();
9669        assert_eq!(response[0].header.ty, FrameType::Error);
9670        let starts = outage_lines(&capture, OUTAGE_START);
9671        assert_eq!(starts.len(), 1, "{starts:?}");
9672        assert_eq!(starts[0].level, tracing::Level::WARN);
9673        assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
9674        assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
9675        assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
9676        handler.route_outages.record_accepted("outage-restart");
9677        assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
9678
9679        let restart = handler
9680            .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
9681            .await
9682            .unwrap();
9683        assert_eq!(
9684            restart[0].header.ty,
9685            FrameType::Response,
9686            "{:?}",
9687            parse_error(&restart[0])
9688        );
9689        handler
9690            .handle_control_frame(
9691                &ctx,
9692                route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
9693            )
9694            .await
9695            .unwrap();
9696        module.stop().await.unwrap();
9697
9698        let starts = outage_lines(&capture, OUTAGE_START);
9699        assert_eq!(starts.len(), 2, "{starts:?}");
9700        assert_eq!(starts[1].level, tracing::Level::INFO);
9701        assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
9702    }
9703
9704    /// A restart refused before it touched the module (here: the module is
9705    /// disabled) must clear its operator mark, so the next real outage is
9706    /// still reported as a warning.
9707    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9708    async fn failed_operator_restart_leaves_no_operator_mark() {
9709        let registry = Arc::new(Registry::default());
9710        let (supervisor_handle, _module) = supervise_stub(&registry, "outage-disabled", false);
9711        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9712        let capture = EventCapture::default();
9713        let _subscriber =
9714            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9715        let (ctx, _rx) = route_ctx(ConnectionId::new(92));
9716        handler.route_outages.record_accepted("outage-disabled");
9717
9718        let restart = handler
9719            .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
9720            .await
9721            .unwrap();
9722        assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
9723        assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
9724
9725        handler
9726            .handle_control_frame(
9727                &ctx,
9728                route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
9729            )
9730            .await
9731            .unwrap();
9732        let starts = outage_lines(&capture, OUTAGE_START);
9733        assert_eq!(starts.len(), 1, "{starts:?}");
9734        assert_eq!(starts[0].level, tracing::Level::WARN);
9735    }
9736
9737    #[tokio::test(flavor = "current_thread")]
9738    async fn route_open_unknown_module_escapes_target_module_id() {
9739        let handler = ControlHandler::new(Arc::new(Registry::default()));
9740        let capture = EventCapture::default();
9741        let _subscriber =
9742            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9743        let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
9744        let (ctx, _rx) = route_ctx(ConnectionId::new(95));
9745        let response = handler
9746            .handle_control_frame(
9747                &ctx,
9748                route_open_frame(
9749                    395,
9750                    hostile_module_id,
9751                    unique_project_root("hostile-target-module-id"),
9752                ),
9753            )
9754            .await
9755            .unwrap();
9756
9757        assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9758        let event = capture
9759            .events()
9760            .into_iter()
9761            .find(|event| {
9762                event.target == "control"
9763                    && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
9764            })
9765            .expect("route.open unknown-module refusal event");
9766        let logged = event.fields.get("module_id").expect("module_id field");
9767        assert!(!logged.bytes().any(|byte| byte < 0x20));
9768        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
9769    }
9770
9771    #[tokio::test(flavor = "current_thread")]
9772    async fn route_open_module_rejection_uses_daemon_counter_key() {
9773        let registry = Arc::new(Registry::default());
9774        let forwarding = Arc::new(ForwardingTable::default());
9775        let handler =
9776            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
9777        let module_connection = ConnectionId::new(95);
9778        let (module_ctx, mut module_rx) = route_ctx(module_connection);
9779        hello_via_sink(
9780            &handler,
9781            &module_ctx,
9782            &mut module_rx,
9783            hello_frame("aft", PROTOCOL_VERSION, 395),
9784        )
9785        .await;
9786
9787        let client_connection = ConnectionId::new(96);
9788        let (client_ctx, _client_rx) = route_ctx(client_connection);
9789        let capture = EventCapture::default();
9790        let _subscriber =
9791            tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9792        let (route_task, bind) = relay_route_open(
9793            &handler,
9794            client_connection,
9795            &client_ctx.egress,
9796            &mut module_rx,
9797            396,
9798            "aft",
9799            "hostile-module-code",
9800        )
9801        .await;
9802        let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
9803        let rejection = Frame::build(
9804            FrameType::Error,
9805            control_flags(),
9806            0,
9807            0,
9808            bind.header.corr,
9809            serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
9810        )
9811        .unwrap();
9812        handler
9813            .handle_control_frame(&module_ctx, rejection)
9814            .await
9815            .unwrap();
9816
9817        let response = route_task.await.unwrap();
9818        assert_eq!(parse_error(&response[0])["code"], hostile_code);
9819        let counters = handler.counters().snapshot();
9820        assert_eq!(
9821            counters["route_open_refused_by_code"],
9822            json!({ "module_rejected": 1 })
9823        );
9824        assert!(counters["route_open_refused_by_code"]
9825            .get(hostile_code)
9826            .is_none());
9827
9828        let event = capture
9829            .events()
9830            .into_iter()
9831            .find(|event| {
9832                event.target == "control"
9833                    && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
9834            })
9835            .expect("route.open module-rejection refusal event");
9836        let logged = event.fields.get("module_code").expect("module_code field");
9837        assert!(!logged.bytes().any(|byte| byte < 0x20));
9838        assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
9839    }
9840
9841    #[tokio::test]
9842    async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
9843        let registry = Arc::new(Registry::default());
9844        let supervisor_handle = SupervisorHandle::new();
9845        let missing_program = std::env::temp_dir().join(format!(
9846            "subc-route-open-missing-program-{}",
9847            std::process::id()
9848        ));
9849        let supervisor =
9850            Supervisor::new(Arc::clone(&registry), RestartPolicy::new(0, Duration::ZERO))
9851                .with_handle(supervisor_handle.clone());
9852        let module = supervisor
9853            .supervise_configured(
9854                ModuleSpec {
9855                    module_id: "failed".to_string(),
9856                    program: missing_program,
9857                    args: Vec::new(),
9858                    env: Vec::new(),
9859                    reserved: false,
9860                    reserved_prefixes: Vec::new(),
9861                    protocol: ModuleProtocol::Subc,
9862                    overlap: Default::default(),
9863                },
9864                true,
9865            )
9866            .unwrap();
9867        assert_eq!(module.state().unwrap(), ModuleState::Failed);
9868
9869        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
9870        let (ctx, _rx) = route_ctx(ConnectionId::new(40));
9871        let response = handler
9872            .handle_control_frame(
9873                &ctx,
9874                route_open_frame(305, "failed", unique_project_root("failed")),
9875            )
9876            .await
9877            .unwrap();
9878
9879        assert_eq!(response[0].header.ty, FrameType::Error);
9880        let error = parse_error(&response[0]);
9881        assert_eq!(error["code"], "target_unavailable");
9882        assert!(error["message"]
9883            .as_str()
9884            .unwrap()
9885            .contains("state=failed, enabled=true, live=false"));
9886    }
9887
9888    #[tokio::test]
9889    async fn route_open_role_mismatch_remains_target_unavailable() {
9890        let registry = Arc::new(Registry::default());
9891        let handler = ControlHandler::new(Arc::clone(&registry));
9892        handler
9893            .handle_control(
9894                ConnectionId::new(41),
9895                non_routable_hello_frame_with_control_ops("health-only", 306, None),
9896            )
9897            .unwrap();
9898
9899        let (ctx, _rx) = route_ctx(ConnectionId::new(42));
9900        let response = handler
9901            .handle_control_frame(
9902                &ctx,
9903                route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
9904            )
9905            .await
9906            .unwrap();
9907
9908        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9909        assert!(parse_error(&response[0])["message"]
9910            .as_str()
9911            .unwrap()
9912            .contains("does not provide the requested target"));
9913    }
9914
9915    #[tokio::test]
9916    async fn route_open_inactive_registration_remains_target_unavailable() {
9917        let registry = Arc::new(Registry::default());
9918        let handler = ControlHandler::new(Arc::clone(&registry));
9919        handler
9920            .handle_control(
9921                ConnectionId::new(43),
9922                hello_frame("inactive", PROTOCOL_VERSION, 308),
9923            )
9924            .unwrap();
9925        assert!(registry
9926            .set_module_state_for_test("inactive", ChannelState::Closed)
9927            .unwrap());
9928
9929        let (ctx, _rx) = route_ctx(ConnectionId::new(44));
9930        let response = handler
9931            .handle_control_frame(
9932                &ctx,
9933                route_open_frame(309, "inactive", unique_project_root("inactive")),
9934            )
9935            .await
9936            .unwrap();
9937
9938        assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9939        assert!(parse_error(&response[0])["message"]
9940            .as_str()
9941            .unwrap()
9942            .contains("is not active"));
9943    }
9944
9945    #[tokio::test]
9946    async fn late_health_reply_is_recorded_through_the_module_response_path() {
9947        let registry = Arc::new(Registry::default());
9948        let forwarding = Arc::new(ForwardingTable::default());
9949        let supervisor_handle = SupervisorHandle::new();
9950        let supervisor = Supervisor::new(Arc::clone(&registry), crate::RestartPolicy::default())
9951            .with_forwarding(Arc::clone(&forwarding))
9952            .with_handle(supervisor_handle.clone());
9953        let module = supervisor
9954            .supervise_configured(
9955                crate::ModuleSpec {
9956                    module_id: "late-health-response".to_string(),
9957                    program: PathBuf::from("disabled-module"),
9958                    args: Vec::new(),
9959                    env: Vec::new(),
9960                    reserved: false,
9961                    reserved_prefixes: Vec::new(),
9962                    protocol: ModuleProtocol::Subc,
9963                    overlap: Default::default(),
9964                },
9965                false,
9966            )
9967            .unwrap();
9968        let handler =
9969            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
9970                .with_supervisor(supervisor_handle);
9971        let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
9972        handler
9973            .handle_control_frame(
9974                &module_ctx,
9975                hello_frame_with_control_ops(
9976                    "late-health-response",
9977                    PROTOCOL_VERSION,
9978                    7,
9979                    Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9980                ),
9981            )
9982            .await
9983            .unwrap();
9984        let probe_started_at = Instant::now() - Duration::from_millis(80);
9985        let pending = forwarding
9986            .begin_health_probe_rpc_for(
9987                "late-health-response",
9988                MODULE_CONTROL_OP_HEALTH_CHECK,
9989                probe_started_at,
9990                Instant::now() - Duration::from_millis(1),
9991            )
9992            .unwrap();
9993        assert!(forwarding
9994            .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
9995            .unwrap());
9996
9997        let responses = handler
9998            .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
9999            .await
10000            .unwrap();
10001
10002        assert!(responses.is_empty());
10003        let health = module.status().unwrap().health;
10004        assert_eq!(health.late_answer_count, 1);
10005        assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
10006    }
10007
10008    #[tokio::test]
10009    async fn health_probe_timeout_and_module_death_are_typed() {
10010        let registry = Arc::new(Registry::default());
10011        let forwarding = Arc::new(ForwardingTable::default());
10012        let handler =
10013            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10014                .with_health_probe_timeout(Duration::from_millis(50));
10015        let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
10016        hello_via_sink(
10017            &handler,
10018            &module_ctx,
10019            &mut module_rx,
10020            hello_frame_with_control_ops(
10021                "aft",
10022                PROTOCOL_VERSION,
10023                7,
10024                Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10025            ),
10026        )
10027        .await;
10028
10029        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10030        let responses = handler
10031            .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
10032            .await
10033            .unwrap();
10034        assert_eq!(responses[0].header.ty, FrameType::Error);
10035        assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
10036        let _ = module_rx.try_recv();
10037
10038        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
10039        let health_handler = handler.clone();
10040        let death_task = tokio::spawn(async move {
10041            health_handler
10042                .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
10043                .await
10044                .unwrap()
10045        });
10046        tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10047            .await
10048            .unwrap()
10049            .unwrap();
10050        handler
10051            .cleanup_connection(module_ctx.connection_id)
10052            .unwrap();
10053        let responses = death_task.await.unwrap();
10054        assert_eq!(responses[0].header.ty, FrameType::Error);
10055        assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
10056    }
10057
10058    #[test]
10059    fn hello_requires_exact_protocol_version() {
10060        for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
10061            let registry = Arc::new(Registry::default());
10062            let handler = ControlHandler::new(Arc::clone(&registry));
10063            let responses = handler
10064                .handle_control(
10065                    ConnectionId::new(connection),
10066                    hello_frame("aft", offered, 9),
10067                )
10068                .unwrap();
10069
10070            assert_eq!(responses.len(), 1);
10071            assert_eq!(responses[0].header.ty, FrameType::Error);
10072            let error = parse_error(&responses[0]);
10073            assert_eq!(error["code"], "version_unsupported");
10074            assert!(registry.get_module("aft").unwrap().is_none());
10075            assert_eq!(registry.active_registration_count().unwrap(), 0);
10076        }
10077    }
10078
10079    #[test]
10080    fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
10081        let registry = Arc::new(Registry::default());
10082        let forwarding = Arc::new(ForwardingTable::default());
10083        let handler =
10084            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10085        let module_connection = ConnectionId::new(301);
10086        let registration = registry
10087            .register_with_control_ops(
10088                manifest("aft-push", PROTOCOL_VERSION),
10089                PROTOCOL_VERSION,
10090                module_connection,
10091                module_baseline_control_ops(),
10092            )
10093            .unwrap();
10094        let (module_tx, _module_rx) = mpsc::channel(8);
10095        let endpoint = forwarding
10096            .register_module_connection(
10097                module_connection,
10098                "aft-push".to_string(),
10099                PROTOCOL_VERSION,
10100                manifest_concurrency(&registration.manifest),
10101                FrameSink::new(module_tx),
10102            )
10103            .unwrap();
10104
10105        // A push op this version does not know is ignored (forward-compat), not errored.
10106        let unknown = Frame::build(
10107            FrameType::Push,
10108            control_flags(),
10109            0,
10110            0,
10111            5,
10112            serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
10113        )
10114        .unwrap();
10115        let out = handler.handle_status_update(endpoint, unknown).unwrap();
10116        assert!(
10117            out.is_empty(),
10118            "unknown push op must be ignored, got {out:?}"
10119        );
10120
10121        // A malformed body for a KNOWN op is a real error worth surfacing.
10122        let malformed = Frame::build(
10123            FrameType::Push,
10124            control_flags(),
10125            0,
10126            0,
10127            6,
10128            serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
10129        )
10130        .unwrap();
10131        let out = handler.handle_status_update(endpoint, malformed).unwrap();
10132        assert_eq!(out.len(), 1);
10133        assert_eq!(out[0].header.ty, FrameType::Error);
10134        assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
10135    }
10136
10137    #[test]
10138    fn hello_rejected_when_connection_already_owns_client_routes() {
10139        let registry = Arc::new(Registry::default());
10140        let forwarding = Arc::new(ForwardingTable::default());
10141        let handler =
10142            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10143        // Commits a client route on connection 202 (bound to a module on conn 101).
10144        let _ = bind_liveness_route(&registry, &forwarding, "aft-module");
10145        let client_connection = ConnectionId::new(202);
10146
10147        // That same connection now tries to register as a module: rejected, so one
10148        // connection never holds both client-route and module-endpoint state.
10149        let responses = handler
10150            .handle_control(
10151                client_connection,
10152                hello_frame("aft-second", PROTOCOL_VERSION, 9),
10153            )
10154            .unwrap();
10155        assert_eq!(responses[0].header.ty, FrameType::Error);
10156        assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
10157        assert!(registry.get_module("aft-second").unwrap().is_none());
10158    }
10159
10160    #[test]
10161    fn reserved_module_hello_requires_matching_launch_nonce() {
10162        let registry = Arc::new(Registry::default());
10163        let supervisor = SupervisorHandle::new();
10164        // The supervisor recorded the nonce it injected when it spawned the reserved
10165        // module; the HELLO verifier checks against the same shared handle.
10166        supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
10167        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
10168
10169        // A HELLO with NO nonce is rejected.
10170        let no_nonce = handler
10171            .handle_control(
10172                ConnectionId::new(1),
10173                hello_frame("vault", PROTOCOL_VERSION, 1),
10174            )
10175            .unwrap();
10176        assert_eq!(no_nonce[0].header.ty, FrameType::Error);
10177        assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
10178        assert!(registry.get_module("vault").unwrap().is_none());
10179
10180        // A HELLO with the WRONG nonce is rejected.
10181        let wrong = handler
10182            .handle_control(
10183                ConnectionId::new(2),
10184                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
10185            )
10186            .unwrap();
10187        assert_eq!(wrong[0].header.ty, FrameType::Error);
10188        assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
10189        assert!(registry.get_module("vault").unwrap().is_none());
10190
10191        // A HELLO with the CORRECT nonce registers.
10192        let ok = handler
10193            .handle_control(
10194                ConnectionId::new(3),
10195                hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
10196            )
10197            .unwrap();
10198        assert_eq!(ok[0].header.ty, FrameType::HelloAck);
10199        assert!(registry.get_module("vault").unwrap().is_some());
10200    }
10201
10202    #[test]
10203    fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
10204        let registry = Arc::new(Registry::default());
10205        let supervisor = SupervisorHandle::new();
10206        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
10207        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
10208        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
10209
10210        let squat = handler
10211            .handle_control(
10212                ConnectionId::new(1),
10213                hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
10214            )
10215            .unwrap();
10216        assert_eq!(squat[0].header.ty, FrameType::Error);
10217        assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
10218        assert!(parse_error(&squat[0])["message"]
10219            .as_str()
10220            .unwrap()
10221            .contains("fed:"));
10222
10223        let accepted_peer = handler
10224            .handle_control(
10225                ConnectionId::new(2),
10226                hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
10227            )
10228            .unwrap();
10229        assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
10230
10231        let accepted_short = handler
10232            .handle_control(
10233                ConnectionId::new(3),
10234                hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
10235            )
10236            .unwrap();
10237        assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
10238
10239        for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
10240            let response = handler
10241                .handle_control(
10242                    ConnectionId::new(conn),
10243                    hello_frame(module_id, PROTOCOL_VERSION, conn),
10244                )
10245                .unwrap();
10246            assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
10247        }
10248    }
10249
10250    #[test]
10251    fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
10252        let registry = Arc::new(Registry::default());
10253        let supervisor = SupervisorHandle::new();
10254        supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
10255        supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
10256        supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
10257        let handler = ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor);
10258
10259        let owner_nonce = handler
10260            .handle_control(
10261                ConnectionId::new(1),
10262                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
10263            )
10264            .unwrap();
10265        assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
10266        assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
10267        assert!(registry.get_module("fed:special").unwrap().is_none());
10268
10269        let exact_nonce = handler
10270            .handle_control(
10271                ConnectionId::new(2),
10272                hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
10273            )
10274            .unwrap();
10275        assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
10276        assert!(registry.get_module("fed:special").unwrap().is_some());
10277    }
10278
10279    #[test]
10280    fn non_reserved_module_ignores_launch_nonce() {
10281        let registry = Arc::new(Registry::default());
10282        // No reserved nonce recorded for these ids: they are not reserved, so HELLO
10283        // registration succeeds whether a spawned process echoes a nonce or not.
10284        let handler = ControlHandler::new(Arc::clone(&registry));
10285        let no_nonce = handler
10286            .handle_control(
10287                ConnectionId::new(1),
10288                hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
10289            )
10290            .unwrap();
10291        assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
10292        assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
10293
10294        let echoed_nonce = handler
10295            .handle_control(
10296                ConnectionId::new(2),
10297                hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
10298            )
10299            .unwrap();
10300        assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
10301        assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
10302    }
10303
10304    #[test]
10305    fn malformed_hello_returns_error_and_handler_still_answers_ping() {
10306        let handler = ControlHandler::default();
10307        let conn = ConnectionId::new(1);
10308        let malformed = Frame::build(
10309            FrameType::Hello,
10310            control_flags(),
10311            0,
10312            0,
10313            3,
10314            b"{not json".to_vec(),
10315        )
10316        .unwrap();
10317
10318        let error = handler.handle_control(conn, malformed).unwrap();
10319        assert_eq!(error[0].header.ty, FrameType::Error);
10320        assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
10321
10322        let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
10323        let pong = handler.handle_control(conn, ping).unwrap();
10324        assert_eq!(pong[0].header.ty, FrameType::Pong);
10325        assert_eq!(pong[0].header.corr, 4);
10326    }
10327
10328    #[test]
10329    fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
10330        let registry = Arc::new(Registry::default());
10331        let handler = ControlHandler::new(Arc::clone(&registry));
10332
10333        handler
10334            .handle_control(
10335                ConnectionId::new(1),
10336                hello_frame("aft", PROTOCOL_VERSION, 1),
10337            )
10338            .unwrap();
10339        let duplicate = handler
10340            .handle_control(
10341                ConnectionId::new(2),
10342                hello_frame("aft", PROTOCOL_VERSION, 2),
10343            )
10344            .unwrap();
10345
10346        assert_eq!(duplicate[0].header.ty, FrameType::Error);
10347        assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10348        let registration = registry.get_module("aft").unwrap().unwrap();
10349        assert_eq!(registration.connection_id, ConnectionId::new(1));
10350    }
10351
10352    #[test]
10353    fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
10354        let registry = Arc::new(Registry::default());
10355        let forwarding = Arc::new(ForwardingTable::default());
10356        let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
10357        let handler =
10358            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10359                .with_process_liveness(process_liveness);
10360        let (ctx, route_channel, route_epoch) =
10361            bind_liveness_route(&registry, &forwarding, "aft-dead");
10362        let responses = handler
10363            .handle_route_poll(
10364                &ctx,
10365                route_poll_frame(41, PollKind::Liveness, route_channel),
10366                route_channel,
10367                route_epoch,
10368                PollKind::Liveness,
10369            )
10370            .unwrap();
10371
10372        assert_eq!(responses.len(), 1);
10373        assert_eq!(responses[0].header.ty, FrameType::Response);
10374        assert_route_poll_liveness(&responses[0], false);
10375    }
10376
10377    #[test]
10378    fn liveness_poll_without_process_source_uses_bound_route() {
10379        let registry = Arc::new(Registry::default());
10380        let forwarding = Arc::new(ForwardingTable::default());
10381        let handler =
10382            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
10383        let (ctx, route_channel, route_epoch) =
10384            bind_liveness_route(&registry, &forwarding, "aft-bound-only");
10385        let responses = handler
10386            .handle_route_poll(
10387                &ctx,
10388                route_poll_frame(42, PollKind::Liveness, route_channel),
10389                route_channel,
10390                route_epoch,
10391                PollKind::Liveness,
10392            )
10393            .unwrap();
10394
10395        assert_route_poll_liveness(&responses[0], true);
10396    }
10397
10398    #[test]
10399    fn liveness_poll_untracked_process_source_uses_bound_route() {
10400        let registry = Arc::new(Registry::default());
10401        let forwarding = Arc::new(ForwardingTable::default());
10402        let process_liveness = Arc::new(FakeProcessLiveness { live: None });
10403        let handler =
10404            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10405                .with_process_liveness(process_liveness);
10406        let (ctx, route_channel, route_epoch) =
10407            bind_liveness_route(&registry, &forwarding, "aft-untracked");
10408        let responses = handler
10409            .handle_route_poll(
10410                &ctx,
10411                route_poll_frame(43, PollKind::Liveness, route_channel),
10412                route_channel,
10413                route_epoch,
10414                PollKind::Liveness,
10415            )
10416            .unwrap();
10417
10418        assert_route_poll_liveness(&responses[0], true);
10419    }
10420
10421    #[tokio::test]
10422    async fn unknown_op_returns_unknown_control_op() {
10423        let handler = ControlHandler::default();
10424        let (ctx, _rx) = route_ctx(ConnectionId::new(77));
10425        let request = Frame::build(
10426            FrameType::Request,
10427            control_flags(),
10428            0,
10429            0,
10430            55,
10431            br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
10432        )
10433        .unwrap();
10434
10435        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10436
10437        assert_eq!(response.len(), 1);
10438        assert_eq!(response[0].header.ty, FrameType::Error);
10439        assert_eq!(response[0].header.corr, 55);
10440        assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
10441    }
10442
10443    #[tokio::test]
10444    async fn supervisor_provenance_rejects_unknown_exact_module() {
10445        let handler = ControlHandler::default();
10446        let (ctx, _rx) = route_ctx(ConnectionId::new(79));
10447        let request = Frame::build(
10448            FrameType::Request,
10449            control_flags(),
10450            0,
10451            0,
10452            57,
10453            br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
10454        )
10455        .unwrap();
10456
10457        let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10458
10459        assert_eq!(response.len(), 1);
10460        assert_eq!(response[0].header.ty, FrameType::Error);
10461        assert_eq!(response[0].header.corr, 57);
10462        let error = parse_error(&response[0]);
10463        assert_eq!(error["code"], "unknown_module");
10464        assert_eq!(error["message"], "module_id 'missing' is not supervised");
10465    }
10466
10467    #[test]
10468    fn provenance_probe_override_keeps_handler_tests_deterministic() {
10469        let expected = subc_control::RunningImageAgreement::Unavailable {
10470            reason: subc_control::RunningImageUnavailableReason::HashFailed,
10471        };
10472        let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
10473        assert_eq!(handler.provenance_probe_override, Some(expected));
10474    }
10475
10476    #[test]
10477    fn reload_verdict_detects_configured_program_different_from_spawned_path() {
10478        let verdict = reload_verdict(
10479            std::path::Path::new("/bin/new"),
10480            Some(std::path::Path::new("/bin/old")),
10481            subc_control::RunningImageAgreement::Unavailable {
10482                reason: subc_control::RunningImageUnavailableReason::HashFailed,
10483            },
10484        );
10485        assert!(matches!(
10486            verdict.path,
10487            subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
10488                if configured == std::path::Path::new("/bin/new")
10489                    && spawned_from == std::path::Path::new("/bin/old")
10490        ));
10491    }
10492
10493    #[test]
10494    fn reload_verdict_detects_replaced_image_at_same_path() {
10495        let image = subc_control::RunningImageAgreement::Mismatch {
10496            running: subc_control::RunningImageEvidence::LinuxProcSha256 {
10497                digest: "old".into(),
10498            },
10499            disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
10500                digest: "new".into(),
10501            },
10502        };
10503        let verdict = reload_verdict(
10504            std::path::Path::new("/bin/same"),
10505            Some(std::path::Path::new("/bin/same")),
10506            image.clone(),
10507        );
10508        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10509        assert_eq!(verdict.image, image);
10510    }
10511
10512    #[test]
10513    fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
10514        let image = subc_control::RunningImageAgreement::Unavailable {
10515            reason: subc_control::RunningImageUnavailableReason::NotRunning,
10516        };
10517        let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
10518        assert_eq!(
10519            verdict.path,
10520            subc_control::ReloadPathAgreement::Unavailable {
10521                reason: subc_control::ReloadPathUnavailableReason::NotRunning,
10522            }
10523        );
10524        assert_eq!(verdict.image, image);
10525
10526        let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
10527            reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
10528        };
10529        let verdict = reload_verdict(
10530            std::path::Path::new("/bin/same"),
10531            Some(std::path::Path::new("/bin/same")),
10532            unconfirmed.clone(),
10533        );
10534        assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10535        assert_eq!(verdict.image, unconfirmed);
10536    }
10537
10538    #[test]
10539    fn reload_verdict_preserves_each_image_unavailability_reason() {
10540        use subc_control::RunningImageUnavailableReason as Reason;
10541
10542        for reason in [
10543            Reason::NotRunning,
10544            Reason::UnsupportedPlatform,
10545            Reason::RunningExecutableUnreadable,
10546            Reason::SpawnedPathUnreadable,
10547            Reason::HashFailed,
10548            Reason::ProcessIdentityUnconfirmed,
10549            Reason::Unknown("future_probe_reason".to_string()),
10550        ] {
10551            let image = subc_control::RunningImageAgreement::Unavailable {
10552                reason: reason.clone(),
10553            };
10554            let verdict = reload_verdict(
10555                std::path::Path::new("/bin/same"),
10556                Some(std::path::Path::new("/bin/same")),
10557                image.clone(),
10558            );
10559            assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10560            assert_eq!(verdict.image, image, "{reason:?}");
10561        }
10562    }
10563
10564    #[tokio::test]
10565    async fn malformed_control_bodies_return_invalid_control_body() {
10566        let handler = ControlHandler::default();
10567        let (ctx, _rx) = route_ctx(ConnectionId::new(78));
10568
10569        for (corr, body) in [
10570            (56, br#"{"route_channel":1}"#.as_slice()),
10571            (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
10572            (
10573                58,
10574                br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
10575            ),
10576        ] {
10577            let request = Frame::build(
10578                FrameType::Request,
10579                control_flags(),
10580                0,
10581                0,
10582                corr,
10583                body.to_vec(),
10584            )
10585            .unwrap();
10586            let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10587
10588            assert_eq!(response.len(), 1);
10589            assert_eq!(response[0].header.ty, FrameType::Error);
10590            assert_eq!(response[0].header.corr, corr);
10591            assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
10592        }
10593    }
10594
10595    #[tokio::test]
10596    async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
10597        let registry = Arc::new(Registry::default());
10598        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
10599        let router = Router::with_control_handler(Arc::clone(&control));
10600        let connection = router.begin_connection();
10601        let (ctx, mut rx) = route_ctx(connection.id());
10602
10603        router
10604            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
10605            .await
10606            .unwrap();
10607        let response = rx.recv().await.unwrap();
10608        let ack = parse_ack(&response);
10609        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
10610        let channel = 1;
10611
10612        let goodbye =
10613            Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
10614        router.route_for_connection(&ctx, goodbye).await.unwrap();
10615        assert!(rx.try_recv().is_err());
10616        assert!(registry.get_module("aft").unwrap().is_none());
10617
10618        router
10619            .route_for_connection(&ctx, channel_request(channel, 13))
10620            .await
10621            .unwrap();
10622        let error_frame = rx.recv().await.unwrap();
10623        assert_eq!(error_frame.header.ty, FrameType::Error);
10624        assert_eq!(error_frame.header.channel, channel);
10625    }
10626
10627    #[tokio::test]
10628    async fn dropping_router_connection_releases_registration() {
10629        let registry = Arc::new(Registry::default());
10630        let control = Arc::new(ControlHandler::new(Arc::clone(&registry)));
10631        let router = Router::with_control_handler(control);
10632        let connection = router.begin_connection();
10633        let (ctx, mut rx) = route_ctx(connection.id());
10634
10635        router
10636            .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
10637            .await
10638            .unwrap();
10639        let response = rx.recv().await.unwrap();
10640        let ack = parse_ack(&response);
10641        assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
10642        assert!(registry.get_module("aft").unwrap().is_some());
10643
10644        drop(connection);
10645
10646        assert!(registry.get_module("aft").unwrap().is_none());
10647        assert_eq!(registry.active_registration_count().unwrap(), 0);
10648    }
10649
10650    fn capability_manifest(
10651        module_id: &str,
10652        provides: &[&str],
10653        must_never_reach: &[&str],
10654    ) -> ModuleManifest {
10655        let mut manifest = manifest(module_id, PROTOCOL_VERSION);
10656        manifest.capabilities = Some(CapabilityDeclarations {
10657            provides: provides
10658                .iter()
10659                .map(|capability| (*capability).to_string())
10660                .collect(),
10661            requires: Vec::new(),
10662            must_never_reach: must_never_reach
10663                .iter()
10664                .map(|capability| (*capability).to_string())
10665                .collect(),
10666        });
10667        manifest
10668    }
10669
10670    fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
10671        Frame::build(
10672            FrameType::Hello,
10673            control_flags(),
10674            0,
10675            0,
10676            corr,
10677            serde_json::to_vec(&ModuleHelloBody {
10678                protocol_ver: manifest.protocol_ver,
10679                manifest,
10680                control_ops: None,
10681                launch_nonce: None,
10682            })
10683            .expect("capability test HELLO serializes"),
10684        )
10685        .expect("capability test HELLO frame builds")
10686    }
10687
10688    fn catalog_update_with_capabilities_frame(
10689        corr: u64,
10690        capabilities: CapabilityDeclarations,
10691    ) -> Frame {
10692        Frame::build(
10693            FrameType::Request,
10694            control_flags(),
10695            0,
10696            0,
10697            corr,
10698            serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
10699                provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
10700                capabilities: Some(capabilities),
10701                ready: None,
10702            })
10703            .expect("capability catalog.update serializes"),
10704        )
10705        .expect("capability catalog.update frame builds")
10706    }
10707
10708    async fn register_capability_manifest(
10709        handler: &ControlHandler,
10710        ctx: &RouteCtx,
10711        rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10712        manifest: ModuleManifest,
10713        corr: u64,
10714    ) {
10715        hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
10716    }
10717
10718    async fn open_route_for_capability_test(
10719        handler: &ControlHandler,
10720        target_ctx: &RouteCtx,
10721        target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10722        client_connection_id: u64,
10723        corr: u64,
10724        target_module_id: &str,
10725        consumer_identity: Option<ConsumerIdentity>,
10726    ) -> (
10727        mpsc::Receiver<crate::router::OutboundFrame>,
10728        ModuleControlRequest,
10729    ) {
10730        let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
10731        let route_handler = handler.clone();
10732        let target_module_id = target_module_id.to_string();
10733        let route_task = tokio::spawn(async move {
10734            route_handler
10735                .handle_control_frame(
10736                    &client_ctx,
10737                    route_open_frame_with_admission_facts(
10738                        corr,
10739                        &target_module_id,
10740                        unique_project_root("admission-facts"),
10741                        consumer_identity,
10742                        None,
10743                    ),
10744                )
10745                .await
10746                .expect("capability test route.open succeeds")
10747        });
10748        let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
10749            .await
10750            .expect("capability test route.open must reach route.bind")
10751            .expect("target control receiver stays open");
10752        let bind_request: ModuleControlRequest =
10753            serde_json::from_slice(&bind.body).expect("route.bind decodes");
10754        handler
10755            .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
10756            .await
10757            .expect("capability test route.bind ACK succeeds");
10758        assert!(route_task.await.expect("route.open task joins").is_empty());
10759        let opened = client_rx
10760            .recv()
10761            .await
10762            .expect("successful route.open publishes a response");
10763        assert!(matches!(
10764            serde_json::from_slice::<ClientControlResponse>(&opened.body),
10765            Ok(ClientControlResponse::RouteOpen { .. })
10766        ));
10767        (client_rx, bind_request)
10768    }
10769
10770    fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
10771        assert_eq!(frame.header.ty, FrameType::Push);
10772        assert_eq!(frame.header.channel, 0);
10773        let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
10774            .expect("route.closed control push decodes");
10775        let ClientControlPush::RouteClosed { channels, .. } = &push else {
10776            panic!("expected route.closed");
10777        };
10778        assert_eq!(channels.len(), 1, "exactly one violating route closed");
10779        let channels = channels.clone();
10780        assert_eq!(
10781            push,
10782            ClientControlPush::RouteClosed {
10783                module_id: target_module_id.to_string(),
10784                channels,
10785                reason: RouteCloseReason::CapabilityDenied,
10786                drained: false,
10787                abandoned: 0,
10788                excluded_subscriptions: 0,
10789                terminal: Some(false),
10790            }
10791        );
10792    }
10793
10794    #[tokio::test]
10795    async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
10796        let registry = Arc::new(Registry::default());
10797        let forwarding = Arc::new(ForwardingTable::default());
10798        let supervisor = SupervisorHandle::new();
10799        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10800        let handler =
10801            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10802                .with_supervisor(supervisor);
10803        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
10804        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
10805        register_capability_manifest(
10806            &handler,
10807            &target_ctx,
10808            &mut target_rx,
10809            capability_manifest("target", &["credentials-provider/v1"], &[]),
10810            1,
10811        )
10812        .await;
10813        register_capability_manifest(
10814            &handler,
10815            &opener_ctx,
10816            &mut opener_rx,
10817            capability_manifest("opener", &[], &["credentials-provider/v1"]),
10818            2,
10819        )
10820        .await;
10821
10822        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
10823        let replies = handler
10824            .handle_control_frame(
10825                &client_ctx,
10826                route_open_frame_with_admission_facts(
10827                    3,
10828                    "target",
10829                    unique_project_root("admission-facts"),
10830                    Some(ConsumerIdentity {
10831                        module_id: "opener".to_string(),
10832                        launch_nonce: "opener-nonce".to_string(),
10833                    }),
10834                    None,
10835                ),
10836            )
10837            .await
10838            .expect("denied route.open returns a typed frame");
10839        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
10840        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10841        assert!(
10842            target_rx.try_recv().is_err(),
10843            "forbidden route.open must not relay route.bind"
10844        );
10845    }
10846
10847    #[tokio::test]
10848    async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
10849        let registry = Arc::new(Registry::default());
10850        let forwarding = Arc::new(ForwardingTable::default());
10851        let supervisor = SupervisorHandle::new();
10852        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10853        let handler =
10854            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10855                .with_supervisor(supervisor);
10856        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
10857        let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
10858        register_capability_manifest(
10859            &handler,
10860            &target_ctx,
10861            &mut target_rx,
10862            capability_manifest("target", &["credentials-provider/v1"], &[]),
10863            1,
10864        )
10865        .await;
10866        register_capability_manifest(
10867            &handler,
10868            &old_opener_ctx,
10869            &mut old_opener_rx,
10870            capability_manifest("opener", &[], &[]),
10871            2,
10872        )
10873        .await;
10874        let (mut client_rx, _) = open_route_for_capability_test(
10875            &handler,
10876            &target_ctx,
10877            &mut target_rx,
10878            712,
10879            3,
10880            "target",
10881            Some(ConsumerIdentity {
10882                module_id: "opener".to_string(),
10883                launch_nonce: "opener-nonce".to_string(),
10884            }),
10885        )
10886        .await;
10887        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10888
10889        handler
10890            .cleanup_connection(old_opener_ctx.connection_id)
10891            .expect("old opener registration cleans up");
10892        let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
10893        register_capability_manifest(
10894            &handler,
10895            &new_opener_ctx,
10896            &mut new_opener_rx,
10897            capability_manifest("opener", &[], &["credentials-provider/v1"]),
10898            4,
10899        )
10900        .await;
10901
10902        assert_capability_denied_push(
10903            client_rx
10904                .try_recv()
10905                .expect("HELLO deny addition must emit route.closed")
10906                .frame,
10907            "target",
10908        );
10909        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10910        assert!(matches!(
10911            target_rx.try_recv(),
10912            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
10913        ));
10914    }
10915
10916    #[tokio::test]
10917    async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
10918        let registry = Arc::new(Registry::default());
10919        let forwarding = Arc::new(ForwardingTable::default());
10920        let supervisor = SupervisorHandle::new();
10921        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10922        let handler =
10923            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10924                .with_supervisor(supervisor);
10925        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
10926        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
10927        register_capability_manifest(
10928            &handler,
10929            &target_ctx,
10930            &mut target_rx,
10931            capability_manifest("target", &[], &[]),
10932            1,
10933        )
10934        .await;
10935        register_capability_manifest(
10936            &handler,
10937            &opener_ctx,
10938            &mut opener_rx,
10939            capability_manifest("opener", &[], &["credentials-provider/v1"]),
10940            2,
10941        )
10942        .await;
10943        let (mut client_rx, _) = open_route_for_capability_test(
10944            &handler,
10945            &target_ctx,
10946            &mut target_rx,
10947            722,
10948            3,
10949            "target",
10950            Some(ConsumerIdentity {
10951                module_id: "opener".to_string(),
10952                launch_nonce: "opener-nonce".to_string(),
10953            }),
10954        )
10955        .await;
10956        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10957
10958        let replies = handler
10959            .handle_control_frame(
10960                &target_ctx,
10961                catalog_update_with_capabilities_frame(
10962                    4,
10963                    CapabilityDeclarations {
10964                        provides: vec!["credentials-provider/v1".to_string()],
10965                        requires: Vec::new(),
10966                        must_never_reach: Vec::new(),
10967                    },
10968                ),
10969            )
10970            .await
10971            .expect("claim catalog.update succeeds");
10972        assert!(matches!(
10973            serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
10974            Ok(ModuleControlResponseToModule::CatalogUpdate {})
10975        ));
10976        assert_capability_denied_push(
10977            client_rx
10978                .try_recv()
10979                .expect("claim addition must emit route.closed")
10980                .frame,
10981            "target",
10982        );
10983        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10984        assert!(matches!(
10985            target_rx.try_recv(),
10986            Ok(outbound) if outbound.header.ty == FrameType::Goodbye
10987        ));
10988    }
10989
10990    #[tokio::test]
10991    async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
10992        let registry = Arc::new(Registry::default());
10993        let forwarding = Arc::new(ForwardingTable::default());
10994        let supervisor = SupervisorHandle::new();
10995        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10996        let handler =
10997            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
10998                .with_supervisor(supervisor);
10999        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
11000        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
11001        register_capability_manifest(
11002            &handler,
11003            &target_ctx,
11004            &mut target_rx,
11005            capability_manifest("target", &["credentials-provider/v1"], &[]),
11006            1,
11007        )
11008        .await;
11009        register_capability_manifest(
11010            &handler,
11011            &opener_ctx,
11012            &mut opener_rx,
11013            capability_manifest("opener", &[], &[]),
11014            2,
11015        )
11016        .await;
11017        let (mut client_rx, _) = open_route_for_capability_test(
11018            &handler,
11019            &target_ctx,
11020            &mut target_rx,
11021            732,
11022            3,
11023            "target",
11024            Some(ConsumerIdentity {
11025                module_id: "opener".to_string(),
11026                launch_nonce: "opener-nonce".to_string(),
11027            }),
11028        )
11029        .await;
11030        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11031
11032        handler
11033            .handle_control_frame(
11034                &target_ctx,
11035                catalog_update_with_capabilities_frame(
11036                    4,
11037                    CapabilityDeclarations {
11038                        provides: Vec::new(),
11039                        requires: Vec::new(),
11040                        must_never_reach: Vec::new(),
11041                    },
11042                ),
11043            )
11044            .await
11045            .expect("claim removal catalog.update succeeds");
11046        assert_eq!(
11047            forwarding.active_binding_count().unwrap(),
11048            1,
11049            "removing an attested target claim must leave the route census unchanged"
11050        );
11051        assert!(
11052            client_rx.try_recv().is_err(),
11053            "claim removal must not emit route.closed capability_denied"
11054        );
11055        assert!(
11056            target_rx.try_recv().is_err(),
11057            "claim removal must not send the target a route GOODBYE"
11058        );
11059    }
11060
11061    /// A direct client may open a route to a denied capability provider; this
11062    /// policy applies only to attested supervised module origins, not to direct clients.
11063    #[tokio::test]
11064    async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
11065        let registry = Arc::new(Registry::default());
11066        let forwarding = Arc::new(ForwardingTable::default());
11067        let supervisor = SupervisorHandle::new();
11068        supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11069        let handler =
11070            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11071                .with_supervisor(supervisor);
11072        let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
11073        let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
11074        register_capability_manifest(
11075            &handler,
11076            &target_ctx,
11077            &mut target_rx,
11078            capability_manifest("target", &["credentials-provider/v1"], &[]),
11079            1,
11080        )
11081        .await;
11082        register_capability_manifest(
11083            &handler,
11084            &opener_ctx,
11085            &mut opener_rx,
11086            capability_manifest("opener", &[], &["credentials-provider/v1"]),
11087            2,
11088        )
11089        .await;
11090
11091        let (_client_rx, bind) = open_route_for_capability_test(
11092            &handler,
11093            &target_ctx,
11094            &mut target_rx,
11095            742,
11096            3,
11097            "target",
11098            None,
11099        )
11100        .await;
11101        let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11102            panic!("direct scope-honesty route must bind");
11103        };
11104        assert_eq!(principal, Some(Principal::Direct));
11105        assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11106    }
11107
11108    /// A module that denies a capability receives no self-route exemption when it
11109    /// also attestedly provides that capability.
11110    #[tokio::test]
11111    async fn must_never_reach_self_route_is_capability_forbidden() {
11112        let registry = Arc::new(Registry::default());
11113        let forwarding = Arc::new(ForwardingTable::default());
11114        let supervisor = SupervisorHandle::new();
11115        supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
11116        let handler =
11117            ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
11118                .with_supervisor(supervisor);
11119        let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
11120        register_capability_manifest(
11121            &handler,
11122            &self_ctx,
11123            &mut self_rx,
11124            capability_manifest(
11125                "self-provider",
11126                &["credentials-provider/v1"],
11127                &["credentials-provider/v1"],
11128            ),
11129            1,
11130        )
11131        .await;
11132
11133        let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
11134        let replies = handler
11135            .handle_control_frame(
11136                &client_ctx,
11137                route_open_frame_with_admission_facts(
11138                    2,
11139                    "self-provider",
11140                    unique_project_root("admission-facts"),
11141                    Some(ConsumerIdentity {
11142                        module_id: "self-provider".to_string(),
11143                        launch_nonce: "self-nonce".to_string(),
11144                    }),
11145                    None,
11146                ),
11147            )
11148            .await
11149            .expect("self-route refusal returns a typed frame");
11150        assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
11151        assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11152        assert!(
11153            self_rx.try_recv().is_err(),
11154            "self denial must not relay route.bind"
11155        );
11156    }
11157
11158    #[test]
11159    fn unsupported_channel_zero_frame_returns_error() {
11160        let handler = ControlHandler::default();
11161        let request = Frame::build(
11162            FrameType::Request,
11163            control_flags(),
11164            0,
11165            0,
11166            21,
11167            b"opaque".to_vec(),
11168        )
11169        .unwrap();
11170
11171        let response = handler
11172            .handle_control(ConnectionId::new(1), request)
11173            .unwrap();
11174
11175        assert_eq!(response[0].header.ty, FrameType::Error);
11176        assert_eq!(
11177            parse_error(&response[0])["code"],
11178            "unsupported_control_frame"
11179        );
11180    }
11181
11182    /// Blue/green swap at the control-plane boundary. The supervisor that opens
11183    /// a swap is not wired yet, so the candidate is registered here directly
11184    /// into the registry and forwarding candidate slots, the way the swap's
11185    /// HELLO admission will.
11186    mod swap {
11187        use super::*;
11188
11189        const INCUMBENT: ConnectionId = ConnectionId::new(30);
11190        const CANDIDATE: ConnectionId = ConnectionId::new(40);
11191
11192        struct Swap {
11193            registry: Arc<Registry>,
11194            forwarding: Arc<ForwardingTable>,
11195            handler: ControlHandler,
11196            incumbent_ctx: RouteCtx,
11197            incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
11198            candidate_ctx: RouteCtx,
11199            candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
11200        }
11201
11202        async fn swap_with_incumbent() -> Swap {
11203            let registry = Arc::new(Registry::default());
11204            let forwarding = Arc::new(ForwardingTable::default());
11205            let handler =
11206                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding));
11207            let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
11208            hello_via_sink(
11209                &handler,
11210                &incumbent_ctx,
11211                &mut incumbent_rx,
11212                hello_frame("aft", PROTOCOL_VERSION, 7),
11213            )
11214            .await;
11215            let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
11216            Swap {
11217                registry,
11218                forwarding,
11219                handler,
11220                incumbent_ctx,
11221                incumbent_rx,
11222                candidate_ctx,
11223                candidate_rx,
11224            }
11225        }
11226
11227        fn register_candidate(swap: &Swap, ready: Option<bool>) {
11228            let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
11229            candidate_manifest.ready = ready;
11230            let registration = swap
11231                .registry
11232                .register_candidate_with_control_ops(
11233                    candidate_manifest,
11234                    PROTOCOL_VERSION,
11235                    CANDIDATE,
11236                    module_baseline_control_ops(),
11237                )
11238                .unwrap();
11239            swap.forwarding
11240                .register_candidate_module_connection(
11241                    CANDIDATE,
11242                    "aft".to_string(),
11243                    PROTOCOL_VERSION,
11244                    manifest_concurrency(&registration.manifest),
11245                    swap.candidate_ctx.egress.clone(),
11246                )
11247                .unwrap();
11248        }
11249
11250        fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
11251            let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
11252            swap.registry.promote_candidate("aft").unwrap().unwrap();
11253            cutover.incumbent.unwrap()
11254        }
11255
11256        fn keyed_total(counters: &Value, key: &str) -> u64 {
11257            counters[key]
11258                .as_object()
11259                .map(|counts| counts.values().filter_map(Value::as_u64).sum())
11260                .unwrap_or(0)
11261        }
11262
11263        /// An ack from the incumbent for a bind it was sent before cutover,
11264        /// arriving before the incumbent is drained. The incumbent is the live
11265        /// connection carrying every other client's routes, so the ack must
11266        /// not end it: the waiting client is told to retry, the reservation is
11267        /// given back, and the incumbent is told to drop just that binding.
11268        #[tokio::test]
11269        async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
11270            let mut swap = swap_with_incumbent().await;
11271            let handler = swap.handler.clone();
11272
11273            // A co-tenant route, bound on the incumbent before the swap.
11274            let cotenant = ConnectionId::new(31);
11275            let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
11276            let (cotenant_task, cotenant_bind) = relay_route_open(
11277                &handler,
11278                cotenant,
11279                &cotenant_ctx.egress,
11280                &mut swap.incumbent_rx,
11281                100,
11282                "aft",
11283                "swap-cotenant",
11284            )
11285            .await;
11286            handler
11287                .handle_control_frame(
11288                    &swap.incumbent_ctx,
11289                    route_bind_ack(cotenant_bind.header.corr),
11290                )
11291                .await
11292                .unwrap();
11293            assert!(cotenant_task.await.unwrap().is_empty());
11294            let (cotenant_channel, cotenant_epoch) =
11295                published_route(&cotenant_rx.recv().await.unwrap());
11296
11297            // A second route.open, relayed to the incumbent and not yet acked.
11298            let caller = ConnectionId::new(32);
11299            let (caller_ctx, mut caller_rx) = route_ctx(caller);
11300            let (caller_task, caller_bind) = relay_route_open(
11301                &handler,
11302                caller,
11303                &caller_ctx.egress,
11304                &mut swap.incumbent_rx,
11305                101,
11306                "aft",
11307                "swap-caller",
11308            )
11309            .await;
11310            let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
11311
11312            register_candidate(&swap, None);
11313            cutover(&swap);
11314
11315            // The incumbent acks after promotion and before any drain.
11316            let ack = handler
11317                .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
11318                .await;
11319            let module_loop_error = ack.as_ref().err().map(ToString::to_string);
11320            if module_loop_error.is_some() {
11321                // What the connection loop does with an untranslated router
11322                // error: end the connection, releasing every route on it.
11323                handler.cleanup_connection(INCUMBENT).unwrap();
11324            }
11325
11326            // 1. The incumbent's other routes survive.
11327            assert!(
11328                cotenant_rx.try_recv().is_err(),
11329                "the co-tenant route on the incumbent was torn down by one late ack: \
11330                 {module_loop_error:?}"
11331            );
11332            assert!(matches!(
11333                swap.forwarding
11334                    .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
11335                    .unwrap(),
11336                DataRoute::Client(DataRouteState::Bound(_))
11337            ));
11338            assert_eq!(module_loop_error, None);
11339            assert!(swap
11340                .registry
11341                .get_module_by_connection(INCUMBENT)
11342                .unwrap()
11343                .is_some());
11344
11345            // 2. Exactly one channel-scoped GOODBYE to the incumbent.
11346            let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
11347                .await
11348                .expect("the incumbent is told to drop the abandoned binding")
11349                .unwrap()
11350                .frame;
11351            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11352            assert_eq!(goodbye.header.channel, abandoned_channel);
11353            assert_eq!(goodbye.header.epoch, abandoned_epoch);
11354            assert!(swap.incumbent_rx.try_recv().is_err());
11355
11356            // 3. The waiting client gets a retryable refusal and no route.
11357            let response = caller_task.await.unwrap();
11358            assert_eq!(response.len(), 1);
11359            assert_eq!(parse_error(&response[0])["code"], "module_reloading");
11360            assert!(caller_rx.try_recv().is_err());
11361
11362            // 4. The reservation pair is given back, and the pending bind
11363            //    settled exactly once: one accepted open (the co-tenant) and one
11364            //    refused open (the caller), nothing counted twice.
11365            assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
11366            let counters = handler.counters().snapshot();
11367            assert_eq!(
11368                keyed_total(&counters, "route_open_accepted_by_principal"),
11369                1
11370            );
11371            assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
11372            assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
11373        }
11374
11375        /// After cutover the incumbent is drained BY ENDPOINT. Draining by module
11376        /// id would resolve to the promoted candidate and every new route.open
11377        /// would be refused as reloading, leaving neither process routable.
11378        #[tokio::test]
11379        async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
11380            let mut swap = swap_with_incumbent().await;
11381            register_candidate(&swap, None);
11382            let incumbent = cutover(&swap);
11383            swap.forwarding
11384                .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
11385                .unwrap()
11386                .expect("the incumbent is still registered");
11387
11388            let client = ConnectionId::new(33);
11389            let (client_ctx, mut client_rx) = route_ctx(client);
11390            let route_handler = swap.handler.clone();
11391            let open_ctx = RouteCtx {
11392                connection_id: client,
11393                egress: client_ctx.egress.clone(),
11394            };
11395            let mut route_task = tokio::spawn(async move {
11396                route_handler
11397                    .handle_control_frame(
11398                        &open_ctx,
11399                        route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
11400                    )
11401                    .await
11402                    .unwrap()
11403            });
11404            let bind = tokio::select! {
11405                bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
11406                response = &mut route_task => {
11407                    let response = response.unwrap();
11408                    panic!(
11409                        "post-cutover route.open was refused instead of relayed to the candidate: {}",
11410                        parse_error(&response[0])["code"]
11411                    );
11412                }
11413            };
11414            swap.handler
11415                .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
11416                .await
11417                .unwrap();
11418            assert!(route_task.await.unwrap().is_empty());
11419            let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
11420            match swap
11421                .forwarding
11422                .lookup_data_route(client, channel, epoch)
11423                .unwrap()
11424            {
11425                DataRoute::Client(DataRouteState::Bound(route)) => {
11426                    assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
11427                }
11428                other => panic!("expected a bound route on the candidate, got {other:?}"),
11429            }
11430            assert!(swap.incumbent_rx.try_recv().is_err());
11431        }
11432
11433        /// A candidate declares itself ready with `catalog.update` on its own
11434        /// connection. If the connection-keyed registry lookups searched only the
11435        /// active slot, this would answer `not_registered` and the candidate
11436        /// would never become ready.
11437        #[tokio::test]
11438        async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
11439            let swap = swap_with_incumbent().await;
11440            register_candidate(&swap, Some(false));
11441            let update = Frame::build(
11442                FrameType::Request,
11443                control_flags(),
11444                0,
11445                0,
11446                55,
11447                serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
11448                    provides: manifest("aft", PROTOCOL_VERSION).provides,
11449                    capabilities: None,
11450                    ready: Some(true),
11451                })
11452                .unwrap(),
11453            )
11454            .unwrap();
11455
11456            let replies = swap
11457                .handler
11458                .handle_control_frame(&swap.candidate_ctx, update)
11459                .await
11460                .unwrap();
11461
11462            assert_eq!(replies.len(), 1);
11463            assert_eq!(
11464                replies[0].header.ty,
11465                FrameType::Response,
11466                "candidate catalog.update was refused: {:?}",
11467                serde_json::from_slice::<Value>(&replies[0].body).ok()
11468            );
11469            assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
11470            assert_eq!(
11471                swap.registry
11472                    .get_module("aft")
11473                    .unwrap()
11474                    .unwrap()
11475                    .connection_id,
11476                INCUMBENT
11477            );
11478        }
11479    }
11480
11481    /// The HELLO gate while the supervisor has a swap open: only the nonce it
11482    /// minted for the candidate admits a second process, into the candidate
11483    /// slot, and that check runs ahead of the reserved-module gate.
11484    mod swap_admission {
11485        use super::*;
11486
11487        const INCUMBENT_NONCE: &str = "incumbent-nonce";
11488        const CANDIDATE_NONCE: &str = "candidate-nonce";
11489
11490        fn handler_with_incumbent(
11491            module_id: &str,
11492            reserved: bool,
11493        ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
11494            let registry = Arc::new(Registry::default());
11495            let supervisor = SupervisorHandle::new();
11496            supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
11497            if reserved {
11498                supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
11499            }
11500            let handler =
11501                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor.clone());
11502            let incumbent = handler
11503                .handle_control(
11504                    ConnectionId::new(1),
11505                    hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
11506                )
11507                .unwrap();
11508            assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
11509            supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
11510            (registry, supervisor, handler)
11511        }
11512
11513        /// Design mutation arm (ii). On an UNRESERVED id the reserved gate
11514        /// admits every nonce, so while a swap is open the swap gate is the only
11515        /// thing between a key-holder and the candidate slot. A nonce the
11516        /// supervisor did not mint, or none at all, is refused, and neither the
11517        /// incumbent's registration nor the candidate slot moves.
11518        #[test]
11519        fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
11520            let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
11521
11522            for (connection, nonce) in [(2, Some("forged")), (3, None)] {
11523                let replies = handler
11524                    .handle_control(
11525                        ConnectionId::new(connection),
11526                        hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
11527                    )
11528                    .unwrap();
11529                assert_eq!(replies[0].header.ty, FrameType::Error);
11530                assert_eq!(
11531                    parse_error(&replies[0])["code"],
11532                    "swap_token_invalid",
11533                    "nonce {nonce:?}"
11534                );
11535            }
11536            assert!(registry.get_candidate("aft").unwrap().is_none());
11537            assert_eq!(
11538                registry.get_module("aft").unwrap().unwrap().connection_id,
11539                ConnectionId::new(1)
11540            );
11541
11542            // Control: the minted token is admitted, into the candidate slot,
11543            // and only once.
11544            let admitted = handler
11545                .handle_control(
11546                    ConnectionId::new(4),
11547                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
11548                )
11549                .unwrap();
11550            assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
11551            assert_eq!(
11552                registry
11553                    .get_candidate("aft")
11554                    .unwrap()
11555                    .unwrap()
11556                    .connection_id,
11557                ConnectionId::new(4)
11558            );
11559            assert_eq!(
11560                registry.get_module("aft").unwrap().unwrap().connection_id,
11561                ConnectionId::new(1),
11562                "the candidate must not take the active slot"
11563            );
11564            let replayed = handler
11565                .handle_control(
11566                    ConnectionId::new(5),
11567                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
11568                )
11569                .unwrap();
11570            assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
11571
11572            // The case only this gate covers: the incumbent has died mid-swap,
11573            // so its duplicate refusal is gone too, and without the gate a
11574            // key-holder would take the id's ACTIVE slot.
11575            handler.cleanup_connection(ConnectionId::new(1)).unwrap();
11576            let squatter = handler
11577                .handle_control(
11578                    ConnectionId::new(6),
11579                    hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
11580                )
11581                .unwrap();
11582            assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
11583            assert!(
11584                registry.get_module("aft").unwrap().is_none(),
11585                "a squatter took the active slot of an id being swapped"
11586            );
11587        }
11588
11589        /// Design mutation arm (iii). A reserved module's candidate presents a
11590        /// nonce the reserved gate has never seen (that gate holds the
11591        /// incumbent's), so the swap gate must run first or the candidate is
11592        /// refused `reserved_module` and a reserved module can never be swapped.
11593        #[test]
11594        fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
11595            let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
11596
11597            let replies = handler
11598                .handle_control(
11599                    ConnectionId::new(2),
11600                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
11601                )
11602                .unwrap();
11603
11604            assert_eq!(
11605                replies[0].header.ty,
11606                FrameType::HelloAck,
11607                "reserved candidate refused: {:?}",
11608                serde_json::from_slice::<Value>(&replies[0].body).ok()
11609            );
11610            assert_eq!(
11611                registry
11612                    .get_candidate("vault")
11613                    .unwrap()
11614                    .unwrap()
11615                    .connection_id,
11616                ConnectionId::new(2)
11617            );
11618        }
11619
11620        /// With no swap open the gate is inert: the incumbent's reserved gate
11621        /// and duplicate refusal behave exactly as before.
11622        #[test]
11623        fn without_an_open_swap_the_ordinary_gates_decide() {
11624            let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
11625            supervisor.close_swap("vault");
11626
11627            let candidate = handler
11628                .handle_control(
11629                    ConnectionId::new(2),
11630                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
11631                )
11632                .unwrap();
11633            assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
11634            let duplicate = handler
11635                .handle_control(
11636                    ConnectionId::new(3),
11637                    hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
11638                )
11639                .unwrap();
11640            assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11641            assert!(registry.get_candidate("vault").unwrap().is_none());
11642        }
11643    }
11644
11645    /// `scope.sync` and `scope.describe` through the real control handler: who
11646    /// may sync is decided by the registration and launch nonce of the module
11647    /// connection, never by the request body.
11648    mod scopes {
11649        use subc_protocol::scope::{
11650            ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
11651            ScopeStatus,
11652        };
11653
11654        use super::*;
11655
11656        const OWNER: &str = "prefrontal-core";
11657
11658        fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
11659            ScopeRecord {
11660                scope_ref: scope_ref.to_string(),
11661                scope_epoch,
11662                kind: ScopeKind::Head,
11663                parent: None,
11664                child_owners: Vec::new(),
11665                carriers: Vec::new(),
11666                attributes: Default::default(),
11667            }
11668        }
11669
11670        async fn call(
11671            handler: &ControlHandler,
11672            ctx: &RouteCtx,
11673            request: &ModuleControlRequestFromModule,
11674        ) -> Frame {
11675            let body = serde_json::to_vec(request).unwrap();
11676            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
11677            let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
11678            assert_eq!(replies.len(), 1, "{replies:?}");
11679            replies.pop().unwrap()
11680        }
11681
11682        async fn sync(
11683            handler: &ControlHandler,
11684            ctx: &RouteCtx,
11685            generation: u64,
11686            scopes: Vec<ScopeRecord>,
11687        ) -> Result<ModuleControlResponseToModule, String> {
11688            let reply = call(
11689                handler,
11690                ctx,
11691                &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
11692            )
11693            .await;
11694            match reply.header.ty {
11695                FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
11696                _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
11697            }
11698        }
11699
11700        async fn describe(
11701            handler: &ControlHandler,
11702            ctx: &RouteCtx,
11703            owner: &str,
11704            scope_ref: &str,
11705        ) -> ModuleControlResponseToModule {
11706            let reply = call(
11707                handler,
11708                ctx,
11709                &ModuleControlRequestFromModule::ScopeDescribe {
11710                    owner: Principal::Reserved {
11711                        module_id: owner.to_string(),
11712                    },
11713                    scope_ref: scope_ref.to_string(),
11714                },
11715            )
11716            .await;
11717            assert_eq!(
11718                reply.header.ty,
11719                FrameType::Response,
11720                "{:?}",
11721                parse_error(&reply)
11722            );
11723            serde_json::from_slice(&reply.body).unwrap()
11724        }
11725
11726        /// Register `module_id` on `connection` with `nonce`, returning its ctx.
11727        async fn module(
11728            handler: &ControlHandler,
11729            connection: u64,
11730            module_id: &str,
11731            nonce: Option<&str>,
11732        ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
11733            let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
11734            hello_via_sink(
11735                handler,
11736                &ctx,
11737                &mut rx,
11738                hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
11739            )
11740            .await;
11741            (ctx, rx)
11742        }
11743
11744        /// `direct` and every other client connection has no registration, so
11745        /// it can neither sync nor own a scope.
11746        #[tokio::test]
11747        async fn a_client_connection_cannot_sync_or_describe() {
11748            let handler = ControlHandler::new(Arc::new(Registry::default()));
11749            let (ctx, _rx) = route_ctx(ConnectionId::new(9));
11750            for request in [
11751                ModuleControlRequestFromModule::ScopeSync {
11752                    generation: 1,
11753                    scopes: vec![head("s", 1)],
11754                },
11755                ModuleControlRequestFromModule::ScopeDescribe {
11756                    owner: Principal::Direct,
11757                    scope_ref: "s".to_string(),
11758                },
11759            ] {
11760                let reply = call(&handler, &ctx, &request).await;
11761                assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
11762            }
11763            assert!(
11764                !handler
11765                    .scopes
11766                    .read()
11767                    .unwrap()
11768                    .describe(
11769                        &Principal::Reserved {
11770                            module_id: OWNER.to_string()
11771                        },
11772                        "s"
11773                    )
11774                    .owner_synced
11775            );
11776        }
11777
11778        /// A module the supervisor did not spawn registers without a launch
11779        /// nonce, so it is never an owner's current launch.
11780        #[tokio::test]
11781        async fn a_module_without_a_supervised_launch_cannot_sync() {
11782            let handler = ControlHandler::new(Arc::new(Registry::default()));
11783            let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
11784            assert_eq!(
11785                sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
11786                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
11787            );
11788        }
11789
11790        #[tokio::test]
11791        async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
11792            let supervisor = SupervisorHandle::new();
11793            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
11794            let handler = ControlHandler::new(Arc::new(Registry::default()))
11795                .with_supervisor(supervisor.clone());
11796            let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
11797            sync(&handler, &incumbent, 1, vec![head("s", 1)])
11798                .await
11799                .expect("the current launch syncs");
11800
11801            // A swap candidate registers with the swap token and is refused
11802            // while the incumbent keeps syncing.
11803            supervisor.open_swap(OWNER, "n2".to_string());
11804            let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
11805            assert_eq!(
11806                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
11807                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
11808            );
11809            sync(&handler, &incumbent, 2, vec![head("s", 1)])
11810                .await
11811                .expect("the serving owner syncs during the swap");
11812
11813            // The swap fails and is rolled back. The candidate never held sync
11814            // authority, and still cannot sync.
11815            supervisor.close_swap(OWNER);
11816            assert_eq!(
11817                sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
11818                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
11819            );
11820            sync(&handler, &incumbent, 3, vec![head("s", 1)])
11821                .await
11822                .expect("the serving owner syncs after the rollback");
11823            handler.cleanup_connection(candidate.connection_id).unwrap();
11824
11825            // A swap that cuts over. Promotion records the candidate's nonce as
11826            // the module's spawn nonce, which is what `set_spawn_nonce` does
11827            // here; the promoted connection then takes authority at any
11828            // generation and the superseded incumbent is refused.
11829            supervisor.open_swap(OWNER, "n3".to_string());
11830            let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
11831            supervisor.set_spawn_nonce(OWNER, "n3".to_string());
11832            let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
11833                .await
11834                .expect("the promoted launch takes authority");
11835            let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
11836                panic!("unexpected reply {reply:?}");
11837            };
11838            assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
11839            assert_eq!(
11840                sync(&handler, &incumbent, 4, Vec::new()).await,
11841                Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
11842            );
11843        }
11844
11845        /// Authority dies with its connection: the cleanup path releases it,
11846        /// so the owner's next connection takes it at any generation.
11847        #[tokio::test]
11848        async fn closing_the_authority_connection_frees_sync_authority() {
11849            let supervisor = SupervisorHandle::new();
11850            supervisor.set_spawn_nonce(OWNER, "n1".to_string());
11851            let handler = ControlHandler::new(Arc::new(Registry::default()))
11852                .with_supervisor(supervisor.clone());
11853            let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
11854            sync(&handler, &first, 10, vec![head("s", 1)])
11855                .await
11856                .unwrap();
11857            handler.cleanup_connection(first.connection_id).unwrap();
11858
11859            let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
11860            sync(&handler, &second, 1, vec![head("s", 1)])
11861                .await
11862                .expect("the next connection takes the released authority");
11863        }
11864
11865        #[tokio::test]
11866        async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
11867            let registry = Arc::new(Registry::default());
11868            let supervisor_handle = SupervisorHandle::new();
11869            let supervisor = Supervisor::new(Arc::clone(&registry), RestartPolicy::default())
11870                .with_handle(supervisor_handle.clone())
11871                .with_daemon_incarnation("incarnation-7".to_string());
11872            // Configured with enabled: false, so the supervisor lists the
11873            // module without spawning a process for it.
11874            supervisor
11875                .supervise_configured(
11876                    ModuleSpec {
11877                        module_id: OWNER.to_string(),
11878                        program: PathBuf::from("/nonexistent/prefrontal-core"),
11879                        args: Vec::new(),
11880                        env: Vec::new(),
11881                        reserved: false,
11882                        reserved_prefixes: Vec::new(),
11883                        protocol: ModuleProtocol::Subc,
11884                        overlap: Default::default(),
11885                    },
11886                    false,
11887                )
11888                .unwrap();
11889            supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
11890            let handler =
11891                ControlHandler::new(Arc::clone(&registry)).with_supervisor(supervisor_handle);
11892            let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
11893
11894            // Configured but not yet synced: a reader waits for the owner.
11895            let ModuleControlResponseToModule::ScopeDescribe {
11896                status,
11897                daemon_incarnation,
11898                owner_synced,
11899                owner_configured,
11900                scope,
11901                ..
11902            } = describe(&handler, &reader, OWNER, "s").await
11903            else {
11904                panic!("not a describe reply");
11905            };
11906            assert_eq!(status, ScopeStatus::NotLive);
11907            assert_eq!(daemon_incarnation, "incarnation-7");
11908            assert!(!owner_synced);
11909            assert!(owner_configured);
11910            assert!(scope.is_none());
11911
11912            // Not a supervised module: the owner will never sync, and a reader
11913            // refuses rather than waits.
11914            let ModuleControlResponseToModule::ScopeDescribe {
11915                status,
11916                owner_configured,
11917                ..
11918            } = describe(&handler, &reader, "ghost", "s").await
11919            else {
11920                panic!("not a describe reply");
11921            };
11922            assert_eq!(status, ScopeStatus::NotLive);
11923            assert!(!owner_configured);
11924
11925            // Live, with the stamp fields and the computed owner_authorized.
11926            let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
11927            sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
11928            let ModuleControlResponseToModule::ScopeDescribe {
11929                status,
11930                scope_epoch,
11931                owner_synced,
11932                scope,
11933                ..
11934            } = describe(&handler, &reader, OWNER, "s").await
11935            else {
11936                panic!("not a describe reply");
11937            };
11938            assert_eq!(status, ScopeStatus::Live);
11939            assert_eq!(scope_epoch, Some(4));
11940            assert!(owner_synced);
11941            let stamp = scope.expect("a live scope carries its stamp");
11942            assert!(
11943                stamp.owner_authorized,
11944                "prefrontal-core is the default authority"
11945            );
11946            assert_eq!(stamp.kind, ScopeKind::Head);
11947        }
11948
11949        #[tokio::test]
11950        async fn scope_authority_owners_decides_owner_authorized() {
11951            let supervisor = SupervisorHandle::new();
11952            supervisor.set_spawn_nonce("broca", "b1".to_string());
11953            let handler = ControlHandler::new(Arc::new(Registry::default()))
11954                .with_supervisor(supervisor)
11955                .with_scope_authority_owners(vec!["broca".to_string()]);
11956            let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
11957            let mut gated = head("s", 1);
11958            gated.attributes.agent_id = Some("agent".to_string());
11959            sync(&handler, &broca, 1, vec![gated]).await.unwrap();
11960            let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
11961                describe(&handler, &broca, "broca", "s").await
11962            else {
11963                panic!("not a describe reply");
11964            };
11965            assert!(scope.unwrap().owner_authorized);
11966        }
11967
11968        /// With route admission, the stamp, the commit re-check and drains in
11969        /// place, the feature is advertised: the module ops in HELLO_ACK, and
11970        /// `scopes/v1` in HELLO_ACK and `server.describe`.
11971        #[tokio::test]
11972        async fn scope_ops_and_the_scopes_capability_are_advertised() {
11973            let handler = ControlHandler::new(Arc::new(Registry::default()));
11974            let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
11975            let ack = hello_via_sink(
11976                &handler,
11977                &ctx,
11978                &mut rx,
11979                hello_frame("m", PROTOCOL_VERSION, 1),
11980            )
11981            .await;
11982            let ack = parse_ack(&ack);
11983            for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
11984                assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
11985            }
11986            assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
11987
11988            let (client, _client_rx) = route_ctx(ConnectionId::new(2));
11989            let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11990            let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11991            let reply = handler
11992                .handle_control_frame(&client, frame)
11993                .await
11994                .unwrap()
11995                .pop()
11996                .unwrap();
11997            let ClientControlResponse::ServerDescribe { capabilities, .. } =
11998                serde_json::from_slice(&reply.body).unwrap()
11999            else {
12000                panic!("not a server.describe reply");
12001            };
12002            assert!(
12003                capabilities.iter().any(|c| c == CAP_SCOPES_V1),
12004                "{capabilities:?}"
12005            );
12006        }
12007
12008        // ---- route admission, stamps, commit re-check and drains ----------
12009
12010        const PLEXUS: &str = "plexus";
12011        const OTHER: &str = "other";
12012        const AFT: &str = "aft";
12013        const BROCA: &str = "broca";
12014        const MAGIC: &str = "magic-context";
12015
12016        fn nonce(module_id: &str) -> String {
12017            format!("nonce-{module_id}")
12018        }
12019
12020        fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
12021            let (tx, rx) = mpsc::channel(64);
12022            (
12023                RouteCtx {
12024                    connection_id: ConnectionId::new(connection),
12025                    egress: FrameSink::new(tx),
12026                },
12027                rx,
12028            )
12029        }
12030
12031        /// A daemon with a configured owner (prefrontal-core) registered on its
12032        /// own module connection, two routable targets (plexus, other), and
12033        /// launch nonces minted for the modules that open routes as carriers.
12034        struct Rig {
12035            handler: ControlHandler,
12036            forwarding: Arc<ForwardingTable>,
12037            owner: RouteCtx,
12038            _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12039            modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
12040            generation: u64,
12041            next_connection: u64,
12042            _supervisor: Supervisor,
12043        }
12044
12045        async fn rig() -> Rig {
12046            let registry = Arc::new(Registry::default());
12047            let forwarding = Arc::new(ForwardingTable::default());
12048            let supervisor_handle = SupervisorHandle::new();
12049            let supervisor = Supervisor::new(Arc::clone(&registry), RestartPolicy::default())
12050                .with_handle(supervisor_handle.clone());
12051            supervisor
12052                .supervise_configured(
12053                    ModuleSpec {
12054                        module_id: OWNER.to_string(),
12055                        program: PathBuf::from("/nonexistent/prefrontal-core"),
12056                        args: Vec::new(),
12057                        env: Vec::new(),
12058                        reserved: false,
12059                        reserved_prefixes: Vec::new(),
12060                        protocol: ModuleProtocol::Subc,
12061                        overlap: Default::default(),
12062                    },
12063                    false,
12064                )
12065                .unwrap();
12066            for module_id in [OWNER, AFT, BROCA, MAGIC] {
12067                supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
12068            }
12069            let handler =
12070                ControlHandler::with_forwarding(Arc::clone(&registry), Arc::clone(&forwarding))
12071                    .with_supervisor(supervisor_handle);
12072            let (owner, mut owner_rx) = wide_ctx(1);
12073            hello_via_sink(
12074                &handler,
12075                &owner,
12076                &mut owner_rx,
12077                hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
12078            )
12079            .await;
12080            let mut modules = BTreeMap::new();
12081            for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
12082                let (ctx, mut rx) = wide_ctx(connection);
12083                hello_via_sink(
12084                    &handler,
12085                    &ctx,
12086                    &mut rx,
12087                    hello_frame(module_id, PROTOCOL_VERSION, connection),
12088                )
12089                .await;
12090                modules.insert(module_id.to_string(), (ctx, rx));
12091            }
12092            Rig {
12093                handler,
12094                forwarding,
12095                owner,
12096                _owner_rx: owner_rx,
12097                modules,
12098                generation: 0,
12099                next_connection: 100,
12100                _supervisor: supervisor,
12101            }
12102        }
12103
12104        fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
12105            ScopeCarrier {
12106                principal: Principal::Reserved {
12107                    module_id: module_id.to_string(),
12108                },
12109                targets: targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()),
12110            }
12111        }
12112
12113        /// The scope most tests open under: aft carries to any module, broca
12114        /// only to plexus and other, and the owner delegates as agent-1.
12115        fn session(scope_epoch: u64) -> ScopeRecord {
12116            let mut record = head("s", scope_epoch);
12117            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
12118            record.attributes.agent_id = Some("agent-1".to_string());
12119            record.attributes.delegates = true;
12120            record
12121        }
12122
12123        impl Rig {
12124            async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
12125                self.generation += 1;
12126                sync(&self.handler, &self.owner, self.generation, scopes)
12127                    .await
12128                    .expect("the owner's sync is accepted");
12129            }
12130
12131            fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
12132                ScopeSelector {
12133                    owner: Principal::Reserved {
12134                        module_id: OWNER.to_string(),
12135                    },
12136                    scope_ref: scope_ref.to_string(),
12137                    scope_epoch,
12138                }
12139            }
12140
12141            fn open_frame(
12142                &mut self,
12143                opener: Option<&str>,
12144                target: &str,
12145                scope: Option<ScopeSelector>,
12146            ) -> (
12147                RouteCtx,
12148                mpsc::Receiver<crate::router::OutboundFrame>,
12149                Frame,
12150            ) {
12151                self.next_connection += 1;
12152                let (ctx, rx) = wide_ctx(self.next_connection);
12153                let root = unique_project_root("scoped-open");
12154                let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
12155                    target: RouteTarget::ToolProvider {
12156                        module_id: target.to_string(),
12157                    },
12158                    identity: BindIdentity::new(
12159                        root.path().to_path_buf(),
12160                        "unit".to_string(),
12161                        "session".to_string(),
12162                    ),
12163                    consumer_identity: opener.map(|module_id| ConsumerIdentity {
12164                        module_id: module_id.to_string(),
12165                        launch_nonce: nonce(module_id),
12166                    }),
12167                    consumer_capabilities: None,
12168                    role_versions: None,
12169                    admission_facts: None,
12170                    scope,
12171                })
12172                .unwrap();
12173                let frame = Frame::build(
12174                    FrameType::Request,
12175                    control_flags(),
12176                    0,
12177                    0,
12178                    self.next_connection,
12179                    body,
12180                )
12181                .unwrap();
12182                (ctx, rx, frame)
12183            }
12184
12185            /// Open and expect a refusal before anything is relayed.
12186            async fn refused(
12187                &mut self,
12188                opener: Option<&str>,
12189                target: &str,
12190                scope: Option<ScopeSelector>,
12191            ) -> String {
12192                let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
12193                let replies = self
12194                    .handler
12195                    .handle_control_frame(&ctx, frame)
12196                    .await
12197                    .unwrap();
12198                assert_eq!(replies.len(), 1, "{replies:?}");
12199                assert_eq!(replies[0].header.ty, FrameType::Error);
12200                let (_, module_rx) = self.modules.get_mut(target).unwrap();
12201                assert!(
12202                    module_rx.try_recv().is_err(),
12203                    "a refused open relays nothing"
12204                );
12205                parse_error(&replies[0])["code"]
12206                    .as_str()
12207                    .unwrap()
12208                    .to_string()
12209            }
12210
12211            /// Start an open and return its task and the bind the target got.
12212            async fn relayed(
12213                &mut self,
12214                opener: Option<&str>,
12215                target: &str,
12216                scope: Option<ScopeSelector>,
12217            ) -> Relayed {
12218                let (ctx, rx, frame) = self.open_frame(opener, target, scope);
12219                let handler = self.handler.clone();
12220                let task_ctx = ctx.clone();
12221                let task = tokio::spawn(async move {
12222                    handler
12223                        .handle_control_frame(&task_ctx, frame)
12224                        .await
12225                        .unwrap()
12226                });
12227                let (_, module_rx) = self.modules.get_mut(target).unwrap();
12228                let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
12229                    .await
12230                    .expect("the target receives the relayed route.bind")
12231                    .unwrap()
12232                    .frame;
12233                Relayed {
12234                    target: target.to_string(),
12235                    client: ctx,
12236                    client_rx: rx,
12237                    task,
12238                    bind,
12239                }
12240            }
12241
12242            async fn ack(&self, relayed: &Relayed) {
12243                let (module, _) = &self.modules[&relayed.target];
12244                self.handler
12245                    .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
12246                    .await
12247                    .unwrap();
12248            }
12249
12250            /// Open, ack and return the bound route.
12251            async fn bound(
12252                &mut self,
12253                opener: Option<&str>,
12254                target: &str,
12255                scope: Option<ScopeSelector>,
12256            ) -> Bound {
12257                let relayed = self.relayed(opener, target, scope).await;
12258                self.ack(&relayed).await;
12259                let Relayed {
12260                    target,
12261                    client,
12262                    mut client_rx,
12263                    task,
12264                    bind,
12265                } = relayed;
12266                assert!(
12267                    task.await.unwrap().is_empty(),
12268                    "the open is answered by commit"
12269                );
12270                let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
12271                Bound {
12272                    target,
12273                    client,
12274                    client_rx,
12275                    channel,
12276                    epoch,
12277                    bind,
12278                }
12279            }
12280
12281            fn live(&self, route: &Bound) -> bool {
12282                matches!(
12283                    self.forwarding
12284                        .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
12285                        .unwrap(),
12286                    DataRoute::Client(DataRouteState::Bound(_))
12287                )
12288            }
12289        }
12290
12291        struct Relayed {
12292            target: String,
12293            client: RouteCtx,
12294            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12295            task: tokio::task::JoinHandle<Vec<Frame>>,
12296            bind: Frame,
12297        }
12298
12299        struct Bound {
12300            target: String,
12301            client: RouteCtx,
12302            client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12303            channel: u16,
12304            epoch: u32,
12305            bind: Frame,
12306        }
12307
12308        impl Bound {
12309            /// The reason of the `route.closed` this client was sent, after
12310            /// checking it also got a GOODBYE on exactly this route.
12311            fn closed_reason(&mut self) -> RouteCloseReason {
12312                let mut reason = None;
12313                let mut goodbye = false;
12314                while let Ok(outbound) = self.client_rx.try_recv() {
12315                    let frame = outbound.frame;
12316                    match frame.header.ty {
12317                        FrameType::Goodbye => {
12318                            assert_eq!(
12319                                (frame.header.channel, frame.header.epoch),
12320                                (self.channel, self.epoch)
12321                            );
12322                            goodbye = true;
12323                        }
12324                        FrameType::Push => {
12325                            let ClientControlPush::RouteClosed {
12326                                reason: r,
12327                                module_id,
12328                                ..
12329                            } = serde_json::from_slice(&frame.body).unwrap()
12330                            else {
12331                                panic!("unexpected push");
12332                            };
12333                            assert_eq!(module_id, self.target);
12334                            reason = Some(r);
12335                        }
12336                        other => panic!("unexpected frame {other:?}"),
12337                    }
12338                }
12339                assert!(goodbye, "the client is sent a GOODBYE for the closed route");
12340                reason.expect("the client is told why the route closed")
12341            }
12342
12343            fn untouched(&mut self) -> bool {
12344                self.client_rx.try_recv().is_err()
12345            }
12346
12347            fn stamp(&self) -> Option<ScopeStamp> {
12348                match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
12349                    ModuleControlRequest::RouteBind { scope, .. } => scope,
12350                    other => panic!("expected a route.bind, got {other:?}"),
12351                }
12352            }
12353        }
12354
12355        #[tokio::test]
12356        async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
12357        ) {
12358            let mut rig = rig().await;
12359            let mut record = session(1);
12360            record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
12361            record.child_owners = vec![Principal::Reserved {
12362                module_id: MAGIC.to_string(),
12363            }];
12364            rig.sync(vec![record]).await;
12365            let scope = || Some(rig_selector("s", Some(1)));
12366
12367            // Admitted: the owner, a bare carrier to any module, a targeted
12368            // carrier to its listed module.
12369            rig.bound(Some(OWNER), PLEXUS, scope()).await;
12370            rig.bound(Some(AFT), OTHER, scope()).await;
12371            rig.bound(Some(BROCA), PLEXUS, scope()).await;
12372
12373            // Refused scope_not_carrier: a targeted carrier to an unlisted
12374            // module, a module that is not listed at all (a child owner is not
12375            // a carrier), and a direct key-holder.
12376            for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
12377                assert_eq!(
12378                    rig.refused(opener, target, scope()).await,
12379                    error_codes::SCOPE_NOT_CARRIER,
12380                    "{opener:?} -> {target}"
12381                );
12382            }
12383        }
12384
12385        fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
12386            ScopeSelector {
12387                owner: Principal::Reserved {
12388                    module_id: OWNER.to_string(),
12389                },
12390                scope_ref: scope_ref.to_string(),
12391                scope_epoch,
12392            }
12393        }
12394
12395        #[tokio::test]
12396        async fn an_open_without_an_epoch_is_refused_the_owners_included() {
12397            let mut rig = rig().await;
12398            rig.sync(vec![session(1)]).await;
12399            for opener in [OWNER, AFT] {
12400                assert_eq!(
12401                    rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
12402                        .await,
12403                    error_codes::SCOPE_EPOCH_REQUIRED,
12404                    "{opener}"
12405                );
12406            }
12407        }
12408
12409        #[tokio::test]
12410        async fn admission_separates_not_synced_not_live_and_ended() {
12411            let mut rig = rig().await;
12412            // Before the configured owner's first sync: retryable.
12413            let code = rig
12414                .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12415                .await;
12416            assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
12417            assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
12418
12419            // An owner that is not configured will never sync: terminal.
12420            let ghost = ScopeSelector {
12421                owner: Principal::Reserved {
12422                    module_id: "ghost".to_string(),
12423                },
12424                scope_ref: "s".to_string(),
12425                scope_epoch: Some(1),
12426            };
12427            assert_eq!(
12428                rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
12429                error_codes::SCOPE_NOT_LIVE
12430            );
12431
12432            rig.sync(vec![session(2)]).await;
12433            assert_eq!(
12434                rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
12435                    .await,
12436                error_codes::SCOPE_NOT_LIVE
12437            );
12438            for epoch in [1, 3] {
12439                assert_eq!(
12440                    rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
12441                        .await,
12442                    error_codes::SCOPE_ENDED,
12443                    "epoch {epoch}"
12444                );
12445            }
12446            // Control: the live epoch is admitted.
12447            rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
12448                .await;
12449        }
12450
12451        #[tokio::test]
12452        async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
12453            let mut rig = rig().await;
12454            rig.sync(vec![session(1)]).await;
12455            let route = rig
12456                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12457                .await;
12458            let stamp = route.stamp().expect("a scoped bind carries the stamp");
12459            assert_eq!(stamp.scope_ref, "s");
12460            assert_eq!(stamp.scope_epoch, 1);
12461            assert_eq!(stamp.kind, ScopeKind::Head);
12462            assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
12463            assert!(stamp.attributes.delegates);
12464            assert!(stamp.owner_authorized);
12465            let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
12466            assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
12467
12468            // broca owns a scope of its own on its own module connection; it is
12469            // not in scope_authority_owners, so its stamp is not authorized.
12470            let (broca, mut broca_rx) = wide_ctx(50);
12471            hello_via_sink(
12472                &rig.handler,
12473                &broca,
12474                &mut broca_rx,
12475                hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
12476            )
12477            .await;
12478            sync(&rig.handler, &broca, 1, vec![head("b", 1)])
12479                .await
12480                .unwrap();
12481            let own = ScopeSelector {
12482                owner: Principal::Reserved {
12483                    module_id: BROCA.to_string(),
12484                },
12485                scope_ref: "b".to_string(),
12486                scope_epoch: Some(1),
12487            };
12488            let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
12489            assert!(!route.stamp().unwrap().owner_authorized);
12490        }
12491
12492        /// The owner's sync lands between admission and the module's ack. The
12493        /// open is refused by name, the module's other routes stay up, and the
12494        /// reserved pair is released. Changed content is retryable; an ended
12495        /// scope is not.
12496        #[tokio::test]
12497        async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
12498            let mut rig = rig().await;
12499            rig.sync(vec![session(1)]).await;
12500            let mut cotenant = rig
12501                .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
12502                .await;
12503
12504            let mut changed = session(1);
12505            changed.child_owners.push(Principal::Reserved {
12506                module_id: MAGIC.to_string(),
12507            });
12508            let mut ended = None;
12509            for (code, next) in [
12510                (error_codes::SCOPE_CHANGED, vec![changed]),
12511                (error_codes::SCOPE_ENDED, Vec::new()),
12512            ] {
12513                let relayed = rig
12514                    .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12515                    .await;
12516                let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
12517                ended = Some(next.is_empty());
12518                rig.sync(next).await;
12519                rig.ack(&relayed).await;
12520                let replies = relayed.task.await.unwrap();
12521                assert_eq!(replies.len(), 1, "{replies:?}");
12522                assert_eq!(parse_error(&replies[0])["code"], code);
12523                assert_eq!(
12524                    subc_protocol::error_codes::is_retryable_route_open(code),
12525                    code == error_codes::SCOPE_CHANGED
12526                );
12527                assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
12528                // The module is told to drop just the binding it created.
12529                let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
12530                // Collected, because ending the scope also closes the co-tenant
12531                // route, whose GOODBYE comes first.
12532                let mut goodbyes = Vec::new();
12533                while let Ok(outbound) = plexus_rx.try_recv() {
12534                    assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
12535                    goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
12536                }
12537                assert!(
12538                    goodbyes.contains(&(bind_channel, bind_epoch)),
12539                    "{goodbyes:?}"
12540                );
12541                assert!(rig
12542                    .handler
12543                    .registry
12544                    .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
12545                    .unwrap()
12546                    .is_some());
12547            }
12548            assert_eq!(ended, Some(true));
12549            // The co-tenant stayed up through the change, and closed only when
12550            // the scope ended, by the drain rule rather than by the commit.
12551            assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
12552        }
12553
12554        /// Each row of the drain table on one set of routes: the owner's, a
12555        /// bare carrier's, and a targeted carrier's to each of its targets.
12556        #[tokio::test]
12557        async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
12558            struct Case {
12559                name: &'static str,
12560                change: fn(&mut ScopeRecord),
12561                /// Closed routes by index: owner->plexus, aft->plexus,
12562                /// broca->plexus, broca->other.
12563                closed: [Option<RouteCloseReason>; 4],
12564            }
12565            use RouteCloseReason::*;
12566            let cases = [
12567                Case {
12568                    name: "a carrier entry removed",
12569                    change: |r| {
12570                        r.carriers.retain(|c| {
12571                            c.principal
12572                                != Principal::Reserved {
12573                                    module_id: AFT.to_string(),
12574                                }
12575                        })
12576                    },
12577                    closed: [None, Some(ScopeCarrierRemoved), None, None],
12578                },
12579                Case {
12580                    name: "a target removed from a carrier",
12581                    change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
12582                    closed: [None, None, None, Some(ScopeCarrierRemoved)],
12583                },
12584                Case {
12585                    name: "a bare carrier narrowed to targets",
12586                    change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
12587                    closed: [None, Some(ScopeCarrierRemoved), None, None],
12588                },
12589                Case {
12590                    name: "delegates turned off",
12591                    change: |r| r.attributes.delegates = false,
12592                    closed: [Some(ScopeDelegationChanged); 4],
12593                },
12594                Case {
12595                    name: "agent_id changed",
12596                    change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
12597                    closed: [Some(ScopeDelegationChanged); 4],
12598                },
12599                Case {
12600                    name: "a carrier added, child owners changed, the record re-sent",
12601                    change: |r| {
12602                        r.carriers.push(carrier(MAGIC, None));
12603                        r.child_owners.push(Principal::Reserved {
12604                            module_id: MAGIC.to_string(),
12605                        });
12606                    },
12607                    closed: [None; 4],
12608                },
12609                Case {
12610                    name: "a target added",
12611                    change: |r| {
12612                        r.carriers[1]
12613                            .targets
12614                            .as_mut()
12615                            .unwrap()
12616                            .push("third".to_string())
12617                    },
12618                    closed: [None; 4],
12619                },
12620                Case {
12621                    name: "delegates turned on",
12622                    change: |r| r.attributes.delegates = true,
12623                    closed: [None; 4],
12624                },
12625            ];
12626            for case in cases {
12627                let mut rig = rig().await;
12628                rig.sync(vec![session(1)]).await;
12629                let scope = || Some(rig_selector("s", Some(1)));
12630                let mut routes = [
12631                    rig.bound(Some(OWNER), PLEXUS, scope()).await,
12632                    rig.bound(Some(AFT), PLEXUS, scope()).await,
12633                    rig.bound(Some(BROCA), PLEXUS, scope()).await,
12634                    rig.bound(Some(BROCA), OTHER, scope()).await,
12635                ];
12636                let mut record = session(1);
12637                (case.change)(&mut record);
12638                rig.sync(vec![record]).await;
12639                for (index, expected) in case.closed.iter().enumerate() {
12640                    let route = &mut routes[index];
12641                    match expected {
12642                        Some(reason) => {
12643                            assert!(!rig.live(route), "{}: route {index} still live", case.name);
12644                            assert_eq!(
12645                                route.closed_reason(),
12646                                *reason,
12647                                "{}: route {index}",
12648                                case.name
12649                            );
12650                        }
12651                        None => {
12652                            assert!(rig.live(route), "{}: route {index} closed", case.name);
12653                            assert!(
12654                                route.untouched(),
12655                                "{}: route {index} was told something",
12656                                case.name
12657                            );
12658                        }
12659                    }
12660                }
12661            }
12662        }
12663
12664        #[tokio::test]
12665        async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
12666            // Removed, and replaced by a higher epoch.
12667            for next in [Vec::new(), vec![session(2)]] {
12668                let mut rig = rig().await;
12669                rig.sync(vec![session(1)]).await;
12670                let mut route = rig
12671                    .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12672                    .await;
12673                rig.sync(next).await;
12674                assert!(!rig.live(&route));
12675                assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
12676            }
12677
12678            // A child whose parent ends: its routes close as parent-ended, the
12679            // child stays live, and routes under the parent close as ended.
12680            let mut rig = rig().await;
12681            let mut child = session(1);
12682            child.scope_ref = "child".to_string();
12683            child.kind = ScopeKind::Worker;
12684            child.parent = Some(ScopeParent {
12685                owner: Principal::Reserved {
12686                    module_id: OWNER.to_string(),
12687                },
12688                scope_ref: "s".to_string(),
12689                scope_epoch: 1,
12690            });
12691            rig.sync(vec![session(1), child.clone()]).await;
12692            let mut child_route = rig
12693                .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
12694                .await;
12695            assert_eq!(
12696                child_route.stamp().unwrap().parent_state,
12697                Some(ParentState::Linked)
12698            );
12699            rig.sync(vec![child]).await;
12700            assert!(!rig.live(&child_route));
12701            assert_eq!(
12702                child_route.closed_reason(),
12703                RouteCloseReason::ScopeParentEnded
12704            );
12705        }
12706
12707        #[tokio::test]
12708        async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
12709        ) {
12710            let mut rig = rig().await;
12711            rig.sync(vec![session(1)]).await;
12712            let mut route = rig
12713                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12714                .await;
12715            let before = rig.forwarding.published_scope_tag(OWNER, "s");
12716            rig.sync(vec![session(1)]).await;
12717            assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
12718            assert!(rig.live(&route) && route.untouched());
12719
12720            // A call in flight on the route when another carrier is added. A
12721            // forwarded REQUEST holds one credit on the route's flow until the
12722            // module answers; the router takes it exactly like this.
12723            let DataRoute::Client(DataRouteState::Bound(binding)) = rig
12724                .forwarding
12725                .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
12726                .unwrap()
12727            else {
12728                panic!("the route is bound");
12729            };
12730            binding.flow.acquire_tagged(9, false).await.unwrap();
12731            let mut widened = session(1);
12732            widened.carriers.push(carrier(MAGIC, None));
12733            rig.sync(vec![widened]).await;
12734            assert!(rig.live(&route) && route.untouched());
12735            let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
12736            assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
12737            // The call's credit is still held on an open flow, so its answer
12738            // will be delivered: closing the route would have closed the flow.
12739            assert_eq!(binding.flow.in_flight(), 1);
12740            binding
12741                .flow
12742                .acquire_tagged(10, false)
12743                .await
12744                .expect("the flow is still open");
12745        }
12746
12747        /// A swap's superseded endpoint keeps its routes until drained; ending
12748        /// the scope closes them there too.
12749        #[tokio::test]
12750        async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
12751            let mut rig = rig().await;
12752            rig.sync(vec![session(1)]).await;
12753            let mut on_incumbent = rig
12754                .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12755                .await;
12756
12757            // Swap plexus: register a candidate and cut over, leaving the
12758            // incumbent superseded with the route still on it.
12759            let (candidate, _candidate_rx) = wide_ctx(9);
12760            let registration = rig
12761                .handler
12762                .registry
12763                .register_candidate_with_control_ops(
12764                    manifest(PLEXUS, PROTOCOL_VERSION),
12765                    PROTOCOL_VERSION,
12766                    candidate.connection_id,
12767                    module_baseline_control_ops(),
12768                )
12769                .unwrap();
12770            rig.forwarding
12771                .register_candidate_module_connection(
12772                    candidate.connection_id,
12773                    PLEXUS.to_string(),
12774                    PROTOCOL_VERSION,
12775                    manifest_concurrency(&registration.manifest),
12776                    candidate.egress.clone(),
12777                )
12778                .unwrap();
12779            rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
12780            rig.handler
12781                .registry
12782                .promote_candidate(PLEXUS)
12783                .unwrap()
12784                .unwrap();
12785            assert!(rig.live(&on_incumbent), "cutover alone does not drain");
12786
12787            rig.sync(Vec::new()).await;
12788            assert!(!rig.live(&on_incumbent));
12789            assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
12790            let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
12791            let goodbye = incumbent_rx
12792                .try_recv()
12793                .expect("the superseded endpoint is told")
12794                .frame;
12795            assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12796        }
12797    }
12798}
12799
12800#[cfg(test)]
12801mod concurrency_default_exposure_tests {
12802    use super::*;
12803
12804    fn hello_body(role_json: &str) -> Vec<u8> {
12805        format!(
12806            r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
12807        )
12808        .into_bytes()
12809    }
12810
12811    fn manifest_from(body: &[u8]) -> ModuleManifest {
12812        let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
12813        serde_json::from_value(value.get("manifest").expect("manifest key").clone())
12814            .expect("manifest parses")
12815    }
12816
12817    const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
12818
12819    #[test]
12820    fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
12821        let body = hello_body(&format!(
12822            r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
12823        ));
12824        let manifest = manifest_from(&body);
12825        // Precondition: serde really resolved it to the default, so the typed
12826        // manifest alone cannot answer the question this probe exists for.
12827        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
12828        assert!(manifest_concurrency_was_defaulted(&body, &manifest));
12829    }
12830
12831    #[test]
12832    fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
12833        let body = hello_body(&format!(
12834            r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
12835        ));
12836        let manifest = manifest_from(&body);
12837        assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
12838        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
12839    }
12840
12841    #[test]
12842    fn non_management_roles_are_never_reported() {
12843        let body = hello_body(
12844            r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
12845        );
12846        let manifest = manifest_from(&body);
12847        assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
12848    }
12849}